diff --git a/README.md b/README.md index a8fb32d..002bdd3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![workflow status](https://github.com/hyparam/hyparquet/actions/workflows/ci.yml/badge.svg)](https://github.com/hyparam/hyparquet/actions) [![mit license](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) [![dependencies](https://img.shields.io/badge/Dependencies-0-blueviolet)](https://www.npmjs.com/package/hyparquet?activeTab=dependencies) -![coverage](https://img.shields.io/badge/Coverage-95-darkred) +![coverage](https://img.shields.io/badge/Coverage-96-darkred) Dependency free since 2023! diff --git a/demo/App.tsx b/demo/App.tsx index 3d046da..68c1a32 100644 --- a/demo/App.tsx +++ b/demo/App.tsx @@ -1,15 +1,13 @@ import HighTable, { DataFrame } from 'hightable' -import { compressors } from 'hyparquet-compressors' import React, { useEffect, useState } from 'react' import { FileMetaData, parquetMetadataAsync, parquetSchema } from '../src/metadata.js' -import { parquetQuery } from '../src/query.js' -import type { AsyncBuffer } from '../src/types.js' -import { asyncBufferFromUrl } from '../src/utils.js' +import { byteLengthFromUrl } from '../src/utils.js' import Dropdown from './Dropdown.js' import Dropzone from './Dropzone.js' import Layout from './Layout.js' import ParquetLayout from './ParquetLayout.js' import ParquetMetadata from './ParquetMetadata.js' +import { AsyncBufferFrom, asyncBufferFrom, parquetQueryWorker } from './workers/parquetWorkerClient.js' type Lens = 'table' | 'metadata' | 'layout' @@ -30,29 +28,35 @@ export default function App({ url }: { url?: string }) { useEffect(() => { if (!df && url) { - asyncBufferFromUrl(url).then(asyncBuffer => setAsyncBuffer(url, asyncBuffer)) + onUrlDrop(url) } }, [ url ]) async function onFileDrop(file: File) { // Clear query string history.pushState({}, '', location.pathname) - setAsyncBuffer(file.name, await file.arrayBuffer()) + setAsyncBuffer(file.name, { file, byteLength: file.size }) } async function onUrlDrop(url: string) { // Add key=url to query string const params = new URLSearchParams(location.search) params.set('key', url) history.pushState({}, '', `${location.pathname}?${params}`) - setAsyncBuffer(url, await asyncBufferFromUrl(url)) + try { + const byteLength = await byteLengthFromUrl(url) + setAsyncBuffer(url, { url, byteLength }) + } catch (e) { + setError(e as Error) + } } - async function setAsyncBuffer(name: string, asyncBuffer: AsyncBuffer) { + async function setAsyncBuffer(name: string, from: AsyncBufferFrom) { // TODO: Replace welcome with spinner + const asyncBuffer = await asyncBufferFrom(from) const metadata = await parquetMetadataAsync(asyncBuffer) setMetadata(metadata) setName(name) - setByteLength(asyncBuffer.byteLength) - const df = parquetDataFrame(asyncBuffer, metadata) + setByteLength(from.byteLength) + const df = parquetDataFrame(from, metadata) setDf(df) document.getElementById('welcome')?.remove() } @@ -83,12 +87,8 @@ export default function App({ url }: { url?: string }) { /** * Convert a parquet file into a dataframe. - * - * @param {AsyncBuffer} file - parquet file asyncbuffer - * @param {FileMetaData} metadata - parquet file metadata - * @returns {DataFrame} dataframe */ -function parquetDataFrame(file: AsyncBuffer, metadata: FileMetaData): DataFrame { +function parquetDataFrame(from: AsyncBufferFrom, metadata: FileMetaData): DataFrame { const { children } = parquetSchema(metadata) return { header: children.map(child => child.element.name), @@ -101,7 +101,7 @@ function parquetDataFrame(file: AsyncBuffer, metadata: FileMetaData): DataFrame */ rows(rowStart, rowEnd, orderBy) { console.log(`reading rows ${rowStart}-${rowEnd}`, orderBy) - return parquetQuery({ file, compressors, rowStart, rowEnd, orderBy }) + return parquetQueryWorker({ asyncBuffer: from, rowStart, rowEnd, orderBy }) }, sortable: true, } diff --git a/demo/bundle.min.js b/demo/bundle.min.js index c4c53e2..a2b95fc 100644 --- a/demo/bundle.min.js +++ b/demo/bundle.min.js @@ -1,4 +1,4 @@ -!function(e){"function"==typeof define&&define.amd?define(e):e()}((function(){"use strict";function e(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var n,t,r={exports:{}},l={};function a(){if(n)return l;n=1;var e=Symbol.for("react.element"),t=Symbol.for("react.portal"),r=Symbol.for("react.fragment"),a=Symbol.for("react.strict_mode"),o=Symbol.for("react.profiler"),i=Symbol.for("react.provider"),u=Symbol.for("react.context"),s=Symbol.for("react.forward_ref"),c=Symbol.for("react.suspense"),f=Symbol.for("react.memo"),d=Symbol.for("react.lazy"),p=Symbol.iterator;var h={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},g=Object.assign,m={};function A(e,n,t){this.props=e,this.context=n,this.refs=m,this.updater=t||h}function w(){}function y(e,n,t){this.props=e,this.context=n,this.refs=m,this.updater=t||h}A.prototype.isReactComponent={},A.prototype.setState=function(e,n){if("object"!=typeof e&&"function"!=typeof e&&null!=e)throw Error("setState(...): takes an object of state variables to update or a function which returns an object of state variables.");this.updater.enqueueSetState(this,e,n,"setState")},A.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this,e,"forceUpdate")},w.prototype=A.prototype;var v=y.prototype=new w;v.constructor=y,g(v,A.prototype),v.isPureReactComponent=!0;var b=Array.isArray,E=Object.prototype.hasOwnProperty,I={current:null},S={key:!0,ref:!0,__self:!0,__source:!0};function C(n,t,r){var l,a={},o=null,i=null;if(null!=t)for(l in void 0!==t.ref&&(i=t.ref),void 0!==t.key&&(o=""+t.key),t)E.call(t,l)&&!S.hasOwnProperty(l)&&(a[l]=t[l]);var u=arguments.length-2;if(1===u)a.children=r;else if(1>>1,a=e[r];if(!(0>>1;rl(u,t))sl(c,u)?(e[r]=c,e[s]=t,r=s):(e[r]=u,e[i]=t,r=i);else{if(!(sl(c,t)))break e;e[r]=c,e[s]=t,r=s}}}return n}function l(e,n){var t=e.sortIndex-n.sortIndex;return 0!==t?t:e.id-n.id}if("object"==typeof performance&&"function"==typeof performance.now){var a=performance;e.unstable_now=function(){return a.now()}}else{var o=Date,i=o.now();e.unstable_now=function(){return o.now()-i}}var u=[],s=[],c=1,f=null,d=3,p=!1,h=!1,g=!1,m="function"==typeof setTimeout?setTimeout:null,A="function"==typeof clearTimeout?clearTimeout:null,w="undefined"!=typeof setImmediate?setImmediate:null;function y(e){for(var l=t(s);null!==l;){if(null===l.callback)r(s);else{if(!(l.startTime<=e))break;r(s),l.sortIndex=l.expirationTime,n(u,l)}l=t(s)}}function v(e){if(g=!1,y(e),!h)if(null!==t(u))h=!0,P(b);else{var n=t(s);null!==n&&Q(v,n.startTime-e)}}function b(n,l){h=!1,g&&(g=!1,A(C),C=-1),p=!0;var a=d;try{for(y(l),f=t(u);null!==f&&(!(f.expirationTime>l)||n&&!B());){var o=f.callback;if("function"==typeof o){f.callback=null,d=f.priorityLevel;var i=o(f.expirationTime<=l);l=e.unstable_now(),"function"==typeof i?f.callback=i:f===t(u)&&r(u),y(l)}else r(u);f=t(u)}if(null!==f)var c=!0;else{var m=t(s);null!==m&&Q(v,m.startTime-l),c=!1}return c}finally{f=null,d=a,p=!1}}"undefined"!=typeof navigator&&void 0!==navigator.scheduling&&void 0!==navigator.scheduling.isInputPending&&navigator.scheduling.isInputPending.bind(navigator.scheduling);var E,I=!1,S=null,C=-1,k=5,x=-1;function B(){return!(e.unstable_now()-xe||125o?(r.sortIndex=a,n(s,r),null===t(u)&&r===t(s)&&(g?(A(C),C=-1):g=!0,Q(v,a-o))):(r.sortIndex=i,n(u,r),h||p||(h=!0,P(b))),r},e.unstable_shouldYield=B,e.unstable_wrapCallback=function(e){var n=d;return function(){var t=d;d=n;try{return e.apply(this,arguments)}finally{d=t}}}}(w)),w)),A.exports} +!function(e){"function"==typeof define&&define.amd?define(e):e()}((function(){"use strict";var e="undefined"!=typeof document?document.currentScript:null;function t(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var n,r,l={exports:{}},a={};function o(){if(n)return a;n=1;var e=Symbol.for("react.element"),t=Symbol.for("react.portal"),r=Symbol.for("react.fragment"),l=Symbol.for("react.strict_mode"),o=Symbol.for("react.profiler"),i=Symbol.for("react.provider"),u=Symbol.for("react.context"),s=Symbol.for("react.forward_ref"),c=Symbol.for("react.suspense"),f=Symbol.for("react.memo"),d=Symbol.for("react.lazy"),p=Symbol.iterator;var m={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},h=Object.assign,g={};function v(e,t,n){this.props=e,this.context=t,this.refs=g,this.updater=n||m}function y(){}function b(e,t,n){this.props=e,this.context=t,this.refs=g,this.updater=n||m}v.prototype.isReactComponent={},v.prototype.setState=function(e,t){if("object"!=typeof e&&"function"!=typeof e&&null!=e)throw Error("setState(...): takes an object of state variables to update or a function which returns an object of state variables.");this.updater.enqueueSetState(this,e,t,"setState")},v.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this,e,"forceUpdate")},y.prototype=v.prototype;var w=b.prototype=new y;w.constructor=b,h(w,v.prototype),w.isPureReactComponent=!0;var _=Array.isArray,k=Object.prototype.hasOwnProperty,S={current:null},E={key:!0,ref:!0,__self:!0,__source:!0};function x(t,n,r){var l,a={},o=null,i=null;if(null!=n)for(l in void 0!==n.ref&&(i=n.ref),void 0!==n.key&&(o=""+n.key),n)k.call(n,l)&&!E.hasOwnProperty(l)&&(a[l]=n[l]);var u=arguments.length-2;if(1===u)a.children=r;else if(1>>1,a=e[r];if(!(0>>1;rl(u,n))sl(c,u)?(e[r]=c,e[s]=n,r=s):(e[r]=u,e[i]=n,r=i);else{if(!(sl(c,n)))break e;e[r]=c,e[s]=n,r=s}}}return t}function l(e,t){var n=e.sortIndex-t.sortIndex;return 0!==n?n:e.id-t.id}if("object"==typeof performance&&"function"==typeof performance.now){var a=performance;e.unstable_now=function(){return a.now()}}else{var o=Date,i=o.now();e.unstable_now=function(){return o.now()-i}}var u=[],s=[],c=1,f=null,d=3,p=!1,m=!1,h=!1,g="function"==typeof setTimeout?setTimeout:null,v="function"==typeof clearTimeout?clearTimeout:null,y="undefined"!=typeof setImmediate?setImmediate:null;function b(e){for(var l=n(s);null!==l;){if(null===l.callback)r(s);else{if(!(l.startTime<=e))break;r(s),l.sortIndex=l.expirationTime,t(u,l)}l=n(s)}}function w(e){if(h=!1,b(e),!m)if(null!==n(u))m=!0,z(_);else{var t=n(s);null!==t&&I(w,t.startTime-e)}}function _(t,l){m=!1,h&&(h=!1,v(x),x=-1),p=!0;var a=d;try{for(b(l),f=n(u);null!==f&&(!(f.expirationTime>l)||t&&!L());){var o=f.callback;if("function"==typeof o){f.callback=null,d=f.priorityLevel;var i=o(f.expirationTime<=l);l=e.unstable_now(),"function"==typeof i?f.callback=i:f===n(u)&&r(u),b(l)}else r(u);f=n(u)}if(null!==f)var c=!0;else{var g=n(s);null!==g&&I(w,g.startTime-l),c=!1}return c}finally{f=null,d=a,p=!1}}"undefined"!=typeof navigator&&void 0!==navigator.scheduling&&void 0!==navigator.scheduling.isInputPending&&navigator.scheduling.isInputPending.bind(navigator.scheduling);var k,S=!1,E=null,x=-1,C=5,N=-1;function L(){return!(e.unstable_now()-Ne||125o?(r.sortIndex=a,t(s,r),null===n(u)&&r===n(s)&&(h?(v(x),x=-1):h=!0,I(w,a-o))):(r.sortIndex=i,t(u,r),m||p||(m=!0,z(_))),r},e.unstable_shouldYield=L,e.unstable_wrapCallback=function(e){var t=d;return function(){var n=d;d=t;try{return e.apply(this,arguments)}finally{d=n}}}}(b)),b)),y.exports} /** * @license React * react-dom.production.min.js @@ -7,5 +7,5 @@ * * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. - */function v(){if(s)return m;s=1;var e=o(),n=y();function t(e){for(var n="https://reactjs.org/docs/error-decoder.html?invariant="+e,t=1;t