diff --git a/demo/App.tsx b/demo/App.tsx index 8614961..0ed9aa9 100644 --- a/demo/App.tsx +++ b/demo/App.tsx @@ -5,8 +5,13 @@ import { FileMetaData, parquetMetadata, parquetMetadataAsync, parquetSchema } fr import { parquetRead } from '../src/read.js' import type { AsyncBuffer } from '../src/types.js' import { asyncBufferFromUrl } from '../src/utils.js' +import Dropdown from './Dropdown.js' import Dropzone from './Dropzone.js' import Layout from './Layout.js' +import ParquetLayout from './ParquetLayout.js' +import ParquetMetadata from './ParquetMetadata.js' + +type Lens = 'table' | 'metadata' | 'layout' /** * Hyparquet demo viewer page @@ -17,11 +22,14 @@ export default function App() { const [error, setError] = useState() const [df, setDf] = useState() const [name, setName] = useState() + const [lens, setLens] = useState('table') + const [metadata, setMetadata] = useState() const [byteLength, setByteLength] = useState() async function onFileDrop(file: File) { const arrayBuffer = await file.arrayBuffer() const metadata = parquetMetadata(arrayBuffer) + setMetadata(metadata) setName(file.name) setByteLength(file.size) setDf(parquetDataFrame(arrayBuffer, metadata)) @@ -30,6 +38,7 @@ export default function App() { async function onUrlDrop(url: string) { const asyncBuffer = await asyncBufferFromUrl(url) const metadata = await parquetMetadataAsync(asyncBuffer) + setMetadata(metadata) setName(url) setByteLength(asyncBuffer.byteLength) setDf(parquetDataFrame(asyncBuffer, metadata)) @@ -41,13 +50,20 @@ export default function App() { onError={(e) => setError(e)} onFileDrop={onFileDrop} onUrlDrop={onUrlDrop}> - {df && <> + {metadata && df && <>
{name}
{byteLength !== undefined && {formatFileSize(byteLength)}} {df.numRows.toLocaleString()} rows + + + + +
- + {lens === 'table' && } + {lens === 'metadata' && } + {lens === 'layout' && } } @@ -80,7 +96,6 @@ function parquetDataFrame(file: AsyncBuffer, metadata: FileMetaData): DataFrame } } - /** * Returns the file size in human readable format. * diff --git a/demo/Dropdown.tsx b/demo/Dropdown.tsx new file mode 100644 index 0000000..6f87449 --- /dev/null +++ b/demo/Dropdown.tsx @@ -0,0 +1,73 @@ +import React from 'react' +import { ReactNode, useEffect, useRef, useState } from 'react' +import { cn } from './Layout.js' + +interface DropdownProps { + label?: string + className?: string + children: ReactNode +} + +/** + * Dropdown menu component. + * + * @param {Object} props + * @param {string} props.label - button label + * @param {string} props.className - custom class name for the dropdown container + * @param {ReactNode} props.children - dropdown menu items + * @returns {ReactNode} + * @example + * + * + * + * + */ +export default function Dropdown({ label, className, children }: DropdownProps) { + const [isOpen, setIsOpen] = useState(false) + const dropdownRef = useRef(null) + const menuRef = useRef(null) + + function toggleDropdown() { + setIsOpen(!isOpen) + } + + useEffect(() => { + function handleClickInside(event: MouseEvent) { + const target = event.target as Element + if (menuRef.current && menuRef.current.contains(target) && target?.tagName !== 'INPUT') { + setIsOpen(false) + } + } + function handleClickOutside(event: MouseEvent) { + if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) { + setIsOpen(false) + } + } + function handleEscape(event: KeyboardEvent) { + if (event.key === 'Escape') { + setIsOpen(false) + } + } + document.addEventListener('click', handleClickInside) + document.addEventListener('keydown', handleEscape) + document.addEventListener('mousedown', handleClickOutside) + return () => { + document.removeEventListener('click', handleClickInside) + document.removeEventListener('keydown', handleEscape) + document.removeEventListener('mousedown', handleClickOutside) + } + }, []) + + return ( +
+ +
+ {children} +
+
+ ) +} diff --git a/demo/ParquetLayout.tsx b/demo/ParquetLayout.tsx new file mode 100644 index 0000000..c7996e8 --- /dev/null +++ b/demo/ParquetLayout.tsx @@ -0,0 +1,113 @@ +import React, { ReactNode } from 'react' +import { getColumnRange } from '../src/column.js' +import type { FileMetaData } from '../src/metadata.js' +import { ColumnChunk } from '../src/types.js' + +interface LayoutProps { + byteLength: number + metadata: FileMetaData +} + +export default function ParquetLayout({ byteLength, metadata }: LayoutProps) { + const metadataStart = byteLength - metadata.metadata_length - 4 + const metadataEnd = byteLength - 4 + + return
+
+ + + + + +
+
+} + + +function Cell({ name, start, end }: { name: string, start: N, end: N }) { + const bytes = end - start + return
+ +
    +
  • start {start.toLocaleString()}
  • +
  • bytes {bytes.toLocaleString()}
  • +
  • end {end.toLocaleString()}
  • +
+
+} + +function Group({ children, name, bytes }: { children: ReactNode, name?: string, bytes?: bigint }) { + return
+
+ + {bytes === undefined ? '' : `bytes ${bytes.toLocaleString()}`} +
+ {children} +
+} + +function RowGroups({ metadata }: { metadata: FileMetaData }) { + return <> + {metadata.row_groups.map((rowGroup, i) => ( + + {rowGroup.columns.map((column, j) => ( + + ))} + + ))} + +} + +function Column({ key, column }: { key: number, column: ColumnChunk }) { + if (!column.meta_data) return null + const end = getColumnRange(column.meta_data)[1] + const pages = [ + { name: 'Dictionary', offset: column.meta_data.dictionary_page_offset }, + { name: 'Data', offset: column.meta_data.data_page_offset }, + { name: 'Index', offset: column.meta_data.index_page_offset }, + { name: 'End', offset: end }, + ] + .filter(({ offset }) => offset !== undefined) + .sort((a, b) => Number(a.offset) - Number(b.offset)) + + const children = pages.slice(0, -1).map(({ name, offset }, index) => ( + + )) + + + return + {children} + +} + +function ColumnIndexes({ metadata }: { metadata: FileMetaData }) { + const indexPages = [] + for (const rowGroup of metadata.row_groups) { + for (const column of rowGroup.columns) { + const columnName = column.meta_data?.path_in_schema.join('.') + if (column.column_index_offset) { + indexPages.push({ + name: `ColumnIndex ${columnName}`, + start: column.column_index_offset, + end: column.column_index_offset + BigInt(column.column_index_length || 0), + }) + } + if (column.offset_index_offset) { + indexPages.push({ + name: `OffsetIndex ${columnName}`, + start: column.offset_index_offset, + end: column.offset_index_offset + BigInt(column.offset_index_length || 0), + }) + } + } + } + + return + {indexPages.map(({ name, start, end }, index) => ( + + ))} + +} diff --git a/demo/ParquetMetadata.tsx b/demo/ParquetMetadata.tsx new file mode 100644 index 0000000..84f0f0f --- /dev/null +++ b/demo/ParquetMetadata.tsx @@ -0,0 +1,13 @@ +import React from 'react' +import type { FileMetaData } from '../src/metadata.js' +import { toJson } from '../src/utils.js' + +interface MetadataProps { + metadata: FileMetaData +} + +export default function ParquetMetadata({ metadata }: MetadataProps) { + return + {JSON.stringify(toJson(metadata), null, ' ')} + +} diff --git a/demo/bundle.min.js b/demo/bundle.min.js index bba23bb..5918537 100644 --- a/demo/bundle.min.js +++ b/demo/bundle.min.js @@ -1,4 +1,4 @@ -!function(e){"function"==typeof define&&define.amd?define(e):e()}((function(){"use strict";function e(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var n,t={exports:{}},r={};t.exports=function(){if(n)return r;n=1;var e=Symbol.for("react.element"),t=Symbol.for("react.portal"),l=Symbol.for("react.fragment"),o=Symbol.for("react.strict_mode"),a=Symbol.for("react.profiler"),i=Symbol.for("react.provider"),u=Symbol.for("react.context"),s=Symbol.for("react.forward_ref"),c=Symbol.for("react.suspense"),f=Symbol.for("react.memo"),d=Symbol.for("react.lazy"),p=Symbol.iterator,h={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},g=Object.assign,m={};function A(e,n,t){this.props=e,this.context=n,this.refs=m,this.updater=t||h}function w(){}function y(e,n,t){this.props=e,this.context=n,this.refs=m,this.updater=t||h}A.prototype.isReactComponent={},A.prototype.setState=function(e,n){if("object"!=typeof e&&"function"!=typeof e&&null!=e)throw Error("setState(...): takes an object of state variables to update or a function which returns an object of state variables.");this.updater.enqueueSetState(this,e,n,"setState")},A.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this,e,"forceUpdate")},w.prototype=A.prototype;var v=y.prototype=new w;v.constructor=y,g(v,A.prototype),v.isPureReactComponent=!0;var b=Array.isArray,E=Object.prototype.hasOwnProperty,I={current:null},S={key:!0,ref:!0,__self:!0,__source:!0};function C(n,t,r){var l,o={},a=null,i=null;if(null!=t)for(l in void 0!==t.ref&&(i=t.ref),void 0!==t.key&&(a=""+t.key),t)E.call(t,l)&&!S.hasOwnProperty(l)&&(o[l]=t[l]);var u=arguments.length-2;if(1===u)o.children=r;else if(1>>1,o=e[r];if(!(0>>1;rl(u,t))sl(c,u)?(e[r]=c,e[s]=t,r=s):(e[r]=u,e[i]=t,r=i);else{if(!(sl(c,t)))break e;e[r]=c,e[s]=t,r=s}}}return n}function l(e,n){var t=e.sortIndex-n.sortIndex;return 0!==t?t:e.id-n.id}if("object"==typeof performance&&"function"==typeof performance.now){var o=performance;e.unstable_now=function(){return o.now()}}else{var a=Date,i=a.now();e.unstable_now=function(){return a.now()-i}}var u=[],s=[],c=1,f=null,d=3,p=!1,h=!1,g=!1,m="function"==typeof setTimeout?setTimeout:null,A="function"==typeof clearTimeout?clearTimeout:null,w="undefined"!=typeof setImmediate?setImmediate:null;function y(e){for(var l=t(s);null!==l;){if(null===l.callback)r(s);else{if(!(l.startTime<=e))break;r(s),l.sortIndex=l.expirationTime,n(u,l)}l=t(s)}}function v(e){if(g=!1,y(e),!h)if(null!==t(u))h=!0,P(b);else{var n=t(s);null!==n&&Q(v,n.startTime-e)}}function b(n,l){h=!1,g&&(g=!1,A(C),C=-1),p=!0;var o=d;try{for(y(l),f=t(u);null!==f&&(!(f.expirationTime>l)||n&&!B());){var a=f.callback;if("function"==typeof a){f.callback=null,d=f.priorityLevel;var i=a(f.expirationTime<=l);l=e.unstable_now(),"function"==typeof i?f.callback=i:f===t(u)&&r(u),y(l)}else r(u);f=t(u)}if(null!==f)var c=!0;else{var m=t(s);null!==m&&Q(v,m.startTime-l),c=!1}return c}finally{f=null,d=o,p=!1}}"undefined"!=typeof navigator&&void 0!==navigator.scheduling&&void 0!==navigator.scheduling.isInputPending&&navigator.scheduling.isInputPending.bind(navigator.scheduling);var E,I=!1,S=null,C=-1,k=5,x=-1;function B(){return!(e.unstable_now()-xe||125a?(r.sortIndex=o,n(s,r),null===t(u)&&r===t(s)&&(g?(A(C),C=-1):g=!0,Q(v,o-a))):(r.sortIndex=i,n(u,r),h||p||(h=!0,P(b))),r},e.unstable_shouldYield=B,e.unstable_wrapCallback=function(e){var n=d;return function(){var t=d;d=n;try{return e.apply(this,arguments)}finally{d=t}}}}(p)),p)),d.exports} +!function(e){"function"==typeof define&&define.amd?define(e):e()}((function(){"use strict";function e(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var n,t={exports:{}},r={};t.exports=function(){if(n)return r;n=1;var e=Symbol.for("react.element"),t=Symbol.for("react.portal"),l=Symbol.for("react.fragment"),a=Symbol.for("react.strict_mode"),o=Symbol.for("react.profiler"),i=Symbol.for("react.provider"),u=Symbol.for("react.context"),s=Symbol.for("react.forward_ref"),c=Symbol.for("react.suspense"),f=Symbol.for("react.memo"),d=Symbol.for("react.lazy"),p=Symbol.iterator,h={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},g=Object.assign,m={};function A(e,n,t){this.props=e,this.context=n,this.refs=m,this.updater=t||h}function y(){}function w(e,n,t){this.props=e,this.context=n,this.refs=m,this.updater=t||h}A.prototype.isReactComponent={},A.prototype.setState=function(e,n){if("object"!=typeof e&&"function"!=typeof e&&null!=e)throw Error("setState(...): takes an object of state variables to update or a function which returns an object of state variables.");this.updater.enqueueSetState(this,e,n,"setState")},A.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this,e,"forceUpdate")},y.prototype=A.prototype;var v=w.prototype=new y;v.constructor=w,g(v,A.prototype),v.isPureReactComponent=!0;var b=Array.isArray,E=Object.prototype.hasOwnProperty,I={current:null},S={key:!0,ref:!0,__self:!0,__source:!0};function C(n,t,r){var l,a={},o=null,i=null;if(null!=t)for(l in void 0!==t.ref&&(i=t.ref),void 0!==t.key&&(o=""+t.key),t)E.call(t,l)&&!S.hasOwnProperty(l)&&(a[l]=t[l]);var u=arguments.length-2;if(1===u)a.children=r;else if(1>>1,a=e[r];if(!(0>>1;rl(u,t))sl(c,u)?(e[r]=c,e[s]=t,r=s):(e[r]=u,e[i]=t,r=i);else{if(!(sl(c,t)))break e;e[r]=c,e[s]=t,r=s}}}return n}function l(e,n){var t=e.sortIndex-n.sortIndex;return 0!==t?t:e.id-n.id}if("object"==typeof performance&&"function"==typeof performance.now){var a=performance;e.unstable_now=function(){return a.now()}}else{var o=Date,i=o.now();e.unstable_now=function(){return o.now()-i}}var u=[],s=[],c=1,f=null,d=3,p=!1,h=!1,g=!1,m="function"==typeof setTimeout?setTimeout:null,A="function"==typeof clearTimeout?clearTimeout:null,y="undefined"!=typeof setImmediate?setImmediate:null;function w(e){for(var l=t(s);null!==l;){if(null===l.callback)r(s);else{if(!(l.startTime<=e))break;r(s),l.sortIndex=l.expirationTime,n(u,l)}l=t(s)}}function v(e){if(g=!1,w(e),!h)if(null!==t(u))h=!0,P(b);else{var n=t(s);null!==n&&Q(v,n.startTime-e)}}function b(n,l){h=!1,g&&(g=!1,A(C),C=-1),p=!0;var a=d;try{for(w(l),f=t(u);null!==f&&(!(f.expirationTime>l)||n&&!B());){var o=f.callback;if("function"==typeof o){f.callback=null,d=f.priorityLevel;var i=o(f.expirationTime<=l);l=e.unstable_now(),"function"==typeof i?f.callback=i:f===t(u)&&r(u),w(l)}else r(u);f=t(u)}if(null!==f)var c=!0;else{var m=t(s);null!==m&&Q(v,m.startTime-l),c=!1}return c}finally{f=null,d=a,p=!1}}"undefined"!=typeof navigator&&void 0!==navigator.scheduling&&void 0!==navigator.scheduling.isInputPending&&navigator.scheduling.isInputPending.bind(navigator.scheduling);var E,I=!1,S=null,C=-1,k=5,x=-1;function B(){return!(e.unstable_now()-xe||125o?(r.sortIndex=a,n(s,r),null===t(u)&&r===t(s)&&(g?(A(C),C=-1):g=!0,Q(v,a-o))):(r.sortIndex=i,n(u,r),h||p||(h=!0,P(b))),r},e.unstable_shouldYield=B,e.unstable_wrapCallback=function(e){var n=d;return function(){var t=d;d=n;try{return e.apply(this,arguments)}finally{d=t}}}}(p)),p)),d.exports} /** * @license React * react-dom.production.min.js @@ -7,5 +7,5 @@ * * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. - */!function e(){if("undefined"!=typeof __REACT_DEVTOOLS_GLOBAL_HOOK__&&"function"==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE)try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(e)}catch(e){console.error(e)}}(),c.exports=function(){if(a)return f;a=1;var e=i,n=h();function t(e){for(var n="https://reactjs.org/docs/error-decoder.html?invariant="+e,t=1;t