demo: upgrade hightable

demo/App.tsx

@@ -1,4 +1,4 @@
-import HighTable, { DataFrame } from 'hightable'
+import HighTable, { DataFrame, rowCache } from 'hightable'
 import React, { useEffect, useState } from 'react'
 import { FileMetaData, parquetMetadataAsync, parquetSchema } from '../src/metadata.js'
 import { byteLengthFromUrl } from '../src/utils.js'
@@ -56,7 +56,8 @@ export default function App({ url }: { url?: string }) {
     setMetadata(metadata)
     setName(name)
     setByteLength(from.byteLength)
-    const df = parquetDataFrame(from, metadata)
+    let df = parquetDataFrame(from, metadata)
+    df = rowCache(df)
     setDf(df)
     document.getElementById('welcome')?.remove()
   }

demo/demo.css

@@ -425,7 +425,43 @@ table:focus-visible {
   white-space: nowrap;
 }
 
-/* pending state */
+/* pending cell state */
+.table td.pending {
+  position: relative;
+}
+.table td.pending::after {
+  content: '';
+  position: absolute;
+  top: 8px;
+  left: 8px;
+  right: 8px;
+  bottom: 8px;
+  border-radius: 4px;
+  background: linear-gradient(
+    60deg,
+    rgba(0, 0, 0, 0.05) 25%,
+    rgba(0, 0, 0, 0.08) 50%,
+    rgba(0, 0, 0, 0.05) 75%
+  );
+  background-size: 120px 100%;
+  animation: textshimmer 3s infinite linear;
+}
+/* stagger row shimmering */
+.table tr:nth-child(2n) td.pending::after { animation-delay: -1s; }
+.table tr:nth-child(2n+1) td.pending::after { animation-delay: -3s; }
+.table tr:nth-child(3n) td.pending::after { animation-delay: -2s; }
+.table tr:nth-child(5n) td.pending::after { animation-delay: -4s; }
+.table tr:nth-child(7n) td.pending::after { animation-delay: -1.5s; }
+@keyframes textshimmer {
+  0% {
+    background-position: -120px 0;
+  }
+  100% {
+    background-position: 120px 0;
+  }
+}
+
+/* pending table state */
 .table th::before {
   content: '';
   position: absolute;

demo/workers/parquetWorker.js

@@ -3,14 +3,19 @@ import { parquetQuery } from '../../src/query.js'
 import { asyncBufferFrom } from './parquetWorkerClient.js'
 
 self.onmessage = async ({ data }) => {
-  const { metadata, asyncBuffer, rowStart, rowEnd, orderBy } = data
+  const { metadata, asyncBuffer, rowStart, rowEnd, orderBy, queryId, chunks } = data
   const file = await asyncBufferFrom(asyncBuffer)
+  /**
+   * @typedef {import('../../src/hyparquet.js').ColumnData} ColumnData
+   * @type {((chunk: ColumnData) => void) | undefined}
+   */
+  const onChunk = chunks ? chunk => self.postMessage({ chunk, queryId }) : undefined
   try {
     const result = await parquetQuery({
-      metadata, file, rowStart, rowEnd, orderBy, compressors,
+      metadata, file, rowStart, rowEnd, orderBy, compressors, onChunk,
     })
-    self.postMessage({ result })
+    self.postMessage({ result, queryId })
   } catch (error) {
-    self.postMessage({ error })
+    self.postMessage({ error, queryId })
   }
 }

demo/workers/parquetWorkerClient.ts

@@ -1,5 +1,5 @@
 import { cachedAsyncBuffer } from '../../src/asyncBuffer.js'
-import type { AsyncBuffer, FileMetaData } from '../../src/hyparquet.js'
+import type { AsyncBuffer, ParquetReadOptions } from '../../src/hyparquet.js'
 import { asyncBufferFromUrl } from '../../src/utils.js'
 
 // Serializable constructors for AsyncBuffers
@@ -14,16 +14,14 @@ interface AsyncBufferFromUrl {
 export type AsyncBufferFrom = AsyncBufferFromFile | AsyncBufferFromUrl
 
 // Same as ParquetReadOptions, but AsyncBufferFrom instead of AsyncBuffer
-interface ParquetReadWorkerOptions {
+interface ParquetReadWorkerOptions extends Omit<ParquetReadOptions, 'file'> {
   asyncBuffer: AsyncBufferFrom
-  metadata?: FileMetaData // parquet metadata, will be parsed if not provided
-  columns?: number[] // columns to read, all columns if undefined
-  rowStart?: number // inclusive
-  rowEnd?: number // exclusive
-  orderBy?: string // column to sort by
+  orderBy?: string
 }
 
 let worker: Worker | undefined
+let nextQueryId = 0
+const pending = new Map<number, { resolve: (value: any) => void, reject: (error: any) => void }>()
 
 /**
  * Presents almost the same interface as parquetRead, but runs in a worker.
@@ -31,25 +29,34 @@ let worker: Worker | undefined
  * Instead of taking an AsyncBuffer, it takes a FileContent, because it needs
  * to be serialized to the worker.
  */
-export function parquetQueryWorker({
-  metadata, asyncBuffer, rowStart, rowEnd, orderBy }: ParquetReadWorkerOptions
+export function parquetQueryWorker(
+  { metadata, asyncBuffer, rowStart, rowEnd, orderBy, onChunk }: ParquetReadWorkerOptions
 ): Promise<Record<string, any>[]> {
   return new Promise((resolve, reject) => {
+    const queryId = nextQueryId++
+    pending.set(queryId, { resolve, reject })
     // Create a worker
     if (!worker) {
       worker = new Worker(new URL('demo/workers/worker.min.js', import.meta.url))
-    }
-    worker.onmessage = ({ data }) => {
-      // Convert postmessage data to callbacks
-      if (data.error) {
-        reject(data.error)
-      } else if (data.result) {
-        resolve(data.result)
-      } else {
-        reject(new Error('Unexpected message from worker'))
+      worker.onmessage = ({ data }) => {
+        const { resolve, reject } = pending.get(data.queryId)!
+        // Convert postmessage data to callbacks
+        if (data.error) {
+          reject(data.error)
+        } else if (data.result) {
+          resolve(data.result)
+        } else if (data.chunk) {
+          onChunk?.(data.chunk)
+        } else {
+          reject(new Error('Unexpected message from worker'))
+        }
       }
     }
-    worker.postMessage({ metadata, asyncBuffer, rowStart, rowEnd, orderBy })
+    // If caller provided an onChunk callback, worker will send chunks as they are parsed
+    const chunks = onChunk !== undefined
+    worker.postMessage({
+      queryId, metadata, asyncBuffer, rowStart, rowEnd, orderBy, chunks
+    })
   })
 }
 

package.json

@@ -32,20 +32,20 @@
     "@rollup/plugin-replace": "6.0.1",
     "@rollup/plugin-terser": "0.4.4",
     "@rollup/plugin-typescript": "12.1.1",
-    "@types/node": "22.7.5",
+    "@types/node": "22.7.7",
     "@types/react": "18.3.11",
     "@types/react-dom": "18.3.1",
     "@vitest/coverage-v8": "2.1.3",
-    "eslint": "9.12.0",
-    "eslint-plugin-jsdoc": "50.4.1",
-    "hightable": "0.5.3",
+    "eslint": "9.13.0",
+    "eslint-plugin-jsdoc": "50.4.3",
+    "hightable": "0.5.5",
     "http-server": "14.1.1",
     "hyparquet-compressors": "0.1.4",
     "react": "18.3.1",
     "react-dom": "18.3.1",
     "rollup": "4.24.0",
     "typescript": "5.6.3",
-    "typescript-eslint": "8.9.0",
+    "typescript-eslint": "8.10.0",
     "vitest": "2.1.3"
   }
 }