demo: cached asyncbuffer

demo/asyncBuffer.ts

@@ -0,0 +1,50 @@
+import type { AsyncBuffer, Awaitable } from "../src/types.js"
+
+/**
+ * Returns a caches layer on top of an AsyncBuffer.
+ * This is useful for caching slices of a file that are read multiple times,
+ * possibly over a network.
+ *
+ * TODO: require data to be loaded with preload(), reads outside of preload rejected.
+ *
+ * @param {AsyncBuffer} file file-like object to cache
+ * @returns {AsyncBuffer} cached file-like object
+ */
+export function cachedAsyncBuffer(file: AsyncBuffer): AsyncBuffer {
+  // indexed by 'start,end'
+  const cache = new Map<string, Awaitable<ArrayBuffer>>()
+  return {
+    byteLength: file.byteLength,
+    slice(start: number, end?: number): Awaitable<ArrayBuffer> {
+      // ensure both "100-200" and "100-" are both cached the same
+      const key = cacheKey(start, end, file.byteLength)
+      const cached = cache.get(key)
+      if (cached) return cached
+      // cache miss, read from file
+      const promise = file.slice(start, end)
+      cache.set(key, promise)
+      return promise
+    },
+  }
+}
+
+
+/**
+ * Returns canonical cache key for a byte range.
+ * Cache key is a string of the form 'start,end'.
+ * Attempts to normalize int-range and suffix-range requests to the same key.
+ */
+function cacheKey(start: number, end: number | undefined, fileSize: number | undefined): string {
+  if (start < 0) {
+    if (end !== undefined) throw new Error(`invalid suffix range [${start}, ${end}]`)
+    if (fileSize === undefined) return `${start},`
+    return `${fileSize + start},${fileSize}`
+  } else if (end !== undefined) {
+    if (start > end) throw new Error(`invalid empty range [${start}, ${end}]`)
+    return `${start},${end}`
+  } else if (fileSize === undefined) {
+    return `${start},`
+  } else {
+    return `${start},${fileSize}`
+  }
+}

demo/workers/parquetWorkerClient.ts

@@ -1,5 +1,6 @@
 import type { AsyncBuffer, FileMetaData } from '../../src/hyparquet.js'
 import { asyncBufferFromUrl } from '../../src/utils.js'
+import { cachedAsyncBuffer } from '../asyncBuffer.js'
 
 // Serializable constructors for AsyncBuffers
 interface AsyncBufferFromFile {
@@ -56,9 +57,11 @@ export function parquetQueryWorker({
  * Convert AsyncBufferFrom to AsyncBuffer.
  */
 export async function asyncBufferFrom(from: AsyncBufferFrom): Promise<AsyncBuffer> {
-  if ('url' in from) {
-    return asyncBufferFromUrl(from.url)
-  } else {
-    return from.file.arrayBuffer()
-  }
+  const key = JSON.stringify(from)
+  const cached = cache.get(key)
+  if (cached) return cached
+  const asyncBuffer = 'url' in from ? asyncBufferFromUrl(from.url) : from.file.arrayBuffer()
+  cache.set(key, asyncBuffer.then(cachedAsyncBuffer))
+  return asyncBuffer
 }
+const cache = new Map<string, Promise<AsyncBuffer>>()

package.json

@@ -32,20 +32,20 @@
     "@rollup/plugin-replace": "6.0.1",
     "@rollup/plugin-terser": "0.4.4",
     "@rollup/plugin-typescript": "12.1.0",
-    "@types/node": "22.7.4",
+    "@types/node": "22.7.5",
     "@types/react": "18.3.11",
     "@types/react-dom": "18.3.0",
     "@vitest/coverage-v8": "2.1.2",
     "eslint": "9.12.0",
     "eslint-plugin-jsdoc": "50.3.1",
-    "hightable": "0.4.4",
+    "hightable": "0.5.2",
     "http-server": "14.1.1",
     "hyparquet-compressors": "0.1.4",
     "react": "18.3.1",
     "react-dom": "18.3.1",
     "rollup": "4.24.0",
     "typescript": "5.6.2",
-    "typescript-eslint": "8.8.0",
+    "typescript-eslint": "8.8.1",
     "vitest": "2.1.2"
   }
 }