diff --git a/package.json b/package.json index 8d1af0f..969cfbf 100644 --- a/package.json +++ b/package.json @@ -37,9 +37,9 @@ "@vitest/coverage-v8": "3.0.9", "eslint": "9.22.0", "eslint-plugin-jsdoc": "50.6.8", - "hyparquet-compressors": "1.0.0", + "hyparquet-compressors": "1.1.1", "typescript": "5.8.2", - "typescript-eslint": "8.26.1", + "typescript-eslint": "8.27.0", "vitest": "3.0.9" } } diff --git a/src/metadata.js b/src/metadata.js index adefe61..76e30e5 100644 --- a/src/metadata.js +++ b/src/metadata.js @@ -3,6 +3,8 @@ import { parseDecimal, parseFloat16 } from './convert.js' import { getSchemaPath } from './schema.js' import { deserializeTCompactProtocol } from './thrift.js' +export const defaultInitialFetchSize = 1 << 19 // 512kb + /** * Read parquet metadata from an async buffer. * @@ -27,7 +29,7 @@ import { deserializeTCompactProtocol } from './thrift.js' * @param {number} initialFetchSize initial fetch size in bytes (default 512kb) * @returns {Promise} parquet metadata object */ -export async function parquetMetadataAsync(asyncBuffer, initialFetchSize = 1 << 19 /* 512kb */) { +export async function parquetMetadataAsync(asyncBuffer, initialFetchSize = defaultInitialFetchSize) { if (!asyncBuffer || !(asyncBuffer.byteLength >= 0)) throw new Error('parquetMetadataAsync expected AsyncBuffer') // fetch last bytes (footer) of the file diff --git a/src/utils.js b/src/utils.js index 1571076..22c4498 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,3 +1,5 @@ +import { defaultInitialFetchSize } from './metadata.js' + /** * Replace bigint, date, etc with legal JSON types. * When parsing parquet files, bigints are used to represent 64-bit integers. @@ -169,7 +171,7 @@ function readStreamToArrayBuffer(input) { * @param {{ minSize?: number }} [options] * @returns {AsyncBuffer} cached file-like object */ -export function cachedAsyncBuffer({ byteLength, slice }, { minSize = 50000 } = {}) { +export function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) { if (byteLength < minSize) { // Cache whole file if it's small const buffer = slice(0, byteLength)