Use defaultInitialFetchSize for both metadata and cachedAsyncBuffer

This commit is contained in:
Kenny Daniel 2025-03-20 16:05:34 -07:00
parent 4b094178b3
commit 9c201e00e5
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
3 changed files with 8 additions and 4 deletions

@ -37,9 +37,9 @@
"@vitest/coverage-v8": "3.0.9",
"eslint": "9.22.0",
"eslint-plugin-jsdoc": "50.6.8",
"hyparquet-compressors": "1.0.0",
"hyparquet-compressors": "1.1.1",
"typescript": "5.8.2",
"typescript-eslint": "8.26.1",
"typescript-eslint": "8.27.0",
"vitest": "3.0.9"
}
}

@ -3,6 +3,8 @@ import { parseDecimal, parseFloat16 } from './convert.js'
import { getSchemaPath } from './schema.js'
import { deserializeTCompactProtocol } from './thrift.js'
export const defaultInitialFetchSize = 1 << 19 // 512kb
/**
* Read parquet metadata from an async buffer.
*
@ -27,7 +29,7 @@ import { deserializeTCompactProtocol } from './thrift.js'
* @param {number} initialFetchSize initial fetch size in bytes (default 512kb)
* @returns {Promise<FileMetaData>} parquet metadata object
*/
export async function parquetMetadataAsync(asyncBuffer, initialFetchSize = 1 << 19 /* 512kb */) {
export async function parquetMetadataAsync(asyncBuffer, initialFetchSize = defaultInitialFetchSize) {
if (!asyncBuffer || !(asyncBuffer.byteLength >= 0)) throw new Error('parquetMetadataAsync expected AsyncBuffer')
// fetch last bytes (footer) of the file

@ -1,3 +1,5 @@
import { defaultInitialFetchSize } from './metadata.js'
/**
* Replace bigint, date, etc with legal JSON types.
* When parsing parquet files, bigints are used to represent 64-bit integers.
@ -169,7 +171,7 @@ function readStreamToArrayBuffer(input) {
* @param {{ minSize?: number }} [options]
* @returns {AsyncBuffer} cached file-like object
*/
export function cachedAsyncBuffer({ byteLength, slice }, { minSize = 50000 } = {}) {
export function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) {
if (byteLength < minSize) {
// Cache whole file if it's small
const buffer = slice(0, byteLength)