mirror of
https://github.com/asadbek064/hyparquet.git
synced 2025-12-05 22:41:55 +00:00
Use defaultInitialFetchSize for both metadata and cachedAsyncBuffer
This commit is contained in:
parent
4b094178b3
commit
9c201e00e5
@ -37,9 +37,9 @@
|
||||
"@vitest/coverage-v8": "3.0.9",
|
||||
"eslint": "9.22.0",
|
||||
"eslint-plugin-jsdoc": "50.6.8",
|
||||
"hyparquet-compressors": "1.0.0",
|
||||
"hyparquet-compressors": "1.1.1",
|
||||
"typescript": "5.8.2",
|
||||
"typescript-eslint": "8.26.1",
|
||||
"typescript-eslint": "8.27.0",
|
||||
"vitest": "3.0.9"
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,6 +3,8 @@ import { parseDecimal, parseFloat16 } from './convert.js'
|
||||
import { getSchemaPath } from './schema.js'
|
||||
import { deserializeTCompactProtocol } from './thrift.js'
|
||||
|
||||
export const defaultInitialFetchSize = 1 << 19 // 512kb
|
||||
|
||||
/**
|
||||
* Read parquet metadata from an async buffer.
|
||||
*
|
||||
@ -27,7 +29,7 @@ import { deserializeTCompactProtocol } from './thrift.js'
|
||||
* @param {number} initialFetchSize initial fetch size in bytes (default 512kb)
|
||||
* @returns {Promise<FileMetaData>} parquet metadata object
|
||||
*/
|
||||
export async function parquetMetadataAsync(asyncBuffer, initialFetchSize = 1 << 19 /* 512kb */) {
|
||||
export async function parquetMetadataAsync(asyncBuffer, initialFetchSize = defaultInitialFetchSize) {
|
||||
if (!asyncBuffer || !(asyncBuffer.byteLength >= 0)) throw new Error('parquetMetadataAsync expected AsyncBuffer')
|
||||
|
||||
// fetch last bytes (footer) of the file
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
import { defaultInitialFetchSize } from './metadata.js'
|
||||
|
||||
/**
|
||||
* Replace bigint, date, etc with legal JSON types.
|
||||
* When parsing parquet files, bigints are used to represent 64-bit integers.
|
||||
@ -169,7 +171,7 @@ function readStreamToArrayBuffer(input) {
|
||||
* @param {{ minSize?: number }} [options]
|
||||
* @returns {AsyncBuffer} cached file-like object
|
||||
*/
|
||||
export function cachedAsyncBuffer({ byteLength, slice }, { minSize = 50000 } = {}) {
|
||||
export function cachedAsyncBuffer({ byteLength, slice }, { minSize = defaultInitialFetchSize } = {}) {
|
||||
if (byteLength < minSize) {
|
||||
// Cache whole file if it's small
|
||||
const buffer = slice(0, byteLength)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user