diff --git a/src/encoding.js b/src/encoding.js index 303df7e..3cc0eac 100644 --- a/src/encoding.js +++ b/src/encoding.js @@ -1,5 +1,5 @@ import { readVarInt } from './thrift.js' -import { concat } from './utils.js' +import { splice } from './utils.js' /** * Read `count` boolean values. @@ -197,14 +197,13 @@ export function widthFromMaxInt(value) { * @returns {any[]} array of values */ export function readData(reader, encoding, count, bitWidth) { - /** @type {any[]} */ - const values = [] + const values = new Array(count) if (encoding === 'RLE') { let seen = 0 while (seen < count) { const rle = readRleBitPackedHybrid(reader, bitWidth, 0, count) if (!rle.length) break // EOF - concat(values, rle) + splice(values, rle, seen) seen += rle.length } } else { @@ -232,21 +231,24 @@ export function readRleBitPackedHybrid(reader, width, length, numValues) { if (length < 0) throw new Error(`parquet invalid rle/bitpack length ${length}`) } /** @type {number[]} */ - const values = [] + const values = new Array(numValues) + let seen = 0 const startOffset = reader.offset - while (reader.offset - startOffset < length && values.length < numValues) { + while (reader.offset - startOffset < length && seen < numValues) { const [header, newOffset] = readVarInt(reader.view, reader.offset) reader.offset = newOffset if ((header & 1) === 0) { // rle const rle = readRle(reader, header, width) - concat(values, rle) + splice(values, rle, seen) + seen += rle.length } else { // bit-packed const bitPacked = readBitPacked( - reader, header, width, numValues - values.length + reader, header, width, numValues - seen ) - concat(values, bitPacked) + splice(values, bitPacked, seen) + seen += bitPacked.length } } diff --git a/src/utils.js b/src/utils.js index e1160b0..304773a 100644 --- a/src/utils.js +++ b/src/utils.js @@ -33,3 +33,15 @@ export function concat(aaa, bbb) { aaa.push(...bbb.slice(i, i + chunk)) } } + +/** + * Splice one array into another fast. + * @param {any[]} aaa first array + * @param {any[]} bbb second array + * @param {number} index where to insert in aaa + */ +export function splice(aaa, bbb, index) { + for (let i = 0; i < bbb.length; i++) { + aaa[index + i] = bbb[i] + } +}