asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2026-01-09 04:26:38 +00:00
Code Issues Actions Projects Releases Wiki Activity

Splice into fixed length array

Browse Source
This commit is contained in:
Kenny Daniel 2024-04-30 12:27:10 -07:00
parent 28b2562af5
commit 195ee9bc80
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
2 changed files with 23 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/encoding.js
View File

@ -1,5 +1,5 @@
import { readVarInt } from './thrift.js'
import { concat } from './utils.js'
import { splice } from './utils.js'
/**
* Read `count` boolean values.
@ -197,14 +197,13 @@ export function widthFromMaxInt(value) {
* @returns {any[]} array of values
*/
export function readData(reader, encoding, count, bitWidth) {
/** @type {any[]} */
const values = []
const values = new Array(count)
if (encoding === 'RLE') {
let seen = 0
while (seen < count) {
const rle = readRleBitPackedHybrid(reader, bitWidth, 0, count)
if (!rle.length) break // EOF
concat(values, rle)
splice(values, rle, seen)
seen += rle.length
}
} else {
@ -232,21 +231,24 @@ export function readRleBitPackedHybrid(reader, width, length, numValues) {
if (length < 0) throw new Error(`parquet invalid rle/bitpack length ${length}`)
}
/** @type {number[]} */
const values = []
const values = new Array(numValues)
let seen = 0
const startOffset = reader.offset
while (reader.offset - startOffset < length && values.length < numValues) {
while (reader.offset - startOffset < length && seen < numValues) {
const [header, newOffset] = readVarInt(reader.view, reader.offset)
reader.offset = newOffset
if ((header & 1) === 0) {
// rle
const rle = readRle(reader, header, width)
concat(values, rle)
splice(values, rle, seen)
seen += rle.length
} else {
// bit-packed
const bitPacked = readBitPacked(
reader, header, width, numValues - values.length
reader, header, width, numValues - seen
)
concat(values, bitPacked)
splice(values, bitPacked, seen)
seen += bitPacked.length
}
}

12
src/utils.js
View File

@ -33,3 +33,15 @@ export function concat(aaa, bbb) {
aaa.push(...bbb.slice(i, i + chunk))
}
}
/**
* Splice one array into another fast.
* @param {any[]} aaa first array
* @param {any[]} bbb second array
* @param {number} index where to insert in aaa
*/
export function splice(aaa, bbb, index) {
for (let i = 0; i < bbb.length; i++) {
aaa[index + i] = bbb[i]
}
}