asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2025-12-06 06:51:54 +00:00
Code Issues Actions Projects Releases Wiki Activity

Fix 3-byte RLE

Browse Source
This commit is contained in:
Kenny Daniel 2024-05-28 13:58:02 -07:00
parent dbebcde118
commit 490d1ec800
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
2 changed files with 19 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/encoding.js
View File

@ -55,17 +55,10 @@ export function readRleBitPackedHybrid(reader, width, length, output) {
function readRle(reader, count, bitWidth, output, seen) {
const width = bitWidth + 7 >> 3
let value = 0
if (width === 1) {
value = reader.view.getUint8(reader.offset)
// assert(value < 1 << bitWidth)
} else if (width === 2) {
value = reader.view.getUint16(reader.offset, true)
} else if (width === 4) {
value = reader.view.getUint32(reader.offset, true)
} else if (width) {
throw new Error(`parquet invalid rle width ${width}`)
for (let i = 0; i < width; i++) {
value |= reader.view.getUint8(reader.offset++) << (i << 3)
}
reader.offset += width
// assert(value < 1 << bitWidth)
// repeat value count times
for (let i = 0; i < count; i++) {

27
test/encoding.test.js
View File

@ -33,6 +33,22 @@ describe('readRleBitPackedHybrid', () => {
expect(values).toEqual([65535, 65535, 65535])
})
it('reads RLE values with bitwidth=24', () => {
const buffer = new ArrayBuffer(4)
const view = new DataView(buffer)
// RLE 2x 16777215
view.setUint8(0, 0b00000100)
view.setUint8(1, 255)
view.setUint8(2, 255)
view.setUint8(3, 255)
const reader = { view, offset: 0 }
const values = new Array(2)
readRleBitPackedHybrid(reader, 24, 4, values)
expect(reader.offset).toBe(4)
expect(values).toEqual([16777215, 16777215])
})
it('reads RLE values with bitwidth=32', () => {
const buffer = new ArrayBuffer(5)
const view = new DataView(buffer)
@ -47,17 +63,6 @@ describe('readRleBitPackedHybrid', () => {
expect(values).toEqual([234000, 234000, 234000])
})
it('throws for invalid bitwidth', () => {
const buffer = new ArrayBuffer(1)
const view = new DataView(buffer)
view.setUint8(0, 0b00000110)
const reader = { view, offset: 0 }
const values = new Array(3)
expect(() => readRleBitPackedHybrid(reader, 24, 3, values))
.toThrow('parquet invalid rle width 3')
})
it('reads bit-packed values with implicit length', () => {
// Bit-packed values: false, false, true
const buffer = new ArrayBuffer(8)