asadbek064/hyparquet-writer
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet-writer.git synced 2025-12-05 23:31:54 +00:00
Code Issues Actions Projects Releases Wiki Activity

Find bitwidth faster for large arrays

Browse Source
This commit is contained in:
Kenny Daniel 2025-04-15 23:45:29 -07:00
parent c38151e32f
commit c15bcbe2b8
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/encoding.js
View File

@ -10,7 +10,11 @@ import { ByteWriter } from './bytewriter.js'
export function writeRleBitPackedHybrid(writer, values) {
const offsetStart = writer.offset
// find max bitwidth
const bitWidth = Math.ceil(Math.log2(Math.max(...values) + 1))
let max = 0
for (const v of values) {
if (v > max) max = v
}
const bitWidth = Math.ceil(Math.log2(max + 1))
// try both RLE and bit-packed and choose the best
const rle = new ByteWriter()