asadbek064/hyparquet-writer
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet-writer.git synced 2026-01-04 19:06:37 +00:00
Code Issues Actions Projects Releases Wiki Activity

Fix statistics writing for dates and decimals

Browse Source
This commit is contained in:
Kenny Daniel 2025-04-19 00:32:08 -07:00
parent 85ca43374a
commit fa9006221d
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
2 changed files with 19 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -57,7 +57,7 @@
"@babel/eslint-parser": "7.27.0",
"@types/node": "22.14.1",
"@vitest/coverage-v8": "3.1.1",
"eslint": "9.24.0",
"eslint": "9.25.0",
"eslint-plugin-jsdoc": "50.6.9",
"typescript": "5.8.3",
"vitest": "3.1.1"

21
src/unconvert.js
View File

@ -11,8 +11,7 @@ const dayMillis = 86400000 // 1 day in milliseconds
export function unconvert(element, values) {
const ctype = element.converted_type
if (ctype === 'DECIMAL') {
const scale = element.scale || 0
const factor = 10 ** scale
const factor = 10 ** (element.scale || 0)
return values.map(v => {
if (v === null || v === undefined) return v
if (typeof v !== 'number') throw new Error('DECIMAL must be a number')
@ -52,6 +51,22 @@ export function unconvertMinMax(value, element) {
if (value === undefined || value === null) return undefined
const { type, converted_type } = element
if (type === 'BOOLEAN') return new Uint8Array([value ? 1 : 0])
if (converted_type === 'DECIMAL') {
if (typeof value !== 'number') throw new Error('DECIMAL must be a number')
const factor = 10 ** (element.scale || 0)
const out = unconvertDecimal(element, BigInt(Math.round(value * factor)))
if (out instanceof Uint8Array) return out
if (typeof out === 'number') {
const buffer = new ArrayBuffer(4)
new DataView(buffer).setFloat32(0, out, true)
return new Uint8Array(buffer)
}
if (typeof out === 'bigint') {
const buffer = new ArrayBuffer(8)
new DataView(buffer).setBigInt64(0, out, true)
return new Uint8Array(buffer)
}
}
if (type === 'BYTE_ARRAY' || type === 'FIXED_LEN_BYTE_ARRAY') {
// truncate byte arrays to 16 bytes for statistics
if (value instanceof Uint8Array) return value.slice(0, 16)
@ -78,7 +93,7 @@ export function unconvertMinMax(value, element) {
return new Uint8Array(buffer)
}
if (type === 'INT32' && converted_type === 'DATE' && value instanceof Date) {
const buffer = new ArrayBuffer(8)
const buffer = new ArrayBuffer(4)
new DataView(buffer).setInt32(0, Math.floor(value.getTime() / dayMillis), true)
return new Uint8Array(buffer)
}