asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2026-01-03 10:26:36 +00:00
Code Issues Actions Projects Releases Wiki Activity

Actually fix conversion of decimals

Browse Source
This commit is contained in:
Kenny Daniel 2024-05-12 20:41:39 -07:00
parent fe7e19b2a4
commit bd4e7c1699
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
3 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
demo.js
View File

@ -127,6 +127,7 @@ async function render(asyncBuffer, metadata, name) {
function renderSidebar(asyncBuffer, metadata, name) {
label.innerText = name
// render file layout
layout.innerHTML = ''
layout.appendChild(fileLayout(metadata, asyncBuffer.byteLength))
// display metadata
metadataDiv.innerHTML = ''

20
src/convert.js
View File

@ -9,7 +9,6 @@ const dayMillis = 86400000 // 1 day in milliseconds
* @returns {DecodedArray} series of rich types
*/
export function convert(data, schemaElement) {
if (!Array.isArray(data)) return data
const ctype = schemaElement.converted_type
if (ctype === 'UTF8') {
const decoder = new TextDecoder()
@ -17,24 +16,27 @@ export function convert(data, schemaElement) {
}
if (ctype === 'DECIMAL') {
const scale = schemaElement.scale || 0
const precision = schemaElement.precision || 0
const factor = Math.pow(10, scale - precision)
const factor = Math.pow(10, -scale)
if (typeof data[0] === 'number') {
return factor === 1 ? data : data.map(v => v * factor)
if (factor === 1) return data
return Array.from(data).map(v => v * factor)
} else if (typeof data[0] === 'bigint') {
return factor === 1 ? data : data.map(v => v * BigInt(factor))
if (factor === 1) return data
// @ts-expect-error data is either BigInt64Array or bigint[]
if (factor > 1) return data.map(v => v * BigInt(factor))
return Array.from(data).map(v => Number(v) * factor)
} else {
return data.map(v => parseDecimal(v) * factor)
return Array.from(data).map(v => parseDecimal(v) * factor)
}
}
if (ctype === 'DATE') {
return data.map(v => new Date(v * dayMillis))
return Array.from(data).map(v => new Date(v * dayMillis))
}
if (ctype === undefined && schemaElement.type === 'INT96') {
return data.map(parseInt96Date)
return Array.from(data).map(parseInt96Date)
}
if (ctype === 'TIME_MILLIS') {
return data.map(v => new Date(v))
return Array.from(data).map(v => new Date(v))
}
if (ctype === 'JSON') {
return data.map(v => JSON.parse(v))

8
test/convert.test.js
View File

@ -31,7 +31,7 @@ describe('convert function', () => {
const data = [100, 200]
/** @type {SchemaElement} */
const schemaElement = { name, converted_type: 'DECIMAL', scale: 2 }
expect(convert(data, schemaElement)).toEqual([10000, 20000])
expect(convert(data, schemaElement)).toEqual([1, 2])
})
it('converts bigint to DECIMAL', () => {
@ -42,10 +42,10 @@ describe('convert function', () => {
})
it('converts bigint to DECIMAL with scale', () => {
const data = [BigInt(1000), BigInt(2000)]
const data = [BigInt(10), BigInt(20)]
/** @type {SchemaElement} */
const schemaElement = { name, converted_type: 'DECIMAL', scale: 3 }
expect(convert(data, schemaElement)).toEqual([1000000n, 2000000n])
const schemaElement = { name, converted_type: 'DECIMAL', scale: 2 }
expect(convert(data, schemaElement)).toEqual([0.1, 0.2])
})
it('converts byte arrays to DECIMAL', () => {