From a7bfab0e99a8d68b45e35c6a2d8f085719f4bf03 Mon Sep 17 00:00:00 2001 From: Kenny Daniel Date: Mon, 1 Sep 2025 11:24:20 -0700 Subject: [PATCH] Fix high-precision decimal parsing (#116) --- src/convert.js | 14 ++-- test/files/issue115decimal.json | 3 + test/files/issue115decimal.metadata.json | 83 +++++++++++++++++++++++ test/files/issue115decimal.parquet | Bin 0 -> 557 bytes 4 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 test/files/issue115decimal.json create mode 100644 test/files/issue115decimal.metadata.json create mode 100644 test/files/issue115decimal.parquet diff --git a/src/convert.js b/src/convert.js index be5d897..271d52f 100644 --- a/src/convert.js +++ b/src/convert.js @@ -62,7 +62,7 @@ export function convert(data, columnDecoder) { const factor = 10 ** -scale const arr = new Array(data.length) for (let i = 0; i < arr.length; i++) { - if (data[0] instanceof Uint8Array) { + if (data[i] instanceof Uint8Array) { arr[i] = parseDecimal(data[i]) * factor } else { arr[i] = Number(data[i]) * factor @@ -155,18 +155,20 @@ export function convert(data, columnDecoder) { * @returns {number} */ export function parseDecimal(bytes) { - let value = 0 + if (!bytes.length) return 0 + + let value = 0n for (const byte of bytes) { - value = value * 256 + byte + value = value * 256n + BigInt(byte) } // handle signed const bits = bytes.length * 8 - if (value >= 2 ** (bits - 1)) { - value -= 2 ** bits + if (value >= 2n ** BigInt(bits - 1)) { + value -= 2n ** BigInt(bits) } - return value + return Number(value) } /** diff --git a/test/files/issue115decimal.json b/test/files/issue115decimal.json new file mode 100644 index 0000000..36f350d --- /dev/null +++ b/test/files/issue115decimal.json @@ -0,0 +1,3 @@ +[ + [-12345.67] +] diff --git a/test/files/issue115decimal.metadata.json b/test/files/issue115decimal.metadata.json new file mode 100644 index 0000000..9f99cfb --- /dev/null +++ b/test/files/issue115decimal.metadata.json @@ -0,0 +1,83 @@ +{ + "version": 2, + "schema": [ + { + "repetition_type": "REQUIRED", + "name": "schema", + "num_children": 1 + }, + { + "type": "FIXED_LEN_BYTE_ARRAY", + "type_length": 13, + "repetition_type": "OPTIONAL", + "name": "amount", + "converted_type": "DECIMAL", + "scale": 2, + "precision": 29, + "logical_type": { + "type": "DECIMAL", + "scale": 2, + "precision": 29 + } + } + ], + "num_rows": 1, + "row_groups": [ + { + "columns": [ + { + "file_offset": 0, + "meta_data": { + "type": "FIXED_LEN_BYTE_ARRAY", + "encodings": [ + "PLAIN", + "RLE", + "RLE_DICTIONARY" + ], + "path_in_schema": [ + "amount" + ], + "codec": "SNAPPY", + "num_values": 1, + "total_uncompressed_size": 117, + "total_compressed_size": 121, + "data_page_offset": 33, + "dictionary_page_offset": 4, + "statistics": { + "max": -12345.67, + "min": -12345.67, + "null_count": 0, + "max_value": -12345.67, + "min_value": -12345.67 + }, + "encoding_stats": [ + { + "page_type": "DICTIONARY_PAGE", + "encoding": "PLAIN", + "count": 1 + }, + { + "page_type": "DATA_PAGE", + "encoding": "RLE_DICTIONARY", + "count": 1 + } + ] + } + } + ], + "total_byte_size": 117, + "num_rows": 1, + "file_offset": 4, + "total_compressed_size": 121, + "ordinal": 0 + } + ], + "key_value_metadata": [ + { + "key": "ARROW:schema", + "value": "/////4AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAEAAAAUAAAAEAAUAAgABgAHAAwAAAAQABAAAAAAAAEHEAAAACAAAAAEAAAAAAAAAAYAAABhbW91bnQAAAgADAAEAAgACAAAAB0AAAACAAAAAAAAAA==" + } + ], + "created_by": "parquet-cpp-arrow version 19.0.1", + "metadata_length": 424 +} diff --git a/test/files/issue115decimal.parquet b/test/files/issue115decimal.parquet new file mode 100644 index 0000000000000000000000000000000000000000..157948dd516bcd76151f07048365566807024760 GIT binary patch literal 557 zcmbtS%}N4M6h3p2!6HSZJIsws%%-7WGzW5_;NpxKMnx)w1Z_r_bZ8FbO2P2b}DednC}-OKG@&zBi_B#&>UkWd83RW>_(oxMqrMQM3N7NjXnZRd9E zhUj0Z1*d=V0I=r>vcfO~$c*x;V4I8Kcr*zm74oD+*k(AHt)|P9eNtBUp4ifeC6rd$ z1ugTc3T8oV`=qdh^=Vk&h8wuq|0+}-od+xi=*p~>cMCU5`wsz>whyq^yYJUi)%J&R z&bUBEH?fE}h-(ZYyh4ny870*qK)mCqiFnPP@~x+=#ia)}6p@d3zRi0GU2aa%YA2nv zgph=3{gGTS9`tMeV9GKgyi9mHpBsV-+jNQ}Y&5`j=3)G@8ZAr1`Meay@oeq9j^f4h VZ0h*6a;5Amt?2>Y4S)`Plpp;)!rK4< literal 0 HcmV?d00001