From c68256575b06206299ad57519b4234bc705f7b18 Mon Sep 17 00:00:00 2001 From: Kenny Daniel Date: Thu, 23 May 2024 18:46:12 -0700 Subject: [PATCH] Convert logical timestamp --- src/convert.js | 7 + test/files/duckdb5533.json | 30 ++++ test/files/duckdb5533.metadata.json | 241 ++++++++++++++++++++++++++++ test/files/duckdb5533.parquet | Bin 0 -> 1939 bytes 4 files changed, 278 insertions(+) create mode 100644 test/files/duckdb5533.json create mode 100644 test/files/duckdb5533.metadata.json create mode 100644 test/files/duckdb5533.parquet diff --git a/src/convert.js b/src/convert.js index f5dc61a..9108475 100644 --- a/src/convert.js +++ b/src/convert.js @@ -71,6 +71,13 @@ export function convert(data, schemaElement, utf8 = true) { if (logicalType === 'FLOAT16') { return Array.from(data).map(parseFloat16) } + if (logicalType === 'TIMESTAMP') { + const arr = new Array(data.length) + for (let i = 0; i < arr.length; i++) { + arr[i] = new Date(Number(data[i])) + } + return arr + } return data } diff --git a/test/files/duckdb5533.json b/test/files/duckdb5533.json new file mode 100644 index 0000000..9bb3e31 --- /dev/null +++ b/test/files/duckdb5533.json @@ -0,0 +1,30 @@ +[ + [ + "2022-11-27T17:42:43.514Z", + 2448, + null, + 1, + 343 + ], + [ + "2022-11-27T17:42:43.514Z", + 85016, + null, + -1, + 343 + ], + [ + "2022-11-27T17:42:44.280Z", + 1184, + null, + 1, + 343 + ], + [ + "2022-11-27T17:42:44.280Z", + 85016, + null, + -1, + 343 + ] +] diff --git a/test/files/duckdb5533.metadata.json b/test/files/duckdb5533.metadata.json new file mode 100644 index 0000000..a9ff5e8 --- /dev/null +++ b/test/files/duckdb5533.metadata.json @@ -0,0 +1,241 @@ +{ + "version": 2, + "schema": [ + { + "name": "root", + "num_children": 5 + }, + { + "type": "INT64", + "repetition_type": "REQUIRED", + "name": "timestamp", + "logical_type": { + "type": "TIMESTAMP", + "isAdjustedToUTC": false, + "unit": { + "field_1": {} + } + } + }, + { + "type": "INT32", + "repetition_type": "OPTIONAL", + "name": "item_id", + "converted_type": "UINT_32", + "logical_type": { + "type": "INTEGER", + "bitWidth": 32, + "isSigned": false + } + }, + { + "type": "INT32", + "repetition_type": "OPTIONAL", + "name": "currency_id", + "converted_type": "UINT_32", + "logical_type": { + "type": "INTEGER", + "bitWidth": 32, + "isSigned": false + } + }, + { + "type": "INT64", + "repetition_type": "REQUIRED", + "name": "count" + }, + { + "type": "INT32", + "repetition_type": "REQUIRED", + "name": "mf", + "converted_type": "UINT_16", + "logical_type": { + "type": "INTEGER", + "bitWidth": 16, + "isSigned": false + } + } + ], + "num_rows": 4, + "row_groups": [ + { + "columns": [ + { + "file_offset": 77, + "meta_data": { + "type": "INT64", + "encodings": [ + "PLAIN", + "RLE" + ], + "path_in_schema": [ + "timestamp" + ], + "codec": "ZSTD", + "num_values": 4, + "total_uncompressed_size": 78, + "total_compressed_size": 73, + "data_page_offset": 4, + "statistics": { + "null_count": 0, + "max_value": 1669570964280, + "min_value": 1669570963514 + } + }, + "offset_index_offset": 656, + "offset_index_length": 11, + "column_index_offset": 531, + "column_index_length": 31, + "crypto_metadata": 31 + }, + { + "file_offset": 197, + "meta_data": { + "type": "INT32", + "encodings": [ + "PLAIN", + "RLE" + ], + "path_in_schema": [ + "item_id" + ], + "codec": "ZSTD", + "num_values": 4, + "total_uncompressed_size": 56, + "total_compressed_size": 65, + "data_page_offset": 132, + "statistics": { + "null_count": 0, + "max_value": 85016, + "min_value": 1184 + } + }, + "offset_index_offset": 667, + "offset_index_length": 12, + "column_index_offset": 562, + "column_index_length": 23, + "crypto_metadata": 23 + }, + { + "file_offset": 279, + "meta_data": { + "type": "INT32", + "encodings": [ + "PLAIN", + "RLE" + ], + "path_in_schema": [ + "currency_id" + ], + "codec": "ZSTD", + "num_values": 4, + "total_uncompressed_size": 28, + "total_compressed_size": 37, + "data_page_offset": 242, + "statistics": { + "null_count": 4 + } + }, + "offset_index_offset": 679, + "offset_index_length": 11, + "column_index_offset": 585, + "column_index_length": 17, + "crypto_metadata": 17 + }, + { + "file_offset": 383, + "meta_data": { + "type": "INT64", + "encodings": [ + "PLAIN", + "RLE" + ], + "path_in_schema": [ + "count" + ], + "codec": "ZSTD", + "num_values": 4, + "total_uncompressed_size": 78, + "total_compressed_size": 68, + "data_page_offset": 315, + "statistics": { + "null_count": 0, + "max_value": 1, + "min_value": -1 + } + }, + "offset_index_offset": 690, + "offset_index_length": 12, + "column_index_offset": 602, + "column_index_length": 31, + "crypto_metadata": 31 + }, + { + "file_offset": 492, + "meta_data": { + "type": "INT32", + "encodings": [ + "PLAIN", + "RLE" + ], + "path_in_schema": [ + "mf" + ], + "codec": "ZSTD", + "num_values": 4, + "total_uncompressed_size": 54, + "total_compressed_size": 57, + "data_page_offset": 435, + "statistics": { + "null_count": 0, + "max_value": 343, + "min_value": 343 + } + }, + "offset_index_offset": 702, + "offset_index_length": 11, + "column_index_offset": 633, + "column_index_length": 23, + "crypto_metadata": 23 + } + ], + "total_byte_size": 294, + "num_rows": 4, + "file_offset": 4, + "total_compressed_size": 300, + "ordinal": 0 + } + ], + "key_value_metadata": [ + { + "key": "addon_version", + "value": "1.8.0" + }, + { + "key": "map", + "value": "18" + }, + { + "key": "character", + "value": "Undeadscarlight" + }, + { + "key": "level", + "value": "80" + }, + { + "key": "start", + "value": "2022-11-27T17:42:35.480941703Z" + }, + { + "key": "end", + "value": "2022-11-27T17:43:01.105178597Z" + }, + { + "key": "ARROW:schema", + "value": "/////5oBAAAEAAAA8v///xQAAAAEAAEAAAAKAAsACAAKAAQA+P///wwAAAAIAAgAAAAEAAUAAAAoAQAA4AAAAJQAAABIAAAABAAAAOz///80AAAAIAAAABgAAAACAAAAEAARAAQAAAAQAAgAAAAMAAAAAAD2////EAAAAAAABgAIAAQAAgAAAG1mAADs////OAAAACAAAAAYAAAAAgAAABAAEQAEAAAAEAAIAAAADAAAAAAA9P///0AAAAABAAAACAAJAAQACAAFAAAAY291bnQAAADs////NAAAACAAAAAYAAAAAQIAABAAEgAEABAAEQAIAAAADAAAAAAA9v///yAAAAAAAAYACAAEAAsAAABjdXJyZW5jeV9pZADs////NAAAACAAAAAYAAAAAQIAABAAEgAEABAAEQAIAAAADAAAAAAA9v///yAAAAAAAAYACAAEAAcAAABpdGVtX2lkAOz///8wAAAAIAAAABgAAAAKAAAAEAARAAQAAAAQAAgAAAAMAAAAAAD6////AQAGAAYABAAJAAAAdGltZXN0YW1wAA==" + } + ], + "created_by": "Arrow2 - Native Rust implementation of Arrow", + "metadata_length": 1218 +} diff --git a/test/files/duckdb5533.parquet b/test/files/duckdb5533.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1096778780e9ee2f9cc708bac54f0afa895625ed GIT binary patch literal 1939 zcmb_dOKclO7@l3PlXWO6DV@i!={as6XXbx=-~W<# zS=4PhLeJ8<%T%XCK7^9->>QcUoyqB6K2-?edUYZ`5ke+@oc^0^e@V!4B6KEIa`umV zf2ve7a%9BdnW*J^y~y>u^cd6G*9tpSEdBJ;Ac6nQVScU8uknit;o5BtSP4lTkC{Iq z&l~z--cG)e(I#7}I5g=gGD8p>=N8p0l>c}8-GYMpzAimVt z2}7^b*iTR$R2L5t!{b7H>H4ff~fEDsEMP|lt-Wa`^PAZme1UsE5 zsn&sFJS)gH8V_Y@< z>o^>fF`&d?gh;h)ios=J6R~1IYWSc)kL?B(0GK3~#2#2=u>Gws&N7`ID!@aGEIUx? zt>k=^qP4#GfkqD!@!b&`q8KXKepd^FAfjV%$&sNcY=3?tn*|%}Yfk0KnB-o&`0cCa zi9|fy>me0F#qB|F!l8OO4vw#&Bh!tnaoPGx$tInA!P#fG$0%dp==3dOeU{25ElNUs z<|MP9#wce;I?WU7J8j@QoQL!unpyTc(j8{#8^ro2JqY?m#*ZB#>kk=>_|E8n^*KYu z{zBsW8HknrrZL^p*&Xd9H2DU9*=;t1&Xrv+?6rapPuqDXUm!f?yIro@4m7f{?S^h6 z@%~H`IAz;Y#qtMsxjI{{&Xn@A zPN6bumkTqS0PS^}k6F!B3wGWvlf5d29)McR;;DxI^Pq<}zLq`)C#QhN zSSbAfQZm#L7Z$vv{C&I#ajpnX(=r0M&2OP=-(XTx*C6`#Wf|QLNZkI~Y%lzlg$N{1t#h{4Kzbc^<{b`t73=t_^gB z5PEQ)zusK0?QgD?u6vg%-Oc|Mzk&GO=E9|Dz1aS+U+<(zvUp$s{d;D;=dhkoiUo`t zQiHk((OhUpo9l~(jWrwa&YU6qlnBFMuV_x0i*D4~_00NCFEU$xx9#~}Cjwc}F@tNS OG$X&l*K8R+$NvJrTc&mZ literal 0 HcmV?d00001