diff --git a/package.json b/package.json index be5cc93..a540459 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "@vitest/coverage-v8": "1.6.0", "eslint": "8.57.0", "eslint-plugin-import": "2.29.1", - "eslint-plugin-jsdoc": "48.2.5", + "eslint-plugin-jsdoc": "48.2.6", "http-server": "14.1.1", "hyparquet-compressors": "0.1.3", "typescript": "5.4.5", diff --git a/src/convert.js b/src/convert.js index 579503d..f5dc61a 100644 --- a/src/convert.js +++ b/src/convert.js @@ -34,8 +34,23 @@ export function convert(data, schemaElement, utf8 = true) { } return arr } + if (ctype === 'TIMESTAMP_MILLIS') { + const arr = new Array(data.length) + for (let i = 0; i < arr.length; i++) { + arr[i] = new Date(Number(data[i])) + } + return arr + } + if (ctype === 'TIMESTAMP_MICROS') { + const arr = new Array(data.length) + for (let i = 0; i < arr.length; i++) { + arr[i] = new Date(Number(data[i] / 1000n)) + } + return arr + } if (ctype === 'JSON') { - return data.map(v => JSON.parse(v)) + const decoder = new TextDecoder() + return data.map(v => JSON.parse(decoder.decode(v))) } if (ctype === 'BSON') { throw new Error('parquet bson not supported') diff --git a/test/convert.test.js b/test/convert.test.js index ee034e4..c7f9bc2 100644 --- a/test/convert.test.js +++ b/test/convert.test.js @@ -86,13 +86,40 @@ describe('convert function', () => { expect(convert(data, schemaElement)).toEqual([new Date('2009-03-01T00:00:00.000Z'), new Date('2009-03-01T00:01:00.000Z')]) }) + it('converts epoch time to TIMESTAMP_MILLIS', () => { + const data = [1716506900000n, 1716507000000n] + /** @type {SchemaElement} */ + const schemaElement = { name, converted_type: 'TIMESTAMP_MILLIS' } + expect(convert(data, schemaElement)).toEqual([ + new Date('2024-05-23T23:28:20.000Z'), new Date('2024-05-23T23:30:00.000Z'), + ]) + }) + + it('converts epoch time to TIMESTAMP_MICROS', () => { + const data = [1716506900000000n, 1716507000000000n] + /** @type {SchemaElement} */ + const schemaElement = { name, converted_type: 'TIMESTAMP_MICROS' } + expect(convert(data, schemaElement)).toEqual([ + new Date('2024-05-23T23:28:20.000Z'), new Date('2024-05-23T23:30:00.000Z'), + ]) + }) + it('parses strings to JSON', () => { + const encoder = new TextEncoder() const data = ['{"key": true}', '{"quay": 314}'] + .map(str => encoder.encode(str)) /** @type {SchemaElement} */ const schemaElement = { name, converted_type: 'JSON' } expect(convert(data, schemaElement)).toEqual([{ key: true }, { quay: 314 }]) }) + it('converts to float16', () => { + const data = [new Uint8Array([0x00, 0x3c]), new Uint8Array([0x00, 0x40])] + /** @type {SchemaElement} */ + const schemaElement = { name, logical_type: { type: 'FLOAT16' } } + expect(convert(data, schemaElement)).toEqual([1, 2]) + }) + it('throws error for BSON conversion', () => { const data = [{}] /** @type {SchemaElement} */ diff --git a/test/files/nested_structs.rust.json b/test/files/nested_structs.rust.json index 192bd86..683e963 100644 --- a/test/files/nested_structs.rust.json +++ b/test/files/nested_structs.rust.json @@ -274,11 +274,11 @@ }, { "count": 495, - "max": 1608822900000000000, - "mean": 0, - "min": 1608822900000000000, - "sum": 0, - "variance": 0 + "max": "+052951-07-27T10:00:00.000Z", + "mean": "1970-01-01T00:00:00.000Z", + "min": "+052951-07-27T10:00:00.000Z", + "sum": "1970-01-01T00:00:00.000Z", + "variance": "1970-01-01T00:00:00.000Z" }, { "count": 495,