From 2cd582ea5a5b112f559f3e2c5d9abd421ed6cc8a Mon Sep 17 00:00:00 2001 From: Kenny Daniel Date: Mon, 10 Mar 2025 19:32:31 -0700 Subject: [PATCH] Remove unnecessary toJson in tests --- README.md | 2 +- test/query.test.js | 40 +++++++++++----------- test/read.test.js | 78 +++++++++++++++++++----------------------- test/readFiles.test.js | 2 +- 4 files changed, 58 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 2434ac4..e4bf394 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Hyparquet aims to be the world's most compliant parquet parser. And it runs in t 1. **Browser-native**: Built to work seamlessly in the browser, opening up new possibilities for web-based data applications and visualizations. 2. **Performant**: Designed to efficiently process large datasets by only loading the required data, making it suitable for big data and machine learning applications. 3. **TypeScript**: Includes TypeScript definitions. -4. **Dependency-free**: Hyparquet has zero dependencies, making it lightweight and easy to use in any JavaScript project. Only 9.2kb min.gz! +4. **Dependency-free**: Hyparquet has zero dependencies, making it lightweight and easy to use in any JavaScript project. Only 9.7kb min.gz! 5. **Highly Compliant:** Supports all parquet encodings, compression codecs, and can open more parquet files than any other library. ## Why hyparquet? diff --git a/test/query.test.js b/test/query.test.js index 4393460..aff7e3a 100644 --- a/test/query.test.js +++ b/test/query.test.js @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { parquetQuery } from '../src/query.js' -import { asyncBufferFromFile, toJson } from '../src/utils.js' +import { asyncBufferFromFile } from '../src/utils.js' describe('parquetQuery', () => { it('throws error for undefined file', async () => { @@ -12,7 +12,7 @@ describe('parquetQuery', () => { it('reads data without orderBy', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] }, { a: 'abc', b: 2, c: 3, d: true }, { a: 'abc', b: 3, c: 4, d: true }, @@ -24,7 +24,7 @@ describe('parquetQuery', () => { it('reads data with orderBy', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, orderBy: 'c' }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { __index__: 0, a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] }, { __index__: 4, a: 'abc', b: 5, c: 2, d: true, e: [1, 2] }, { __index__: 1, a: 'abc', b: 2, c: 3, d: true }, @@ -36,7 +36,7 @@ describe('parquetQuery', () => { it('reads data with orderBy and limits', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, orderBy: 'c', rowStart: 1, rowEnd: 4 }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { __index__: 4, a: 'abc', b: 5, c: 2, d: true, e: [1, 2] }, { __index__: 1, a: 'abc', b: 2, c: 3, d: true }, { __index__: 2, a: 'abc', b: 3, c: 4, d: true }, @@ -46,7 +46,7 @@ describe('parquetQuery', () => { it('reads data with rowStart and rowEnd without orderBy', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, rowStart: 1, rowEnd: 4 }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 2, c: 3, d: true }, { a: 'abc', b: 3, c: 4, d: true }, { a: null, b: 4, c: 5, d: false, e: [1, 2, 3] }, @@ -62,7 +62,7 @@ describe('parquetQuery', () => { it('reads data with filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { c: 2 } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [ 1, 2, 3 ] }, { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] }, ]) @@ -71,13 +71,13 @@ describe('parquetQuery', () => { it('reads data with filter and rowStart/rowEnd', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { c: 2 }, rowStart: 1, rowEnd: 5 }) - expect(toJson(rows)).toEqual([ { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] } ]) + expect(rows).toEqual([ { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] } ]) }) it('reads data with filter and orderBy', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { c: 2 }, orderBy: 'b' }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [ 1, 2, 3 ] }, { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] }, ]) @@ -86,13 +86,13 @@ describe('parquetQuery', () => { it('reads data with filter, orderBy, and rowStart/rowEnd', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { c: 2 }, orderBy: 'b', rowStart: 1, rowEnd: 2 }) - expect(toJson(rows)).toEqual([ { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] } ]) + expect(rows).toEqual([ { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] } ]) }) it('reads data with $and filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { $and: [{ c: 2 }, { e: [1, 2, 3] }] } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] }, ]) }) @@ -100,7 +100,7 @@ describe('parquetQuery', () => { it('reads data with $or filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { $or: [{ c: 2 }, { d: false }] } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] }, { a: null, b: 4, c: 5, d: false, e: [1, 2, 3] }, { a: 'abc', b: 5, c: 2, d: true, e: [1, 2] }, @@ -110,7 +110,7 @@ describe('parquetQuery', () => { it('reads data with $not filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { $not: { c: 2 } } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 2, c: 3, d: true }, { a: 'abc', b: 3, c: 4, d: true }, { a: null, b: 4, c: 5, d: false, e: [1, 2, 3] }, @@ -120,7 +120,7 @@ describe('parquetQuery', () => { it('reads data with $not value filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { c: { $not: 2 } } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 2, c: 3, d: true }, { a: 'abc', b: 3, c: 4, d: true }, { a: null, b: 4, c: 5, d: false, e: [1, 2, 3] }, @@ -130,7 +130,7 @@ describe('parquetQuery', () => { it('reads data with $gt filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { b: { $gt: 3 } } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: null, b: 4, c: 5, d: false, e: [1, 2, 3] }, { a: 'abc', b: 5, c: 2, d: true, e: [1, 2] }, ]) @@ -140,7 +140,7 @@ describe('parquetQuery', () => { it('reads data with $gte filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { b: { $gte: 3 } } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 3, c: 4, d: true }, { a: null, b: 4, c: 5, d: false, e: [1, 2, 3] }, { a: 'abc', b: 5, c: 2, d: true, e: [1, 2] }, @@ -150,7 +150,7 @@ describe('parquetQuery', () => { it('reads data with $lt filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { b: { $lt: 3 } } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] }, { a: 'abc', b: 2, c: 3, d: true }, ]) @@ -159,7 +159,7 @@ describe('parquetQuery', () => { it('reads data with $lte filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { b: { $lte: 3 } } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] }, { a: 'abc', b: 2, c: 3, d: true }, { a: 'abc', b: 3, c: 4, d: true }, @@ -169,7 +169,7 @@ describe('parquetQuery', () => { it('reads data with $ne filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { b: { $ne: 3 } } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] }, { a: 'abc', b: 2, c: 3, d: true }, { a: null, b: 4, c: 5, d: false, e: [1, 2, 3] }, @@ -180,7 +180,7 @@ describe('parquetQuery', () => { it('reads data with $in filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { b: { $in: [2, 4] } } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 2, c: 3, d: true }, { a: null, b: 4, c: 5, d: false, e: [1, 2, 3] }, ]) @@ -189,7 +189,7 @@ describe('parquetQuery', () => { it('reads data with $nin filter', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetQuery({ file, filter: { b: { $nin: [2, 4] } } }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] }, { a: 'abc', b: 3, c: 4, d: true }, { a: 'abc', b: 5, c: 2, d: true, e: [1, 2] }, diff --git a/test/read.test.js b/test/read.test.js index ade5c50..02a6db5 100644 --- a/test/read.test.js +++ b/test/read.test.js @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { parquetRead, parquetReadObjects } from '../src/hyparquet.js' -import { asyncBufferFromFile, toJson } from '../src/utils.js' +import { asyncBufferFromFile } from '../src/utils.js' describe('parquetRead', () => { it('throws error for undefined file', async () => { @@ -22,8 +22,8 @@ describe('parquetRead', () => { file, rowStart: 2, rowEnd: 4, - onComplete: rows => { - expect(toJson(rows)).toEqual([[3], [4]]) + onComplete(rows) { + expect(rows).toEqual([[3n], [4n]]) }, }) }) @@ -33,8 +33,10 @@ describe('parquetRead', () => { await parquetRead({ file, rowEnd: 100, - onComplete: rows => { - expect(toJson(rows)).toEqual([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15]]) + onComplete(rows) { + expect(rows).toEqual([ + [1n], [2n], [3n], [4n], [5n], [6n], [7n], [8n], [9n], [10n], [11n], [12n], [13n], [14n], [15n], + ]) }, }) }) @@ -43,23 +45,15 @@ describe('parquetRead', () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') await parquetRead({ file, - columns: ['c'], - onChunk: chunk => { - expect(toJson(chunk)).toEqual({ - columnName: 'c', - columnData: [2, 3, 4, 5, 2], + columns: ['b'], + onChunk(chunk) { + expect(chunk).toEqual({ + columnName: 'b', + columnData: [1, 2, 3, 4, 5], rowStart: 0, rowEnd: 5, }) - }, - onComplete: (rows) => { - expect(toJson(rows)).toEqual([ - [2], - [3], - [4], - [5], - [2], - ]) + expect(chunk.columnData).toBeInstanceOf(Array) }, }) }) @@ -69,19 +63,19 @@ describe('parquetRead', () => { await parquetRead({ file, columns: ['e'], - onChunk: chunk => { - expect(toJson(chunk)).toEqual({ + onChunk(chunk) { + expect(chunk).toEqual({ columnName: 'e', - columnData: [[1, 2, 3], null, null, [1, 2, 3], [1, 2]], + columnData: [[1, 2, 3], undefined, undefined, [1, 2, 3], [1, 2]], rowStart: 0, rowEnd: 5, }) }, - onComplete: rows => { - expect(toJson(rows)).toEqual([ + onComplete(rows) { + expect(rows).toEqual([ [[1, 2, 3]], - [null], - [null], + [undefined], + [undefined], [[1, 2, 3]], [[1, 2]], ]) @@ -94,8 +88,8 @@ describe('parquetRead', () => { await parquetRead({ file, columns: ['int_map'], - onChunk: chunk => { - expect(toJson(chunk)).toEqual({ + onChunk(chunk) { + expect(chunk).toEqual({ columnName: 'int_map', columnData: [ { k1: 1, k2: 100 }, @@ -103,21 +97,21 @@ describe('parquetRead', () => { { }, { }, { }, - null, + undefined, { k1: null, k3: null }, ], rowStart: 0, rowEnd: 7, }) }, - onComplete: rows => { - expect(toJson(rows)).toEqual([ + onComplete(rows) { + expect(rows).toEqual([ [{ k1: 1, k2: 100 }], [{ k1: 2, k2: null }], [{ }], [{ }], [{ }], - [null], + [undefined], [{ k1: null, k3: null }], ]) }, @@ -130,8 +124,8 @@ describe('parquetRead', () => { file, columns: ['c'], rowFormat: 'object', - onComplete: (rows) => { - expect(toJson(rows)).toEqual([ + onComplete(rows) { + expect(rows).toEqual([ { c: 2 }, { c: 3 }, { c: 4 }, @@ -147,13 +141,13 @@ describe('parquetRead', () => { await parquetRead({ file, columns: ['c', 'missing', 'b', 'c'], - onComplete: (rows) => { - expect(toJson(rows)).toEqual([ - [2, null, 1, 2], - [3, null, 2, 3], - [4, null, 3, 4], - [5, null, 4, 5], - [2, null, 5, 2], + onComplete(rows) { + expect(rows).toEqual([ + [2, undefined, 1, 2], + [3, undefined, 2, 3], + [4, undefined, 3, 4], + [5, undefined, 4, 5], + [2, undefined, 5, 2], ]) }, }) @@ -162,7 +156,7 @@ describe('parquetRead', () => { it('read objects and return a promise', async () => { const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet') const rows = await parquetReadObjects({ file }) - expect(toJson(rows)).toEqual([ + expect(rows).toEqual([ { a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] }, { a: 'abc', b: 2, c: 3, d: true }, { a: 'abc', b: 3, c: 4, d: true }, diff --git a/test/readFiles.test.js b/test/readFiles.test.js index 5dc0e02..3e9f505 100644 --- a/test/readFiles.test.js +++ b/test/readFiles.test.js @@ -14,7 +14,7 @@ describe('parquetRead test files', () => { await parquetRead({ file, compressors, - onComplete: (rows) => { + onComplete(rows) { const base = filename.replace('.parquet', '') const expected = fileToJson(`test/files/${base}.json`) // stringify and parse to make legal json