asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2025-12-31 09:26:37 +00:00
Code Issues Actions Projects Releases Wiki Activity

Remove unnecessary toJson in tests

Browse Source
This commit is contained in:
Kenny Daniel 2025-03-10 19:32:31 -07:00
parent 613cd0e639
commit 2cd582ea5a
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
4 changed files with 58 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -28,7 +28,7 @@ Hyparquet aims to be the world's most compliant parquet parser. And it runs in t
1. **Browser-native**: Built to work seamlessly in the browser, opening up new possibilities for web-based data applications and visualizations.
2. **Performant**: Designed to efficiently process large datasets by only loading the required data, making it suitable for big data and machine learning applications.
3. **TypeScript**: Includes TypeScript definitions.
4. **Dependency-free**: Hyparquet has zero dependencies, making it lightweight and easy to use in any JavaScript project. Only 9.2kb min.gz!
4. **Dependency-free**: Hyparquet has zero dependencies, making it lightweight and easy to use in any JavaScript project. Only 9.7kb min.gz!
5. **Highly Compliant:** Supports all parquet encodings, compression codecs, and can open more parquet files than any other library.
## Why hyparquet?

40
test/query.test.js
View File

@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest'
import { parquetQuery } from '../src/query.js'
import { asyncBufferFromFile, toJson } from '../src/utils.js'
import { asyncBufferFromFile } from '../src/utils.js'
describe('parquetQuery', () => {
it('throws error for undefined file', async () => {
@ -12,7 +12,7 @@ describe('parquetQuery', () => {
it('reads data without orderBy', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] },
{ a: 'abc', b: 2, c: 3, d: true },
{ a: 'abc', b: 3, c: 4, d: true },
@ -24,7 +24,7 @@ describe('parquetQuery', () => {
it('reads data with orderBy', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, orderBy: 'c' })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ __index__: 0, a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] },
{ __index__: 4, a: 'abc', b: 5, c: 2, d: true, e: [1, 2] },
{ __index__: 1, a: 'abc', b: 2, c: 3, d: true },
@ -36,7 +36,7 @@ describe('parquetQuery', () => {
it('reads data with orderBy and limits', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, orderBy: 'c', rowStart: 1, rowEnd: 4 })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ __index__: 4, a: 'abc', b: 5, c: 2, d: true, e: [1, 2] },
{ __index__: 1, a: 'abc', b: 2, c: 3, d: true },
{ __index__: 2, a: 'abc', b: 3, c: 4, d: true },
@ -46,7 +46,7 @@ describe('parquetQuery', () => {
it('reads data with rowStart and rowEnd without orderBy', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, rowStart: 1, rowEnd: 4 })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 2, c: 3, d: true },
{ a: 'abc', b: 3, c: 4, d: true },
{ a: null, b: 4, c: 5, d: false, e: [1, 2, 3] },
@ -62,7 +62,7 @@ describe('parquetQuery', () => {
it('reads data with filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { c: 2 } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [ 1, 2, 3 ] },
{ a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] },
])
@ -71,13 +71,13 @@ describe('parquetQuery', () => {
it('reads data with filter and rowStart/rowEnd', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { c: 2 }, rowStart: 1, rowEnd: 5 })
expect(toJson(rows)).toEqual([ { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] } ])
expect(rows).toEqual([ { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] } ])
})
it('reads data with filter and orderBy', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { c: 2 }, orderBy: 'b' })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [ 1, 2, 3 ] },
{ a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] },
])
@ -86,13 +86,13 @@ describe('parquetQuery', () => {
it('reads data with filter, orderBy, and rowStart/rowEnd', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { c: 2 }, orderBy: 'b', rowStart: 1, rowEnd: 2 })
expect(toJson(rows)).toEqual([ { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] } ])
expect(rows).toEqual([ { a: 'abc', b: 5, c: 2, d: true, e: [ 1, 2 ] } ])
})
it('reads data with $and filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { $and: [{ c: 2 }, { e: [1, 2, 3] }] } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] },
])
})
@ -100,7 +100,7 @@ describe('parquetQuery', () => {
it('reads data with $or filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { $or: [{ c: 2 }, { d: false }] } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] },
{ a: null, b: 4, c: 5, d: false, e: [1, 2, 3] },
{ a: 'abc', b: 5, c: 2, d: true, e: [1, 2] },
@ -110,7 +110,7 @@ describe('parquetQuery', () => {
it('reads data with $not filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { $not: { c: 2 } } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 2, c: 3, d: true },
{ a: 'abc', b: 3, c: 4, d: true },
{ a: null, b: 4, c: 5, d: false, e: [1, 2, 3] },
@ -120,7 +120,7 @@ describe('parquetQuery', () => {
it('reads data with $not value filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { c: { $not: 2 } } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 2, c: 3, d: true },
{ a: 'abc', b: 3, c: 4, d: true },
{ a: null, b: 4, c: 5, d: false, e: [1, 2, 3] },
@ -130,7 +130,7 @@ describe('parquetQuery', () => {
it('reads data with $gt filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { b: { $gt: 3 } } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: null, b: 4, c: 5, d: false, e: [1, 2, 3] },
{ a: 'abc', b: 5, c: 2, d: true, e: [1, 2] },
])
@ -140,7 +140,7 @@ describe('parquetQuery', () => {
it('reads data with $gte filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { b: { $gte: 3 } } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 3, c: 4, d: true },
{ a: null, b: 4, c: 5, d: false, e: [1, 2, 3] },
{ a: 'abc', b: 5, c: 2, d: true, e: [1, 2] },
@ -150,7 +150,7 @@ describe('parquetQuery', () => {
it('reads data with $lt filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { b: { $lt: 3 } } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] },
{ a: 'abc', b: 2, c: 3, d: true },
])
@ -159,7 +159,7 @@ describe('parquetQuery', () => {
it('reads data with $lte filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { b: { $lte: 3 } } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] },
{ a: 'abc', b: 2, c: 3, d: true },
{ a: 'abc', b: 3, c: 4, d: true },
@ -169,7 +169,7 @@ describe('parquetQuery', () => {
it('reads data with $ne filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { b: { $ne: 3 } } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] },
{ a: 'abc', b: 2, c: 3, d: true },
{ a: null, b: 4, c: 5, d: false, e: [1, 2, 3] },
@ -180,7 +180,7 @@ describe('parquetQuery', () => {
it('reads data with $in filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { b: { $in: [2, 4] } } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 2, c: 3, d: true },
{ a: null, b: 4, c: 5, d: false, e: [1, 2, 3] },
])
@ -189,7 +189,7 @@ describe('parquetQuery', () => {
it('reads data with $nin filter', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetQuery({ file, filter: { b: { $nin: [2, 4] } } })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] },
{ a: 'abc', b: 3, c: 4, d: true },
{ a: 'abc', b: 5, c: 2, d: true, e: [1, 2] },

78
test/read.test.js
View File

@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest'
import { parquetRead, parquetReadObjects } from '../src/hyparquet.js'
import { asyncBufferFromFile, toJson } from '../src/utils.js'
import { asyncBufferFromFile } from '../src/utils.js'
describe('parquetRead', () => {
it('throws error for undefined file', async () => {
@ -22,8 +22,8 @@ describe('parquetRead', () => {
file,
rowStart: 2,
rowEnd: 4,
onComplete: rows => {
expect(toJson(rows)).toEqual([[3], [4]])
onComplete(rows) {
expect(rows).toEqual([[3n], [4n]])
},
})
})
@ -33,8 +33,10 @@ describe('parquetRead', () => {
await parquetRead({
file,
rowEnd: 100,
onComplete: rows => {
expect(toJson(rows)).toEqual([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14], [15]])
onComplete(rows) {
expect(rows).toEqual([
[1n], [2n], [3n], [4n], [5n], [6n], [7n], [8n], [9n], [10n], [11n], [12n], [13n], [14n], [15n],
])
},
})
})
@ -43,23 +45,15 @@ describe('parquetRead', () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
await parquetRead({
file,
columns: ['c'],
onChunk: chunk => {
expect(toJson(chunk)).toEqual({
columnName: 'c',
columnData: [2, 3, 4, 5, 2],
columns: ['b'],
onChunk(chunk) {
expect(chunk).toEqual({
columnName: 'b',
columnData: [1, 2, 3, 4, 5],
rowStart: 0,
rowEnd: 5,
})
},
onComplete: (rows) => {
expect(toJson(rows)).toEqual([
[2],
[3],
[4],
[5],
[2],
])
expect(chunk.columnData).toBeInstanceOf(Array)
},
})
})
@ -69,19 +63,19 @@ describe('parquetRead', () => {
await parquetRead({
file,
columns: ['e'],
onChunk: chunk => {
expect(toJson(chunk)).toEqual({
onChunk(chunk) {
expect(chunk).toEqual({
columnName: 'e',
columnData: [[1, 2, 3], null, null, [1, 2, 3], [1, 2]],
columnData: [[1, 2, 3], undefined, undefined, [1, 2, 3], [1, 2]],
rowStart: 0,
rowEnd: 5,
})
},
onComplete: rows => {
expect(toJson(rows)).toEqual([
onComplete(rows) {
expect(rows).toEqual([
[[1, 2, 3]],
[null],
[null],
[undefined],
[undefined],
[[1, 2, 3]],
[[1, 2]],
])
@ -94,8 +88,8 @@ describe('parquetRead', () => {
await parquetRead({
file,
columns: ['int_map'],
onChunk: chunk => {
expect(toJson(chunk)).toEqual({
onChunk(chunk) {
expect(chunk).toEqual({
columnName: 'int_map',
columnData: [
{ k1: 1, k2: 100 },
@ -103,21 +97,21 @@ describe('parquetRead', () => {
{ },
{ },
{ },
null,
undefined,
{ k1: null, k3: null },
],
rowStart: 0,
rowEnd: 7,
})
},
onComplete: rows => {
expect(toJson(rows)).toEqual([
onComplete(rows) {
expect(rows).toEqual([
[{ k1: 1, k2: 100 }],
[{ k1: 2, k2: null }],
[{ }],
[{ }],
[{ }],
[null],
[undefined],
[{ k1: null, k3: null }],
])
},
@ -130,8 +124,8 @@ describe('parquetRead', () => {
file,
columns: ['c'],
rowFormat: 'object',
onComplete: (rows) => {
expect(toJson(rows)).toEqual([
onComplete(rows) {
expect(rows).toEqual([
{ c: 2 },
{ c: 3 },
{ c: 4 },
@ -147,13 +141,13 @@ describe('parquetRead', () => {
await parquetRead({
file,
columns: ['c', 'missing', 'b', 'c'],
onComplete: (rows) => {
expect(toJson(rows)).toEqual([
[2, null, 1, 2],
[3, null, 2, 3],
[4, null, 3, 4],
[5, null, 4, 5],
[2, null, 5, 2],
onComplete(rows) {
expect(rows).toEqual([
[2, undefined, 1, 2],
[3, undefined, 2, 3],
[4, undefined, 3, 4],
[5, undefined, 4, 5],
[2, undefined, 5, 2],
])
},
})
@ -162,7 +156,7 @@ describe('parquetRead', () => {
it('read objects and return a promise', async () => {
const file = await asyncBufferFromFile('test/files/datapage_v2.snappy.parquet')
const rows = await parquetReadObjects({ file })
expect(toJson(rows)).toEqual([
expect(rows).toEqual([
{ a: 'abc', b: 1, c: 2, d: true, e: [1, 2, 3] },
{ a: 'abc', b: 2, c: 3, d: true },
{ a: 'abc', b: 3, c: 4, d: true },

2
test/readFiles.test.js
View File

@ -14,7 +14,7 @@ describe('parquetRead test files', () => {
await parquetRead({
file,
compressors,
onComplete: (rows) => {
onComplete(rows) {
const base = filename.replace('.parquet', '')
const expected = fileToJson(`test/files/${base}.json`)
// stringify and parse to make legal json