From 4494260015cac62a5610d4bcd7c0855cfcbee4cc Mon Sep 17 00:00:00 2001 From: Kenny Daniel Date: Sun, 20 Apr 2025 23:20:32 -0700 Subject: [PATCH] Round-trip tests --- README.md | 2 +- eslint.config.js | 2 +- test/files/fixed_length_decimal.parquet | Bin 0 -> 677 bytes test/files/signs.parquet | Bin 0 -> 2443 bytes test/write.roundtrip.test.js | 34 ++++++++++++++++++++++++ 5 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 test/files/fixed_length_decimal.parquet create mode 100644 test/files/signs.parquet create mode 100644 test/write.roundtrip.test.js diff --git a/README.md b/README.md index 4d55321..69e3749 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![minzipped](https://img.shields.io/bundlephobia/minzip/hyparquet-writer)](https://www.npmjs.com/package/hyparquet-writer) [![workflow status](https://github.com/hyparam/hyparquet-writer/actions/workflows/ci.yml/badge.svg)](https://github.com/hyparam/hyparquet-writer/actions) [![mit license](https://img.shields.io/badge/License-MIT-orange.svg)](https://opensource.org/licenses/MIT) -![coverage](https://img.shields.io/badge/Coverage-95-darkred) +![coverage](https://img.shields.io/badge/Coverage-96-darkred) [![dependencies](https://img.shields.io/badge/Dependencies-1-blueviolet)](https://www.npmjs.com/package/hyparquet-writer?activeTab=dependencies) Hyparquet Writer is a JavaScript library for writing [Apache Parquet](https://parquet.apache.org) files. It is designed to be lightweight, fast and store data very efficiently. It is a companion to the [hyparquet](https://github.com/hyparam/hyparquet) library, which is a JavaScript library for reading parquet files. diff --git a/eslint.config.js b/eslint.config.js index becee16..19ce990 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -43,7 +43,7 @@ export default [ 'jsdoc/require-returns-type': 'error', 'jsdoc/sort-tags': 'error', 'no-constant-condition': 'off', - 'no-extra-parens': 'error', + 'no-extra-parens': 'warn', 'no-multi-spaces': 'error', 'no-trailing-spaces': 'error', 'no-undef': 'error', diff --git a/test/files/fixed_length_decimal.parquet b/test/files/fixed_length_decimal.parquet new file mode 100644 index 0000000000000000000000000000000000000000..69fce531e4d8e40890e28d9bcbb74d835971be75 GIT binary patch literal 677 zcmZY7OG?8)7y#gDtTjI9$|(tC5e6(R#E?YO7cO+C;G(z^MeHP*XmQfkJbX~P^8hYg zxby%XzyrAS03N^txD;_(+L?l30$=9;=bxX+wD*o|2;g&`4Fg(?1t@FEDwsXXow*P2 zUK-C!?6q>U3;+Vbcg~z4_$c_zDIE*GaW3eB4}xEu`9r~1&c$`XJHb!R!mi*8=aMW) z1wT0DZNVqOAI{|tlvHDXEg9hW>OAo;Nra&UU=fy}q%7WGnvsnan1>E1S*oULWm$to z)s+iCorWQclz#JUl;xed)Bb<-s>Xg0VC4ZAFR*~BDb;>_Vd4R1JDZax%;cJytefkM zq-NZ|H6xPZ0H;{-hA6!s5Y$3R8fSir3>0<=4HDEk9ikqN#-T~aEHcduh~JGctvIgX zRM~VdqZE6TWwH{)K(pegH5`r3MnE~3ZoL`B`VEPbZojA7X2W#!ikEe1p!+q~aq5jm m&1n+1X<>JZc=cwZzU?*Lu;y=tp>2nD&98zJcE{T6g8l(ebZuw= literal 0 HcmV?d00001 diff --git a/test/files/signs.parquet b/test/files/signs.parquet new file mode 100644 index 0000000000000000000000000000000000000000..100e64de1a7a7c8ef99e1f138d48ed8a84ca8979 GIT binary patch literal 2443 zcmb7GOK%cU6uv_#4z*a+&SWse%%V(8Y)helt+7cj&=z7J6t%S77+R1%DMBfRO}jQ- zxil^sm&Uj-F1s>u;}0+{To~i33s)|i;5m0@V1Vhvlg^zv559Z8@7|L%L6bhgD#$|L zw7?7G1|iM|1Z*Fk19)oi2t?==q<+Xd1)E@VhsYf(#w9B(XkC34Sb5IU@tc@|)iksk z{K^yz;+F@jsce=XeNlfngLWE$!c~UXB`bo zk$NkrWW}gt#X1qhY$2a);8$2wER1hIeCn$|#IvT*bGA^+SED+Myc=WEocz!t%%xd?DYn z_(2PlW>UeXzGkYRXuOE~Gu5iP{%7fpRk4L~yG{`}bFY_)09OG{4}h0wxT4-Gh^q(O z=x-DIQEv@?QKZw>A2dCMdO;n$^{X^CMkKkAE { + const files = fs.readdirSync('test/files').filter(f => f.endsWith('.parquet')) + + files.forEach(filename => { + it(`round-trips data from ${filename}`, async () => { + const file = await asyncBufferFromFile(`test/files/${filename}`) + const metadata = await parquetMetadataAsync(file) + const rows = await parquetReadObjects({ file }) + + // transpose the row data + const schema = parquetSchema(metadata) + const columnData = schema.children.map(({ element }) => ({ + ...element, + data: /** @type {any[]} */ ([]), + })) + for (const row of rows) { + for (const { name, data } of columnData) { + data.push(row[name]) + } + } + + const buffer = parquetWriteBuffer({ columnData }) + const output = await parquetReadObjects({ file: buffer }) + + expect(output.length).toBe(rows.length) + expect(output).toEqual(rows) + }) + }) +})