diff --git a/README.md b/README.md index 2f943ea..0e6e185 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,20 @@ [![minzipped](https://img.shields.io/bundlephobia/minzip/hyparquet-writer)](https://www.npmjs.com/package/hyparquet-writer) [![workflow status](https://github.com/hyparam/hyparquet-writer/actions/workflows/ci.yml/badge.svg)](https://github.com/hyparam/hyparquet-writer/actions) [![mit license](https://img.shields.io/badge/License-MIT-orange.svg)](https://opensource.org/licenses/MIT) -![coverage](https://img.shields.io/badge/Coverage-96-darkred) +![coverage](https://img.shields.io/badge/Coverage-97-darkred) [![dependencies](https://img.shields.io/badge/Dependencies-1-blueviolet)](https://www.npmjs.com/package/hyparquet-writer?activeTab=dependencies) Hyparquet Writer is a JavaScript library for writing [Apache Parquet](https://parquet.apache.org) files. It is designed to be lightweight, fast and store data very efficiently. It is a companion to the [hyparquet](https://github.com/hyparam/hyparquet) library, which is a JavaScript library for reading parquet files. ## Usage -Call `parquetWrite` with a list of columns, each column is an object with a `name` and `data` field. The `data` field should be an array of same-type values. +Call `parquetWrite` with argument `columnData`. Each column in `columnData` should contain: + +- `name`: the column name +- `data`: an array of same-type values +- `type`: the parquet schema type (optional, type guessed from data if not provided) + +Example: ```javascript import { parquetWrite } from 'hyparquet-writer' @@ -28,11 +34,31 @@ const arrayBuffer = parquetWrite({ ## Options +Options can be passed to `parquetWrite` to change parquet file properties: + - `compression`: use snappy compression (default true) - `statistics`: write column statistics (default true) - `rowGroupSize`: number of rows in each row group (default 100000) - `kvMetadata`: extra key-value metadata +```javascript +import { parquetWrite } from 'hyparquet-writer' + +const arrayBuffer = parquetWrite({ + columnData: [ + { name: 'name', data: ['Alice', 'Bob', 'Charlie'], type: 'STRING' }, + { name: 'age', data: [25, 30, 35], type: 'INT32' }, + ], + compression: false, + statistics: false, + rowGroupSize: 1000, + kvMetadata: { + 'key1': 'value1', + 'key2': 'value2', + }, +}) +``` + ## References - https://github.com/hyparam/hyparquet diff --git a/package.json b/package.json index 4fb9f50..191c9fb 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "test": "vitest run" }, "dependencies": { - "hyparquet": "1.10.4" + "hyparquet": "1.11.0" }, "devDependencies": { "@babel/eslint-parser": "7.27.0", diff --git a/src/writer.js b/src/bytewriter.js similarity index 72% rename from src/writer.js rename to src/bytewriter.js index b83d593..58f5037 100644 --- a/src/writer.js +++ b/src/bytewriter.js @@ -1,10 +1,11 @@ /** - * Self-expanding buffer view + * Generic buffered writer. * - * @returns {import('../src/types.js').Writer} + * @import {Writer} from '../src/types.js' + * @returns {Writer} */ -export function Writer() { +export function ByteWriter() { this.buffer = new ArrayBuffer(1024) this.offset = 0 this.view = new DataView(this.buffer) @@ -14,24 +15,29 @@ export function Writer() { /** * @param {number} size */ -Writer.prototype.ensure = function(size) { +ByteWriter.prototype.ensure = function(size) { + // auto-expanding buffer if (this.offset + size > this.buffer.byteLength) { const newSize = Math.max(this.buffer.byteLength * 2, this.offset + size) const newBuffer = new ArrayBuffer(newSize) + // TODO: save buffers until later and merge once? new Uint8Array(newBuffer).set(new Uint8Array(this.buffer)) this.buffer = newBuffer this.view = new DataView(this.buffer) } } -Writer.prototype.getBuffer = function() { +ByteWriter.prototype.finish = function() { +} + +ByteWriter.prototype.getBuffer = function() { return this.buffer.slice(0, this.offset) } /** * @param {number} value */ -Writer.prototype.appendUint8 = function(value) { +ByteWriter.prototype.appendUint8 = function(value) { this.ensure(this.offset + 1) this.view.setUint8(this.offset, value) this.offset++ @@ -40,7 +46,7 @@ Writer.prototype.appendUint8 = function(value) { /** * @param {number} value */ -Writer.prototype.appendUint32 = function(value) { +ByteWriter.prototype.appendUint32 = function(value) { this.ensure(this.offset + 4) this.view.setUint32(this.offset, value, true) this.offset += 4 @@ -49,7 +55,7 @@ Writer.prototype.appendUint32 = function(value) { /** * @param {number} value */ -Writer.prototype.appendInt32 = function(value) { +ByteWriter.prototype.appendInt32 = function(value) { this.ensure(this.offset + 4) this.view.setInt32(this.offset, value, true) this.offset += 4 @@ -58,7 +64,7 @@ Writer.prototype.appendInt32 = function(value) { /** * @param {bigint} value */ -Writer.prototype.appendInt64 = function(value) { +ByteWriter.prototype.appendInt64 = function(value) { this.ensure(this.offset + 8) this.view.setBigInt64(this.offset, BigInt(value), true) this.offset += 8 @@ -67,7 +73,7 @@ Writer.prototype.appendInt64 = function(value) { /** * @param {number} value */ -Writer.prototype.appendFloat64 = function(value) { +ByteWriter.prototype.appendFloat64 = function(value) { this.ensure(this.offset + 8) this.view.setFloat64(this.offset, value, true) this.offset += 8 @@ -76,14 +82,14 @@ Writer.prototype.appendFloat64 = function(value) { /** * @param {ArrayBuffer} value */ -Writer.prototype.appendBuffer = function(value) { +ByteWriter.prototype.appendBuffer = function(value) { this.appendBytes(new Uint8Array(value)) } /** * @param {Uint8Array} value */ -Writer.prototype.appendBytes = function(value) { +ByteWriter.prototype.appendBytes = function(value) { this.ensure(this.offset + value.length) new Uint8Array(this.buffer, this.offset, value.length).set(value) this.offset += value.length @@ -95,7 +101,7 @@ Writer.prototype.appendBytes = function(value) { * * @param {number} value */ -Writer.prototype.appendVarInt = function(value) { +ByteWriter.prototype.appendVarInt = function(value) { while (true) { if ((value & ~0x7f) === 0) { // fits in 7 bits @@ -114,7 +120,7 @@ Writer.prototype.appendVarInt = function(value) { * * @param {bigint} value */ -Writer.prototype.appendVarBigInt = function(value) { +ByteWriter.prototype.appendVarBigInt = function(value) { while (true) { if ((value & ~0x7fn) === 0n) { // fits in 7 bits diff --git a/src/column.js b/src/column.js index 8a3aeac..fc575e7 100644 --- a/src/column.js +++ b/src/column.js @@ -5,10 +5,11 @@ import { writePlain } from './plain.js' import { getMaxDefinitionLevel, getMaxRepetitionLevel } from './schema.js' import { snappyCompress } from './snappy.js' import { serializeTCompactProtocol } from './thrift.js' -import { Writer } from './writer.js' +import { ByteWriter } from './bytewriter.js' /** * @import {ColumnMetaData, DecodedArray, PageHeader, ParquetType, SchemaElement, Statistics} from 'hyparquet' + * @import {Writer} from '../src/types.js' * @param {Writer} writer * @param {SchemaElement[]} schemaPath * @param {DecodedArray} values @@ -50,7 +51,7 @@ export function writeColumn(writer, schemaPath, values, compressed, stats) { } // Write levels to temp buffer - const levels = new Writer() + const levels = new ByteWriter() const { definition_levels_byte_length, repetition_levels_byte_length, num_nulls } = writeLevels(levels, schemaPath, values) // dictionary encoding @@ -81,7 +82,7 @@ export function writeColumn(writer, schemaPath, values, compressed, stats) { } // write page data to temp buffer - const page = new Writer() + const page = new ByteWriter() /** @type {import('hyparquet').Encoding} */ const encoding = dictionary ? 'RLE_DICTIONARY' : 'PLAIN' if (dictionary) { @@ -95,7 +96,7 @@ export function writeColumn(writer, schemaPath, values, compressed, stats) { // compress page data let compressedPage = page if (compressed) { - compressedPage = new Writer() + compressedPage = new ByteWriter() snappyCompress(compressedPage, new Uint8Array(page.getBuffer())) } @@ -187,13 +188,13 @@ function useDictionary(values, type) { * @param {boolean} compressed */ function writeDictionaryPage(writer, dictionary, type, compressed) { - const dictionaryPage = new Writer() + const dictionaryPage = new ByteWriter() writePlain(dictionaryPage, dictionary, type) // compress dictionary page data let compressedDictionaryPage = dictionaryPage if (compressed) { - compressedDictionaryPage = new Writer() + compressedDictionaryPage = new ByteWriter() snappyCompress(compressedDictionaryPage, new Uint8Array(dictionaryPage.getBuffer())) } diff --git a/src/encoding.js b/src/encoding.js index 661fc1e..1df397d 100644 --- a/src/encoding.js +++ b/src/encoding.js @@ -1,7 +1,8 @@ -import { Writer } from './writer.js' +import { ByteWriter } from './bytewriter.js' /** * @import {DecodedArray} from 'hyparquet' + * @import {Writer} from '../src/types.js' * @param {Writer} writer * @param {DecodedArray} values * @returns {number} bytes written @@ -12,9 +13,9 @@ export function writeRleBitPackedHybrid(writer, values) { const bitWidth = Math.ceil(Math.log2(Math.max(...values) + 1)) // try both RLE and bit-packed and choose the best - const rle = new Writer() + const rle = new ByteWriter() writeRle(rle, values, bitWidth) - const bitPacked = new Writer() + const bitPacked = new ByteWriter() writeBitPacked(bitPacked, values, bitWidth) if (rle.offset < bitPacked.offset) { diff --git a/src/metadata.js b/src/metadata.js index f6a8456..198fd29 100644 --- a/src/metadata.js +++ b/src/metadata.js @@ -4,7 +4,7 @@ import { unconvertMetadata } from './unconvert.js' /** * @import {FileMetaData} from 'hyparquet' - * @import {Writer} from './writer.js' + * @import {Writer} from '../src/types.js' * @param {Writer} writer * @param {FileMetaData} metadata */ diff --git a/src/parquet-writer.js b/src/parquet-writer.js index de3305c..82c3e24 100644 --- a/src/parquet-writer.js +++ b/src/parquet-writer.js @@ -1,28 +1,29 @@ import { writeColumn } from './column.js' -import { Writer } from './writer.js' import { writeMetadata } from './metadata.js' /** * Create a new ParquetWriter. * - * @import {ColumnChunk, FileMetaData, RowGroup, SchemaElement, SchemaTree} from 'hyparquet' + * @import {ColumnChunk, FileMetaData, RowGroup, SchemaElement} from 'hyparquet' * @import {KeyValue} from 'hyparquet/src/types.js' - * @import {ColumnData} from './types.js' + * @import {ColumnData, Writer} from '../src/types.js' * @param {object} options + * @param {Writer} options.writer * @param {SchemaElement[]} options.schema * @param {boolean} [options.compressed] * @param {boolean} [options.statistics] * @param {KeyValue[]} [options.kvMetadata] */ -export function ParquetWriter({ schema, compressed = true, statistics = true, kvMetadata }) { - /** @type {RowGroup[]} */ - this.row_groups = [] +export function ParquetWriter({ writer, schema, compressed = true, statistics = true, kvMetadata }) { + this.writer = writer this.schema = schema this.compressed = compressed this.statistics = statistics this.kvMetadata = kvMetadata + + /** @type {RowGroup[]} */ + this.row_groups = [] this.num_rows = BigInt(0) - this.writer = new Writer() // write header PAR1 this.writer.appendUint32(0x31524150) @@ -72,9 +73,7 @@ ParquetWriter.prototype.write = function({ columnData, rowGroupSize = 100000 }) } /** - * Finish writing the file and return the buffer. - * - * @returns {ArrayBuffer} + * Finish writing the file. */ ParquetWriter.prototype.finish = function() { // write metadata @@ -94,6 +93,5 @@ ParquetWriter.prototype.finish = function() { // write footer PAR1 this.writer.appendUint32(0x31524150) - - return this.writer.getBuffer() + this.writer.finish() } diff --git a/src/plain.js b/src/plain.js index a2f42ef..6cfdd1f 100644 --- a/src/plain.js +++ b/src/plain.js @@ -1,7 +1,7 @@ /** * @import {DecodedArray, ParquetType} from 'hyparquet/src/types.js' - * @import {Writer} from './writer.js' + * @import {Writer} from '../src/types.js' * @param {Writer} writer * @param {DecodedArray} values * @param {ParquetType} type diff --git a/src/schema.js b/src/schema.js index 2692773..df4fe51 100644 --- a/src/schema.js +++ b/src/schema.js @@ -95,3 +95,32 @@ export function getMaxDefinitionLevel(schemaPath) { } return maxLevel } + +/** + * Convert column data to schema. + * + * @import {ColumnData} from '../src/types.js' + * @param {ColumnData[]} columnData + * @returns {SchemaElement[]} + */ +export function schemaFromColumnData(columnData) { + /** @type {SchemaElement[]} */ + const schema = [{ + name: 'root', + num_children: columnData.length, + }] + let num_rows = 0 + for (const { name, data, type } of columnData) { + // check if all columns have the same length + if (num_rows === 0) { + num_rows = data.length + } else if (num_rows !== data.length) { + throw new Error('columns must have the same length') + } + // auto-detect type + const schemaElement = getSchemaElementForValues(name, data, type) + if (!schemaElement.type) throw new Error(`column ${name} cannot determine type`) + schema.push(schemaElement) + } + return schema +} diff --git a/src/snappy.js b/src/snappy.js index d83b4a1..85b32be 100644 --- a/src/snappy.js +++ b/src/snappy.js @@ -14,7 +14,7 @@ const globalHashTables = new Array(MAX_HASH_TABLE_BITS + 1) * Compress snappy data. * Writes Snappy-compressed bytes into a writer. * - * @import {Writer} from '../src/writer.js' + * @import {Writer} from '../src/types.js' * @param {Writer} writer * @param {Uint8Array} input - uncompressed data */ diff --git a/src/types.d.ts b/src/types.d.ts index 793dd90..edfa57f 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -9,6 +9,9 @@ export interface ColumnData { export interface Writer { buffer: ArrayBuffer offset: number + view: DataView + ensure(size: number): void + finish(): void getBuffer(): ArrayBuffer appendUint8(value: number): void appendUint32(value: number): void diff --git a/src/write.js b/src/write.js index 8cc3887..63c666e 100644 --- a/src/write.js +++ b/src/write.js @@ -1,10 +1,10 @@ -import { getSchemaElementForValues } from './schema.js' import { ParquetWriter } from './parquet-writer.js' +import { schemaFromColumnData } from './schema.js' +import { ByteWriter } from './bytewriter.js' /** * Write data as parquet to an ArrayBuffer * - * @import {ColumnChunk, DecodedArray, FileMetaData, RowGroup, SchemaElement, SchemaTree} from 'hyparquet' * @import {KeyValue} from 'hyparquet/src/types.js' * @import {ColumnData} from '../src/types.js' * @param {object} options @@ -17,45 +17,18 @@ import { ParquetWriter } from './parquet-writer.js' */ export function parquetWrite({ columnData, compressed = true, statistics = true, rowGroupSize = 100000, kvMetadata }) { const schema = schemaFromColumnData(columnData) - const writer = new ParquetWriter({ + const writer = new ByteWriter() + const pq = new ParquetWriter({ + writer, schema, compressed, statistics, kvMetadata, }) - - writer.write({ + pq.write({ columnData, rowGroupSize, }) - - return writer.finish() -} - -/** - * Convert column data to schema. - * - * @param {ColumnData[]} columnData - * @returns {SchemaElement[]} - */ -function schemaFromColumnData(columnData) { - /** @type {SchemaElement[]} */ - const schema = [{ - name: 'root', - num_children: columnData.length, - }] - let num_rows = 0 - for (const { name, data, type } of columnData) { - // check if all columns have the same length - if (num_rows === 0) { - num_rows = data.length - } else if (num_rows !== data.length) { - throw new Error('columns must have the same length') - } - // auto-detect type - const schemaElement = getSchemaElementForValues(name, data, type) - if (!schemaElement.type) throw new Error(`column ${name} cannot determine type`) - schema.push(schemaElement) - } - return schema + pq.finish() + return writer.getBuffer() } diff --git a/test/bytewriter.test.js b/test/bytewriter.test.js new file mode 100644 index 0000000..6e8cbc0 --- /dev/null +++ b/test/bytewriter.test.js @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest' +import { ByteWriter } from '../src/bytewriter.js' + +describe('ByteWriter', () => { + it('initializes with correct defaults', () => { + const writer = new ByteWriter() + expect(writer.offset).toBe(0) + expect(writer.buffer.byteLength).toBe(1024) + }) + + it('appendUint8 writes single byte', () => { + const writer = new ByteWriter() + writer.appendUint8(255) + expect(new Uint8Array(writer.getBuffer())).toEqual(new Uint8Array([0xff])) + }) + + it('appendUint32 writes a 32-bit integer in little-endian', () => { + const writer = new ByteWriter() + writer.appendUint32(0x12345678) + expect(new Uint8Array(writer.getBuffer())).toEqual( + new Uint8Array([0x78, 0x56, 0x34, 0x12]) + ) + }) + + it('appendInt32 writes signed 32-bit integer in little-endian', () => { + const writer = new ByteWriter() + writer.appendInt32(-1) + expect(new Uint8Array(writer.getBuffer())).toEqual( + new Uint8Array([0xff, 0xff, 0xff, 0xff]) + ) + }) + + it('appendInt64 writes a 64-bit bigint in little-endian', () => { + const writer = new ByteWriter() + writer.appendInt64(0x1122334455667788n) + expect(new Uint8Array(writer.getBuffer())).toEqual( + new Uint8Array([0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11]) + ) + }) + + it('appendFloat64 writes a 64-bit float in little-endian', () => { + const writer = new ByteWriter() + writer.appendFloat64(1.0) + expect(new Uint8Array(writer.getBuffer())).toEqual( + new Uint8Array([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x3f]) + ) + }) + + it('appendBytes writes raw Uint8Array data', () => { + const writer = new ByteWriter() + writer.appendBytes(new Uint8Array([1, 2, 3, 4])) + expect(new Uint8Array(writer.getBuffer())).toEqual(new Uint8Array([1, 2, 3, 4])) + }) + + it('appendBuffer writes raw ArrayBuffer data', () => { + const writer = new ByteWriter() + const buf = new Uint8Array([10, 20, 30]).buffer + writer.appendBuffer(buf) + expect(new Uint8Array(writer.getBuffer())).toEqual(new Uint8Array([10, 20, 30])) + }) + + it('appendVarInt encodes 32-bit varint', () => { + const writer = new ByteWriter() + writer.appendVarInt(127) + writer.appendVarInt(128) + writer.appendVarInt(300) + + expect(new Uint8Array(writer.getBuffer())).toEqual( + new Uint8Array([ + 0x7f, // 127 + 0x80, 0x01, // 128 + 0xac, 0x02, // 300 + ]) + ) + }) + + it('appendVarBigInt encodes bigint varint', () => { + const writer = new ByteWriter() + writer.appendVarBigInt(127n) + writer.appendVarBigInt(128n) + writer.appendVarBigInt(300n) + + expect(new Uint8Array(writer.getBuffer())).toEqual( + new Uint8Array([ + 0x7f, // 127 + 0x80, 0x01, // 128 + 0xac, 0x02, // 300 + ]) + ) + }) + + it('expands buffer automatically when needed', () => { + const writer = new ByteWriter() + // force expansion by writing more than initial 1024 bytes + const largeArray = new Uint8Array(2000).fill(0xaa) + writer.appendBytes(largeArray) + expect(writer.buffer.byteLength).toBeGreaterThanOrEqual(2000) + expect(new Uint8Array(writer.getBuffer()).length).toBe(2000) + }) + + it('finish does nothing but is callable', () => { + const writer = new ByteWriter() + writer.finish() + expect(writer.getBuffer().byteLength).toBe(0) + }) +}) diff --git a/test/encoding.test.js b/test/encoding.test.js index 71b728f..7aa134e 100644 --- a/test/encoding.test.js +++ b/test/encoding.test.js @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { Writer } from '../src/writer.js' +import { ByteWriter } from '../src/bytewriter.js' import { writeRleBitPackedHybrid } from '../src/encoding.js' import { readRleBitPackedHybrid } from 'hyparquet/src/encoding.js' @@ -13,7 +13,7 @@ function roundTripDeserialize(values) { const bitWidth = Math.ceil(Math.log2(Math.max(...values) + 1)) // Serialize the values using writeRleBitPackedHybrid - const writer = new Writer() + const writer = new ByteWriter() writeRleBitPackedHybrid(writer, values) const buffer = writer.getBuffer() const reader = { view: new DataView(buffer), offset: 0 } diff --git a/test/metadata.test.js b/test/metadata.test.js index 3a3d2f5..4ef528d 100644 --- a/test/metadata.test.js +++ b/test/metadata.test.js @@ -1,6 +1,6 @@ import { parquetMetadata } from 'hyparquet' import { describe, expect, it } from 'vitest' -import { Writer } from '../src/writer.js' +import { ByteWriter } from '../src/bytewriter.js' import { writeMetadata } from '../src/metadata.js' /** @@ -145,7 +145,7 @@ export const exampleMetadata = { describe('writeMetadata', () => { it('writes metadata and parses in hyparquet', () => { - const writer = new Writer() + const writer = new ByteWriter() // Write header PAR1 writer.appendUint32(0x31524150) diff --git a/test/plain.test.js b/test/plain.test.js index 78113f7..96719bf 100644 --- a/test/plain.test.js +++ b/test/plain.test.js @@ -1,10 +1,10 @@ import { describe, expect, it } from 'vitest' -import { Writer } from '../src/writer.js' +import { ByteWriter } from '../src/bytewriter.js' import { writePlain } from '../src/plain.js' describe('writePlain', () => { it('writes BOOLEAN (multiple of 8 bits, plus leftover)', () => { - const writer = new Writer() + const writer = new ByteWriter() const booleans = [true, false, true, true, false, false, false, true, true] writePlain(writer, booleans, 'BOOLEAN') @@ -14,7 +14,7 @@ describe('writePlain', () => { }) it('writes INT32', () => { - const writer = new Writer() + const writer = new ByteWriter() const ints = [0, 1, 255, 256, 65535, -1, -2147483648, 2147483647] writePlain(writer, ints, 'INT32') @@ -28,7 +28,7 @@ describe('writePlain', () => { }) it('writes INT64', () => { - const writer = new Writer() + const writer = new ByteWriter() const bigints = [0n, 1n, 42n, BigInt(2 ** 53 - 1)] writePlain(writer, bigints, 'INT64') @@ -42,7 +42,7 @@ describe('writePlain', () => { }) it('writes DOUBLE', () => { - const writer = new Writer() + const writer = new ByteWriter() const doubles = [0, 3.14, -2.71, Infinity, -Infinity, NaN] writePlain(writer, doubles, 'DOUBLE') @@ -60,7 +60,7 @@ describe('writePlain', () => { }) it('writes BYTE_ARRAY', () => { - const writer = new Writer() + const writer = new ByteWriter() const strings = ['a', 'b', 'c', 'd'] // strings must be pre-converted to Uint8Array const encoder = new TextEncoder() @@ -81,7 +81,7 @@ describe('writePlain', () => { }) it('throws error on unsupported type', () => { - const writer = new Writer() + const writer = new ByteWriter() expect(() => writePlain(writer, [1, 2, 3], 'INT96')) .toThrow(/parquet unsupported type/i) }) diff --git a/test/snappy.test.js b/test/snappy.test.js index 264c33d..b4a3322 100644 --- a/test/snappy.test.js +++ b/test/snappy.test.js @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { snappyCompress } from '../src/snappy.js' -import { Writer } from '../src/writer.js' +import { ByteWriter } from '../src/bytewriter.js' import { snappyUncompress } from 'hyparquet' describe('snappy compress', () => { @@ -42,7 +42,7 @@ describe('snappy compress', () => { uncompressed: new Uint8Array([5]), }, ])('compresses valid input %p', ({ compressed, uncompressed }) => { - const writer = new Writer() + const writer = new ByteWriter() const encoder = new TextEncoder() const input = typeof uncompressed === 'string' ? encoder.encode(uncompressed) : new Uint8Array(uncompressed) snappyCompress(writer, input) diff --git a/test/thrift.test.js b/test/thrift.test.js index 6e07a19..d54114a 100644 --- a/test/thrift.test.js +++ b/test/thrift.test.js @@ -1,7 +1,7 @@ import { deserializeTCompactProtocol } from 'hyparquet/src/thrift.js' import { describe, expect, it } from 'vitest' import { serializeTCompactProtocol } from '../src/thrift.js' -import { Writer } from '../src/writer.js' +import { ByteWriter } from '../src/bytewriter.js' /** * Utility to decode a Thrift-serialized buffer and return the parsed object. @@ -28,7 +28,7 @@ describe('serializeTCompactProtocol', () => { field_9: new TextEncoder().encode('Hello, Thrift!'), } - const writer = new Writer() + const writer = new ByteWriter() serializeTCompactProtocol(writer, data) const buf = writer.buffer.slice(0, writer.offset) const result = roundTripDeserialize(buf) @@ -59,7 +59,7 @@ describe('serializeTCompactProtocol', () => { field_2: [true, false, true, false], } - const writer = new Writer() + const writer = new ByteWriter() serializeTCompactProtocol(writer, data) const buf = writer.buffer.slice(0, writer.offset) const result = roundTripDeserialize(buf) @@ -72,7 +72,7 @@ describe('serializeTCompactProtocol', () => { it('handles empty object (only STOP)', () => { const data = {} - const writer = new Writer() + const writer = new ByteWriter() serializeTCompactProtocol(writer, data) const buf = writer.buffer.slice(0, writer.offset) const arr = new Uint8Array(buf) @@ -89,7 +89,7 @@ describe('serializeTCompactProtocol', () => { field_2: 2, field_1: 1, // field_1 is out of order (less than field_2) } - const writer = new Writer() + const writer = new ByteWriter() expect(() => serializeTCompactProtocol(writer, invalidData)).toThrow() }) })