mirror of
https://github.com/asadbek064/hyparquet.git
synced 2026-02-23 04:41:33 +00:00
Use hyparquet-compressors in benchmark
This commit is contained in:
parent
3ea58cb0b9
commit
8762256d9a
@ -1,5 +1,5 @@
|
||||
import { createReadStream, createWriteStream, promises as fs } from 'fs'
|
||||
import { snappyUncompressor } from 'hysnappy'
|
||||
import { compressors } from 'hyparquet-compressors'
|
||||
import { parquetRead } from './src/hyparquet.js'
|
||||
|
||||
const url = 'https://huggingface.co/datasets/wikimedia/wikipedia/resolve/main/20231101.en/train-00000-of-00041.parquet'
|
||||
@ -37,7 +37,7 @@ console.log('parsing example.parquet data...')
|
||||
// read parquet file
|
||||
await parquetRead({
|
||||
file,
|
||||
compressors: { SNAPPY: snappyUncompressor() }, // hysnappy wasm
|
||||
compressors,
|
||||
})
|
||||
const ms = performance.now() - startTime
|
||||
console.log(`parsed ${stat.size.toLocaleString()} bytes in ${ms.toFixed(0)} ms`)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user