asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2026-02-23 04:41:33 +00:00
Code Issues Actions Projects Releases Wiki Activity

Await benchmark example.parquet download

Browse Source
This commit is contained in:
Kenny Daniel 2024-07-23 07:08:14 -07:00 committed by Kenny
parent cb1263564e
commit 5ec5121f31
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
benchmark.js
View File

@ -1,5 +1,6 @@
import { createReadStream, createWriteStream, promises as fs } from 'fs'
import { compressors } from 'hyparquet-compressors'
import { pipeline } from 'stream/promises'
import { parquetRead } from './src/hyparquet.js'
const url = 'https://huggingface.co/datasets/wikimedia/wikipedia/resolve/main/20231101.en/train-00000-of-00041.parquet'
@ -12,13 +13,9 @@ if (!stat) {
const res = await fetch(url)
if (!res.ok) throw new Error(res.statusText)
// write to file async
const writeStream = createWriteStream(filename)
for await (const chunk of res.body) {
writeStream.write(chunk)
}
writeStream.end()
console.log('downloaded example.parquet')
await pipeline(res.body, createWriteStream(filename))
stat = await fs.stat(filename).catch(() => undefined)
console.log('downloaded example.parquet', stat.size)
}
// asyncBuffer