asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2025-12-05 22:41:55 +00:00
Code Issues Actions Projects Releases Wiki Activity

Export toJson

Browse Source
This commit is contained in:
Kenny Daniel 2024-01-05 01:39:59 -08:00
parent 02c25051f2
commit 954671f810
No known key found for this signature in database
GPG Key ID: 6A3C5E318BE71391
8 changed files with 45 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
index.html
View File

@ -31,7 +31,7 @@
<body>
<div id="dropzone">Drop .parquet file here</div>
<script type="module">
import { parquetMetadata, toJson } from './src/metadata.js'
import { parquetMetadata, toJson } from './src/hyparquet.js'
const dropZone = document.getElementById('dropzone')

4
package.json
View File

@ -4,7 +4,9 @@
"description": "parquet file parser for javascript",
"keywords": [
"parquet",
"parser"
"parser",
"snappy",
"thrift"
],
"license": "MIT",
"repository": {

10
src/hyparquet.d.ts vendored
View File

@ -25,3 +25,13 @@ export function parquetMetadata(arrayBuffer: ArrayBuffer): any
* @returns {boolean} true if successful
*/
export function snappyUncompress(inputArray: Uint8Array, outputArray: Uint8Array): boolean
/**
* Replace bigints with numbers.
* When parsing parquet files, bigints are used to represent 64-bit integers.
* However, JSON does not support bigints, so it's helpful to convert to numbers.
*
* @param {any} obj object to convert
* @returns {unknown} converted object
*/
export function toJson(obj: any): unknown

3
src/hyparquet.js
View File

@ -4,6 +4,9 @@ export { parquetMetadata }
import { snappyUncompress } from './snappy.js'
export { snappyUncompress }
import { toJson } from './toJson.js'
export { toJson }
/**
* Read parquet data rows from a file
*

25
src/metadata.js
View File

@ -97,28 +97,3 @@ export function parquetMetadata(arrayBuffer) {
created_by,
}
}
/**
* Replace bigints with numbers.
* When parsing parquet files, bigints are used to represent 64-bit integers.
* However, JSON does not support bigints, so it's helpful to convert to numbers.
*
* @param {any} obj object to convert
* @returns {unknown} converted object
*/
export function toJson(obj) {
if (typeof obj === 'bigint') {
return Number(obj)
} else if (Array.isArray(obj)) {
return obj.map(toJson)
} else if (obj instanceof Object) {
/** @type {Record<string, unknown>} */
const newObj = {}
for (const key of Object.keys(obj)) {
newObj[key] = toJson(obj[key])
}
return newObj
} else {
return obj
}
}

24
src/toJson.js Normal file
View File

@ -0,0 +1,24 @@
/**
* Replace bigints with numbers.
* When parsing parquet files, bigints are used to represent 64-bit integers.
* However, JSON does not support bigints, so it's helpful to convert to numbers.
*
* @param {any} obj object to convert
* @returns {unknown} converted object
*/
export function toJson(obj) {
if (typeof obj === 'bigint') {
return Number(obj)
} else if (Array.isArray(obj)) {
return obj.map(toJson)
} else if (obj instanceof Object) {
/** @type {Record<string, unknown>} */
const newObj = {}
for (const key of Object.keys(obj)) {
newObj[key] = toJson(obj[key])
}
return newObj
} else {
return obj
}
}

4
src/types.d.ts vendored
View File

@ -33,7 +33,7 @@ export enum ParquetType {
BOOLEAN = 0,
INT32 = 1,
INT64 = 2,
INT96 = 3,
INT96 = 3, // deprecated
FLOAT = 4,
DOUBLE = 5,
BYTE_ARRAY = 6,
@ -93,7 +93,7 @@ export enum Encoding {
PLAIN = 0,
PLAIN_DICTIONARY = 2,
RLE = 3,
BIT_PACKED = 4,
BIT_PACKED = 4, // deprecated
DELTA_BINARY_PACKED = 5,
DELTA_LENGTH_BYTE_ARRAY = 6,
DELTA_BYTE_ARRAY = 7,

3
test/metadata.test.js
View File

@ -1,6 +1,7 @@
import { promises as fs } from 'fs'
import { describe, expect, it } from 'vitest'
import { parquetMetadata, toJson } from '../src/metadata.js'
import { parquetMetadata } from '../src/metadata.js'
import { toJson } from '../src/toJson.js'
/**
* Helper function to read .parquet file into ArrayBuffer