asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2025-12-28 07:56:38 +00:00
Code Issues Actions Projects Releases Wiki Activity

Update geospatial and variant metadata from thrift spec

Browse Source
This commit is contained in:
Kenny Daniel 2025-10-23 18:44:55 -07:00
parent 2a91b8a118
commit acdbb22828
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
3 changed files with 34 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
package.json
View File

@ -54,12 +54,12 @@
"test": "vitest run"
},
"devDependencies": {
"@types/node": "24.7.2",
"@vitest/coverage-v8": "3.2.4",
"eslint": "9.37.0",
"eslint-plugin-jsdoc": "61.1.4",
"@types/node": "24.9.1",
"@vitest/coverage-v8": "4.0.2",
"eslint": "9.38.0",
"eslint-plugin-jsdoc": "61.1.7",
"hyparquet-compressors": "1.1.1",
"typescript": "5.9.3",
"vitest": "3.2.4"
"vitest": "4.0.2"
}
}

16
src/metadata.js
View File

@ -138,7 +138,10 @@ export function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}
num_values: column.field_3.field_5,
total_uncompressed_size: column.field_3.field_6,
total_compressed_size: column.field_3.field_7,
key_value_metadata: column.field_3.field_8,
key_value_metadata: column.field_3.field_8?.map((/** @type {any} */ kv) => ({
key: decode(kv.field_1),
value: decode(kv.field_2),
})),
data_page_offset: column.field_3.field_9,
index_page_offset: column.field_3.field_10,
dictionary_page_offset: column.field_3.field_11,
@ -188,9 +191,9 @@ export function parquetMetadata(arrayBuffer, { parsers, geoparquet = true } = {}
ordinal: rowGroup.field_7,
}))
/** @type {KeyValue[] | undefined} */
const key_value_metadata = metadata.field_5?.map((/** @type {any} */ keyValue) => ({
key: decode(keyValue.field_1),
value: decode(keyValue.field_2),
const key_value_metadata = metadata.field_5?.map((/** @type {any} */ kv) => ({
key: decode(kv.field_1),
value: decode(kv.field_2),
}))
const created_by = decode(metadata.field_6)
@ -254,7 +257,10 @@ function logicalType(logicalType) {
if (logicalType?.field_13) return { type: 'BSON' }
if (logicalType?.field_14) return { type: 'UUID' }
if (logicalType?.field_15) return { type: 'FLOAT16' }
if (logicalType?.field_16) return { type: 'VARIANT' }
if (logicalType?.field_16) return {
type: 'VARIANT',
specification_version: logicalType.field_16.field_1,
}
if (logicalType?.field_17) return {
type: 'GEOMETRY',
crs: decode(logicalType.field_17.field_1),

19
src/types.d.ts vendored
View File

@ -186,11 +186,11 @@ export type LogicalType =
| { type: 'BSON' }
| { type: 'UUID' }
| { type: 'FLOAT16' }
| { type: 'VARIANT' }
| { type: 'DECIMAL', precision: number, scale: number }
| { type: 'TIME', isAdjustedToUTC: boolean, unit: TimeUnit }
| { type: 'TIMESTAMP', isAdjustedToUTC: boolean, unit: TimeUnit }
| { type: 'INTEGER', bitWidth: number, isSigned: boolean }
| { type: 'VARIANT', specification_version?: number }
| { type: 'GEOMETRY', crs?: string }
| { type: 'GEOGRAPHY', crs?: string, algorithm?: EdgeInterpolationAlgorithm }
@ -235,6 +235,7 @@ export interface ColumnMetaData {
bloom_filter_offset?: bigint
bloom_filter_length?: number
size_statistics?: SizeStatistics
geospatial_statistics?: GeospatialStatistics
}
type ColumnCryptoMetaData = Record<string, never>
@ -289,6 +290,22 @@ interface SizeStatistics {
definition_level_histogram?: bigint[]
}
export interface GeospatialStatistics {
bbox?: BoundingBox
geospatial_types?: number[]
}
export interface BoundingBox {
xmin: number
xmax: number
ymin: number
ymax: number
zmin?: number
zmax?: number
mmin?: number
mmax?: number
}
interface PageEncodingStats {
page_type: PageType
encoding: Encoding