mirror of
https://github.com/asadbek064/hyparquet.git
synced 2025-12-29 00:16:38 +00:00
Simplify imports
This commit is contained in:
parent
59db67f8d8
commit
bf1b8d79c7
@ -34,7 +34,7 @@
|
||||
"eslint-plugin-import": "2.29.1",
|
||||
"eslint-plugin-jsdoc": "48.2.3",
|
||||
"http-server": "14.1.1",
|
||||
"hysnappy": "0.3.0",
|
||||
"hysnappy": "0.3.1",
|
||||
"typescript": "5.4.5",
|
||||
"vitest": "1.5.2"
|
||||
}
|
||||
|
||||
@ -8,13 +8,13 @@
|
||||
* @param {number[] | undefined} definitionLevels definition levels, max 3
|
||||
* @param {number[]} repetitionLevels repetition levels, max 1
|
||||
* @param {ArrayLike<any>} values values to process
|
||||
* @param {boolean} isNull can an entry be null?
|
||||
* @param {boolean} isNullable can entries be null?
|
||||
* @param {number} maxDefinitionLevel definition level that corresponds to non-null
|
||||
* @param {number} maxRepetitionLevel repetition level that corresponds to a new row
|
||||
* @returns {any[]} array of values
|
||||
*/
|
||||
export function assembleObjects(
|
||||
definitionLevels, repetitionLevels, values, isNull, maxDefinitionLevel, maxRepetitionLevel
|
||||
definitionLevels, repetitionLevels, values, isNullable, maxDefinitionLevel, maxRepetitionLevel
|
||||
) {
|
||||
let valueIndex = 0
|
||||
/** @type {any[]} */
|
||||
@ -53,7 +53,7 @@ export function assembleObjects(
|
||||
throw new Error('parquet assembleObjects: currentContainer is undefined')
|
||||
}
|
||||
currentContainer.push(values[valueIndex++])
|
||||
} else if (isNull) {
|
||||
} else if (isNullable) {
|
||||
if (def) {
|
||||
// TODO: Go up maxDefinitionLevel - def - 1 levels to add null
|
||||
for (let j = def; j < maxDefinitionLevel - 1; j++) {
|
||||
|
||||
@ -169,11 +169,9 @@ export function getColumnOffset(columnMetadata) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {import('./types.js').PageHeader} PageHeader
|
||||
* @typedef {import('./types.js').CompressionCodec} CompressionCodec
|
||||
* @param {Uint8Array} compressedBytes
|
||||
* @param {number} uncompressed_page_size
|
||||
* @param {CompressionCodec} codec
|
||||
* @param {import('./types.js').CompressionCodec} codec
|
||||
* @param {Compressors | undefined} compressors
|
||||
* @returns {Uint8Array}
|
||||
*/
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
/**
|
||||
* @typedef {import('./types.js').ParquetType} ParquetTypeType
|
||||
* @type {ParquetTypeType[]}
|
||||
* @type {import('./types.js').ParquetType[]}
|
||||
*/
|
||||
export const ParquetType = [
|
||||
'BOOLEAN',
|
||||
@ -33,8 +32,7 @@ export const FieldRepetitionType = [
|
||||
]
|
||||
|
||||
/**
|
||||
* @typedef {import('./types.js').ConvertedType} ConvertedTypeType
|
||||
* @type {ConvertedTypeType[]}
|
||||
* @type {import('./types.js').ConvertedType[]}
|
||||
*/
|
||||
export const ConvertedType = [
|
||||
'UTF8',
|
||||
@ -62,8 +60,7 @@ export const ConvertedType = [
|
||||
]
|
||||
|
||||
/**
|
||||
* @typedef {import('./types.js').LogicalTypeType} LogicalTypeType
|
||||
* @type {LogicalTypeType[]}
|
||||
* @type {import('./types.js').LogicalTypeType[]}
|
||||
*/
|
||||
export const logicalTypeType = [
|
||||
'NULL',
|
||||
@ -95,8 +92,7 @@ export const CompressionCodec = [
|
||||
]
|
||||
|
||||
/**
|
||||
* @typedef {import('./types.js').PageType} PageType
|
||||
* @type {PageType[]}
|
||||
* @type {import('./types.js').PageType[]}
|
||||
*/
|
||||
export const PageType = [
|
||||
'DATA_PAGE',
|
||||
|
||||
@ -1,14 +1,10 @@
|
||||
/**
|
||||
* @typedef {import('./types.js').SchemaElement} SchemaElement
|
||||
*/
|
||||
|
||||
const dayMillis = 86400000000000 // 1 day in milliseconds
|
||||
const dayMillis = 86400000000000 // 1 day in ms
|
||||
|
||||
/**
|
||||
* Convert known types from primitive to rich.
|
||||
*
|
||||
* @param {any[]} data series of primitive types
|
||||
* @param {SchemaElement} schemaElement schema element for the data
|
||||
* @param {import('./types.js').SchemaElement} schemaElement schema element for the data
|
||||
* @returns {any[]} series of rich types
|
||||
*/
|
||||
export function convert(data, schemaElement) {
|
||||
|
||||
@ -10,10 +10,9 @@ import { readVarInt, readZigZag } from './thrift.js'
|
||||
* @typedef {import("./types.d.ts").ColumnMetaData} ColumnMetaData
|
||||
* @typedef {import("./types.d.ts").Compressors} Compressors
|
||||
* @typedef {import("./types.d.ts").DataPageHeaderV2} DataPageHeaderV2
|
||||
* @typedef {import("./types.d.ts").PageHeader} PageHeader
|
||||
* @typedef {import("./types.d.ts").SchemaElement} SchemaElement
|
||||
* @param {Uint8Array} compressedBytes raw page data (should already be decompressed)
|
||||
* @param {PageHeader} ph page header
|
||||
* @param {import("./types.d.ts").PageHeader} ph page header
|
||||
* @param {SchemaElement[]} schema schema for the file
|
||||
* @param {ColumnMetaData} columnMetadata metadata for the column
|
||||
* @param {Compressors | undefined} compressors
|
||||
|
||||
@ -168,9 +168,8 @@ export function parquetMetadata(arrayBuffer) {
|
||||
/**
|
||||
* Return a tree of schema elements from parquet metadata.
|
||||
*
|
||||
* @typedef {import("./types.d.ts").SchemaTree} SchemaTree
|
||||
* @param {FileMetaData} metadata parquet metadata object
|
||||
* @returns {SchemaTree} tree of schema elements
|
||||
* @returns {import("./types.d.ts").SchemaTree} tree of schema elements
|
||||
*/
|
||||
export function parquetSchema(metadata) {
|
||||
return schemaElement(metadata.schema, [])
|
||||
|
||||
@ -14,9 +14,8 @@ export async function readFileToArrayBuffer(filePath) {
|
||||
/**
|
||||
* Wrap .parquet file in an AsyncBuffer
|
||||
*
|
||||
* @typedef {import('../src/types.js').AsyncBuffer} AsyncBuffer
|
||||
* @param {string} filePath
|
||||
* @returns {AsyncBuffer}
|
||||
* @returns {import('../src/types.js').AsyncBuffer}
|
||||
*/
|
||||
export function fileToAsyncBuffer(filePath) {
|
||||
return {
|
||||
|
||||
@ -6,8 +6,7 @@ import { toJson } from '../src/utils.js'
|
||||
import { fileToAsyncBuffer, fileToJson } from './helpers.js'
|
||||
|
||||
/**
|
||||
* @typedef {import('../src/types.js').Compressors} Compressors
|
||||
* @type {Compressors}
|
||||
* @type {import('../src/types.js').Compressors}
|
||||
*/
|
||||
const compressors = {
|
||||
GZIP: (/** @type {Uint8Array} */ input, /** @type {number} */ outputLength) => {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user