asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2026-01-04 10:36:37 +00:00
Code Issues Actions Projects Releases Wiki Activity

Fix nested optional from duckdb#3734 🦆

Browse Source
This commit is contained in:
Kenny Daniel 2024-05-23 18:07:36 -07:00
parent af7bab33f8
commit ed3b525a27
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
4 changed files with 334 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/column.js
View File

@ -64,7 +64,7 @@ export function readColumn(reader, rowGroup, columnMetadata, schemaPath, { compr
// wrap nested flat data by depth
for (let i = 2; i < schemaPath.length; i++) {
if (schemaPath[i].element.repetition_type !== 'REQUIRED') {
values = [values]
values = Array.from(values, e => [e])
}
}
}

102
test/files/duckdb3734.json Normal file
View File

@ -0,0 +1,102 @@
[
[
"tt0000001",
{
"category": "self",
"characters": ["[\"Self\"]"],
"job": "\\N",
"nconst": "nm1588970",
"ordering": 1
}
],
[
"tt0000001",
{
"category": "director",
"characters": ["\\N"],
"job": "\\N",
"nconst": "nm0005690",
"ordering": 2
}
],
[
"tt0000001",
{
"category": "cinematographer",
"characters": ["\\N"],
"job": "director of photography",
"nconst": "nm0374658",
"ordering": 3
}
],
[
"tt0000002",
{
"category": "director",
"characters": ["\\N"],
"job": "\\N",
"nconst": "nm0721526",
"ordering": 1
}
],
[
"tt0000002",
{
"category": "composer",
"characters": ["\\N"],
"job": "\\N",
"nconst": "nm1335271",
"ordering": 2
}
],
[
"tt0000003",
{
"category": "director",
"characters": ["\\N"],
"job": "\\N",
"nconst": "nm0721526",
"ordering": 1
}
],
[
"tt0000003",
{
"category": "producer",
"characters": ["\\N"],
"job": "producer",
"nconst": "nm1770680",
"ordering": 2
}
],
[
"tt0000003",
{
"category": "composer",
"characters": ["\\N"],
"job": "\\N",
"nconst": "nm1335271",
"ordering": 3
}
],
[
"tt0000003",
{
"category": "editor",
"characters": ["\\N"],
"job": "\\N",
"nconst": "nm5442200",
"ordering": 4
}
],
[
"tt0000004",
{
"category": "director",
"characters": ["\\N"],
"job": "\\N",
"nconst": "nm0721526",
"ordering": 1
}
]
]

231
test/files/duckdb3734.metadata.json Normal file
View File

@ -0,0 +1,231 @@
{
"version": 1,
"schema": [
{
"repetition_type": "REQUIRED",
"name": "duckdb_schema",
"num_children": 2
},
{
"type": "BYTE_ARRAY",
"repetition_type": "OPTIONAL",
"name": "tconst",
"num_children": 0,
"converted_type": "UTF8"
},
{
"repetition_type": "OPTIONAL",
"name": "principals",
"num_children": 5
},
{
"type": "INT32",
"repetition_type": "OPTIONAL",
"name": "ordering",
"num_children": 0,
"converted_type": "INT_32"
},
{
"type": "BYTE_ARRAY",
"repetition_type": "OPTIONAL",
"name": "nconst",
"num_children": 0,
"converted_type": "UTF8"
},
{
"type": "BYTE_ARRAY",
"repetition_type": "OPTIONAL",
"name": "category",
"num_children": 0,
"converted_type": "UTF8"
},
{
"type": "BYTE_ARRAY",
"repetition_type": "OPTIONAL",
"name": "job",
"num_children": 0,
"converted_type": "UTF8"
},
{
"repetition_type": "OPTIONAL",
"name": "characters",
"num_children": 1,
"converted_type": "LIST"
},
{
"repetition_type": "REPEATED",
"name": "list",
"num_children": 1
},
{
"type": "BYTE_ARRAY",
"repetition_type": "OPTIONAL",
"name": "element",
"num_children": 0,
"converted_type": "UTF8"
}
],
"num_rows": 10,
"row_groups": [
{
"columns": [
{
"file_offset": 0,
"meta_data": {
"type": "BYTE_ARRAY",
"encodings": [
"PLAIN",
"RLE_DICTIONARY"
],
"path_in_schema": [
"tconst"
],
"codec": "SNAPPY",
"num_values": 10,
"total_uncompressed_size": 0,
"total_compressed_size": 83,
"data_page_offset": 4,
"statistics": {
"max": "tt0000004",
"min": "tt0000001",
"null_count": 0,
"max_value": "tt0000004",
"min_value": "tt0000001"
}
}
},
{
"file_offset": 0,
"meta_data": {
"type": "INT32",
"encodings": [
"PLAIN"
],
"path_in_schema": [
"principals",
"ordering"
],
"codec": "SNAPPY",
"num_values": 10,
"total_uncompressed_size": 0,
"total_compressed_size": 53,
"data_page_offset": 87,
"statistics": {
"max": 4,
"min": 1,
"null_count": 0,
"max_value": 4,
"min_value": 1
}
}
},
{
"file_offset": 0,
"meta_data": {
"type": "BYTE_ARRAY",
"encodings": [
"PLAIN"
],
"path_in_schema": [
"principals",
"nconst"
],
"codec": "SNAPPY",
"num_values": 10,
"total_uncompressed_size": 0,
"total_compressed_size": 116,
"data_page_offset": 140,
"statistics": {
"max": "nm5442200",
"min": "nm0005690",
"null_count": 0,
"max_value": "nm5442200",
"min_value": "nm0005690"
}
}
},
{
"file_offset": 0,
"meta_data": {
"type": "BYTE_ARRAY",
"encodings": [
"PLAIN"
],
"path_in_schema": [
"principals",
"category"
],
"codec": "SNAPPY",
"num_values": 10,
"total_uncompressed_size": 0,
"total_compressed_size": 121,
"data_page_offset": 256,
"statistics": {
"max": "self",
"min": "cinematographer",
"null_count": 0,
"max_value": "self",
"min_value": "cinematographer"
}
}
},
{
"file_offset": 0,
"meta_data": {
"type": "BYTE_ARRAY",
"encodings": [
"PLAIN"
],
"path_in_schema": [
"principals",
"job"
],
"codec": "SNAPPY",
"num_values": 10,
"total_uncompressed_size": 0,
"total_compressed_size": 90,
"data_page_offset": 377,
"statistics": {
"max": "producer",
"min": "\\N",
"null_count": 0,
"max_value": "producer",
"min_value": "\\N"
}
}
},
{
"file_offset": 0,
"meta_data": {
"type": "BYTE_ARRAY",
"encodings": [
"PLAIN"
],
"path_in_schema": [
"principals",
"characters",
"list",
"element"
],
"codec": "SNAPPY",
"num_values": 10,
"total_uncompressed_size": 0,
"total_compressed_size": 50,
"data_page_offset": 467,
"statistics": {
"max": "\\N",
"min": "[\"Self\"]",
"max_value": "\\N",
"min_value": "[\"Self\"]"
}
}
}
],
"total_byte_size": 0,
"num_rows": 10,
"file_offset": 4
}
],
"created_by": "DuckDB",
"metadata_length": 690
}

BIN
test/files/duckdb3734.parquet Normal file
View File

Binary file not shown.