asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2025-12-26 23:26:38 +00:00
Code Issues Actions Projects Releases Wiki Activity

Fix optional structs!

Browse Source
This commit is contained in:
Kenny Daniel 2024-05-20 04:59:30 -07:00
parent 9cd09b8eed
commit d453313dca
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
2 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/assemble.js
View File

@ -151,9 +151,6 @@ export function assembleNested(subcolumnData, schema, depth = 0) {
assembleNested(subcolumnData, child, nextDepth)
const childData = subcolumnData.get(child.path.join('.'))
if (!childData) throw new Error('parquet struct-like column missing child data')
if (child.element.repetition_type === 'OPTIONAL') {
flattenAtDepth(childData, depth)
}
struct[child.element.name] = childData
}
// remove children
@ -161,7 +158,9 @@ export function assembleNested(subcolumnData, schema, depth = 0) {
subcolumnData.delete(child.path.join('.'))
}
// invert struct by depth
subcolumnData.set(path, invertStruct(struct, depth))
const inverted = invertStruct(struct, nextDepth)
if (optional) flattenAtDepth(inverted, depth)
subcolumnData.set(path, inverted)
return
}
// assert(schema.element.repetition_type !== 'REPEATED')

8
test/files/nullable.impala.json
View File

@ -13,7 +13,7 @@
[[null, 1, 2, null], [3, null, 4], [], null],
{"k1": 2, "k2": null},
[{"k1": 1, "k3": null}, null, {}],
{"A":null,"b":[null],"C":{"d":[[{"E":null,"F":null},{"E":10,"F":"aaa"},{"E":null,"F":null},{"E":-10,"F":"bbb"},{"E":null,"F":null}],[{"E":11,"F":"c"},{}],[],null]},"g":{"g1":{"H":{"i":[2.2,null]}},"g2":{"H":{"i":[]}},"g3":{},"g4":{"H":{}},"g5":{"H":{}}}}
{"A":null,"b":[null],"C":{"d":[[{"E":null,"F":null},{"E":10,"F":"aaa"},{"E":null,"F":null},{"E":-10,"F":"bbb"},{"E":null,"F":null}],[{"E":11,"F":"c"},null],[],null]},"g":{"g1":{"H":{"i":[2.2,null]}},"g2":{"H":{"i":[]}},"g3":null,"g4":{"H":{}},"g5":{}}}
],
[
3,
@ -37,7 +37,7 @@
null,
{},
null,
{"A":null,"C":{},"g":{"foo":{"H":{"i":[2.2,3.3]}}}}
{"A":null,"g":{"foo":{"H":{"i":[2.2,3.3]}}}}
],
[
6,
@ -45,7 +45,7 @@
null,
null,
null,
{}
null
],
[
7,
@ -53,6 +53,6 @@
[null, [5, 6]],
{"k1": null, "k3": null},
null,
{"A":7,"b":[2,3,null],"C":{"d":[[],[{}],null]}}
{"A":7,"b":[2,3,null],"C":{"d":[[],[null],null]}}
]
]