asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2025-12-05 22:41:55 +00:00
Code Issues Actions Projects Releases Wiki Activity

Fix page continuation issue #81

Browse Source
This commit is contained in:
Kenny Daniel 2025-05-24 23:35:12 -07:00
parent 5d8f17903e
commit 5e846e6b13
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
2 changed files with 15 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/assemble.js
View File

@ -28,12 +28,13 @@ export function assembleLists(output, definitionLevels, repetitionLevels, values
if (repetitionLevels[0]) {
// continue previous row
while (currentDepth < repetitionPath.length - 2 && currentRepLevel < repetitionLevels[0]) {
if (!currentContainer) throw new Error('parquet cannot resume previous page')
// go into last list
currentContainer = currentContainer.at(-1)
containerStack.push(currentContainer)
currentDepth++
if (repetitionPath[currentDepth] !== 'REQUIRED') currentDefLevel++
if (repetitionPath[currentDepth] !== 'REQUIRED') {
// go into last list
currentContainer = currentContainer.at(-1)
containerStack.push(currentContainer)
currentDefLevel++
}
if (repetitionPath[currentDepth] === 'REPEATED') currentRepLevel++
}
}

10
test/assemble.test.js
View File

@ -66,7 +66,7 @@ describe('assembleLists', () => {
expect(result).toEqual([[['a', 'b']], [], [['d', 'e']]])
})
it('should handle continuing a row from the previous page', () => {
it('should continue from the previous page', () => {
const definitionLevels = [3, 3, 3, 1]
const repetitionLevels = [1, 0, 1, 0]
const values = ['b', 'c', 'd', 'e']
@ -75,6 +75,14 @@ describe('assembleLists', () => {
expect(result).toEqual([[['a', 'b']], [['c', 'd']], [[]]])
})
it('should continue from the previous page (depth 2)', () => {
const repetitionLevels = [2, 0, 2, 0]
const values = ['b', 'c', 'd', 'e']
const prev = [[['a']]]
const result = assembleLists(prev, [], repetitionLevels, values, nestedRequired)
expect(result).toEqual([[['a', 'b']], [['c', 'd']], [['e']]])
})
it('should handle nested arrays', () => {
// from nullable.impala.parquet
const repetitionLevels = [0, 2, 1, 2]