From af7bab33f8e83cfc4ccc4c8366f26f744c58a096 Mon Sep 17 00:00:00 2001 From: Kenny Daniel Date: Thu, 23 May 2024 17:27:43 -0700 Subject: [PATCH] Handle top level repeated from duckdb#2557 :duck: --- src/assemble.js | 5 +- test/files/duckdb2557.json | 502 ++++++++++++++++++++++++++++ test/files/duckdb2557.metadata.json | 91 +++++ test/files/duckdb2557.parquet | Bin 0 -> 7917 bytes 4 files changed, 597 insertions(+), 1 deletion(-) create mode 100644 test/files/duckdb2557.json create mode 100644 test/files/duckdb2557.metadata.json create mode 100644 test/files/duckdb2557.parquet diff --git a/src/assemble.js b/src/assemble.js index 7eebebf..f63017e 100644 --- a/src/assemble.js +++ b/src/assemble.js @@ -49,7 +49,10 @@ export function assembleLists( currentContainer = containerStack.at(-1) // Go deeper to end of definition level - while (currentDepth < repetitionPath.length - 2 && (currentDefLevel < def || repetitionPath[currentDepth + 1] === 'REQUIRED')) { + while ( + (currentDepth < repetitionPath.length - 2 || repetitionPath[currentDepth + 1] === 'REPEATED') && + (currentDefLevel < def || repetitionPath[currentDepth + 1] === 'REQUIRED') + ) { currentDepth++ if (repetitionPath[currentDepth] !== 'REQUIRED') { /** @type {any[]} */ diff --git a/test/files/duckdb2557.json b/test/files/duckdb2557.json new file mode 100644 index 0000000..77f998f --- /dev/null +++ b/test/files/duckdb2557.json @@ -0,0 +1,502 @@ +[ + [ + ["adipiscing","elit"], + [267], + null + ], + [ + ["adipiscing","elit"], + [58,146], + [3105.7357307755087,7332.144961207942,2693.4596589664648,2058.8303474455493] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [26,701], + [2252.3150410275202] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [763], + [4318.131164063701,703.3323215551796] + ], + [ + ["consectetur","adipiscing","elit"], + null, + [4921.065813194329] + ], + [ + ["ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [503], + [3143.3117237229058] + ], + [ + ["amet","consectetur","adipiscing","elit"], + [981], + [1556.8447821660177,5388.780545775566] + ], + [ + ["consectetur","adipiscing","elit"], + [822,843,702,469], + null + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [698,385], + [3591.160509216094] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [597,719], + [1218.8037401553586] + ], + [ + ["consectetur","adipiscing","elit"], + [982,61,421], + [3340.170661764955] + ], + [ + ["amet","consectetur","adipiscing","elit"], + [755,871,74], + [1501.3693785848154,422.14793963826895,5024.141281463876,6237.271801038096] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [211], + [4786.395338954335,1496.226058736663,6779.587474868914] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [315,936], + null + ], + [ + ["consectetur","adipiscing","elit"], + [844,175,911,241], + null + ], + [ + ["adipiscing","elit"], + [47,435,911,139], + [2441.714990790491,2271.880165736814,5973.072651226163] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [423], + [9570.98997705378,3552.7055353566743,5578.739767267185,5553.47138198093] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + null, + [7888.953864907877] + ], + [ + ["elit"], + [989,708], + [8329.09939011241,3308.3460598640904,5222.748347325276,8592.378127392156] + ], + [ + ["adipiscing","elit"], + [17,940], + null + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [150,457,406,128], + null + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [562,980,810], + [9313.463136275077] + ], + [ + ["amet","consectetur","adipiscing","elit"], + [934,565], + [4726.017684669396,915.9554253555252,43.23578448203644] + ], + [ + ["amet","consectetur","adipiscing","elit"], + null, + [5480.171753257772,8836.074587570927] + ], + [ + ["consectetur","adipiscing","elit"], + [224,462,286], + [4317.835809902944] + ], + [ + ["amet","consectetur","adipiscing","elit"], + [500,909,743,590], + [9290.81696266973,7393.76235627646,4353.930925060115,9508.516094740304] + ], + [ + ["adipiscing","elit"], + [194,381,324], + [647.950302311957] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [36,579], + [6885.930540160353,5917.361343514306,4548.728302237076] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [991,987,308], + [2266.123776139482,7305.037486237143,9191.592087337953,6673.36239870585] + ], + [ + ["consectetur","adipiscing","elit"], + null, + [2464.480077569758,8697.46909254981] + ], + [ + ["elit"], + [840,349], + [5121.80139067144,946.8387655923967,3432.1676015554935] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [3,794,11], + null + ], + [ + ["amet","consectetur","adipiscing","elit"], + [198,803,153,587], + [3070.004665972744,5480.698857029653,1148.961165054546,9768.062749170045] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + null, + null + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [437,512,307], + [5972.458856658998,6845.457547027043] + ], + [ + ["elit"], + [178,534,960], + [2721.449073524091,8461.352080454199,2583.2797872288115,7712.395605491593] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + null, + [5269.5337341809945,6144.28443776808,3284.873639079399,5904.365808300479] + ], + [ + ["adipiscing","elit"], + [330,897,939,953], + [3496.179648030784,8192.087649092551,7517.135901762508,7393.634782817085] + ], + [ + ["ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [117], + [6812.592153780877,9995.317895016125,7743.313278458503] + ], + [ + ["elit"], + null, + null + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [160,58], + [7678.085059607664,6539.924624358436] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [912,577], + [4619.958927937337,6038.533932343868,5773.6529333645,3733.3680753703707] + ], + [ + ["ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [327,634], + [5889.477126190288,9561.750382454742,1879.139746783595] + ], + [ + ["ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [689,573,154], + null + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [243], + [8367.528047322694,3549.7693351040607,4839.1473151311475] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [605], + null + ], + [ + ["elit"], + [112,386], + [1498.5598914421823,3543.2882763573275,9012.492304553425] + ], + [ + ["adipiscing","elit"], + [930,794,556], + [6047.020269076377,408.94005261710765] + ], + [ + ["consectetur","adipiscing","elit"], + [17,88,67,644], + [4895.379151378348,4367.513870792818] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [383,913,353], + [4670.973882056124,9942.983668528092,6566.851385809874,598.7621764005469] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [901,650,656], + [6250.305851205317,2810.3723884937854,9011.871765756341,1549.2422033034238] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + null, + [4125.175201983808,1696.1916192518433,7171.897025370562] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [533,132,478,258], + null + ], + [ + ["ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [91,994,707,492], + [5990.85669667847,7696.302079435442] + ], + [ + ["elit"], + [844], + [3897.439630516024,3361.9126261341476,3044.4396230158686,3568.6693921816473] + ], + [ + ["amet","consectetur","adipiscing","elit"], + [998,812,607], + [7445.490550294864,149.58593241554266,7273.297118952501,4475.322175038508] + ], + [ + ["adipiscing","elit"], + [970], + null + ], + [ + ["adipiscing","elit"], + [417,93,831,239], + [5334.980793738933] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [531], + null + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [136,454,647,548], + [3077.3392305678976,3282.994362444506,5615.484087460349] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + null, + [682.3737905128879,3021.2450897276713,3922.2218598695717,6382.788491011993] + ], + [ + ["elit"], + [355,88], + [9682.360811914446,602.7431101350196,2684.1004577653393,8782.542226659194] + ], + [ + ["consectetur","adipiscing","elit"], + [629,761,789,986], + [5251.1906293895745,5251.405751079003,6630.454545816088,4905.487033906431] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [805,223,377,716], + [1451.5808348010034,5523.483513273464] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + null, + [1882.1189849132475,7050.232591848215,6808.016315248813] + ], + [ + ["consectetur","adipiscing","elit"], + [337,290], + [7217.2520157941335,895.6717735572475,1325.0868250721135] + ], + [ + ["elit"], + [828,20,434,104], + [2923.6903455939873,5637.36807908733,2944.3389848562906,3543.9481597338477] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + null, + [267.2495106375208,3087.341650241528,9858.741302383121] + ], + [ + ["ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [92], + [3063.9841438239237] + ], + [ + ["adipiscing","elit"], + [780,356,177,783], + [5033.837904885062] + ], + [ + ["adipiscing","elit"], + [672,527], + [8821.83264234605,5579.802523534555,3500.2512473704664,5880.503553442674] + ], + [ + ["elit"], + [235], + [3230.27633188766] + ], + [ + ["amet","consectetur","adipiscing","elit"], + [362], + [9738.047844761331] + ], + [ + ["amet","consectetur","adipiscing","elit"], + [341,101,341], + null + ], + [ + ["ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [362,174], + [2792.2996177048203,2666.952450305537] + ], + [ + ["consectetur","adipiscing","elit"], + [946,496], + [481.38726845186375] + ], + [ + ["elit"], + [305,108], + [5099.123113081872,5839.2964468180935,2468.42273496024,9903.152610040035] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [711,710], + null + ], + [ + ["adipiscing","elit"], + [618,240,426], + [5852.928755649495,998.3541856358428] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [356,811], + null + ], + [ + ["adipiscing","elit"], + [62,952], + [469.02139625265846,8025.423231510394,7390.735678118863,8817.788129733082] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [716,347,577,65], + null + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [269,496], + [8238.480884176539,6226.275986271456] + ], + [ + ["adipiscing","elit"], + [321,4], + [7760.885932310369,1012.6424901267686,8042.562421074255,3128.0840479746207] + ], + [ + ["adipiscing","elit"], + [349,298], + null + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [301,32], + [8304.31871764813,375.28234034370615,8513.262037880933,8573.798838034425] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [480,325,546], + [9361.738552926725,7936.206856812588] + ], + [ + ["elit"], + [420,236,397,232], + null + ], + [ + ["amet","consectetur","adipiscing","elit"], + [132,946,269,446], + [2225.9740143331824] + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [494,231,752], + [2649.6845028896682] + ], + [ + ["elit"], + [765,811,89,988], + [9390.536056348445,2072.754849436884] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [317,351,804,750], + [1771.8131458649202,6846.017812163365,9399.01819527325,243.64103179547746] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [906,333,985,944], + [5085.78003539902] + ], + [ + ["dolor","sit","amet","consectetur","adipiscing","elit"], + [491,324,927,928], + null + ], + [ + ["Lorem","ipsum","dolor","sit","amet","consectetur","adipiscing","elit"], + [230], + null + ], + [ + ["amet","consectetur","adipiscing","elit"], + [468], + [2434.0862346139543,1631.8833707524257,3720.706865499823,5200.343932621845] + ], + [ + ["sit","amet","consectetur","adipiscing","elit"], + [33,406,706], + [8530.57558590904] + ], + [ + ["adipiscing","elit"], + [181,228], + null + ], + [ + ["consectetur","adipiscing","elit"], + [560,501,792], + [9528.16519708566,5994.038668308305,7634.964147522635] + ], + [ + ["amet","consectetur","adipiscing","elit"], + null, + [2785.6660943419297,7838.997264980804,8751.793158344097] + ] +] diff --git a/test/files/duckdb2557.metadata.json b/test/files/duckdb2557.metadata.json new file mode 100644 index 0000000..20c4926 --- /dev/null +++ b/test/files/duckdb2557.metadata.json @@ -0,0 +1,91 @@ +{ + "version": 1, + "schema": [ + { + "name": "root", + "num_children": 3 + }, + { + "type": "BYTE_ARRAY", + "repetition_type": "REPEATED", + "name": "stringArray", + "converted_type": "UTF8" + }, + { + "type": "INT32", + "repetition_type": "REPEATED", + "name": "intArray" + }, + { + "type": "DOUBLE", + "repetition_type": "REPEATED", + "name": "doubleArray" + } + ], + "num_rows": 100, + "row_groups": [ + { + "columns": [ + { + "file_offset": 4802, + "meta_data": { + "type": "BYTE_ARRAY", + "encodings": [ + "RLE", + "PLAIN" + ], + "path_in_schema": [ + "stringArray" + ], + "codec": "UNCOMPRESSED", + "num_values": 449, + "total_uncompressed_size": 4798, + "total_compressed_size": 4798, + "data_page_offset": 4 + } + }, + { + "file_offset": 5874, + "meta_data": { + "type": "INT32", + "encodings": [ + "RLE", + "PLAIN" + ], + "path_in_schema": [ + "intArray" + ], + "codec": "UNCOMPRESSED", + "num_values": 237, + "total_uncompressed_size": 1038, + "total_compressed_size": 1038, + "data_page_offset": 4836 + } + }, + { + "file_offset": 7663, + "meta_data": { + "type": "DOUBLE", + "encodings": [ + "RLE", + "PLAIN" + ], + "path_in_schema": [ + "doubleArray" + ], + "codec": "UNCOMPRESSED", + "num_values": 225, + "total_uncompressed_size": 1757, + "total_compressed_size": 1757, + "data_page_offset": 5906 + } + } + ], + "total_byte_size": 7694, + "num_rows": 100 + } + ], + "key_value_metadata": [], + "created_by": "parquet.js", + "metadata_length": 211 +} diff --git a/test/files/duckdb2557.parquet b/test/files/duckdb2557.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9f1c9b759610997c100e0f1c3f5c904a6e720065 GIT binary patch literal 7917 zcmd5>2~-qk67F$kKw+X7(nP%C0l1Jr`&${1+-c)g(^H%tG2ebY5`BRy}s>SO;4V%z6H)!th>4Z z=iWfn4lYu}6E zb|Sl#g7+>Hxl_}n}Vwew2gHmI0y1o!0R*_S-$C}!7+TSbArN}hCdltt%{xJ&Pzf(uoi zyHM~JxaH*jJSnC$I-ZLHI(Fxe_76yz0!vxJyX(kim+JX%P=H<9XU9#+yWjWk-JMrf z=WNGryH(KL3RcXE_lKvx^YkdV;Qy41$WwToZ>)1xp7psr%H6u32 zz7Gy7_Bx;K_inyr!S3hgTNqpkF^zJ@fJ+RSgfUfEP1KgjN<3lGJZ|cRG1ZMo{E2Qr z=pfK*ECoE}N5{7ynnQyU!(T7L2uv5?q#8F4!x#)Bt#%z?Fn~Cp!#p0JQROkrG`vR# zf&M^24~BUSyaZo`Bu@qw0)^nJz~w>ig0zJE9=HPEOq`bh?iuX0uuWiJ2Bw2cf^7<< z!486Kh8zn>;p6Om&=aB8KtF@?`@%i}Jq+?Y@L_;2^zrch2woRB226r|3;ad+2LsQ6 z1K{t2Uj)eD^9DB;(1LCR?k4;{0TaP3fV75$VEW?^R|RFjaOjbc3lX0MKLa@dQUX;2 zdNKSauy;YG0H**C_+kJqk6*(!MBIGDctK_WGl3}ho&na--5~woKZ;oH+M~g7_q_rz z0{mg`gg+QQTgW|-d%=ATuz){eYQgsaRzbfFe<}DUu(@2<1ET>OaOWUn!CND4HuOeF zb?||ZW#In_T??>-&RyRF`Y!;B*z15ca1;DC_%DE80yzwr0sbTK66ot7tpE`?d)OJ! z`#_IDd>`O3Y-jM6fCF?+a&{_o3F4Zdj{`QqE=PPB;0!duJ`SE+a}?rc!On%`_OS{a zmuEw8A3*2UFa)?J!aY}AU<@z>_AN+bU>M@#A)kOd0Qmrt%O$r5ZjC~q0k{vY39)Se zciqon-+-J6pADb|y#cadcm1NLlZ&{cSzPC9M62P@IlGYS7XAZpSg|rbtM{&3*t5Rxx`l9rpMck^?4qp! zM5;ke4Y~p#l;+``KL+$h2Wd!sS~e8#{9QN?ftv%}4f+6nJ2xzc^#+~djTT1aTz5c3 z!*5-pbqoL0?w-eSGRi|URFM`^`KDgIP(sIhe} zWhysLbdX5sTYmHQ`(K39q;HGu!?x$qgd4`{F6wc#(wa?>6~)pTbK}SlEpzCOOK^@) zhK$DOG)#9YNukNp?Q9FqWs%U_`^46aEE*Q!t^d5TkVa=z=I)fPWn}ued}L`|J{eT@U(&$Kpd+64edZjL(dP!5Q=?}4 zljYpHDGm4XC|JM8E~`N~bgh~1o_8jLCiD)onVlI+p%O_%uP^gyQ>g!U=bx2Of%V8T ztG;=(cvRkn7K)?6{syY~Z;EMzL(_-~UM5uy%RiSDc7P_mt^MAz@;Ifch1&IalSoa= z7Cw+*pO#OLO~3Xmm?HIcd-nXOl%CCx_4HVpLxE-$Lp=)es9EXdlY4U#X!N63Y9|AV zY1BSr3et0ei%(C^?yQZ$>9rtL2d3=;hW` z<|!noW2dSz#k9%5Uo0@~y{WY-zj2;neAv%mC!h&b7_HLIrLT|UMKNDHaWeq5$Efkq?IF@wY+#aw8$t> zbnAqSMEA-ygRBnGPwgLsmo6`+$*v~#U!G2(9c7!cweL%)F+p$XOy_W_+1L^hlA1zG z9#!#YN0rdX6D#$FM{?-r#%Qrh&^DTRP)*JD6Db9J+UvmXJ{dISkfr%Avm`V$t@?rY z`cg`dUH`oL*eU9Lsj0DFwja&UH!@x1oJ%hDnaPFE6DW4PFsw1Mgi7{n30_T%pko2O zu8$s+P8SznnR3QAie@{CeE>B!V==07DeN3J5>GdQvFN^INbakC6`}#^I zRSRR&PO2o4NLhVQTS78L-5&9Ddt(+E+tux9v)Dtmk&ZjOqY`QIgG1q);!{Z~+<*PM zhFqGT&5qT3D5KlIoVwFo^}il?D>Q`Z`gD51Jd8WsAxGwG<0+0C__TsFvB zTz`smMTV(6MoOvV_^L9C)G*492`Jwll0{pr^sa7OcZeKM4&OI@?`ir-JY$R?B9H2v z2KdvsdNtROCETzy#(@JP;nx1>b z`7Fx4=o9C=B8Aj%-Fo2rDx1;*ZMGknfxP@&_KA|Pl=%0(*G5(*P)3j7#QRn8IejZ)B5^<$Rl}@j+B^ zB_eLQxi^)2*00#`c@foTI$Gw}=9B3f*Sfn?OGsKBePHIt#UwK6l{n8Lmv)`@xNzIN zkj|^uTpFGbM1I#Uw+1;E(Kn)HJx94EQMyyX`F~6;re_ZpdHRbzX^Wc2-q@I8S~}!2 zsxK_2&)Vt(YZu5!xH0(oM5c(cvUrl8?PBTE<0Cy+OXDfypwFSX%ViX=8MEDQTnt^E zvpAX3{zXG}FAEYB=202nV5#3K8A+1{TJ9FR(C#lB)5~k5&Hd;ns2kF-La?YWdvuu8N@k0U4H(R1& zS(-phdx`1W!-ZtvGCAPh$0fw7{Pttn-qmBx28WLCpNy}OHvorDycPF-G+iKYcXbuB zN(?K~7W~gIYi;fW3hqB&Pli?DLjG$#M&m3`bD-u79sh*`fB(lhqjPuFz~B2OGdi`N k1G}(yMk9JFuFhzR`l-0tx^MDu5bHT_{`KdvIsVc8Z~Q>@ZU6uP literal 0 HcmV?d00001