asadbek064/hyparquet
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet.git synced 2026-01-05 10:56:38 +00:00
Code Issues Actions Projects Releases Wiki Activity

Fix thrift parsing of crypto_metadata

Browse Source
This commit is contained in:
Kenny Daniel 2025-03-25 15:42:48 -07:00
parent 74e8cc8f41
commit 85e1af66c1
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
11 changed files with 171 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/metadata.js
View File

@ -153,8 +153,8 @@ export function parquetMetadata(arrayBuffer) {
offset_index_length: column.field_5,
column_index_offset: column.field_6,
column_index_length: column.field_7,
crypto_metadata: column.field_7,
encrypted_column_metadata: column.field_8,
crypto_metadata: column.field_8,
encrypted_column_metadata: column.field_9,
})),
total_byte_size: rowGroup.field_2,
num_rows: rowGroup.field_3,

2
src/thrift.js
View File

@ -1,5 +1,5 @@
// TCompactProtocol types
const CompactType = {
export const CompactType = {
STOP: 0,
TRUE: 1,
FALSE: 2,

7
src/types.d.ts vendored
View File

@ -131,7 +131,10 @@ type LogicalTypeSimple =
'JSON' |
'BSON' |
'UUID' |
'FLOAT16'
'FLOAT16' |
'VARIANT' |
'GEOMETRY' |
'GEOGRAPHY'
export type LogicalTypeType = LogicalTypeSimple |
'TIME' | // convertedType TIME_MILLIS or TIME_MICROS
@ -276,7 +279,7 @@ export interface DictionaryPageHeader {
is_sorted?: boolean
}
interface DataPageHeaderV2 {
export interface DataPageHeaderV2 {
num_values: number
num_nulls: number
num_rows: number

273
test/files/adam_genotypes.metadata.json
View File

@ -795,8 +795,7 @@
"offset_index_offset": 6051,
"offset_index_length": 10,
"column_index_offset": 4672,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 54,
@ -830,8 +829,7 @@
"offset_index_offset": 6061,
"offset_index_length": 10,
"column_index_offset": 4687,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 104,
@ -865,8 +863,7 @@
"offset_index_offset": 6071,
"offset_index_length": 11,
"column_index_offset": 4702,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 154,
@ -904,8 +901,7 @@
"offset_index_offset": 6082,
"offset_index_length": 11,
"column_index_offset": 4717,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 213,
@ -943,8 +939,7 @@
"offset_index_offset": 6093,
"offset_index_length": 11,
"column_index_offset": 4740,
"column_index_length": 17,
"crypto_metadata": 17
"column_index_length": 17
},
{
"file_offset": 264,
@ -978,8 +973,7 @@
"offset_index_offset": 6104,
"offset_index_length": 11,
"column_index_offset": 4757,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 314,
@ -1013,8 +1007,7 @@
"offset_index_offset": 6115,
"offset_index_length": 11,
"column_index_offset": 4772,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 364,
@ -1048,8 +1041,7 @@
"offset_index_offset": 6126,
"offset_index_length": 11,
"column_index_offset": 4787,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 414,
@ -1083,8 +1075,7 @@
"offset_index_offset": 6137,
"offset_index_length": 11,
"column_index_offset": 4802,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 464,
@ -1118,8 +1109,7 @@
"offset_index_offset": 6148,
"offset_index_length": 11,
"column_index_offset": 4817,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 514,
@ -1153,8 +1143,7 @@
"offset_index_offset": 6159,
"offset_index_length": 11,
"column_index_offset": 4832,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 567,
@ -1189,8 +1178,7 @@
"offset_index_offset": 6170,
"offset_index_length": 11,
"column_index_offset": 4847,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 617,
@ -1225,8 +1213,7 @@
"offset_index_offset": 6181,
"offset_index_length": 11,
"column_index_offset": 4862,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 667,
@ -1261,8 +1248,7 @@
"offset_index_offset": 6192,
"offset_index_length": 11,
"column_index_offset": 4877,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 717,
@ -1297,8 +1283,7 @@
"offset_index_offset": 6203,
"offset_index_length": 11,
"column_index_offset": 4892,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 767,
@ -1333,8 +1318,7 @@
"offset_index_offset": 6214,
"offset_index_length": 11,
"column_index_offset": 4907,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 817,
@ -1369,8 +1353,7 @@
"offset_index_offset": 6225,
"offset_index_length": 11,
"column_index_offset": 4922,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 867,
@ -1405,8 +1388,7 @@
"offset_index_offset": 6236,
"offset_index_length": 11,
"column_index_offset": 4937,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 917,
@ -1441,8 +1423,7 @@
"offset_index_offset": 6247,
"offset_index_length": 11,
"column_index_offset": 4952,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 967,
@ -1477,8 +1458,7 @@
"offset_index_offset": 6258,
"offset_index_length": 11,
"column_index_offset": 4967,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1017,
@ -1513,8 +1493,7 @@
"offset_index_offset": 6269,
"offset_index_length": 11,
"column_index_offset": 4982,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1067,
@ -1549,8 +1528,7 @@
"offset_index_offset": 6280,
"offset_index_length": 11,
"column_index_offset": 4997,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1117,
@ -1585,8 +1563,7 @@
"offset_index_offset": 6291,
"offset_index_length": 11,
"column_index_offset": 5012,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1167,
@ -1621,8 +1598,7 @@
"offset_index_offset": 6302,
"offset_index_length": 11,
"column_index_offset": 5027,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1217,
@ -1657,8 +1633,7 @@
"offset_index_offset": 6313,
"offset_index_length": 11,
"column_index_offset": 5042,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1267,
@ -1693,8 +1668,7 @@
"offset_index_offset": 6324,
"offset_index_length": 11,
"column_index_offset": 5057,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1317,
@ -1729,8 +1703,7 @@
"offset_index_offset": 6335,
"offset_index_length": 11,
"column_index_offset": 5072,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1367,
@ -1766,8 +1739,7 @@
"offset_index_offset": 6346,
"offset_index_length": 11,
"column_index_offset": 5087,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1421,
@ -1804,8 +1776,7 @@
"offset_index_offset": 6357,
"offset_index_length": 11,
"column_index_offset": 5102,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1476,
@ -1841,8 +1812,7 @@
"offset_index_offset": 6368,
"offset_index_length": 11,
"column_index_offset": 5117,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1530,
@ -1878,8 +1848,7 @@
"offset_index_offset": 6379,
"offset_index_length": 11,
"column_index_offset": 5132,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1584,
@ -1915,8 +1884,7 @@
"offset_index_offset": 6390,
"offset_index_length": 11,
"column_index_offset": 5147,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1638,
@ -1952,8 +1920,7 @@
"offset_index_offset": 6401,
"offset_index_length": 11,
"column_index_offset": 5162,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1692,
@ -1989,8 +1956,7 @@
"offset_index_offset": 6412,
"offset_index_length": 11,
"column_index_offset": 5177,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1746,
@ -2026,8 +1992,7 @@
"offset_index_offset": 6423,
"offset_index_length": 11,
"column_index_offset": 5192,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1800,
@ -2063,8 +2028,7 @@
"offset_index_offset": 6434,
"offset_index_length": 11,
"column_index_offset": 5207,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1854,
@ -2100,8 +2064,7 @@
"offset_index_offset": 6445,
"offset_index_length": 11,
"column_index_offset": 5222,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1908,
@ -2137,8 +2100,7 @@
"offset_index_offset": 6456,
"offset_index_length": 11,
"column_index_offset": 5237,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 1962,
@ -2174,8 +2136,7 @@
"offset_index_offset": 6467,
"offset_index_length": 11,
"column_index_offset": 5252,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2016,
@ -2211,8 +2172,7 @@
"offset_index_offset": 6478,
"offset_index_length": 11,
"column_index_offset": 5267,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2070,
@ -2248,8 +2208,7 @@
"offset_index_offset": 6489,
"offset_index_length": 11,
"column_index_offset": 5282,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2124,
@ -2285,8 +2244,7 @@
"offset_index_offset": 6500,
"offset_index_length": 11,
"column_index_offset": 5297,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2178,
@ -2322,8 +2280,7 @@
"offset_index_offset": 6511,
"offset_index_length": 11,
"column_index_offset": 5312,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2232,
@ -2359,8 +2316,7 @@
"offset_index_offset": 6522,
"offset_index_length": 11,
"column_index_offset": 5327,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2286,
@ -2396,8 +2352,7 @@
"offset_index_offset": 6533,
"offset_index_length": 11,
"column_index_offset": 5342,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2340,
@ -2433,8 +2388,7 @@
"offset_index_offset": 6544,
"offset_index_length": 11,
"column_index_offset": 5357,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2394,
@ -2470,8 +2424,7 @@
"offset_index_offset": 6555,
"offset_index_length": 11,
"column_index_offset": 5372,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2448,
@ -2507,8 +2460,7 @@
"offset_index_offset": 6566,
"offset_index_length": 11,
"column_index_offset": 5387,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2502,
@ -2544,8 +2496,7 @@
"offset_index_offset": 6577,
"offset_index_length": 11,
"column_index_offset": 5402,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2556,
@ -2581,8 +2532,7 @@
"offset_index_offset": 6588,
"offset_index_length": 11,
"column_index_offset": 5417,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2610,
@ -2618,8 +2568,7 @@
"offset_index_offset": 6599,
"offset_index_length": 11,
"column_index_offset": 5432,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2664,
@ -2655,8 +2604,7 @@
"offset_index_offset": 6610,
"offset_index_length": 11,
"column_index_offset": 5447,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2718,
@ -2693,8 +2641,7 @@
"offset_index_offset": 6621,
"offset_index_length": 11,
"column_index_offset": 5462,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2773,
@ -2730,8 +2677,7 @@
"offset_index_offset": 6632,
"offset_index_length": 11,
"column_index_offset": 5477,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2826,
@ -2767,8 +2713,7 @@
"offset_index_offset": 6643,
"offset_index_length": 11,
"column_index_offset": 5492,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2879,
@ -2801,8 +2746,7 @@
"offset_index_offset": 6654,
"offset_index_length": 11,
"column_index_offset": 5507,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2928,
@ -2835,8 +2779,7 @@
"offset_index_offset": 6665,
"offset_index_length": 11,
"column_index_offset": 5522,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 2977,
@ -2869,8 +2812,7 @@
"offset_index_offset": 6676,
"offset_index_length": 11,
"column_index_offset": 5537,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3026,
@ -2904,8 +2846,7 @@
"offset_index_offset": 6687,
"offset_index_length": 11,
"column_index_offset": 5552,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3076,
@ -2939,8 +2880,7 @@
"offset_index_offset": 6698,
"offset_index_length": 11,
"column_index_offset": 5567,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3126,
@ -2974,8 +2914,7 @@
"offset_index_offset": 6709,
"offset_index_length": 11,
"column_index_offset": 5582,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3177,
@ -3009,8 +2948,7 @@
"offset_index_offset": 6720,
"offset_index_length": 11,
"column_index_offset": 5597,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3227,
@ -3044,8 +2982,7 @@
"offset_index_offset": 6731,
"offset_index_length": 11,
"column_index_offset": 5612,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3277,
@ -3079,8 +3016,7 @@
"offset_index_offset": 6742,
"offset_index_length": 11,
"column_index_offset": 5627,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3327,
@ -3114,8 +3050,7 @@
"offset_index_offset": 6753,
"offset_index_length": 11,
"column_index_offset": 5642,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3377,
@ -3149,8 +3084,7 @@
"offset_index_offset": 6764,
"offset_index_length": 11,
"column_index_offset": 5657,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3427,
@ -3184,8 +3118,7 @@
"offset_index_offset": 6775,
"offset_index_length": 11,
"column_index_offset": 5672,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3477,
@ -3219,8 +3152,7 @@
"offset_index_offset": 6786,
"offset_index_length": 11,
"column_index_offset": 5687,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3527,
@ -3254,8 +3186,7 @@
"offset_index_offset": 6797,
"offset_index_length": 11,
"column_index_offset": 5702,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3578,
@ -3289,8 +3220,7 @@
"offset_index_offset": 6808,
"offset_index_length": 11,
"column_index_offset": 5717,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3629,
@ -3324,8 +3254,7 @@
"offset_index_offset": 6819,
"offset_index_length": 11,
"column_index_offset": 5732,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3679,
@ -3359,8 +3288,7 @@
"offset_index_offset": 6830,
"offset_index_length": 11,
"column_index_offset": 5747,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3729,
@ -3395,8 +3323,7 @@
"offset_index_offset": 6841,
"offset_index_length": 11,
"column_index_offset": 5762,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3780,
@ -3431,8 +3358,7 @@
"offset_index_offset": 6852,
"offset_index_length": 11,
"column_index_offset": 5777,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3831,
@ -3465,8 +3391,7 @@
"offset_index_offset": 6863,
"offset_index_length": 11,
"column_index_offset": 5792,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3880,
@ -3499,8 +3424,7 @@
"offset_index_offset": 6874,
"offset_index_length": 11,
"column_index_offset": 5807,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3929,
@ -3533,8 +3457,7 @@
"offset_index_offset": 6885,
"offset_index_length": 11,
"column_index_offset": 5822,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 3978,
@ -3567,8 +3490,7 @@
"offset_index_offset": 6896,
"offset_index_length": 11,
"column_index_offset": 5837,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4029,
@ -3601,8 +3523,7 @@
"offset_index_offset": 6907,
"offset_index_length": 11,
"column_index_offset": 5852,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4078,
@ -3635,8 +3556,7 @@
"offset_index_offset": 6918,
"offset_index_length": 11,
"column_index_offset": 5867,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4127,
@ -3669,8 +3589,7 @@
"offset_index_offset": 6929,
"offset_index_length": 11,
"column_index_offset": 5882,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4176,
@ -3703,8 +3622,7 @@
"offset_index_offset": 6940,
"offset_index_length": 11,
"column_index_offset": 5897,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4225,
@ -3737,8 +3655,7 @@
"offset_index_offset": 6951,
"offset_index_length": 11,
"column_index_offset": 5912,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4274,
@ -3771,8 +3688,7 @@
"offset_index_offset": 6962,
"offset_index_length": 11,
"column_index_offset": 5927,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4323,
@ -3805,8 +3721,7 @@
"offset_index_offset": 6973,
"offset_index_length": 11,
"column_index_offset": 5942,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4374,
@ -3839,8 +3754,7 @@
"offset_index_offset": 6984,
"offset_index_length": 11,
"column_index_offset": 5957,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4425,
@ -3873,8 +3787,7 @@
"offset_index_offset": 6995,
"offset_index_length": 11,
"column_index_offset": 5972,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4476,
@ -3911,8 +3824,7 @@
"offset_index_offset": 7006,
"offset_index_length": 11,
"column_index_offset": 5987,
"column_index_length": 17,
"crypto_metadata": 17
"column_index_length": 17
},
{
"file_offset": 4525,
@ -3949,8 +3861,7 @@
"offset_index_offset": 7017,
"offset_index_length": 11,
"column_index_offset": 6004,
"column_index_length": 17,
"crypto_metadata": 17
"column_index_length": 17
},
{
"file_offset": 4574,
@ -3983,8 +3894,7 @@
"offset_index_offset": 7028,
"offset_index_length": 11,
"column_index_offset": 6021,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
},
{
"file_offset": 4623,
@ -4017,8 +3927,7 @@
"offset_index_offset": 7039,
"offset_index_length": 11,
"column_index_offset": 6036,
"column_index_length": 15,
"crypto_metadata": 15
"column_index_length": 15
}
],
"total_byte_size": 2976,

1
test/files/boolean_rle.metadata.json
View File

@ -14,7 +14,6 @@
{
"column_index_length": 17,
"column_index_offset": 89,
"crypto_metadata": 17,
"file_offset": 47,
"meta_data": {
"codec": "SNAPPY",

51
test/files/delta_encoding_required_column.metadata.json
View File

@ -160,8 +160,7 @@
"offset_index_offset": 9714,
"offset_index_length": 10,
"column_index_offset": 9233,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 54,
@ -196,8 +195,7 @@
"offset_index_offset": 9724,
"offset_index_length": 11,
"column_index_offset": 9256,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 442,
@ -232,8 +230,7 @@
"offset_index_offset": 9735,
"offset_index_length": 12,
"column_index_offset": 9279,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 703,
@ -268,8 +265,7 @@
"offset_index_offset": 9747,
"offset_index_length": 12,
"column_index_offset": 9302,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 1010,
@ -304,8 +300,7 @@
"offset_index_offset": 9759,
"offset_index_length": 12,
"column_index_offset": 9325,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 1257,
@ -340,8 +335,7 @@
"offset_index_offset": 9771,
"offset_index_length": 12,
"column_index_offset": 9348,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 1504,
@ -376,8 +370,7 @@
"offset_index_offset": 9783,
"offset_index_length": 12,
"column_index_offset": 9371,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 1635,
@ -412,8 +405,7 @@
"offset_index_offset": 9795,
"offset_index_length": 12,
"column_index_offset": 9394,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 1750,
@ -448,8 +440,7 @@
"offset_index_offset": 9807,
"offset_index_length": 12,
"column_index_offset": 9417,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 1894,
@ -482,8 +473,7 @@
"offset_index_offset": 9819,
"offset_index_length": 12,
"column_index_offset": 9440,
"column_index_length": 47,
"crypto_metadata": 47
"column_index_length": 47
},
{
"file_offset": 2827,
@ -516,8 +506,7 @@
"offset_index_offset": 9831,
"offset_index_length": 12,
"column_index_offset": 9487,
"column_index_length": 21,
"crypto_metadata": 21
"column_index_length": 21
},
{
"file_offset": 3205,
@ -550,8 +539,7 @@
"offset_index_offset": 9843,
"offset_index_length": 12,
"column_index_offset": 9508,
"column_index_length": 28,
"crypto_metadata": 28
"column_index_length": 28
},
{
"file_offset": 3912,
@ -584,8 +572,7 @@
"offset_index_offset": 9855,
"offset_index_length": 12,
"column_index_offset": 9536,
"column_index_length": 25,
"crypto_metadata": 25
"column_index_length": 25
},
{
"file_offset": 4663,
@ -618,8 +605,7 @@
"offset_index_offset": 9867,
"offset_index_length": 12,
"column_index_offset": 9561,
"column_index_length": 17,
"crypto_metadata": 17
"column_index_length": 17
},
{
"file_offset": 4817,
@ -652,8 +638,7 @@
"offset_index_offset": 9879,
"offset_index_length": 12,
"column_index_offset": 9578,
"column_index_length": 43,
"crypto_metadata": 43
"column_index_length": 43
},
{
"file_offset": 5971,
@ -686,8 +671,7 @@
"offset_index_offset": 9891,
"offset_index_length": 12,
"column_index_offset": 9621,
"column_index_length": 64,
"crypto_metadata": 64
"column_index_length": 64
},
{
"file_offset": 8828,
@ -720,8 +704,7 @@
"offset_index_offset": 9903,
"offset_index_length": 13,
"column_index_offset": 9685,
"column_index_length": 29,
"crypto_metadata": 29
"column_index_length": 29
}
],
"total_byte_size": 9229,

3
test/files/dict-page-offset-zero.metadata.json
View File

@ -67,8 +67,7 @@
"offset_index_offset": 67,
"offset_index_length": 10,
"column_index_offset": 44,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
}
],
"total_byte_size": 180,

15
test/files/duckdb5533.metadata.json
View File

@ -83,8 +83,7 @@
"offset_index_offset": 656,
"offset_index_length": 11,
"column_index_offset": 531,
"column_index_length": 31,
"crypto_metadata": 31
"column_index_length": 31
},
{
"file_offset": 197,
@ -111,8 +110,7 @@
"offset_index_offset": 667,
"offset_index_length": 12,
"column_index_offset": 562,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 279,
@ -137,8 +135,7 @@
"offset_index_offset": 679,
"offset_index_length": 11,
"column_index_offset": 585,
"column_index_length": 17,
"crypto_metadata": 17
"column_index_length": 17
},
{
"file_offset": 383,
@ -165,8 +162,7 @@
"offset_index_offset": 690,
"offset_index_length": 12,
"column_index_offset": 602,
"column_index_length": 31,
"crypto_metadata": 31
"column_index_length": 31
},
{
"file_offset": 492,
@ -193,8 +189,7 @@
"offset_index_offset": 702,
"offset_index_length": 11,
"column_index_offset": 633,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
}
],
"total_byte_size": 294,

6
test/files/incorrect_map_schema.metadata.json
View File

@ -77,8 +77,7 @@
"offset_index_offset": 198,
"offset_index_length": 11,
"column_index_offset": 145,
"column_index_length": 25,
"crypto_metadata": 25
"column_index_length": 25
},
{
"file_offset": 73,
@ -114,8 +113,7 @@
"offset_index_offset": 209,
"offset_index_length": 12,
"column_index_offset": 170,
"column_index_length": 28,
"crypto_metadata": 28
"column_index_length": 28
}
],
"total_byte_size": 111,

135
test/files/issue23.metadata.json
View File

@ -615,8 +615,7 @@
"offset_index_offset": 5551,
"offset_index_length": 13,
"column_index_offset": 3575,
"column_index_length": 39,
"crypto_metadata": 39
"column_index_length": 39
},
{
"file_offset": 46,
@ -659,8 +658,7 @@
"offset_index_offset": 5564,
"offset_index_length": 13,
"column_index_offset": 3614,
"column_index_length": 29,
"crypto_metadata": 29
"column_index_length": 29
},
{
"file_offset": 88,
@ -703,8 +701,7 @@
"offset_index_offset": 5577,
"offset_index_length": 14,
"column_index_offset": 3643,
"column_index_length": 45,
"crypto_metadata": 45
"column_index_length": 45
},
{
"file_offset": 138,
@ -753,8 +750,7 @@
"offset_index_offset": 5591,
"offset_index_length": 14,
"column_index_offset": 3688,
"column_index_length": 41,
"crypto_metadata": 41
"column_index_length": 41
},
{
"file_offset": 223,
@ -804,8 +800,7 @@
"offset_index_offset": 5605,
"offset_index_length": 14,
"column_index_offset": 3729,
"column_index_length": 31,
"crypto_metadata": 31
"column_index_length": 31
},
{
"file_offset": 262,
@ -847,8 +842,7 @@
"offset_index_offset": 5619,
"offset_index_length": 15,
"column_index_offset": 3760,
"column_index_length": 57,
"crypto_metadata": 57
"column_index_length": 57
},
{
"file_offset": 338,
@ -890,8 +884,7 @@
"offset_index_offset": 5634,
"offset_index_length": 14,
"column_index_offset": 3817,
"column_index_length": 43,
"crypto_metadata": 43
"column_index_length": 43
},
{
"file_offset": 431,
@ -940,8 +933,7 @@
"offset_index_offset": 5648,
"offset_index_length": 14,
"column_index_offset": 3860,
"column_index_length": 33,
"crypto_metadata": 33
"column_index_length": 33
},
{
"file_offset": 536,
@ -990,8 +982,7 @@
"offset_index_offset": 5662,
"offset_index_length": 15,
"column_index_offset": 3893,
"column_index_length": 67,
"crypto_metadata": 67
"column_index_length": 67
},
{
"file_offset": 576,
@ -1046,8 +1037,7 @@
"offset_index_offset": 5677,
"offset_index_length": 14,
"column_index_offset": 3960,
"column_index_length": 45,
"crypto_metadata": 45
"column_index_length": 45
},
{
"file_offset": 663,
@ -1113,8 +1103,7 @@
"offset_index_offset": 5691,
"offset_index_length": 14,
"column_index_offset": 4005,
"column_index_length": 35,
"crypto_metadata": 35
"column_index_length": 35
},
{
"file_offset": 704,
@ -1172,8 +1161,7 @@
"offset_index_offset": 5705,
"offset_index_length": 15,
"column_index_offset": 4040,
"column_index_length": 64,
"crypto_metadata": 64
"column_index_length": 64
},
{
"file_offset": 784,
@ -1229,8 +1217,7 @@
"offset_index_offset": 5720,
"offset_index_length": 11,
"column_index_offset": 4104,
"column_index_length": 31,
"crypto_metadata": 31
"column_index_length": 31
},
{
"file_offset": 842,
@ -1292,8 +1279,7 @@
"offset_index_offset": 5731,
"offset_index_length": 11,
"column_index_offset": 4135,
"column_index_length": 37,
"crypto_metadata": 37
"column_index_length": 37
},
{
"file_offset": 882,
@ -1351,8 +1337,7 @@
"offset_index_offset": 5742,
"offset_index_length": 15,
"column_index_offset": 4172,
"column_index_length": 47,
"crypto_metadata": 47
"column_index_length": 47
},
{
"file_offset": 1046,
@ -1418,8 +1403,7 @@
"offset_index_offset": 5757,
"offset_index_length": 14,
"column_index_offset": 4219,
"column_index_length": 46,
"crypto_metadata": 46
"column_index_length": 46
},
{
"file_offset": 1126,
@ -1488,8 +1472,7 @@
"offset_index_offset": 5771,
"offset_index_length": 14,
"column_index_offset": 4265,
"column_index_length": 38,
"crypto_metadata": 38
"column_index_length": 38
},
{
"file_offset": 1299,
@ -1558,8 +1541,7 @@
"offset_index_offset": 5785,
"offset_index_length": 15,
"column_index_offset": 4303,
"column_index_length": 52,
"crypto_metadata": 52
"column_index_length": 52
},
{
"file_offset": 1376,
@ -1625,8 +1607,7 @@
"offset_index_offset": 5800,
"offset_index_length": 14,
"column_index_offset": 4355,
"column_index_length": 35,
"crypto_metadata": 35
"column_index_length": 35
},
{
"file_offset": 1444,
@ -1693,8 +1674,7 @@
"offset_index_offset": 5814,
"offset_index_length": 11,
"column_index_offset": 4390,
"column_index_length": 40,
"crypto_metadata": 40
"column_index_length": 40
},
{
"file_offset": 1486,
@ -1756,8 +1736,7 @@
"offset_index_offset": 5825,
"offset_index_length": 14,
"column_index_offset": 4430,
"column_index_length": 35,
"crypto_metadata": 35
"column_index_length": 35
},
{
"file_offset": 1569,
@ -1824,8 +1803,7 @@
"offset_index_offset": 5839,
"offset_index_length": 11,
"column_index_offset": 4465,
"column_index_length": 40,
"crypto_metadata": 40
"column_index_length": 40
},
{
"file_offset": 1610,
@ -1885,8 +1863,7 @@
"offset_index_offset": 5850,
"offset_index_length": 14,
"column_index_offset": 4505,
"column_index_length": 33,
"crypto_metadata": 33
"column_index_length": 33
},
{
"file_offset": 1681,
@ -1947,8 +1924,7 @@
"offset_index_offset": 5864,
"offset_index_length": 14,
"column_index_offset": 4538,
"column_index_length": 33,
"crypto_metadata": 33
"column_index_length": 33
},
{
"file_offset": 1721,
@ -2003,8 +1979,7 @@
"offset_index_offset": 5878,
"offset_index_length": 16,
"column_index_offset": 4571,
"column_index_length": 66,
"crypto_metadata": 66
"column_index_length": 66
},
{
"file_offset": 1946,
@ -2065,8 +2040,7 @@
"offset_index_offset": 5894,
"offset_index_length": 14,
"column_index_offset": 4637,
"column_index_length": 33,
"crypto_metadata": 33
"column_index_length": 33
},
{
"file_offset": 2009,
@ -2129,8 +2103,7 @@
"offset_index_offset": 5908,
"offset_index_length": 14,
"column_index_offset": 4670,
"column_index_length": 29,
"crypto_metadata": 29
"column_index_length": 29
},
{
"file_offset": 2071,
@ -2190,8 +2163,7 @@
"offset_index_offset": 5922,
"offset_index_length": 14,
"column_index_offset": 4699,
"column_index_length": 31,
"crypto_metadata": 31
"column_index_length": 31
},
{
"file_offset": 2133,
@ -2255,8 +2227,7 @@
"offset_index_offset": 5936,
"offset_index_length": 11,
"column_index_offset": 4730,
"column_index_length": 38,
"crypto_metadata": 38
"column_index_length": 38
},
{
"file_offset": 2173,
@ -2315,8 +2286,7 @@
"offset_index_offset": 5947,
"offset_index_length": 14,
"column_index_offset": 4768,
"column_index_length": 33,
"crypto_metadata": 33
"column_index_length": 33
},
{
"file_offset": 2244,
@ -2380,8 +2350,7 @@
"offset_index_offset": 5961,
"offset_index_length": 11,
"column_index_offset": 4801,
"column_index_length": 38,
"crypto_metadata": 38
"column_index_length": 38
},
{
"file_offset": 2308,
@ -2448,8 +2417,7 @@
"offset_index_offset": 5972,
"offset_index_length": 14,
"column_index_offset": 4839,
"column_index_length": 33,
"crypto_metadata": 33
"column_index_length": 33
},
{
"file_offset": 2371,
@ -2508,8 +2476,7 @@
"offset_index_offset": 5986,
"offset_index_length": 11,
"column_index_offset": 4872,
"column_index_length": 35,
"crypto_metadata": 35
"column_index_length": 35
},
{
"file_offset": 2410,
@ -2562,8 +2529,7 @@
"offset_index_offset": 5997,
"offset_index_length": 11,
"column_index_offset": 4907,
"column_index_length": 29,
"crypto_metadata": 29
"column_index_length": 29
},
{
"file_offset": 2554,
@ -2622,8 +2588,7 @@
"offset_index_offset": 6008,
"offset_index_length": 15,
"column_index_offset": 4936,
"column_index_length": 49,
"crypto_metadata": 49
"column_index_length": 49
},
{
"file_offset": 2595,
@ -2680,8 +2645,7 @@
"offset_index_offset": 6023,
"offset_index_length": 16,
"column_index_offset": 4985,
"column_index_length": 157,
"crypto_metadata": 157
"column_index_length": 157
},
{
"file_offset": 2759,
@ -2739,8 +2703,7 @@
"offset_index_offset": 6039,
"offset_index_length": 15,
"column_index_offset": 5142,
"column_index_length": 51,
"crypto_metadata": 51
"column_index_length": 51
},
{
"file_offset": 2839,
@ -2798,8 +2761,7 @@
"offset_index_offset": 6054,
"offset_index_length": 16,
"column_index_offset": 5193,
"column_index_length": 159,
"crypto_metadata": 159
"column_index_length": 159
},
{
"file_offset": 3299,
@ -2848,8 +2810,7 @@
"offset_index_offset": 6070,
"offset_index_length": 14,
"column_index_offset": 5352,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 3333,
@ -2900,8 +2861,7 @@
"offset_index_offset": 6084,
"offset_index_length": 14,
"column_index_offset": 5375,
"column_index_length": 29,
"crypto_metadata": 29
"column_index_length": 29
},
{
"file_offset": 3375,
@ -2952,8 +2912,7 @@
"offset_index_offset": 6098,
"offset_index_length": 14,
"column_index_offset": 5404,
"column_index_length": 45,
"crypto_metadata": 45
"column_index_length": 45
},
{
"file_offset": 3425,
@ -3000,8 +2959,7 @@
"offset_index_offset": 6112,
"offset_index_length": 14,
"column_index_offset": 5449,
"column_index_length": 28,
"crypto_metadata": 28
"column_index_length": 28
},
{
"file_offset": 3479,
@ -3046,8 +3004,7 @@
"offset_index_offset": 6126,
"offset_index_length": 14,
"column_index_offset": 5477,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 3509,
@ -3092,8 +3049,7 @@
"offset_index_offset": 6140,
"offset_index_length": 14,
"column_index_offset": 5500,
"column_index_length": 23,
"crypto_metadata": 23
"column_index_length": 23
},
{
"file_offset": 3539,
@ -3140,8 +3096,7 @@
"offset_index_offset": 6154,
"offset_index_length": 14,
"column_index_offset": 5523,
"column_index_length": 28,
"crypto_metadata": 28
"column_index_length": 28
}
],
"total_byte_size": 3571,

6
test/files/plain-dict-uncompressed-checksum.metadata.json
View File

@ -60,8 +60,7 @@
"offset_index_offset": 262,
"offset_index_length": 10,
"column_index_offset": 144,
"column_index_length": 31,
"crypto_metadata": 31
"column_index_length": 31
},
{
"file_offset": 117,
@ -103,8 +102,7 @@
"offset_index_offset": 272,
"offset_index_length": 11,
"column_index_offset": 175,
"column_index_length": 87,
"crypto_metadata": 87
"column_index_length": 87
}
],
"total_byte_size": 140,