diff --git a/package.json b/package.json index cba229e..1142709 100644 --- a/package.json +++ b/package.json @@ -27,13 +27,13 @@ }, "devDependencies": { "@babel/eslint-parser": "7.24.5", - "@types/node": "20.12.11", + "@types/node": "20.12.12", "@types/pako": "2.0.3", "@vitest/coverage-v8": "1.6.0", "eslint": "8.57.0", "eslint-plugin-import": "2.29.1", - "eslint-plugin-jsdoc": "48.2.3", - "hyparquet": "0.9.0", + "eslint-plugin-jsdoc": "48.2.5", + "hyparquet": "0.9.3", "pako": "2.1.0", "typescript": "5.4.5", "vitest": "1.6.0" diff --git a/src/index.js b/src/index.js index 8aa96c7..d671fb6 100644 --- a/src/index.js +++ b/src/index.js @@ -6,7 +6,7 @@ import pako from 'pako' */ export const compressors = { SNAPPY: snappyUncompressor(), - GZIP: pako.ungzip, + GZIP: (/** @type {Uint8Array} */ input) => pako.ungzip(input), BROTLI: () => new Uint8Array(), // TODO ZSTD: () => new Uint8Array(), // TODO } diff --git a/test/files/concatenated_gzip_members.json b/test/files/concatenated_gzip_members.json new file mode 100644 index 0000000..6f9ddd2 --- /dev/null +++ b/test/files/concatenated_gzip_members.json @@ -0,0 +1,54 @@ +[ + [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], + [11], [12], [13], [14], [15], [16], [17], [18], [19], [20], + [21], [22], [23], [24], [25], [26], [27], [28], [29], [30], + [31], [32], [33], [34], [35], [36], [37], [38], [39], [40], + [41], [42], [43], [44], [45], [46], [47], [48], [49], [50], + [51], [52], [53], [54], [55], [56], [57], [58], [59], [60], + [61], [62], [63], [64], [65], [66], [67], [68], [69], [70], + [71], [72], [73], [74], [75], [76], [77], [78], [79], [80], + [81], [82], [83], [84], [85], [86], [87], [88], [89], [90], + [91], [92], [93], [94], [95], [96], [97], [98], [99], [100], + [101], [102], [103], [104], [105], [106], [107], [108], [109], [110], + [111], [112], [113], [114], [115], [116], [117], [118], [119], [120], + [121], [122], [123], [124], [125], [126], [127], [128], [129], [130], + [131], [132], [133], [134], [135], [136], [137], [138], [139], [140], + [141], [142], [143], [144], [145], [146], [147], [148], [149], [150], + [151], [152], [153], [154], [155], [156], [157], [158], [159], [160], + [161], [162], [163], [164], [165], [166], [167], [168], [169], [170], + [171], [172], [173], [174], [175], [176], [177], [178], [179], [180], + [181], [182], [183], [184], [185], [186], [187], [188], [189], [190], + [191], [192], [193], [194], [195], [196], [197], [198], [199], [200], + [201], [202], [203], [204], [205], [206], [207], [208], [209], [210], + [211], [212], [213], [214], [215], [216], [217], [218], [219], [220], + [221], [222], [223], [224], [225], [226], [227], [228], [229], [230], + [231], [232], [233], [234], [235], [236], [237], [238], [239], [240], + [241], [242], [243], [244], [245], [246], [247], [248], [249], [250], + [251], [252], [253], [254], [255], [256], [257], [258], [259], [260], + [261], [262], [263], [264], [265], [266], [267], [268], [269], [270], + [271], [272], [273], [274], [275], [276], [277], [278], [279], [280], + [281], [282], [283], [284], [285], [286], [287], [288], [289], [290], + [291], [292], [293], [294], [295], [296], [297], [298], [299], [300], + [301], [302], [303], [304], [305], [306], [307], [308], [309], [310], + [311], [312], [313], [314], [315], [316], [317], [318], [319], [320], + [321], [322], [323], [324], [325], [326], [327], [328], [329], [330], + [331], [332], [333], [334], [335], [336], [337], [338], [339], [340], + [341], [342], [343], [344], [345], [346], [347], [348], [349], [350], + [351], [352], [353], [354], [355], [356], [357], [358], [359], [360], + [361], [362], [363], [364], [365], [366], [367], [368], [369], [370], + [371], [372], [373], [374], [375], [376], [377], [378], [379], [380], + [381], [382], [383], [384], [385], [386], [387], [388], [389], [390], + [391], [392], [393], [394], [395], [396], [397], [398], [399], [400], + [401], [402], [403], [404], [405], [406], [407], [408], [409], [410], + [411], [412], [413], [414], [415], [416], [417], [418], [419], [420], + [421], [422], [423], [424], [425], [426], [427], [428], [429], [430], + [431], [432], [433], [434], [435], [436], [437], [438], [439], [440], + [441], [442], [443], [444], [445], [446], [447], [448], [449], [450], + [451], [452], [453], [454], [455], [456], [457], [458], [459], [460], + [461], [462], [463], [464], [465], [466], [467], [468], [469], [470], + [471], [472], [473], [474], [475], [476], [477], [478], [479], [480], + [481], [482], [483], [484], [485], [486], [487], [488], [489], [490], + [491], [492], [493], [494], [495], [496], [497], [498], [499], [500], + [501], [502], [503], [504], [505], [506], [507], [508], [509], [510], + [511], [512], [513] +] diff --git a/test/files/concatenated_gzip_members.parquet b/test/files/concatenated_gzip_members.parquet new file mode 100644 index 0000000..9cdd999 Binary files /dev/null and b/test/files/concatenated_gzip_members.parquet differ diff --git a/test/gzip.test.js b/test/gzip.test.js new file mode 100644 index 0000000..2547651 --- /dev/null +++ b/test/gzip.test.js @@ -0,0 +1,17 @@ +import fs from 'fs' +import { parquetRead, toJson } from 'hyparquet' +import { describe, expect, it } from 'vitest' +import { compressors } from '../src/index.js' + +describe('gzip compressor', () => { + it('should read gzip compressed file', async () => { + const buffer = fs.readFileSync('test/files/concatenated_gzip_members.parquet') + const file = new Uint8Array(buffer).buffer + const expected = fs.readFileSync('test/files/concatenated_gzip_members.json').toString() + + await parquetRead({ file, compressors, onComplete: data => { + expect(data.length).toBe(513) + expect(toJson(data)).toEqual(JSON.parse(expected)) + } }) + }) +})