asadbek064/hyparquet-compressors
1
Fork 0
mirror of https://github.com/asadbek064/hyparquet-compressors.git synced 2026-01-12 05:36:37 +00:00
Code Issues Actions Projects Releases Wiki Activity

Handle raw gzip deflate stream

Browse Source
This commit is contained in:
Kenny Daniel 2025-03-20 01:21:06 -07:00
parent ab97c0149a
commit a78636f84c
No known key found for this signature in database
GPG Key ID: 90AB653A8CAD7E45
2 changed files with 46 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/gzip.js
View File

@ -57,10 +57,12 @@ function shft(p) {
* @returns {number}
*/
function gzipStart(input, i) {
if (input[i++] !== 31 || input[i++] !== 139 || input[i++] !== 8) throw new Error('invalid gzip data')
// if missing gzip header, assume raw deflate stream
if (input[i++] !== 31 || input[i++] !== 139 || input[i++] !== 8) return 0
const flag = input[i++]
i += 6
if (flag & 4) i += (input[i + 10] | input[i + 11] << 8) + 2
i += 6 // skip header
if (flag & 4) i += (input[i + 10] | input[i + 11] << 8) + 2 // skip extra
// skip name and comment
for (let zs = (flag >> 3 & 1) + (flag >> 4 & 1); zs > 0; zs -= Number(!input[i++]));
return i + (flag & 2)
}

41
test/gzip.test.js
View File

@ -60,4 +60,45 @@ describe('gzip compressor', () => {
const resized = gunzip(input)
expect(resized).toEqual(new Uint8Array([11, 11, 22, 33]))
})
it('gzip raw deflate stream (from avro)', () => {
const input = new Uint8Array([
13, 200, 193, 13, 130, 48, 20, 0,
208, 210, 113, 76, 190, 191, 41, 173,
5, 183, 249, 45, 63, 194, 161, 208,
180, 128, 113, 19, 227, 213, 33, 188,
58, 5, 94, 117, 3, 71, 208, 119, 124,
207, 170, 212, 116, 68, 236, 47, 137,
115, 162, 76, 17, 134, 192, 158, 243,
9, 207, 148, 185, 159, 150, 194, 232,
151, 113, 28, 184, 96, 228, 153, 58,
154, 9, 169, 177, 173, 171, 217, 128,
81, 170, 5, 19, 60, 65, 227, 44,
131, 213, 174, 33, 29, 14, 74, 187,
14, 162, 218, 211, 154, 167, 45, 9,
33, 229, 245, 241, 250, 188, 111, 219,
253, 91, 73, 33, 118, 255, 17, 63,
])
const expected = new Uint8Array([
192, 1, 115, 51, 97, 58, 47, 47,
104, 121, 112, 101, 114, 112, 97, 114,
97, 109, 45, 105, 99, 101, 98, 101,
114, 103, 47, 119, 97, 114, 101, 104,
111, 117, 115, 101, 47, 98, 117, 110,
110, 105, 101, 115, 47, 109, 101, 116,
97, 100, 97, 116, 97, 47, 97, 56,
53, 57, 55, 51, 101, 52, 45, 52,
48, 48, 57, 45, 52, 99, 98, 97,
45, 56, 55, 53, 101, 45, 53, 50,
55, 56, 97, 50, 99, 54, 48, 50,
55, 100, 45, 109, 48, 46, 97, 118,
114, 111, 214, 112, 0, 0, 2, 2,
152, 183, 215, 225, 224, 154, 214, 163,
240, 1, 2, 0, 0, 42, 0, 0,
2, 0,
])
const output = gunzip(input)
expect(output).toEqual(expected)
})
})