diff --git a/package.json b/package.json index b0164b0..3aaa053 100644 --- a/package.json +++ b/package.json @@ -25,14 +25,12 @@ }, "dependencies": { "hysnappy": "0.3.1", - "lz4": "0.6.5", "pako": "2.1.0" }, "devDependencies": { "@babel/eslint-parser": "7.24.5", "@rollup/plugin-node-resolve": "15.2.3", "@rollup/plugin-terser": "0.4.4", - "@types/lz4": "0.6.4", "@types/node": "20.12.12", "@types/pako": "2.0.3", "@vitest/coverage-v8": "1.6.0", diff --git a/src/index.js b/src/index.js index c0611e9..3d02e80 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,6 @@ import { snappyUncompressor } from 'hysnappy' -import lz4 from 'lz4' import pako from 'pako' +import { LZ4 } from './lz4.js' /** * @type {import('hyparquet').Compressors} @@ -10,9 +10,5 @@ export const compressors = { GZIP: input => pako.ungzip(input), BROTLI: () => new Uint8Array(), // TODO ZSTD: () => new Uint8Array(), // TODO - LZ4: (input, outputLength) => { - const out = Buffer.alloc(outputLength) - lz4.decodeBlock(Buffer.from(input), out) - return out - }, + LZ4, } diff --git a/src/lz4.js b/src/lz4.js new file mode 100644 index 0000000..7844118 --- /dev/null +++ b/src/lz4.js @@ -0,0 +1,39 @@ +/** + * LZ4 decompression + * + * @param {Uint8Array} input + * @param {number} outputLength + * @returns {Uint8Array} + */ +export function LZ4(input, outputLength) { + const output = new Uint8Array(outputLength) + let len = 0 // output position + for (let i = 0; i < input.length;) { + const token = input[i++] + + let literals = token >> 4 + if (literals) { + // literal length + let byte = literals + 240 + while (byte === 255) literals += byte = input[i++] + // copy literals + output.set(input.subarray(i, i + literals), len) + len += literals + i += literals + if (i >= input.length) return output + } + + const offset = input[i++] | input[i++] << 8 + if (!offset || offset > len) throw new Error('lz4 offset out of range') + // match length + let matchLength = (token & 0xf) + 4 + let byte = matchLength + 240 + while (byte === 255) matchLength += byte = input[i++] + // copy match + let pos = len - offset + const end = len + matchLength + while (len < end) output[len++] = output[pos++] + } + + return output +}