mirror of
https://github.com/asadbek064/hyparquet-compressors.git
synced 2026-01-11 21:26:38 +00:00
Cleanup brotli prefix
This commit is contained in:
Kenny Daniel
parent
820072ca45
commit
c668a53dce
@ -219,15 +219,15 @@ function brotli(input, output) {
|
||||
} else {
|
||||
distance_code = 0
|
||||
}
|
||||
const insert_code = kInsertRangeLut[range_idx] + (cmd_code >> 3 & 7)
|
||||
const copy_code = kCopyRangeLut[range_idx] + (cmd_code & 7)
|
||||
const insert_length = kInsertLengthPrefixCode[insert_code].offset +
|
||||
br.readBits(kInsertLengthPrefixCode[insert_code].nbits)
|
||||
const copy_length = kCopyLengthPrefixCode[copy_code].offset +
|
||||
br.readBits(kCopyLengthPrefixCode[copy_code].nbits)
|
||||
const insertIndex = kInsertRangeLut[range_idx] + (cmd_code >> 3 & 7)
|
||||
const insertPrefix = kInsertLengthPrefixCode[insertIndex]
|
||||
const insertLength = insertPrefix.offset + br.readBits(insertPrefix.nbits)
|
||||
const copyIndex = kCopyRangeLut[range_idx] + (cmd_code & 7)
|
||||
const copyCode = kCopyLengthPrefixCode[copyIndex]
|
||||
const copyLength = copyCode.offset + br.readBits(copyCode.nbits)
|
||||
prev_byte1 = ringbuffer[pos - 1 & ringbuffer_mask]
|
||||
prev_byte2 = ringbuffer[pos - 2 & ringbuffer_mask]
|
||||
for (let j = 0; j < insert_length; j++) {
|
||||
for (let j = 0; j < insertLength; j++) {
|
||||
br.readMoreInput()
|
||||
|
||||
if (block_length[0] === 0) {
|
||||
@ -253,7 +253,7 @@ function brotli(input, output) {
|
||||
}
|
||||
pos++
|
||||
}
|
||||
meta_block_remaining_len -= insert_length
|
||||
meta_block_remaining_len -= insertLength
|
||||
if (meta_block_remaining_len <= 0) break
|
||||
|
||||
if (distance_code < 0) {
|
||||
@ -266,7 +266,7 @@ function brotli(input, output) {
|
||||
dist_context_map_slice = block_type[2] << kDistanceContextBits
|
||||
}
|
||||
block_length[2]--
|
||||
const context = (copy_length > 4 ? 3 : copy_length - 2) & 0xff
|
||||
const context = (copyLength > 4 ? 3 : copyLength - 2) & 0xff
|
||||
const dist_htree_index = dist_context_map[dist_context_map_slice + context]
|
||||
distance_code = readSymbol(hgroup[2].codes, hgroup[2].htrees[dist_htree_index], br)
|
||||
if (distance_code >= num_direct_distance_codes) {
|
||||
@ -295,16 +295,16 @@ function brotli(input, output) {
|
||||
let copy_dst = pos & ringbuffer_mask
|
||||
|
||||
if (distance > max_distance) {
|
||||
if (copy_length >= minDictionaryWordLength && copy_length <= maxDictionaryWordLength) {
|
||||
let offset = offsetsByLength[copy_length]
|
||||
if (copyLength >= minDictionaryWordLength && copyLength <= maxDictionaryWordLength) {
|
||||
let offset = offsetsByLength[copyLength]
|
||||
const word_id = distance - max_distance - 1
|
||||
const shift = sizeBitsByLength[copy_length]
|
||||
const shift = sizeBitsByLength[copyLength]
|
||||
const mask = (1 << shift) - 1
|
||||
const word_idx = word_id & mask
|
||||
const transform_idx = word_id >> shift
|
||||
offset += word_idx * copy_length
|
||||
offset += word_idx * copyLength
|
||||
if (transform_idx < kNumTransforms) {
|
||||
const len = transformDictionaryWord(ringbuffer, copy_dst, offset, copy_length, transform_idx)
|
||||
const len = transformDictionaryWord(ringbuffer, copy_dst, offset, copyLength, transform_idx)
|
||||
copy_dst += len
|
||||
pos += len
|
||||
meta_block_remaining_len -= len
|
||||
@ -326,11 +326,11 @@ function brotli(input, output) {
|
||||
dist_rb_idx++
|
||||
}
|
||||
|
||||
if (copy_length > meta_block_remaining_len) {
|
||||
if (copyLength > meta_block_remaining_len) {
|
||||
throw new Error('Invalid backward reference')
|
||||
}
|
||||
|
||||
for (let j = 0; j < copy_length; j++) {
|
||||
for (let j = 0; j < copyLength; j++) {
|
||||
ringbuffer[pos & ringbuffer_mask] = ringbuffer[pos - distance & ringbuffer_mask]
|
||||
if ((pos & ringbuffer_mask) === ringbuffer_mask) {
|
||||
output.write(ringbuffer, ringbuffer_size)
|
||||
@ -431,8 +431,8 @@ HuffmanTreeGroup.prototype.decode = function(br) {
|
||||
*/
|
||||
function readBlockLength(table, index, br) {
|
||||
const code = readSymbol(table, index, br)
|
||||
const { nbits } = kBlockLengthPrefixCode[code]
|
||||
return kBlockLengthPrefixCode[code].offset + br.readBits(nbits)
|
||||
const { offset, nbits } = kBlockLengthPrefixCode[code]
|
||||
return offset + br.readBits(nbits)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -1,17 +1,4 @@
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Lookup tables to map prefix codes to value ranges. This is used during
|
||||
decoding of the block lengths, literal insertion lengths and copy lengths.
|
||||
*/
|
||||
@ -19,41 +6,40 @@
|
||||
/**
|
||||
* Represents the range of values belonging to a prefix code:
|
||||
* [offset, offset + 2^nbits)
|
||||
* @param {number} offset
|
||||
* @param {number} nbits
|
||||
* @param {number[]} pair
|
||||
* @returns {{offset: number, nbits: number}}
|
||||
*/
|
||||
function PrefixCodeRange(offset, nbits) {
|
||||
this.offset = offset
|
||||
this.nbits = nbits
|
||||
function prefix([ offset, nbits ]) {
|
||||
return { offset, nbits }
|
||||
}
|
||||
|
||||
export const kBlockLengthPrefixCode = [
|
||||
new PrefixCodeRange(1, 2), new PrefixCodeRange(5, 2), new PrefixCodeRange(9, 2), new PrefixCodeRange(13, 2),
|
||||
new PrefixCodeRange(17, 3), new PrefixCodeRange(25, 3), new PrefixCodeRange(33, 3), new PrefixCodeRange(41, 3),
|
||||
new PrefixCodeRange(49, 4), new PrefixCodeRange(65, 4), new PrefixCodeRange(81, 4), new PrefixCodeRange(97, 4),
|
||||
new PrefixCodeRange(113, 5), new PrefixCodeRange(145, 5), new PrefixCodeRange(177, 5), new PrefixCodeRange(209, 5),
|
||||
new PrefixCodeRange(241, 6), new PrefixCodeRange(305, 6), new PrefixCodeRange(369, 7), new PrefixCodeRange(497, 8),
|
||||
new PrefixCodeRange(753, 9), new PrefixCodeRange(1265, 10), new PrefixCodeRange(2289, 11), new PrefixCodeRange(4337, 12),
|
||||
new PrefixCodeRange(8433, 13), new PrefixCodeRange(16625, 24),
|
||||
]
|
||||
[1, 2], [5, 2], [9, 2], [13, 2],
|
||||
[17, 3], [25, 3], [33, 3], [41, 3],
|
||||
[49, 4], [65, 4], [81, 4], [97, 4],
|
||||
[113, 5], [145, 5], [177, 5], [209, 5],
|
||||
[241, 6], [305, 6], [369, 7], [497, 8],
|
||||
[753, 9], [1265, 10], [2289, 11], [4337, 12],
|
||||
[8433, 13], [16625, 24],
|
||||
].map(prefix)
|
||||
|
||||
export const kInsertLengthPrefixCode = [
|
||||
new PrefixCodeRange(0, 0), new PrefixCodeRange(1, 0), new PrefixCodeRange(2, 0), new PrefixCodeRange(3, 0),
|
||||
new PrefixCodeRange(4, 0), new PrefixCodeRange(5, 0), new PrefixCodeRange(6, 1), new PrefixCodeRange(8, 1),
|
||||
new PrefixCodeRange(10, 2), new PrefixCodeRange(14, 2), new PrefixCodeRange(18, 3), new PrefixCodeRange(26, 3),
|
||||
new PrefixCodeRange(34, 4), new PrefixCodeRange(50, 4), new PrefixCodeRange(66, 5), new PrefixCodeRange(98, 5),
|
||||
new PrefixCodeRange(130, 6), new PrefixCodeRange(194, 7), new PrefixCodeRange(322, 8), new PrefixCodeRange(578, 9),
|
||||
new PrefixCodeRange(1090, 10), new PrefixCodeRange(2114, 12), new PrefixCodeRange(6210, 14), new PrefixCodeRange(22594, 24),
|
||||
]
|
||||
[0, 0], [1, 0], [2, 0], [3, 0],
|
||||
[4, 0], [5, 0], [6, 1], [8, 1],
|
||||
[10, 2], [14, 2], [18, 3], [26, 3],
|
||||
[34, 4], [50, 4], [66, 5], [98, 5],
|
||||
[130, 6], [194, 7], [322, 8], [578, 9],
|
||||
[1090, 10], [2114, 12], [6210, 14], [22594, 24],
|
||||
].map(prefix)
|
||||
|
||||
export const kCopyLengthPrefixCode = [
|
||||
new PrefixCodeRange(2, 0), new PrefixCodeRange(3, 0), new PrefixCodeRange(4, 0), new PrefixCodeRange(5, 0),
|
||||
new PrefixCodeRange(6, 0), new PrefixCodeRange(7, 0), new PrefixCodeRange(8, 0), new PrefixCodeRange(9, 0),
|
||||
new PrefixCodeRange(10, 1), new PrefixCodeRange(12, 1), new PrefixCodeRange(14, 2), new PrefixCodeRange(18, 2),
|
||||
new PrefixCodeRange(22, 3), new PrefixCodeRange(30, 3), new PrefixCodeRange(38, 4), new PrefixCodeRange(54, 4),
|
||||
new PrefixCodeRange(70, 5), new PrefixCodeRange(102, 5), new PrefixCodeRange(134, 6), new PrefixCodeRange(198, 7),
|
||||
new PrefixCodeRange(326, 8), new PrefixCodeRange(582, 9), new PrefixCodeRange(1094, 10), new PrefixCodeRange(2118, 24),
|
||||
]
|
||||
[2, 0], [3, 0], [4, 0], [5, 0],
|
||||
[6, 0], [7, 0], [8, 0], [9, 0],
|
||||
[10, 1], [12, 1], [14, 2], [18, 2],
|
||||
[22, 3], [30, 3], [38, 4], [54, 4],
|
||||
[70, 5], [102, 5], [134, 6], [198, 7],
|
||||
[326, 8], [582, 9], [1094, 10], [2118, 24],
|
||||
].map(prefix)
|
||||
|
||||
export const kInsertRangeLut = [
|
||||
0, 0, 8, 8, 0, 16, 8, 16, 16,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user