mirror of
https://github.com/asadbek064/hyparquet.git
synced 2026-01-03 18:26:38 +00:00
Use .set in case of unaligned read
This commit is contained in:
Kenny Daniel
parent
5412ff9bb7
commit
5291ff1072
286
src/plain.js
286
src/plain.js
@ -1,146 +1,7 @@
|
||||
/**
|
||||
* Read `count` boolean values.
|
||||
*
|
||||
* @typedef {import("./types.d.ts").DataReader} DataReader
|
||||
* @param {DataReader} reader - buffer to read data from
|
||||
* @param {number} count - number of values to read
|
||||
* @returns {boolean[]} array of boolean values
|
||||
*/
|
||||
function readPlainBoolean(reader, count) {
|
||||
const values = new Array(count)
|
||||
for (let i = 0; i < count; i++) {
|
||||
const byteOffset = reader.offset + Math.floor(i / 8)
|
||||
const bitOffset = i % 8
|
||||
const byte = reader.view.getUint8(byteOffset)
|
||||
values[i] = (byte & (1 << bitOffset)) !== 0
|
||||
}
|
||||
reader.offset += Math.ceil(count / 8)
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` int32 values.
|
||||
*
|
||||
* @param {DataReader} reader - buffer to read data from
|
||||
* @param {number} count - number of values to read
|
||||
* @returns {Int32Array} array of int32 values
|
||||
*/
|
||||
function readPlainInt32(reader, count) {
|
||||
if ((reader.view.byteOffset + reader.offset) % 4 === 0) {
|
||||
const values = new Int32Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count)
|
||||
reader.offset += count * 4
|
||||
return values
|
||||
}
|
||||
const values = new Int32Array(count)
|
||||
for (let i = 0; i < count; i++) {
|
||||
values[i] = reader.view.getInt32(reader.offset + i * 4, true)
|
||||
}
|
||||
reader.offset += count * 4
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` int64 values.
|
||||
*
|
||||
* @param {DataReader} reader - buffer to read data from
|
||||
* @param {number} count - number of values to read
|
||||
* @returns {BigInt64Array} array of int64 values
|
||||
*/
|
||||
function readPlainInt64(reader, count) {
|
||||
if ((reader.view.byteOffset + reader.offset) % 8 === 0) {
|
||||
const values = new BigInt64Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count)
|
||||
reader.offset += count * 8
|
||||
return values
|
||||
}
|
||||
const values = new BigInt64Array(count)
|
||||
for (let i = 0; i < count; i++) {
|
||||
values[i] = reader.view.getBigInt64(reader.offset + i * 8, true)
|
||||
}
|
||||
reader.offset += count * 8
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` int96 values.
|
||||
*
|
||||
* @param {DataReader} reader - buffer to read data from
|
||||
* @param {number} count - number of values to read
|
||||
* @returns {bigint[]} array of int96 values
|
||||
*/
|
||||
function readPlainInt96(reader, count) {
|
||||
const values = new Array(count)
|
||||
for (let i = 0; i < count; i++) {
|
||||
const low = reader.view.getBigInt64(reader.offset + i * 12, true)
|
||||
const high = reader.view.getInt32(reader.offset + i * 12 + 8, true)
|
||||
values[i] = (BigInt(high) << BigInt(32)) | low
|
||||
}
|
||||
reader.offset += count * 12
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` float values.
|
||||
*
|
||||
* @param {DataReader} reader - buffer to read data from
|
||||
* @param {number} count - number of values to read
|
||||
* @returns {Float32Array} array of float values
|
||||
*/
|
||||
function readPlainFloat(reader, count) {
|
||||
const values = new Float32Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count)
|
||||
reader.offset += count * 4
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` double values.
|
||||
*
|
||||
* @param {DataReader} reader - buffer to read data from
|
||||
* @param {number} count - number of values to read
|
||||
* @returns {Float64Array} array of double values
|
||||
*/
|
||||
function readPlainDouble(reader, count) {
|
||||
const values = new Float64Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count)
|
||||
reader.offset += count * 8
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` byte array values.
|
||||
*
|
||||
* @param {DataReader} reader - buffer to read data from
|
||||
* @param {number} count - number of values to read
|
||||
* @returns {Uint8Array[]} array of byte arrays
|
||||
*/
|
||||
function readPlainByteArray(reader, count) {
|
||||
const values = new Array(count)
|
||||
for (let i = 0; i < count; i++) {
|
||||
const length = reader.view.getInt32(reader.offset, true)
|
||||
reader.offset += 4
|
||||
values[i] = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length)
|
||||
reader.offset += length
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a fixed length byte array.
|
||||
*
|
||||
* @param {DataReader} reader - buffer to read data from
|
||||
* @param {number} fixedLength - length of each fixed length byte array
|
||||
* @returns {Uint8Array} array of fixed length byte arrays
|
||||
*/
|
||||
function readPlainByteArrayFixed(reader, fixedLength) {
|
||||
reader.offset += fixedLength
|
||||
return new Uint8Array(
|
||||
reader.view.buffer,
|
||||
reader.view.byteOffset + reader.offset - fixedLength,
|
||||
fixedLength
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` values of the given type from the reader.view.
|
||||
*
|
||||
* @typedef {import("./types.d.ts").DataReader} DataReader
|
||||
* @typedef {import("./types.d.ts").DecodedArray} DecodedArray
|
||||
* @typedef {import("./types.d.ts").ParquetType} ParquetType
|
||||
* @param {DataReader} reader - buffer to read data from
|
||||
@ -176,3 +37,148 @@ export function readPlain(reader, type, count, utf8) {
|
||||
throw new Error(`parquet unhandled type: ${type}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` boolean values.
|
||||
*
|
||||
* @param {DataReader} reader
|
||||
* @param {number} count
|
||||
* @returns {boolean[]}
|
||||
*/
|
||||
function readPlainBoolean(reader, count) {
|
||||
const values = new Array(count)
|
||||
for (let i = 0; i < count; i++) {
|
||||
const byteOffset = reader.offset + Math.floor(i / 8)
|
||||
const bitOffset = i % 8
|
||||
const byte = reader.view.getUint8(byteOffset)
|
||||
values[i] = (byte & (1 << bitOffset)) !== 0
|
||||
}
|
||||
reader.offset += Math.ceil(count / 8)
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` int32 values.
|
||||
*
|
||||
* @param {DataReader} reader
|
||||
* @param {number} count
|
||||
* @returns {Int32Array}
|
||||
*/
|
||||
function readPlainInt32(reader, count) {
|
||||
const values = (reader.view.byteOffset + reader.offset) % 4
|
||||
? new Int32Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 4))
|
||||
: new Int32Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count)
|
||||
reader.offset += count * 4
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` int64 values.
|
||||
*
|
||||
* @param {DataReader} reader
|
||||
* @param {number} count
|
||||
* @returns {BigInt64Array}
|
||||
*/
|
||||
function readPlainInt64(reader, count) {
|
||||
const values = (reader.view.byteOffset + reader.offset) % 8
|
||||
? new BigInt64Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 8))
|
||||
: new BigInt64Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count)
|
||||
reader.offset += count * 8
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` int96 values.
|
||||
*
|
||||
* @param {DataReader} reader
|
||||
* @param {number} count
|
||||
* @returns {bigint[]}
|
||||
*/
|
||||
function readPlainInt96(reader, count) {
|
||||
const values = new Array(count)
|
||||
for (let i = 0; i < count; i++) {
|
||||
const low = reader.view.getBigInt64(reader.offset + i * 12, true)
|
||||
const high = reader.view.getInt32(reader.offset + i * 12 + 8, true)
|
||||
values[i] = (BigInt(high) << BigInt(32)) | low
|
||||
}
|
||||
reader.offset += count * 12
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` float values.
|
||||
*
|
||||
* @param {DataReader} reader
|
||||
* @param {number} count
|
||||
* @returns {Float32Array}
|
||||
*/
|
||||
function readPlainFloat(reader, count) {
|
||||
const values = (reader.view.byteOffset + reader.offset) % 4
|
||||
? new Float32Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 4))
|
||||
: new Float32Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count)
|
||||
reader.offset += count * 4
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` double values.
|
||||
*
|
||||
* @param {DataReader} reader
|
||||
* @param {number} count
|
||||
* @returns {Float64Array}
|
||||
*/
|
||||
function readPlainDouble(reader, count) {
|
||||
const values = (reader.view.byteOffset + reader.offset) % 8
|
||||
? new Float64Array(align(reader.view.buffer, reader.view.byteOffset + reader.offset, count * 8))
|
||||
: new Float64Array(reader.view.buffer, reader.view.byteOffset + reader.offset, count)
|
||||
reader.offset += count * 8
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `count` byte array values.
|
||||
*
|
||||
* @param {DataReader} reader
|
||||
* @param {number} count
|
||||
* @returns {Uint8Array[]}
|
||||
*/
|
||||
function readPlainByteArray(reader, count) {
|
||||
const values = new Array(count)
|
||||
for (let i = 0; i < count; i++) {
|
||||
const length = reader.view.getInt32(reader.offset, true)
|
||||
reader.offset += 4
|
||||
values[i] = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length)
|
||||
reader.offset += length
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a fixed length byte array.
|
||||
*
|
||||
* @param {DataReader} reader
|
||||
* @param {number} fixedLength
|
||||
* @returns {Uint8Array}
|
||||
*/
|
||||
function readPlainByteArrayFixed(reader, fixedLength) {
|
||||
reader.offset += fixedLength
|
||||
return new Uint8Array(
|
||||
reader.view.buffer,
|
||||
reader.view.byteOffset + reader.offset - fixedLength,
|
||||
fixedLength
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new buffer with the offset and size.
|
||||
*
|
||||
* @param {ArrayBuffer} buffer
|
||||
* @param {number} offset
|
||||
* @param {number} size
|
||||
* @returns {ArrayBuffer}
|
||||
*/
|
||||
function align(buffer, offset, size) {
|
||||
const aligned = new ArrayBuffer(size)
|
||||
new Uint8Array(aligned).set(new Uint8Array(buffer, offset, size))
|
||||
return aligned
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user