hyparquet/test/convert.test.js

import { describe, expect, it } from 'vitest'
import { DEFAULT_PARSERS, convert, parseDecimal, parseFloat16 } from '../src/convert.js'

/**
 * @import {ColumnDecoder, SchemaElement} from '../src/types.js'
 */

describe('convert function', () => {
  const name = 'name'
  const parsers = DEFAULT_PARSERS

  it('returns the same data if converted_type is undefined', () => {
    const data = [1, 2, 3]
    const element = { name }
    expect(convert(data, { element, parsers })).toEqual(data)
  })

  it('converts byte arrays to utf8', () => {
    const data = [new TextEncoder().encode('foo'), new TextEncoder().encode('bar')]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'UTF8' }
    expect(convert(data, { element, parsers })).toEqual(['foo', 'bar'])
  })

  it('converts byte arrays to utf8 default true', () => {
    const data = [new TextEncoder().encode('foo'), new TextEncoder().encode('bar')]
    /** @type {SchemaElement} */
    const element = { name, type: 'BYTE_ARRAY' }
    expect(convert(data, { element, parsers })).toEqual(['foo', 'bar'])
  })

  it('preserves byte arrays utf8=false', () => {
    const data = [new TextEncoder().encode('foo'), new TextEncoder().encode('bar')]
    /** @type {SchemaElement} */
    const element = { name, type: 'BYTE_ARRAY' }
    expect(convert(data, { element, parsers, utf8: false })).toEqual([
      new Uint8Array([102, 111, 111]), new Uint8Array([98, 97, 114]),
    ])
  })

  it('converts numbers to DECIMAL', () => {
    const data = [100, 200]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'DECIMAL' }
    expect(convert(data, { element, parsers })).toEqual([100, 200])
  })

  it('converts numbers to DECIMAL with scale', () => {
    const data = [100, 200]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'DECIMAL', scale: 2 }
    expect(convert(data, { element, parsers })).toEqual([1, 2])
  })

  it('converts bigint to DECIMAL', () => {
    const data = [1000n, 2000n]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'DECIMAL' }
    expect(convert(data, { element, parsers })).toEqual([1000, 2000])
  })

  it('converts bigint to DECIMAL with scale', () => {
    const data = [10n, 20n]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'DECIMAL', scale: 2 }
    expect(convert(data, { element, parsers })).toEqual([0.1, 0.2])
  })

  it('converts byte arrays to DECIMAL', () => {
    const data = [new Uint8Array([0, 0, 0, 100]), new Uint8Array([0, 0, 0, 200])]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'DECIMAL', scale: 0 }
    expect(convert(data, { element, parsers })).toEqual([100, 200])
  })

  it('converts byte array from issue #59 to DECIMAL', () => {
    const data = [new Uint8Array([18, 83, 137, 151, 156, 0])]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'DECIMAL', scale: 10, precision: 14 }
    expect(convert(data, { element, parsers })).toEqual([2015])
  })

  it('converts epoch time to DATE', () => {
    const data = [1, 2] // days since epoch
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'DATE' }
    expect(convert(data, { element, parsers })).toEqual([new Date(86400000), new Date(86400000 * 2)])
  })

  it('converts INT96 to DATE', () => {
    // from alltypes_plain.parquet
    const data = [45284764452596988585705472n, 45284764452597048585705472n]
    /** @type {SchemaElement} */
    const element = { name, type: 'INT96' }
    expect(convert(data, { element, parsers })).toEqual([new Date('2009-03-01T00:00:00.000Z'), new Date('2009-03-01T00:01:00.000Z')])
  })

  it('converts epoch time to TIMESTAMP_MILLIS', () => {
    const data = [1716506900000n, 1716507000000n]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'TIMESTAMP_MILLIS' }
    expect(convert(data, { element, parsers })).toEqual([
      new Date('2024-05-23T23:28:20.000Z'), new Date('2024-05-23T23:30:00.000Z'),
    ])
  })

  it('converts epoch time to TIMESTAMP_MICROS', () => {
    const data = [1716506900000000n, 1716507000000000n]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'TIMESTAMP_MICROS' }
    expect(convert(data, { element, parsers })).toEqual([
      new Date('2024-05-23T23:28:20.000Z'), new Date('2024-05-23T23:30:00.000Z'),
    ])
  })

  it('parses strings to JSON', () => {
    const encoder = new TextEncoder()
    const data = ['{"key": true}', '{"quay": 314}']
      .map(str => encoder.encode(str))
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'JSON' }
    expect(convert(data, { element, parsers })).toEqual([{ key: true }, { quay: 314 }])
  })

  it('converts uint64', () => {
    const data = [100n, -100n]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'UINT_64' }
    expect(convert(data, { element, parsers })).toEqual(new BigUint64Array([100n, 18446744073709551516n]))
  })

  it('converts to float16', () => {
    const data = [new Uint8Array([0x00, 0x3c]), new Uint8Array([0x00, 0x40])]
    /** @type {SchemaElement} */
    const element = { name, logical_type: { type: 'FLOAT16' } }
    expect(convert(data, { element, parsers })).toEqual([1, 2])
  })

  it('converts timestamp with units', () => {
    const data = [1716506900000000n, 1716507000000000n]
    /** @type {SchemaElement} */
    const element = { name, logical_type: { type: 'TIMESTAMP', isAdjustedToUTC: true, unit: 'MICROS' } }
    expect(convert(data, { element, parsers })).toEqual([
      new Date('2024-05-23T23:28:20.000Z'), new Date('2024-05-23T23:30:00.000Z'),
    ])
  })

  it('throws error for BSON conversion', () => {
    const data = [{}]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'BSON' }
    expect(() => convert(data, { element, parsers }))
      .toThrow('parquet bson not supported')
  })

  it('throws error for INTERVAL conversion', () => {
    const data = [{}]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'INTERVAL' }
    expect(() => convert(data, { element, parsers }))
      .toThrow('parquet interval not supported')
  })

  it('respects custom parsers - dateFromDays', () => {
    const data = [1, 2] // days since epoch
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'DATE' }
    /** @type {Pick<ColumnDecoder, "element" | "utf8" | "parsers">} */
    const columnParser = {
      element,
      parsers: {
        ...parsers,
        dateFromDays: days => days,
      },
    }

    expect(convert(data, columnParser)).toEqual([ 1, 2 ])
  })

  it('respects custom parsers - timestampFromMilliseconds', () => {
    const data = [1716506900000n, 1716507000000n]
    /** @type {SchemaElement} */
    const element = { name, converted_type: 'TIMESTAMP_MILLIS' }
    /** @type {Pick<ColumnDecoder, "element" | "utf8" | "parsers">} */
    const columnParser = {
      element,
      parsers: {
        ...parsers,
        timestampFromMilliseconds: millis => Number(millis / 100000n),
      },
    }

    expect(convert(data, columnParser)).toEqual([ 17165069, 17165070 ])
  })

  it('respects custom parsers - timestampFromMicroseconds', () => {
    const data = [1716506900000000n, 1716507000000000n]
    /** @type {SchemaElement} */
    const element = { name, logical_type: { type: 'TIMESTAMP', isAdjustedToUTC: true, unit: 'MICROS' } }
    /** @type {Pick<ColumnDecoder, "element" | "utf8" | "parsers">} */
    const columnParser = {
      element,
      parsers: {
        ...parsers,
        timestampFromMicroseconds: micros => Number(micros / 100000000n),
      },
    }

    expect(convert(data, columnParser)).toEqual([ 17165069, 17165070 ])
  })

  it('respects custom parsers - timestampFromNanoseconds', () => {
    // from alltypes_plain.parquet
    const data = [45284764452596988585705472n, 45284764452597048585705472n]
    /** @type {SchemaElement} */
    const element = { name, type: 'INT96' }
    /** @type {Pick<ColumnDecoder, "element" | "utf8" | "parsers">} */
    const columnParser = {
      element,
      parsers: {
        ...parsers,
        timestampFromNanoseconds: micros => Number(micros / 100000000000n),
      },
    }

    expect(convert(data, columnParser)).toEqual([ 12358656, 12358656 ])
  })
})

describe('parseFloat16', () => {
  it('convert float16 numbers', () => {
    expect(parseFloat16(undefined)).toBe(undefined)
    expect(parseFloat16(new Uint8Array([0x00, 0xbc]))).toBe(-1)
    expect(parseFloat16(new Uint8Array([0x00, 0x00]))).toBe(0)
    expect(parseFloat16(new Uint8Array([0x00, 0x38]))).toBe(0.5)
    expect(parseFloat16(new Uint8Array([0x00, 0x3c]))).toBe(1)
    expect(parseFloat16(new Uint8Array([0x00, 0x40]))).toBe(2)
  })

  it('convert float16 -0', () => {
    expect(parseFloat16(new Uint8Array([0x00, 0x80]))).toBe(-0)
    expect(parseFloat16(new Uint8Array([0x00, 0x80]))).not.toBe(0)
  })

  it('convert float16 Infinity', () => {
    expect(parseFloat16(new Uint8Array([0x00, 0x7c]))).toBe(Infinity)
    expect(parseFloat16(new Uint8Array([0x00, 0xfc]))).toBe(-Infinity)
  })

  it('convert float16 NaN', () => {
    expect(parseFloat16(new Uint8Array([0x00, 0x7e]))).toBeNaN()
    expect(parseFloat16(new Uint8Array([0x01, 0x7e]))).toBeNaN()
  })

  it('convert float16 subnormal number', () => {
    expect(parseFloat16(new Uint8Array([0xff, 0x03])))
      .toBeCloseTo(2 ** -14 * (1023 / 1024), 5)
  })
})

describe('parseDecimal', () => {
  it('should return 0 for an empty Uint8Array', () => {
    const result = parseDecimal(new Uint8Array())
    expect(result).toBe(0)
  })

  it('should parse a single byte', () => {
    const result = parseDecimal(new Uint8Array([42]))
    expect(result).toBe(42)
  })

  it('should parse two bytes in big-endian order', () => {
    const result = parseDecimal(new Uint8Array([1, 0]))
    expect(result).toBe(256)
  })

  it('should parse three bytes', () => {
    const result = parseDecimal(new Uint8Array([1, 2, 3]))
    expect(result).toBe(66051)
  })

  it('should parse -1 as a 32-bit number', () => {
    const result = parseDecimal(new Uint8Array([255, 255, 255, 255]))
    expect(result).toBe(-1)
  })
})