mirror of
https://github.com/asadbek064/hyparquet.git
synced 2025-12-05 22:41:55 +00:00
Add minSize parameter to cachedAsyncBuffer
This commit is contained in:
parent
f37b2aea9f
commit
95c47f243d
13
src/utils.js
13
src/utils.js
@ -166,9 +166,20 @@ function readStreamToArrayBuffer(input) {
|
||||
* that are read multiple times, possibly over a network.
|
||||
*
|
||||
* @param {AsyncBuffer} file file-like object to cache
|
||||
* @param {{ minSize?: number }} [options]
|
||||
* @returns {AsyncBuffer} cached file-like object
|
||||
*/
|
||||
export function cachedAsyncBuffer({ byteLength, slice }) {
|
||||
export function cachedAsyncBuffer({ byteLength, slice }, { minSize = 50000 } = {}) {
|
||||
if (byteLength < minSize) {
|
||||
// Cache whole file if it's small
|
||||
const buffer = slice(0, byteLength)
|
||||
return {
|
||||
byteLength,
|
||||
async slice(start, end) {
|
||||
return (await buffer).slice(start, end)
|
||||
},
|
||||
}
|
||||
}
|
||||
const cache = new Map()
|
||||
return {
|
||||
byteLength,
|
||||
|
||||
@ -11,10 +11,10 @@ describe('cachedAsyncBuffer', () => {
|
||||
const buffer = new ArrayBuffer(end - start)
|
||||
return buffer
|
||||
})
|
||||
const cachedFile = cachedAsyncBuffer({
|
||||
byteLength: 1000,
|
||||
slice,
|
||||
})
|
||||
const cachedFile = cachedAsyncBuffer(
|
||||
{ byteLength: 1000, slice },
|
||||
{ minSize: 0 }
|
||||
)
|
||||
|
||||
// Test cache miss
|
||||
const slice1 = await cachedFile.slice(0, 100)
|
||||
@ -33,12 +33,52 @@ describe('cachedAsyncBuffer', () => {
|
||||
|
||||
// Test cache hit for suffix-range
|
||||
const slice4 = await cachedFile.slice(-100)
|
||||
expect(slice).toHaveBeenCalledTimes(2) // Still no additional call
|
||||
expect(slice4).toBe(slice3) // Cached result reused
|
||||
expect(slice).toHaveBeenCalledTimes(2)
|
||||
expect(slice4).toBe(slice3)
|
||||
|
||||
// Verify that asking for the same end implicitly gets from cache
|
||||
const slice5 = await cachedFile.slice(900, 1000)
|
||||
expect(slice).toHaveBeenCalledTimes(2) // Still no additional call
|
||||
expect(slice5).toBe(slice3) // Cached result reused
|
||||
expect(slice).toHaveBeenCalledTimes(2)
|
||||
expect(slice5).toBe(slice3)
|
||||
})
|
||||
|
||||
it('caches whole file if it is smaller than minSize', async () => {
|
||||
const slice = vi.fn(async (start, end) => {
|
||||
// Simulate an async slice operation
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
if (end === undefined) end = 1000
|
||||
if (start < 0) start = Math.max(0, 1000 + start)
|
||||
const buffer = new ArrayBuffer(end - start)
|
||||
return buffer
|
||||
})
|
||||
const cachedFile = cachedAsyncBuffer({ byteLength: 1000, slice })
|
||||
|
||||
// Test cache miss
|
||||
const slice1 = await cachedFile.slice(0, 100)
|
||||
expect(slice).toHaveBeenCalledTimes(1)
|
||||
expect(slice1.byteLength).toBe(100)
|
||||
|
||||
// Test cache hit for the same range
|
||||
const slice2 = await cachedFile.slice(0, 100)
|
||||
expect(slice).toHaveBeenCalledTimes(1) // No additional call
|
||||
expect(slice2).toEqual(slice1) // Same data
|
||||
expect(slice2).not.toBe(slice1) // Different object
|
||||
|
||||
// Test cache with undefined end, should use byteLength as end
|
||||
const slice3 = await cachedFile.slice(900)
|
||||
expect(slice).toHaveBeenCalledTimes(1)
|
||||
expect(slice3.byteLength).toBe(100)
|
||||
|
||||
// Test cache hit for suffix-range
|
||||
const slice4 = await cachedFile.slice(-100)
|
||||
expect(slice).toHaveBeenCalledTimes(1)
|
||||
expect(slice4).toEqual(slice3)
|
||||
expect(slice4).not.toBe(slice3)
|
||||
|
||||
// Verify that asking for the same end implicitly gets from cache
|
||||
const slice5 = await cachedFile.slice(900, 1000)
|
||||
expect(slice).toHaveBeenCalledTimes(1)
|
||||
expect(slice5).toEqual(slice3)
|
||||
expect(slice5).not.toBe(slice3)
|
||||
})
|
||||
})
|
||||
|
||||
Loading…
Reference in New Issue
Block a user