diff --git a/dremel/dremel.js b/dremel/dremel.js deleted file mode 100644 index d1eaae9..0000000 --- a/dremel/dremel.js +++ /dev/null @@ -1,47 +0,0 @@ -import { assembleObjects } from '../src/assemble.js' - -const defs = document.getElementById('defs') -const reps = document.getElementById('reps') -const values = document.getElementById('values') -const valuesWithNulls = document.getElementById('values-with-nulls') -const output = document.getElementById('output') - -// update the output on change -defs.addEventListener('keyup', update) -reps.addEventListener('keyup', update) -values.addEventListener('keyup', update) - -function update() { - let def = defs.value ? defs.value.split(',').map(Number) : [] - const rep = reps.value.split(',').map(Number) - const val = values.value.split(',').map(Number) - const maxDef = Math.max(1, ...def) - const maxRep = Math.max(...rep) - // nullable if any definition level is less than max - const isNullable = def.some(d => d < maxDef) - if (def.length === 0) def = undefined - - // update flattened values with nulls - const withNulls = [] - let valueIndex = 0 - for (let i = 0; i < rep.length; i++) { - if (!isNullable || def[i] === maxDef) withNulls.push(val[valueIndex++]) - else withNulls.push('-') - } - valuesWithNulls.innerText = withNulls.join(', ') - - // update the output - try { - const out = assembleObjects(def, rep, val, isNullable, maxDef, maxRep) - output.innerText = '' - for (const obj of out) { - const row = JSON.stringify(obj, null, 1)?.replace(/\s+/g, ' ') - output.innerText += (row || 'null') + '\n' - } - output.classList.remove('error') - } catch (e) { - output.innerHTML = e.toString() - output.classList.add('error') - } -} -update() diff --git a/dremel/index.html b/dremel/index.html deleted file mode 100644 index 31c1600..0000000 --- a/dremel/index.html +++ /dev/null @@ -1,89 +0,0 @@ - - - - - Dremel assembly - hyparquet - - - - -

Dremel Assembly

-

- Online demo of dremel assembly of lists from definition and repetition levels. -

-

- Google introduced dremel in 2010 as a columnar storage format for nested data. - The format uses repetition levels and definition levels to encode nested data efficiently. - This demo maps definition and repetition levels to assembled lists. -

-

- This demo is developed as a learning and debugging tool as part of hyparquet: a parser for apache parquet files. -

- -
- - -
-
- - -
-
- - -

-    
- -
- -
- -

-    
- - - - diff --git a/package.json b/package.json index 772e269..1c3a1c5 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "eslint-plugin-import": "2.29.1", "eslint-plugin-jsdoc": "48.2.6", "http-server": "14.1.1", - "hyparquet-compressors": "0.1.3", + "hyparquet-compressors": "0.1.4", "typescript": "5.4.5", "vitest": "1.6.0" }