diff --git a/.gitignore b/.gitignore index 7157897..3c16c33 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ node_modules/ .log :.prettierrc SheetJSPGExport.xlsx -pres.numbers \ No newline at end of file +pres.numbers diff --git a/SheetJSPG.js b/SheetJSPG.js deleted file mode 100644 index 81ace3f..0000000 --- a/SheetJSPG.js +++ /dev/null @@ -1,173 +0,0 @@ -const pg = require("pg"), format = require("pg-format"); -const XLSX = require("xlsx"); -const opts = { - database:"SheetJSPG", - host: "127.0.0.1", // localhost - port: 5432, - user: "postgres", - password: "7509" -}; - -function deduceType(cells) { - if (!cells || cells.length === 0) return 'text'; - - const nonEmptyCells = cells.filter(cell => cell && cell.v != null); - if (nonEmptyCells.length === 0) return 'text'; - - // Check for dates by looking at both cell type and formatted value - const isDateCell = cell => cell?.t === 'd' || (cell?.t === 'n' && cell.w && /\d{4}-\d{2}-\d{2}|\d{1,2}\/\d{1,2}\/\d{4}|\d{2}-[A-Za-z]{3}-\d{4}|[A-Za-z]{3}-\d{2}|\d{1,2}-[A-Za-z]{3}/.test(cell.w)); - - if (nonEmptyCells.some(isDateCell)) { return 'date'; } - - // Check for booleans - const allBooleans = nonEmptyCells.every(cell => cell.t === 'b'); - if (allBooleans) { return 'boolean'; } - - // Check for numbers - const allNumbers = nonEmptyCells.every(cell => cell.t === 'n' || (cell.t === 's' && !isNaN(cell.v.replace(/[,$\s%()]/g, '')))); - - if (allNumbers) { - const numbers = nonEmptyCells.map(cell => { - if (cell.t === 'n') return cell.v; - return parseFloat(cell.v.replace(/[,$\s%()]/g, '')); - }); - - const needsPrecision = numbers.some(num => { - const str = num.toString(); - return str.includes('e') || - (str.includes('.') && str.split('.')[1].length > 6) || - Math.abs(num) > 1e15; - }); - - return needsPrecision ? 'numeric' : 'double precision'; - } - return 'text'; // default to string type -} - -function parseValue(cell, type) { - if (!cell || cell.v == null) return null; - - switch (type) { - case 'date': - if (cell.t === 'd') { return cell.v.toISOString().split('T')[0]; } - if (cell.t === 'n') { - const date = new Date((cell.v - 25569) * 86400 * 1000); - return date.toISOString().split('T')[0]; - } - return null; - - case 'numeric': - case 'double precision': - if (cell.t === 'n') return cell.v; - if (cell.t === 's') { - const cleaned = cell.v.replace(/[,$\s%()]/g, ''); - if (!isNaN(cleaned)) return parseFloat(cleaned); - } - return null; - - case 'boolean': - return cell.t === 'b' ? cell.v : null; - - default: - return String(cell.v); - } -} - -/* create table and load data given a worksheet and a PostgreSQL client */ -async function sheet_to_pg_table(client, worksheet, tableName) { - if (!worksheet['!ref']) return; - - const range = XLSX.utils.decode_range(worksheet['!ref']); - - /* Extract headers from first row, clean names for PostgreSQL */ - const headers = []; - for (let col = range.s.c; col <= range.e.c; col++) { - const cellAddress = XLSX.utils.encode_cell({ r: range.s.r, c: col }); - const cell = worksheet[cellAddress]; - const headerValue = cell ? String(cell.v).replace(/[^a-zA-Z0-9_]/g, '_') : `column_${col + 1}`; - headers.push(headerValue.toLowerCase()); - } - - /* Group cell values by column for type deduction */ - const columnValues = headers.map(() => []); - for (let row = range.s.r + 1; row <= range.e.r; row++) { - for (let col = range.s.c; col <= range.e.c; col++) { - const cellAddress = XLSX.utils.encode_cell({ r: row, c: col }); - const cell = worksheet[cellAddress]; - columnValues[col].push(cell); - } - } - - /* Deduce PostgreSQL type for each column */ - const types = {}; - headers.forEach((header, idx) => { - types[header] = deduceType(columnValues[idx]); - }); - - /* Delete table if it exists in the DB */ - await client.query(format('DROP TABLE IF EXISTS %I', tableName)); - - /* Create table */ - const createTableSQL = format( - 'CREATE TABLE %I (%s)', - tableName, - headers.map(header => format('%I %s', header, types[header])).join(', ') - ); - await client.query(createTableSQL); - - /* Insert data row by row */ - for (let row = range.s.r + 1; row <= range.e.r; row++) { - const values = headers.map((header, col) => { - const cellAddress = XLSX.utils.encode_cell({ r: row, c: col }); - const cell = worksheet[cellAddress]; - return parseValue(cell, types[header]); - }); - - const insertSQL = format( - 'INSERT INTO %I (%s) VALUES (%s)', - tableName, - headers.map(h => format('%I', h)).join(', '), - values.map(() => '%L').join(', ') - ); - await client.query(format(insertSQL, ...values)); - } -} - -(async() => { - -/* read file and get first worksheet */ -const oldwb = XLSX.readFile("pres.numbers"); -const oldws = oldwb.Sheets[oldwb.SheetNames[0]]; - -/* import data to postgres */ -let client = new pg.Client(opts); -try { - /* open connection to PostgreSQL database */ - await client.connect(); - - /* create table and load data given a worksheet */ - await sheet_to_pg_table(client, oldws, "Presidents"); -} finally { - /* disconnect */ - await client.end(); -} - -/* export data to xlsx */ -client = new pg.Client(opts); -try { - /* open connection to PostgreSQL database */ - await client.connect(); - - /* fetch all data from specified table */ - const res = await client.query(format(`SELECT * FROM %I`, "Presidents")); - - /* export to file */ - const newws = XLSX.utils.json_to_sheet(res.rows); - const newwb = XLSX.utils.book_new(); - XLSX.utils.book_append_sheet(newwb, newws, "Export"); - XLSX.writeFile(newwb, "SheetJSPGExport.xlsx"); -} finally { - /* disconnect */ - await client.end(); -} -})(); \ No newline at end of file diff --git a/sql-types.js b/index.js similarity index 100% rename from sql-types.js rename to index.js diff --git a/test.js b/test.js index 20e1b7c..d551923 100644 --- a/test.js +++ b/test.js @@ -1,6 +1,6 @@ const XLSX = require('xlsx'); const { Client } = require('pg'); -const { sheet_to_pg_table } = require('./sql-types'); +const { sheet_to_pg_table } = require('./index'); const path = require('path'); async function readExcelAndTest(filename, tableName) { @@ -19,7 +19,7 @@ async function readExcelAndTest(filename, tableName) { const client = new Client({ host: 'localhost', database: 'SheetJSPG', - user: 'postgres', + user: 'dark', password: '7509' }); @@ -67,4 +67,4 @@ async function runAllTests() { } } -runAllTests().catch(console.error); \ No newline at end of file +runAllTests().catch(console.error); diff --git a/test_files/boolean_formats.xlsx b/test_files/boolean_formats.xlsx index 80e5da0..a7b35b9 100644 Binary files a/test_files/boolean_formats.xlsx and b/test_files/boolean_formats.xlsx differ diff --git a/test_files/date_formats.xlsx b/test_files/date_formats.xlsx index 61f26ab..58eb0a8 100644 Binary files a/test_files/date_formats.xlsx and b/test_files/date_formats.xlsx differ diff --git a/gen_test_files.py b/test_files/gen_test_files.py similarity index 89% rename from gen_test_files.py rename to test_files/gen_test_files.py index 06a9b1f..0394cbe 100644 --- a/gen_test_files.py +++ b/test_files/gen_test_files.py @@ -2,11 +2,6 @@ import pandas as pd from datetime import datetime import numpy as np import os - -def create_test_directory(): - """Create a directory for test files if it doesn't exist""" - if not os.path.exists('test_files'): - os.makedirs('test_files') def generate_number_formats_test(): """Test Case 1: Common spreadsheet number formats""" @@ -22,8 +17,7 @@ def generate_number_formats_test(): ] }) - # Create Excel writer with xlsxwriter engine - writer = pd.ExcelWriter('test_files/number_formats.xlsx', engine='xlsxwriter') + writer = pd.ExcelWriter('number_formats.xlsx', engine='xlsxwriter') df.to_excel(writer, index=False, sheet_name='Sheet1') # Get workbook and worksheet objects @@ -55,7 +49,7 @@ def generate_date_formats_test(): ] }) - writer = pd.ExcelWriter('test_files/date_formats.xlsx', engine='xlsxwriter') + writer = pd.ExcelWriter('date_formats.xlsx', engine='xlsxwriter') df.to_excel(writer, index=False, sheet_name='Sheet1') workbook = writer.book @@ -98,7 +92,7 @@ def generate_special_values_test(): ] }) - writer = pd.ExcelWriter('test_files/special_values.xlsx', engine='xlsxwriter') + writer = pd.ExcelWriter('special_values.xlsx', engine='xlsxwriter') df.to_excel(writer, index=False, sheet_name='Sheet1') writer.close() @@ -117,7 +111,7 @@ def generate_precision_test(): ] }) - writer = pd.ExcelWriter('test_files/precision.xlsx', engine='xlsxwriter') + writer = pd.ExcelWriter('precision.xlsx', engine='xlsxwriter') df.to_excel(writer, index=False, sheet_name='Sheet1') workbook = writer.book @@ -156,7 +150,7 @@ def generate_string_formats_test(): ] }) - writer = pd.ExcelWriter('test_files/string_formats.xlsx', engine='xlsxwriter') + writer = pd.ExcelWriter('string_formats.xlsx', engine='xlsxwriter') df.to_excel(writer, index=False, sheet_name='Sheet1') workbook = writer.book @@ -179,7 +173,7 @@ def generate_boolean_formats_test(): ] }) - writer = pd.ExcelWriter('test_files/boolean_formats.xlsx', engine='xlsxwriter') + writer = pd.ExcelWriter('boolean_formats.xlsx', engine='xlsxwriter') df.to_excel(writer, index=False, sheet_name='Sheet1') workbook = writer.book @@ -203,9 +197,6 @@ def generate_boolean_formats_test(): def main(): - """Geneate all test Excel files""" - create_test_directory() - print("Generating test Excel files...") generate_number_formats_test() generate_date_formats_test() @@ -213,7 +204,7 @@ def main(): generate_precision_test() generate_string_formats_test() generate_boolean_formats_test() - print("Test files generated in 'test_files' directory") + print("Test files generated") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/test_files/number_formats.xlsx b/test_files/number_formats.xlsx index 8161803..c1d477d 100644 Binary files a/test_files/number_formats.xlsx and b/test_files/number_formats.xlsx differ diff --git a/test_files/precision.xlsx b/test_files/precision.xlsx index c437017..d24693b 100644 Binary files a/test_files/precision.xlsx and b/test_files/precision.xlsx differ diff --git a/requirements.txt b/test_files/requirements.txt similarity index 100% rename from requirements.txt rename to test_files/requirements.txt diff --git a/test_files/special_values.xlsx b/test_files/special_values.xlsx index 56a0a27..e6934e0 100644 Binary files a/test_files/special_values.xlsx and b/test_files/special_values.xlsx differ diff --git a/test_files/string_formats.xlsx b/test_files/string_formats.xlsx index 59fc2fa..3164982 100644 Binary files a/test_files/string_formats.xlsx and b/test_files/string_formats.xlsx differ