623 lines
20 KiB
JavaScript
623 lines
20 KiB
JavaScript
#!/usr/bin/env node
|
||
|
||
/**
|
||
* 🚀 NETGESCON COMPLETE MDB IMPORTER
|
||
* Import completo da database MDB Gescon con tutte le tabelle
|
||
* Basato sulla configurazione esistente del client
|
||
*/
|
||
|
||
const { exec } = require('child_process');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const axios = require('axios');
|
||
|
||
class CompleteMDBImporter {
|
||
constructor(config) {
|
||
this.config = {
|
||
mdbFile: config.mdbFile || '/mnt/gescon/gescon/dbc/Stabili.mdb',
|
||
apiBaseUrl: config.apiBaseUrl || 'http://localhost:8000/api/v1',
|
||
apiToken: config.apiToken,
|
||
batchSize: config.batchSize || 50,
|
||
validateOnly: config.validateOnly || false,
|
||
verbose: config.verbose || false,
|
||
importStep: config.importStep || 'all', // 'stabili', 'condomin', 'all'
|
||
...config
|
||
};
|
||
|
||
// Carica schema tabelle
|
||
this.schema = this.loadSchema();
|
||
|
||
this.results = {
|
||
tables: {},
|
||
totalProcessed: 0,
|
||
totalImported: 0,
|
||
totalErrors: 0,
|
||
startTime: new Date(),
|
||
endTime: null
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 📋 Carica schema tabelle dal file di configurazione
|
||
*/
|
||
loadSchema() {
|
||
const schemaPath = path.join(__dirname, '../01-GESCON/complete-mdb-schema.json');
|
||
|
||
if (fs.existsSync(schemaPath)) {
|
||
const content = fs.readFileSync(schemaPath, 'utf8');
|
||
return JSON.parse(content);
|
||
}
|
||
|
||
// Schema di fallback basato sulla configurazione fornita
|
||
return {
|
||
tables_mapping: {
|
||
stabili: {
|
||
mdb_table: "Stabili",
|
||
priority: 2,
|
||
depends_on: [],
|
||
fields: {
|
||
cod_stabile: { netgescon_field: "cod_stabile_old", required: true },
|
||
denominazione: { netgescon_field: "denominazione", required: true },
|
||
indirizzo: { netgescon_field: "indirizzo", required: true },
|
||
cap: { netgescon_field: "cap", required: true },
|
||
citta: { netgescon_field: "citta", required: true },
|
||
pr: { netgescon_field: "provincia", required: true },
|
||
codice_fisc: { netgescon_field: "codice_fiscale", required: false },
|
||
cf_amministratore: { netgescon_field: "amministratore_cf", required: false },
|
||
num_condomini: { netgescon_field: "totale_unita_immobiliari", required: false },
|
||
num_scale: { netgescon_field: "numero_scale", required: false },
|
||
nome_directory: { netgescon_field: "directory_name", required: true }
|
||
}
|
||
}
|
||
},
|
||
import_sequence: [
|
||
{ step: 1, table: "stabili", description: "Import stabili principali" }
|
||
]
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 🚀 Avvia import completo o selettivo
|
||
*/
|
||
async run() {
|
||
try {
|
||
console.log('🗃️ NETGESCON COMPLETE MDB IMPORTER');
|
||
console.log(`📁 File MDB: ${this.config.mdbFile}`);
|
||
console.log(`🔗 API Base: ${this.config.apiBaseUrl}`);
|
||
console.log(`📦 Batch Size: ${this.config.batchSize}`);
|
||
console.log(`🎯 Import Step: ${this.config.importStep}`);
|
||
|
||
if (this.config.validateOnly) {
|
||
console.log('🔍 MODALITÀ VALIDAZIONE');
|
||
}
|
||
|
||
// 1. Verifica prerequisiti
|
||
await this.checkPrerequisites();
|
||
|
||
// 2. Determina sequenza import
|
||
const sequence = this.getImportSequence();
|
||
console.log(`📋 Tabelle da importare: ${sequence.map(s => s.table).join(', ')}`);
|
||
|
||
// 3. Esegui import sequenziale
|
||
for (const step of sequence) {
|
||
await this.importTable(step.table, step.step);
|
||
}
|
||
|
||
// 4. Mostra risultati finali
|
||
this.showFinalResults();
|
||
|
||
return this.results;
|
||
|
||
} catch (error) {
|
||
console.error('❌ Errore critico:', error.message);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 📝 Determina sequenza import basata su step richiesto
|
||
*/
|
||
getImportSequence() {
|
||
const allSequence = this.schema.import_sequence || [
|
||
{ step: 1, table: "stabili", description: "Import stabili" }
|
||
];
|
||
|
||
if (this.config.importStep === 'all') {
|
||
return allSequence;
|
||
}
|
||
|
||
// Import singola tabella
|
||
return allSequence.filter(s => s.table === this.config.importStep);
|
||
}
|
||
|
||
/**
|
||
* 🔍 Verifica prerequisiti
|
||
*/
|
||
async checkPrerequisites() {
|
||
console.log('\n🔍 Verifica prerequisiti...');
|
||
|
||
// Verifica file MDB
|
||
if (!fs.existsSync(this.config.mdbFile)) {
|
||
throw new Error(`File MDB non trovato: ${this.config.mdbFile}`);
|
||
}
|
||
|
||
// Test mdb-tools
|
||
try {
|
||
await this.execCommand('mdb-tools --version');
|
||
console.log('✅ mdb-tools disponibile');
|
||
} catch (error) {
|
||
throw new Error('mdb-tools non installato. Eseguire: sudo apt-get install mdbtools');
|
||
}
|
||
|
||
// Test API
|
||
if (!this.config.validateOnly) {
|
||
await this.testApiConnection();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 🔗 Test connessione API
|
||
*/
|
||
async testApiConnection() {
|
||
try {
|
||
console.log('🔗 Test connessione API...');
|
||
|
||
const response = await axios.get(`${this.config.apiBaseUrl}/auth/me`, {
|
||
headers: {
|
||
'Authorization': `Bearer ${this.config.apiToken}`,
|
||
'Accept': 'application/json'
|
||
},
|
||
timeout: 10000
|
||
});
|
||
|
||
if (response.data.success) {
|
||
console.log('✅ API connessa - Utente:', response.data.data.user.name);
|
||
} else {
|
||
throw new Error('Risposta API non valida');
|
||
}
|
||
|
||
} catch (error) {
|
||
if (error.response?.status === 401) {
|
||
throw new Error('Token API non valido o scaduto');
|
||
}
|
||
throw new Error(`Connessione API fallita: ${error.message}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 📦 Import singola tabella
|
||
*/
|
||
async importTable(tableName, step) {
|
||
console.log(`\n📦 STEP ${step}: Import tabella ${tableName.toUpperCase()}`);
|
||
|
||
const tableConfig = this.schema.tables_mapping[tableName];
|
||
if (!tableConfig) {
|
||
throw new Error(`Configurazione tabella ${tableName} non trovata`);
|
||
}
|
||
|
||
this.results.tables[tableName] = {
|
||
processed: 0,
|
||
imported: 0,
|
||
errors: 0,
|
||
duplicates: 0,
|
||
details: []
|
||
};
|
||
|
||
try {
|
||
// 1. Leggi dati da MDB
|
||
const data = await this.readTableFromMDB(tableConfig.mdb_table);
|
||
console.log(`📊 Record trovati: ${data.length}`);
|
||
|
||
if (data.length === 0) {
|
||
console.log(`⚠️ Tabella ${tableConfig.mdb_table} vuota`);
|
||
return;
|
||
}
|
||
|
||
// 2. Processa in batch
|
||
const batches = this.createBatches(data, this.config.batchSize);
|
||
|
||
for (let i = 0; i < batches.length; i++) {
|
||
const batch = batches[i];
|
||
console.log(`🔄 Batch ${i + 1}/${batches.length} (${batch.length} record)`);
|
||
|
||
await this.processBatch(tableName, batch, tableConfig);
|
||
}
|
||
|
||
const result = this.results.tables[tableName];
|
||
console.log(`✅ ${tableName}: ${result.imported} importati, ${result.duplicates} duplicati, ${result.errors} errori`);
|
||
|
||
} catch (error) {
|
||
console.error(`❌ Errore import ${tableName}:`, error.message);
|
||
this.results.tables[tableName].errors++;
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 📖 Legge tabella da MDB
|
||
*/
|
||
async readTableFromMDB(mdbTable) {
|
||
try {
|
||
const command = `mdb-export "${this.config.mdbFile}" "${mdbTable}"`;
|
||
const csvOutput = await this.execCommand(command);
|
||
|
||
const lines = csvOutput.trim().split('\n');
|
||
if (lines.length <= 1) {
|
||
return [];
|
||
}
|
||
|
||
const headers = this.parseCsvLine(lines[0]);
|
||
const data = [];
|
||
|
||
for (let i = 1; i < lines.length; i++) {
|
||
if (lines[i].trim()) {
|
||
const values = this.parseCsvLine(lines[i]);
|
||
const record = {};
|
||
|
||
headers.forEach((header, index) => {
|
||
record[header] = values[index] || null;
|
||
});
|
||
|
||
data.push(record);
|
||
}
|
||
}
|
||
|
||
return data;
|
||
|
||
} catch (error) {
|
||
throw new Error(`Lettura tabella ${mdbTable} fallita: ${error.message}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 🔄 Processa batch
|
||
*/
|
||
async processBatch(tableName, batch, tableConfig) {
|
||
const mappedData = batch.map(record => this.mapTableData(record, tableConfig));
|
||
|
||
if (this.config.validateOnly) {
|
||
// Solo validazione
|
||
mappedData.forEach(data => {
|
||
this.results.tables[tableName].processed++;
|
||
const validation = this.validateData(data, tableConfig);
|
||
|
||
if (validation.valid) {
|
||
this.results.tables[tableName].imported++;
|
||
} else {
|
||
this.results.tables[tableName].errors++;
|
||
if (this.config.verbose) {
|
||
console.log(`❌ Validazione fallita:`, validation.errors);
|
||
}
|
||
}
|
||
});
|
||
return;
|
||
}
|
||
|
||
// Import reale via API
|
||
try {
|
||
const endpoint = this.getApiEndpoint(tableName);
|
||
const payload = this.buildApiPayload(tableName, mappedData);
|
||
|
||
const response = await axios.post(
|
||
`${this.config.apiBaseUrl}${endpoint}`,
|
||
payload,
|
||
{
|
||
headers: {
|
||
'Authorization': `Bearer ${this.config.apiToken}`,
|
||
'Content-Type': 'application/json',
|
||
'Accept': 'application/json'
|
||
},
|
||
timeout: 60000
|
||
}
|
||
);
|
||
|
||
if (response.data.success) {
|
||
const result = response.data.data;
|
||
this.results.tables[tableName].processed += batch.length;
|
||
this.results.tables[tableName].imported += result.importati || 0;
|
||
this.results.tables[tableName].duplicates += result.duplicati || 0;
|
||
this.results.tables[tableName].errors += result.errori || 0;
|
||
} else {
|
||
throw new Error(response.data.message || 'Risposta API non valida');
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error(`❌ Errore API ${tableName}:`, error.response?.data?.message || error.message);
|
||
this.results.tables[tableName].errors += batch.length;
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 🗺️ Mapping dati tabella
|
||
*/
|
||
mapTableData(record, tableConfig) {
|
||
const mapped = {};
|
||
|
||
Object.entries(tableConfig.fields).forEach(([mdbField, config]) => {
|
||
const value = record[mdbField];
|
||
const targetField = config.netgescon_field;
|
||
|
||
if (targetField) {
|
||
mapped[targetField] = this.transformValue(value, config);
|
||
}
|
||
});
|
||
|
||
// Aggiungi metadati migrazione
|
||
mapped.migrazione_data = new Date().toISOString().split('T')[0];
|
||
mapped.migrazione_tabella = tableConfig.mdb_table;
|
||
|
||
return mapped;
|
||
}
|
||
|
||
/**
|
||
* 🔄 Trasforma valore secondo tipo campo
|
||
*/
|
||
transformValue(value, config) {
|
||
if (value === null || value === undefined || value === '') {
|
||
return config.default || null;
|
||
}
|
||
|
||
switch (config.type) {
|
||
case 'Integer':
|
||
return parseInt(value) || 0;
|
||
case 'Currency':
|
||
return parseFloat(value) || 0;
|
||
case 'Boolean':
|
||
return this.parseBoolean(value);
|
||
case 'DateTime':
|
||
return this.parseDate(value);
|
||
default:
|
||
return String(value);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* ✅ Validazione dati
|
||
*/
|
||
validateData(data, tableConfig) {
|
||
const errors = [];
|
||
|
||
Object.entries(tableConfig.fields).forEach(([mdbField, config]) => {
|
||
if (config.required) {
|
||
const value = data[config.netgescon_field];
|
||
if (!value) {
|
||
errors.push(`Campo ${config.netgescon_field} obbligatorio`);
|
||
}
|
||
}
|
||
|
||
if (config.validation) {
|
||
const value = data[config.netgescon_field];
|
||
if (value && !this.validateField(value, config.validation)) {
|
||
errors.push(`Campo ${config.netgescon_field} non valido`);
|
||
}
|
||
}
|
||
});
|
||
|
||
return {
|
||
valid: errors.length === 0,
|
||
errors: errors
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 🔗 Endpoint API per tabella
|
||
*/
|
||
getApiEndpoint(tableName) {
|
||
const endpoints = {
|
||
stabili: '/import/stabili',
|
||
condomin: '/import/unita',
|
||
fornitori: '/import/fornitori',
|
||
rate: '/import/rate',
|
||
incassi: '/import/movimenti',
|
||
parti_comuni_amministratore: '/import/amministratori',
|
||
singolo_anno_assemblee: '/import/assemblee'
|
||
};
|
||
|
||
return endpoints[tableName] || `/import/${tableName}`;
|
||
}
|
||
|
||
/**
|
||
* 📦 Costruisce payload API
|
||
*/
|
||
buildApiPayload(tableName, data) {
|
||
const payloads = {
|
||
stabili: {
|
||
bulk_import: true,
|
||
validate_only: this.config.validateOnly,
|
||
stabili: data
|
||
},
|
||
parti_comuni_amministratore: {
|
||
bulk_import: true,
|
||
validate_only: this.config.validateOnly,
|
||
amministratori: data
|
||
}
|
||
};
|
||
|
||
return payloads[tableName] || {
|
||
bulk_import: true,
|
||
validate_only: this.config.validateOnly,
|
||
data: data
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 📊 Risultati finali
|
||
*/
|
||
showFinalResults() {
|
||
this.results.endTime = new Date();
|
||
const duration = ((this.results.endTime - this.results.startTime) / 1000).toFixed(2);
|
||
|
||
console.log('\n📊 RISULTATI FINALI:');
|
||
console.log(`⏱️ Durata: ${duration}s`);
|
||
|
||
Object.entries(this.results.tables).forEach(([table, result]) => {
|
||
console.log(`\n🏷️ ${table.toUpperCase()}:`);
|
||
console.log(` 📋 Processati: ${result.processed}`);
|
||
console.log(` ✅ Importati: ${result.imported}`);
|
||
console.log(` 🔄 Duplicati: ${result.duplicates}`);
|
||
console.log(` ❌ Errori: ${result.errors}`);
|
||
|
||
this.results.totalProcessed += result.processed;
|
||
this.results.totalImported += result.imported;
|
||
this.results.totalErrors += result.errors;
|
||
});
|
||
|
||
console.log(`\n🎯 TOTALI:`);
|
||
console.log(` 📋 Record processati: ${this.results.totalProcessed}`);
|
||
console.log(` ✅ Record importati: ${this.results.totalImported}`);
|
||
console.log(` ❌ Errori totali: ${this.results.totalErrors}`);
|
||
}
|
||
|
||
// 🛠️ UTILITY METHODS
|
||
|
||
createBatches(array, size) {
|
||
const batches = [];
|
||
for (let i = 0; i < array.length; i += size) {
|
||
batches.push(array.slice(i, i + size));
|
||
}
|
||
return batches;
|
||
}
|
||
|
||
parseCsvLine(line) {
|
||
const result = [];
|
||
let current = '';
|
||
let inQuotes = false;
|
||
|
||
for (let i = 0; i < line.length; i++) {
|
||
const char = line[i];
|
||
|
||
if (char === '"') {
|
||
inQuotes = !inQuotes;
|
||
} else if (char === ',' && !inQuotes) {
|
||
result.push(current.trim());
|
||
current = '';
|
||
} else {
|
||
current += char;
|
||
}
|
||
}
|
||
|
||
result.push(current.trim());
|
||
return result;
|
||
}
|
||
|
||
parseBoolean(value, defaultValue = false) {
|
||
if (value === null || value === undefined || value === '') {
|
||
return defaultValue;
|
||
}
|
||
const str = String(value).toLowerCase();
|
||
return str === 'true' || str === '1' || str === 'yes' || str === 'si';
|
||
}
|
||
|
||
parseDate(dateString) {
|
||
if (!dateString) return null;
|
||
try {
|
||
const date = new Date(dateString);
|
||
return date.toISOString().split('T')[0];
|
||
} catch (e) {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
validateField(value, validation) {
|
||
if (validation === 'email') {
|
||
return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value);
|
||
}
|
||
if (typeof validation === 'string' && validation.startsWith('^')) {
|
||
const regex = new RegExp(validation);
|
||
return regex.test(value);
|
||
}
|
||
return true;
|
||
}
|
||
|
||
execCommand(command) {
|
||
return new Promise((resolve, reject) => {
|
||
exec(command, (error, stdout, stderr) => {
|
||
if (error) {
|
||
reject(new Error(`Command failed: ${error.message}`));
|
||
return;
|
||
}
|
||
if (stderr) {
|
||
console.warn('Warning:', stderr);
|
||
}
|
||
resolve(stdout);
|
||
});
|
||
});
|
||
}
|
||
}
|
||
|
||
// 🚀 ESECUZIONE SCRIPT
|
||
if (require.main === module) {
|
||
const args = process.argv.slice(2);
|
||
|
||
const config = {
|
||
mdbFile: '/mnt/gescon/gescon/dbc/Stabili.mdb' // Default dal mapping_config.json
|
||
};
|
||
|
||
// Parsing argomenti
|
||
for (let i = 0; i < args.length; i += 2) {
|
||
const flag = args[i];
|
||
const value = args[i + 1];
|
||
|
||
switch (flag) {
|
||
case '--mdb-file':
|
||
config.mdbFile = value;
|
||
break;
|
||
case '--api-url':
|
||
config.apiBaseUrl = value;
|
||
break;
|
||
case '--api-token':
|
||
config.apiToken = value;
|
||
break;
|
||
case '--batch-size':
|
||
config.batchSize = parseInt(value);
|
||
break;
|
||
case '--step':
|
||
config.importStep = value; // 'stabili', 'condomin', 'all'
|
||
break;
|
||
case '--validate-only':
|
||
config.validateOnly = true;
|
||
i--; // Flag senza valore
|
||
break;
|
||
case '--verbose':
|
||
config.verbose = true;
|
||
i--; // Flag senza valore
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (!config.apiToken && !config.validateOnly) {
|
||
console.log('❌ Token API richiesto per import reale');
|
||
console.log('');
|
||
console.log('Uso: node complete-mdb-importer.js [options]');
|
||
console.log('');
|
||
console.log('Opzioni:');
|
||
console.log(' --mdb-file <path> Path file MDB (default: /mnt/gescon/gescon/dbc/Stabili.mdb)');
|
||
console.log(' --api-token <token> Token autenticazione API');
|
||
console.log(' --step <name> Tabella da importare (stabili, condomin, all)');
|
||
console.log(' --batch-size <size> Dimensione batch (default: 50)');
|
||
console.log(' --validate-only Solo validazione senza import');
|
||
console.log(' --verbose Output dettagliato');
|
||
console.log('');
|
||
console.log('Esempi:');
|
||
console.log(' node complete-mdb-importer.js --step stabili --api-token abc123');
|
||
console.log(' node complete-mdb-importer.js --step all --validate-only');
|
||
process.exit(1);
|
||
}
|
||
|
||
const importer = new CompleteMDBImporter(config);
|
||
|
||
importer.run()
|
||
.then(results => {
|
||
const success = results.totalErrors === 0;
|
||
console.log(success ? '\n🎉 Import completato con successo!' : '\n⚠️ Import completato con errori');
|
||
process.exit(success ? 0 : 1);
|
||
})
|
||
.catch(error => {
|
||
console.error('\n💥 Import fallito:', error.message);
|
||
process.exit(1);
|
||
});
|
||
}
|
||
|
||
module.exports = CompleteMDBImporter;
|