62 lines
2.2 KiB
Python
62 lines
2.2 KiB
Python
import os
|
|
import pandas as pd
|
|
from sqlalchemy import create_engine, Column, Integer, String, Table, MetaData, inspect, text
|
|
from sqlalchemy.dialects.mysql import LONGTEXT
|
|
|
|
DB_USER = "laravel_user"
|
|
DB_PWD = "P4ssw0rd.96!"
|
|
DB_HOST = "127.0.0.1"
|
|
DB_DB = "laravel_db"
|
|
OUT_PATH = "estratti_serializzati"
|
|
|
|
engine = create_engine(f"mysql+pymysql://{DB_USER}:{DB_PWD}@{DB_HOST}/{DB_DB}?charset=utf8mb4")
|
|
metadata = MetaData()
|
|
|
|
def ensure_vertical_table():
|
|
insp = inspect(engine)
|
|
if insp.has_table("csv_verticale"):
|
|
print(f"[INFO] Tabella verticale esistente.")
|
|
# Mostra la definizione della tabella per debug
|
|
with engine.connect() as conn:
|
|
result = conn.execute(text("SHOW CREATE TABLE csv_verticale"))
|
|
print(result.fetchone()[1])
|
|
return
|
|
Table(
|
|
"csv_verticale", metadata,
|
|
Column('id', Integer, primary_key=True, autoincrement=True),
|
|
Column('tabella_orig', String(128)),
|
|
Column('riga_csv', Integer),
|
|
Column('colonna', String(128)),
|
|
Column('valore', LONGTEXT), # LONGTEXT: nessun limite pratico
|
|
mysql_engine='InnoDB',
|
|
mysql_row_format='DYNAMIC'
|
|
).create(engine)
|
|
print("[OK] Tabella verticale creata.")
|
|
|
|
def import_vertical(tablename, df):
|
|
records = []
|
|
for idx, row in df.iterrows():
|
|
for col in df.columns:
|
|
records.append({
|
|
"tabella_orig": tablename,
|
|
"riga_csv": idx,
|
|
"colonna": col,
|
|
"valore": str(row[col]) if pd.notnull(row[col]) else None
|
|
})
|
|
if not records:
|
|
return
|
|
pd.DataFrame(records).to_sql("csv_verticale", engine, if_exists='append', index=False, method='multi', chunksize=1000)
|
|
print(f"[OK] Importate {len(records)} celle da {tablename}")
|
|
|
|
def main():
|
|
# SOLO ALLA PRIMA ESECUZIONE, PER SICUREZZA:
|
|
# engine.execute("DROP TABLE IF EXISTS csv_verticale")
|
|
ensure_vertical_table()
|
|
for fname in os.listdir(OUT_PATH):
|
|
if fname.endswith(".pkl") and fname != "all_tables.pkl":
|
|
tablename = fname.replace(".pkl", "")
|
|
df = pd.read_pickle(os.path.join(OUT_PATH, fname))
|
|
import_vertical(tablename, df)
|
|
|
|
if __name__ == "__main__":
|
|
main() |