76 lines
3.1 KiB
Python
76 lines
3.1 KiB
Python
# Importing libraries
|
|
#from numpy import genfromtxt
|
|
import pandas as pd
|
|
from time import time
|
|
from datetime import date, datetime
|
|
from models import models
|
|
from models.database import engine, SessionLocal
|
|
|
|
import os, glob
|
|
from pytz import timezone
|
|
|
|
""" def Load_Data(file_name):
|
|
#data = genfromtxt(file_name, delimiter=';', skip_header=1, converters={0: lambda s: str(s)})
|
|
#str2date = lambda x: datetime.strptime(x, '%d-%m-%y')
|
|
data = genfromtxt(file_name, delimiter=';', dtype=None, skip_header=1, encoding='utf-8') #
|
|
return data.tolist()
|
|
"""
|
|
|
|
#use pandas instead of genfromtxt for better performance and handling of csv files
|
|
def Load_Data(file_name):
|
|
df = pd.read_csv(file_name, delimiter=',', encoding='utf-8')
|
|
return df.values.tolist()
|
|
|
|
if __name__ == "__main__":
|
|
t = time()
|
|
|
|
db = SessionLocal()
|
|
models.Base.metadata.create_all(bind=engine)
|
|
|
|
try:
|
|
home_directory = os.path.expanduser('~')
|
|
source_path = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "import_csv", "*_out.csv")
|
|
for source_csv in glob.glob(source_path):
|
|
#source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407_20250529_091945_mod_out.csv")
|
|
datas = Load_Data(source_csv)
|
|
tz = timezone('UTC') # Replace with your desired timezone
|
|
for data in datas:
|
|
dt = datetime.strptime(data[8], '%d.%m.%Y %H:%M:%S:%f') # type: ignore
|
|
ts_timestamp = tz.localize(dt)
|
|
|
|
# Parse year - data[9] is now the year (from convert_payment_export_csv.py)
|
|
# Convert year string to a valid date (January 1st of that year)
|
|
year_value = data[9] # This is '2025' from the CSV
|
|
pc_year = date(int(year_value), 1, 1) # Convert to date: 2025-01-01
|
|
|
|
#TODO: Check if amout is valid!
|
|
record = models.Payments_CSV(**{
|
|
'pc_valuta_date' : datetime.strptime(data[0], '%d.%m.%Y').date(),
|
|
'pc_client' : data[1],
|
|
'pc_reference' : data[2],
|
|
'pc_intended' : data[3],
|
|
'pc_iban' : data[4],
|
|
'pc_entry_date' : datetime.strptime(data[5], '%d.%m.%Y').date(),
|
|
'pc_amount' : float(data[6]),
|
|
'pc_currency' : data[7],
|
|
'pc_timestamp' : ts_timestamp,
|
|
'pc_year' : pc_year
|
|
})
|
|
db.add(record) #Add all the records
|
|
|
|
try:
|
|
db.commit() #Attempt to commit all the records
|
|
#move the processed file to a backup directory
|
|
bak_dir = os.path.join(os.path.dirname(source_csv), "bak")
|
|
os.makedirs(bak_dir, exist_ok=True)
|
|
os.rename(source_csv, os.path.join(bak_dir, os.path.basename(source_csv)))
|
|
except Exception as e:
|
|
print(e)
|
|
|
|
except Exception as e:
|
|
print (e)
|
|
db.rollback() #Rollback the changes on error
|
|
finally:
|
|
db.close() #Close the connection
|
|
|
|
print("Time elapsed: " + str(time() - t) + " s.") #0.091s |