Ongoing enhancements

sustApp/assign_payments.py

@@ -1,8 +1,9 @@
 # 3. assign manually
 
-from database import Base, engine, SessionLocal
+from models import models
+from models.database import Base, engine, SessionLocal
+from models.models import Payments_CSV, Payments, Members
 from time import time
-from models import Payments_CSV, Payments, Members
 import logging, sys
 import datetime
 
@@ -41,7 +42,7 @@ def assign_by_name(payments: Payments_CSV):
             while removeable in name: name.remove(removeable)
         
         # last part is usually surname
-        member = db.query(Members).filter(Members.member_surname.like(name[len(name)-1]), Members.member_firstname.like(name[0])).all()
+        member = db.query(Members).filter(Members.member_surname.like(name[len(name)-1]), Members.member_firstname.like(name[0]), Members.member_active.is_(True)).all()
         if len(member) > 1:
             logging.debug(f'More than one member with same name {payment.pc_client}')
         elif len(member) == 0:
@@ -60,6 +61,18 @@ def assign_by_name(payments: Payments_CSV):
 
 
 def addPayment(payment, member_id, year, amount):
+    # Check if payment already exists
+    exists = db.query(Payments).filter(
+        Payments.payment_date == payment.pc_valuta_date,
+        Payments.member_id == member_id,
+        Payments.payment_year == datetime.date(int(year), 1, 1)
+    ).first()
+    if exists:
+        logging.debug(f"Payment already exists for member_id {member_id}, year {year}, amount {amount}, date {payment.pc_valuta_date}")
+        payment.pc_processed = True
+        return
+
+    logging.debug(f'Adding payment for member_id {member_id}, year {year}, amount {amount}')
     newPayment = Payments(payment_date = payment.pc_valuta_date,
                              member_id = member_id,
                           payment_year = datetime.date(int(year),1,1),
@@ -96,7 +109,9 @@ if __name__ == "__main__":
     Base.metadata.create_all(bind=engine)
 
     # 1. create missing reference by name
-    payments = db.query(Payments_CSV).filter(Payments_CSV.pc_intended!='', Payments_CSV.pc_reference=='', Payments_CSV.pc_processed!=True).all()
+    #TODO: "None" shouldn't be a string
+    #payments = db.query(Payments_CSV).filter(Payments_CSV.pc_reference.is_(None), Payments_CSV.pc_processed!=True).all()
+    payments = db.query(Payments_CSV).filter(Payments_CSV.pc_processed!=True).all()
     assign_by_name(payments)
 
     # 2. assign payment by reference

sustApp/convert_payment_export_csv.py

@@ -2,17 +2,42 @@
 from numpy import genfromtxt, savetxt
 from time import time
 from datetime import datetime
-import models
+from models import models
+from models.database import engine, SessionLocal
+# import sustApp.models.models as models
+# from sustApp.models.database import engine, SessionLocal
+
 import os, re
-from database import engine, SessionLocal
 from pytz import timezone
+import pandas as pd
+import glob
+#import csv
+
+# def Load_Data(file_name):
+#     with open(source_csv, encoding='utf-8-sig') as f:
+#         for line in f:
+#             fields = line.strip().split(';')
+#             print(fields)
 
 def Load_Data(file_name):
-    #data = genfromtxt(file_name, delimiter=';', skip_header=1, converters={0: lambda s: str(s)})
+    df = pd.read_csv(file_name, delimiter=';', encoding='utf-8-sig', dtype=str, header=None)
-    #str2date = lambda x: datetime.strptime(x, '%d-%m-%y')
+    print(df.head())
-    data = genfromtxt(file_name, delimiter=';', dtype=None, skip_header=0, encoding='utf-8-sig')
+    return df.values.tolist()
 
-    return data.tolist()
+# def Load_Data(file_name):
+#     #data = genfromtxt(file_name, delimiter=';', skip_header=1, converters={0: lambda s: str(s)})
+#     #str2date = lambda x: datetime.strptime(x, '%d-%m-%y')
+#     data = genfromtxt(file_name, delimiter=';', dtype=None, skip_header=0, encoding='utf-8-sig')
+
+#     return data.tolist()
+
+# def Load_Data(file_name):
+#     data = []
+#     with open(file_name, encoding='utf-8-sig') as f:
+#         reader = csv.reader(f, delimiter=';')
+#         for row in reader:
+#             data.append(row)
+#     return data
 
 def extract_values(text):
     # Regular expressions to match the required fields
@@ -51,7 +76,10 @@ if __name__ == "__main__":
 
     try:
         home_directory = os.path.expanduser('~')
-        source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407_20241212_095142_mod.csv")
+        source_path = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "import_csv", "*.csv")
+        for source_csv in glob.glob(source_path):
+            if '_out.csv' not in source_csv:
+                #source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "import_csv", "UmsatzListe_AT493443700000113407_20250529_091945_mod.csv")
                 target_csv = source_csv.replace(".csv", "_out.csv")
                 data = Load_Data(source_csv) 
 
@@ -66,6 +94,9 @@ if __name__ == "__main__":
                             outData.append([i[0], new_list[0].get('Auftraggeber'), new_list[0].get('Zahlungsreferenz'), new_list[0].get('Verwendungszweck'), new_list[0].get('IBAN'), i[2], i[3].replace(',','.'), i[4], i[5]])
                 
                 savetxt(target_csv, outData, delimiter=";", encoding='utf-8', header='valuta_date;client;reference;intended;IBAN;entry_date;amount;currency;timestamp', fmt='%s')
+                bak_dir = os.path.join(os.path.dirname(source_csv), "bak")
+                os.makedirs(bak_dir, exist_ok=True)
+                os.rename(source_csv, os.path.join(bak_dir, os.path.basename(source_csv)))
 
     except Exception as e:
         print (e)

sustApp/create_payment_slip.py

@@ -1,4 +1,4 @@
-import models
+import sustApp.models.models as models
 
 from reportlab.lib.pagesizes import A4
 from reportlab.pdfgen import canvas

sustApp/helper_scripts.sql

@@ -58,7 +58,7 @@ CSV HEADER;
 CREATE UNIQUE INDEX payments_ukey_member_year ON sust.payments USING btree (member_id, payment_year)
 
 select count(*) from payments_csv where pc_processed = true
-
+select  from payments_csv where pc_processed = false
 
 select * from payments_csv pc where PC.pc_client like '%Anna%';
 select * from members mem where mem.member_firstname like 'Anna%';
@@ -69,6 +69,8 @@ ALTER TABLE payments_csv ADD PRIMARY KEY (id);
 select * from payments p where p.member_id in (205,206);
 select * from payments p where date_part('year', p.payment_year) = 2025;
 
+select * from payments_csv where pc_processed = false;
+
 select pc.pc_reference from payments_csv pc where pc_processed =false order by pc_reference asc 
 
 select m.member_surname, m.member_firstname, m.member_firstname_spouse from members m
@@ -136,3 +138,14 @@ WHERE member_email is Null
                (92), (93), (95), (96), (97), (98), (99), (203)
     ) AS extracted_members(member_id)
 );
+
+-- Members active in 2025 but haven't paid for 2025
+SELECT m.*
+FROM members_active m
+WHERE m.id NOT IN (
+    SELECT mp.id 
+    FROM members_paid mp 
+    WHERE mp.payment_year = '2025-01-01'
+    AND mp.id IS NOT NULL
+)
+ORDER BY m.member_surname, m.member_firstname;

sustApp/import_transactions.py

@@ -2,9 +2,10 @@
 from numpy import genfromtxt
 from time import time
 from datetime import datetime
-import models
+from models import models
-import os
+from models.database import engine, SessionLocal
-from database import engine, SessionLocal
+
+import os, glob
 from pytz import timezone
 
 def Load_Data(file_name):
@@ -21,29 +22,34 @@ if __name__ == "__main__":
 
     try:
         home_directory = os.path.expanduser('~')
-        source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407_20241212_095142_mod_out.csv")
+        source_path = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "import_csv", "*_out.csv")
+        for source_csv in glob.glob(source_path):
+            #source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407_20250529_091945_mod_out.csv")
             data = Load_Data(source_csv) 
 
             tz = timezone('UTC')  # Replace with your desired timezone
-        for i in data:
+            dt = datetime.strptime(data[8], '%d.%m.%Y %H:%M:%S:%f') # type: ignore
-            dt = datetime.strptime(i[8], '%d.%m.%Y %H:%M:%S:%f')
             ts_timestamp = tz.localize(dt)
 
             record = models.Payments_CSV(**{
-                'pc_valuta_date' : datetime.strptime(i[0], '%d.%m.%Y').date(),
+                'pc_valuta_date' : datetime.strptime(data[0], '%d.%m.%Y').date(),
-                'pc_client' : i[1],
+                'pc_client' : data[1],
-                'pc_reference' : i[2],
+                'pc_reference' : data[2],
-                'pc_intended' : i[3],
+                'pc_intended' : data[3],
-                'pc_iban' : i[4],
+                'pc_iban' : data[4],
-                'pc_entry_date' : datetime.strptime(i[5], '%d.%m.%Y').date(),
+                'pc_entry_date' : datetime.strptime(data[5], '%d.%m.%Y').date(),
-                'pc_amount' : float(i[6]),
+                'pc_amount' : float(data[6]),
-                'pc_currency' : i[7],
+                'pc_currency' : data[7],
                 'pc_timestamp' : ts_timestamp
             })
             db.add(record) #Add all the records
 
             try:
                 db.commit() #Attempt to commit all the records
+                #move the processed file to a backup directory
+                bak_dir = os.path.join(os.path.dirname(source_csv), "bak")
+                os.makedirs(bak_dir, exist_ok=True)
+                os.rename(source_csv, os.path.join(bak_dir, os.path.basename(source_csv)))
             except Exception as e:
                 print(e)
     

sustApp/mail_payment_slips.py

@@ -1,6 +1,6 @@
 from sqlalchemy import extract
-from database import engine, SessionLocal
+from sustApp.models.database import engine, SessionLocal
-import models
+import sustApp.models.models as models
 import logging, sys
 import smtplib
 from email.mime.multipart import MIMEMultipart

sustApp/main.py

@@ -3,8 +3,8 @@ from fastapi.security import OAuth2PasswordBearer
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
 from typing import List, Annotated, Optional
-import models
+from sustApp.models.models import Base, Members, Payments
-from database import engine, SessionLocal
+from sustApp.models.database import engine, SessionLocal
 from sqlalchemy.orm import Session
 from datetime import datetime
 
@@ -12,8 +12,14 @@ from datetime import datetime
 #ngrok http http://localhost:8000
 
 app = FastAPI()
+
+@app.get("/")
+
+def read_root():
+    return {"Hello": "World"}
+
 #oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
-models.Base.metadata.create_all(bind=engine)
+Base.metadata.create_all(bind=engine)
 
 origins = [
     "http://localhost.tiangolo.com",
@@ -75,21 +81,21 @@ db_dependency = Annotated[Session, Depends(get_db)]
 
 @app.get("/members/", response_model=List[MemberBase])
 async def read_members(db: db_dependency, skip: int = 0, limit: int = 100 ):
-    members = db.query(models.Members).offset(skip).limit(limit).all()
+    members = db.query(Members).offset(skip).limit(limit).all()
     if not members:
         raise HTTPException(status_code=400, detail='No members')
     return members
 
 @app.get("/member/{member_id}")
 async def read_member(member_id: int, db: db_dependency): #, token: Annotated[str, Depends(oauth2_scheme)]):
-    result = db.query(models.Members).filter(models.Members.id == member_id).first()
+    result = db.query(Members).filter(Members.id == member_id).first()
     if not result:
         raise HTTPException(status_code=400, detail='Member does not exist')
     return result
 
 @app.get("/payments/{member_id}")
 async def read_payments(member_id: int, db: db_dependency):
-    result = db.query(models.Members).filter(models.Members.id == member_id).all()
+    result = db.query(Members).filter(Members.id == member_id).all()
     if not result:
         raise HTTPException(status_code=400, detail=f'No payments found for member {member_id}')
     return result
@@ -97,7 +103,7 @@ async def read_payments(member_id: int, db: db_dependency):
 @app.post("/members/")
 async def create_member(member: MemberBase, db: db_dependency):
     #db_member = models.Members(**member.model_dump())
-    db_member = models.Members(member_firstname=member.member_firstname,
+    db_member = Members(member_firstname=member.member_firstname,
                         member_surname=member.member_surname,
                         member_active=member.member_active)
     db.add(db_member)
@@ -106,7 +112,7 @@ async def create_member(member: MemberBase, db: db_dependency):
 @app.post("/fullmembers/")
 async def create_members(members: List[MemberBase], db: db_dependency):
     for member in members:
-        db_member = models.Members(member_count=member.member_count,
+        db_member = Members(member_count=member.member_count,
                             member_firstname=member.member_firstname,
                             member_firstname_spouse=member.member_firstname_spouse,
                             member_surname=member.member_surname,
@@ -124,7 +130,7 @@ async def create_members(members: List[MemberBase], db: db_dependency):
         #db.refresh(db_member)
         if not member.payments is None:
             for payment in member.payments:
-                db_payment = models.Payments(payment_date=payment.payment_date, 
+                db_payment = Payments(payment_date=payment.payment_date, 
                                       member_id=db_member.id)
                 db.add(db_payment)
     db.commit()

sustApp/member_map.py

@@ -1,28 +1,28 @@
 # import the library and its Marker clusterization service
 import folium
-from database import SessionLocal
+from models.database import SessionLocal
 from folium.plugins import MarkerCluster
-import models
+from models.models import Base, Members, Payments, Deliveries
 import logging
-from geopy.geocoders import Bing
+from geopy.geocoders import Nominatim
 from sqlalchemy import *
 import logging
 import webbrowser
 import re
 from config_loader import load_config
 
-def payment_slip_delivered(member: models.Members, year: int):
+def payment_slip_delivered(member: Members, year: int):
-    return db.query(models.Deliveries.delivery_date)\
+    return db.query(Deliveries.delivery_date)\
-             .filter(models.Deliveries.member_id == member.id, extract('year', models.Deliveries.delivery_date) == year)\
+             .filter(Deliveries.member_id == member.id, extract('year', Deliveries.delivery_date) == year)\
              .count()
 
-def payment_made(member: models.Members, year: int):
+def payment_made(member: Members, year: int):
-    return db.query(models.Payments.payment_year)\
+    return db.query(Payments.payment_year)\
-             .filter(models.Payments.member_id == member.id, extract('year', models.Payments.payment_year) == year)\
+             .filter(Payments.member_id == member.id, extract('year', Payments.payment_year) == year)\
              .count()
 
 def update_missing_locs():
-    members = db.query(models.Members).filter(models.Members.member_geo_latitude == None).all()
+    members = db.query(Members).filter(Members.member_geo_latitude == None).all()
 
     for member in members:
         address = f'{member.member_street} {member.member_streetnumber}, {member.member_zip} {member.member_city}, Austria'
@@ -57,8 +57,8 @@ year = cfg['year']
 
 logger = logging.getLogger(__name__)
 
-#geolocator = Nominatim(user_agent="sust", timeout=10)
+geolocator = Nominatim(user_agent="sust", timeout=10)
-geolocator = Bing(api_key='ApbTfEJ024nQCu6DC7TPIlqAcRMfZT8lPjAyMnVRcmo9rqxde5x-IS4fW-nGu0FI')
+#geolocator = Bing(api_key='ApbTfEJ024nQCu6DC7TPIlqAcRMfZT8lPjAyMnVRcmo9rqxde5x-IS4fW-nGu0FI')
 
 db = SessionLocal()
 #models.Base.metadata.create_all(bind=engine)
@@ -73,13 +73,13 @@ for member in members:
     for delivery in members.deliveries:
         print(f'{delivery.delivery_date}') """
 
-members = db.query(models.Members.member_geo_latitude, 
+members = db.query(Members.member_geo_latitude, 
-                   models.Members.member_geo_longitude, 
+                   Members.member_geo_longitude, 
-                   models.Members.member_firstname, 
+                   Members.member_firstname, 
-                   models.Members.member_surname, 
+                   Members.member_surname, 
-                   models.Members.id,
+                   Members.id,
-                   models.Members.member_email)\
+                   Members.member_email)\
-            .filter(models.Members.member_geo_latitude != None, models.Members.member_count != 0)\
+            .filter(Members.member_geo_latitude != None, Members.member_count != 0)\
             .all()
 
 # .join(models.Members.member_deliveries)\

sustApp/models/models.py

@@ -2,7 +2,7 @@ from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, DateTime, F
 from sqlalchemy.types import TypeDecorator, VARCHAR, Integer
 from sqlalchemy_utils import EmailType
 from sqlalchemy.orm import relationship
-from database import Base
+from models.database import Base
 
 import datetime
 
@@ -48,6 +48,7 @@ class Members(Base):
     member_geo_latitude = Column(Float, nullable=True)
     member_geo_longitude = Column(Float, nullable=True)
     member_deliveries = relationship('Deliveries', backref='members')
+    member_payments = relationship('Payments', backref='members')
     #member_deliveries: Mapped[List["Deliveries"]] = relationship()
 
     def __repr__(self):

sustApp/print_payment_slips.py

@@ -1,9 +1,9 @@
 from reportlab.lib.pagesizes import A4
 from create_payment_slip import create_payment_slip
-from database import engine, SessionLocal
+from sustApp.models.database import engine, SessionLocal
 from pydantic import BaseModel
 from sqlalchemy.orm import Session
-import models
+import sustApp.models.models as models
 from reportlab.lib.utils import ImageReader
 import logging, sys
 from sqlalchemy import extract

sustApp/sust_schema.hcl

@@ -1,273 +0,0 @@
-table "deliveries" {
-  schema = schema.sust
-  column "id" {
-    null = false
-    type = serial
-  }
-  column "delivery_date" {
-    null = true
-    type = timestamp
-  }
-  column "member_id" {
-    null = true
-    type = integer
-  }
-  column "delivered_by_id" {
-    null = true
-    type = integer
-  }
-  column "delivery_year" {
-    null = true
-    type = date
-  }
-  primary_key {
-    columns = [column.id]
-  }
-  foreign_key "deliveries_member_id_delivered_by_fkey" {
-    columns     = [column.delivered_by_id]
-    ref_columns = [table.members.column.id]
-    on_update   = NO_ACTION
-    on_delete   = NO_ACTION
-  }
-  foreign_key "deliveries_member_id_fkey" {
-    columns     = [column.member_id]
-    ref_columns = [table.members.column.id]
-    on_update   = NO_ACTION
-    on_delete   = NO_ACTION
-  }
-  index "ix_deliveries_delivery_date" {
-    columns = [column.delivery_date]
-  }
-  index "ix_deliveries_id" {
-    columns = [column.id]
-  }
-  unique "deliveries_ukey_member_year" {
-    columns = [column.member_id, column.delivery_year]
-  }
-}
-table "members" {
-  schema = schema.sust
-  column "id" {
-    null = false
-    type = serial
-  }
-  column "member_count" {
-    null    = true
-    type    = integer
-    default = 1
-  }
-  column "member_firstname" {
-    null = true
-    type = character_varying
-  }
-  column "member_firstname_spouse" {
-    null = true
-    type = character_varying
-  }
-  column "member_surname" {
-    null = true
-    type = character_varying
-  }
-  column "member_suffix" {
-    null = true
-    type = character_varying
-  }
-  column "member_street" {
-    null = true
-    type = character_varying
-  }
-  column "member_streetnumber" {
-    null = true
-    type = character_varying
-  }
-  column "member_email" {
-    null = true
-    type = character_varying(255)
-  }
-  column "member_zip" {
-    null = true
-    type = integer
-  }
-  column "member_city" {
-    null = true
-    type = character_varying
-  }
-  column "member_active" {
-    null    = true
-    type    = boolean
-    default = true
-  }
-  column "member_connected" {
-    null = true
-    type = integer
-  }
-  column "created_date" {
-    null    = true
-    type    = timestamp
-    default = sql("CURRENT_TIMESTAMP")
-  }
-  column "updated_date" {
-    null = true
-    type = timestamp
-  }
-  column "member_gender" {
-    null = true
-    type = enum.gender
-  }
-  column "member_died" {
-    null = true
-    type = timestamp
-  }
-  column "member_membership_start" {
-    null = true
-    type = timestamp
-  }
-  column "member_printed" {
-    null = true
-    type = boolean
-  }
-  column "member_coordinates" {
-    null    = true
-    type    = point
-    comment = "UNUSED"
-  }
-  column "member_geo_latitude" {
-    null = true
-    type = numeric
-  }
-  column "member_geo_longitude" {
-    null = true
-    type = numeric
-  }
-  primary_key {
-    columns = [column.id]
-  }
-  foreign_key "members_member_connected_fkey" {
-    columns     = [column.member_connected]
-    ref_columns = [table.members.column.id]
-    on_update   = NO_ACTION
-    on_delete   = NO_ACTION
-  }
-  index "ix_members_id" {
-    columns = [column.id]
-  }
-  index "ix_members_member_city" {
-    columns = [column.member_city]
-  }
-  index "ix_members_member_email" {
-    columns = [column.member_email]
-  }
-  index "ix_members_member_firstname" {
-    columns = [column.member_firstname]
-  }
-  index "ix_members_member_street" {
-    columns = [column.member_street]
-  }
-  index "ix_members_member_surname" {
-    columns = [column.member_surname]
-  }
-  index "ix_members_member_zip" {
-    columns = [column.member_zip]
-  }
-}
-table "payments" {
-  schema = schema.sust
-  column "id" {
-    null = false
-    type = serial
-  }
-  column "payment_date" {
-    null = true
-    type = date
-  }
-  column "member_id" {
-    null = true
-    type = integer
-  }
-  column "payment_amount" {
-    null = true
-    type = numeric
-  }
-  column "payment_year" {
-    null    = true
-    type    = date
-    comment = "Year, for which the payment is intended for."
-  }
-  primary_key {
-    columns = [column.id]
-  }
-  foreign_key "payments_member_id_fkey" {
-    columns     = [column.member_id]
-    ref_columns = [table.members.column.id]
-    on_update   = NO_ACTION
-    on_delete   = NO_ACTION
-  }
-  index "ix_payments_id" {
-    columns = [column.id]
-  }
-  index "ix_payments_payment_date" {
-    columns = [column.payment_date]
-  }
-  index "payments_ukey_member_year" {
-    unique  = true
-    columns = [column.member_id, column.payment_year]
-  }
-}
-table "payments_csv" {
-  schema = schema.sust
-  column "id" {
-    null = false
-    type = serial
-  }
-  column "pc_valuta_date" {
-    null = true
-    type = date
-  }
-  column "pc_client" {
-    null = true
-    type = character_varying(50)
-  }
-  column "pc_reference" {
-    null = true
-    type = character_varying(50)
-  }
-  column "pc_intended" {
-    null = true
-    type = character_varying(100)
-  }
-  column "pc_iban" {
-    null = true
-    type = character_varying(50)
-  }
-  column "pc_entry_date" {
-    null = true
-    type = date
-  }
-  column "pc_amount" {
-    null = true
-    type = numeric
-  }
-  column "pc_currency" {
-    null = true
-    type = character_varying(3)
-  }
-  column "pc_timestamp" {
-    null = true
-    type = timestamp
-  }
-  column "pc_processed" {
-    null    = true
-    type    = boolean
-    default = false
-  }
-  primary_key {
-    columns = [column.id]
-  }
-}
-enum "gender" {
-  schema = schema.sust
-  values = ["male", "female", "divers"]
-}
-schema "sust" {
-  comment = "standard public schema"
-}

sustApp/whatsapp_payment_slips.py

@@ -3,8 +3,8 @@ from heyoo import WhatsApp
 
 #import pywhatkit
 
-from database import engine, SessionLocal
+from sustApp.models.database import engine, SessionLocal
-import models
+import sustApp.models.models as models
 import logging, sys
 import smtplib
 from email.mime.multipart import MIMEMultipart