zisco/SUST
1
1
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases Wiki Activity

Ongoing enhancements

Browse Source
This commit is contained in:
zisco
2025-07-24 14:33:25 +02:00
parent 0b27385a24
commit 4baa544458
16 changed files with 170 additions and 8198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
map.html
-7827
View File
File diff suppressed because it is too large Load Diff
sustApp/assign_payments.py
+22 -7
View File
@@ -1,8 +1,9 @@
# 3. assign manually
from database import Base, engine, SessionLocal
from models import models
from models.database import Base, engine, SessionLocal
from models.models import Payments_CSV, Payments, Members
from time import time
from models import Payments_CSV, Payments, Members
import logging, sys
import datetime
@@ -41,7 +42,7 @@ def assign_by_name(payments: Payments_CSV):
while removeable in name: name.remove(removeable)
# last part is usually surname
member = db.query(Members).filter(Members.member_surname.like(name[len(name)-1]), Members.member_firstname.like(name[0])).all()
member = db.query(Members).filter(Members.member_surname.like(name[len(name)-1]), Members.member_firstname.like(name[0]), Members.member_active.is_(True)).all()
if len(member) > 1:
logging.debug(f'More than one member with same name {payment.pc_client}')
elif len(member) == 0:
@@ -60,10 +61,22 @@ def assign_by_name(payments: Payments_CSV):
def addPayment(payment, member_id, year, amount):
# Check if payment already exists
exists = db.query(Payments).filter(
Payments.payment_date == payment.pc_valuta_date,
Payments.member_id == member_id,
Payments.payment_year == datetime.date(int(year), 1, 1)
).first()
if exists:
logging.debug(f"Payment already exists for member_id {member_id}, year {year}, amount {amount}, date {payment.pc_valuta_date}")
payment.pc_processed = True
return
logging.debug(f'Adding payment for member_id {member_id}, year {year}, amount {amount}')
newPayment = Payments(payment_date = payment.pc_valuta_date,
member_id = member_id,
payment_year = datetime.date(int(year),1,1),
payment_amount = amount)
member_id = member_id,
payment_year = datetime.date(int(year),1,1),
payment_amount = amount)
db.add(newPayment)
payment.pc_processed = True
@@ -96,7 +109,9 @@ if __name__ == "__main__":
Base.metadata.create_all(bind=engine)
# 1. create missing reference by name
payments = db.query(Payments_CSV).filter(Payments_CSV.pc_intended!='', Payments_CSV.pc_reference=='', Payments_CSV.pc_processed!=True).all()
#TODO: "None" shouldn't be a string
#payments = db.query(Payments_CSV).filter(Payments_CSV.pc_reference.is_(None), Payments_CSV.pc_processed!=True).all()
payments = db.query(Payments_CSV).filter(Payments_CSV.pc_processed!=True).all()
assign_by_name(payments)
# 2. assign payment by reference
sustApp/convert_payment_export_csv.py
+50 -19
View File
@@ -2,17 +2,42 @@
from numpy import genfromtxt, savetxt
from time import time
from datetime import datetime
import models
from models import models
from models.database import engine, SessionLocal
# import sustApp.models.models as models
# from sustApp.models.database import engine, SessionLocal
import os, re
from database import engine, SessionLocal
from pytz import timezone
import pandas as pd
import glob
#import csv
# def Load_Data(file_name):
# with open(source_csv, encoding='utf-8-sig') as f:
# for line in f:
# fields = line.strip().split(';')
# print(fields)
def Load_Data(file_name):
#data = genfromtxt(file_name, delimiter=';', skip_header=1, converters={0: lambda s: str(s)})
#str2date = lambda x: datetime.strptime(x, '%d-%m-%y')
data = genfromtxt(file_name, delimiter=';', dtype=None, skip_header=0, encoding='utf-8-sig')
df = pd.read_csv(file_name, delimiter=';', encoding='utf-8-sig', dtype=str, header=None)
print(df.head())
return df.values.tolist()
return data.tolist()
# def Load_Data(file_name):
# #data = genfromtxt(file_name, delimiter=';', skip_header=1, converters={0: lambda s: str(s)})
# #str2date = lambda x: datetime.strptime(x, '%d-%m-%y')
# data = genfromtxt(file_name, delimiter=';', dtype=None, skip_header=0, encoding='utf-8-sig')
# return data.tolist()
# def Load_Data(file_name):
# data = []
# with open(file_name, encoding='utf-8-sig') as f:
# reader = csv.reader(f, delimiter=';')
# for row in reader:
# data.append(row)
# return data
def extract_values(text):
# Regular expressions to match the required fields
@@ -51,21 +76,27 @@ if __name__ == "__main__":
try:
home_directory = os.path.expanduser('~')
source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407_20241212_095142_mod.csv")
target_csv = source_csv.replace(".csv", "_out.csv")
data = Load_Data(source_csv)
source_path = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "import_csv", "*.csv")
for source_csv in glob.glob(source_path):
if '_out.csv' not in source_csv:
#source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "import_csv", "UmsatzListe_AT493443700000113407_20250529_091945_mod.csv")
target_csv = source_csv.replace(".csv", "_out.csv")
data = Load_Data(source_csv)
tz = timezone('UTC') # Replace with your desired timezone
outData = []
for i in data:
if ('Auftraggeber' in i[1]):
# extract key-value pairs Auftraggeber, Verwendungszweck, Zahlungsreferenz, IBAN
new_list = extract_values(i[1])
tz = timezone('UTC') # Replace with your desired timezone
outData = []
for i in data:
if ('Auftraggeber' in i[1]):
# extract key-value pairs Auftraggeber, Verwendungszweck, Zahlungsreferenz, IBAN
new_list = extract_values(i[1])
if len(new_list) > 0:
outData.append([i[0], new_list[0].get('Auftraggeber'), new_list[0].get('Zahlungsreferenz'), new_list[0].get('Verwendungszweck'), new_list[0].get('IBAN'), i[2], i[3].replace(',','.'), i[4], i[5]])
savetxt(target_csv, outData, delimiter=";", encoding='utf-8', header='valuta_date;client;reference;intended;IBAN;entry_date;amount;currency;timestamp', fmt='%s')
if len(new_list) > 0:
outData.append([i[0], new_list[0].get('Auftraggeber'), new_list[0].get('Zahlungsreferenz'), new_list[0].get('Verwendungszweck'), new_list[0].get('IBAN'), i[2], i[3].replace(',','.'), i[4], i[5]])
savetxt(target_csv, outData, delimiter=";", encoding='utf-8', header='valuta_date;client;reference;intended;IBAN;entry_date;amount;currency;timestamp', fmt='%s')
bak_dir = os.path.join(os.path.dirname(source_csv), "bak")
os.makedirs(bak_dir, exist_ok=True)
os.rename(source_csv, os.path.join(bak_dir, os.path.basename(source_csv)))
except Exception as e:
print (e)
sustApp/create_payment_slip.py
+1 -1
View File
@@ -1,4 +1,4 @@
import models
import sustApp.models.models as models
from reportlab.lib.pagesizes import A4
from reportlab.pdfgen import canvas
sustApp/helper_scripts.sql
+14 -1
View File
@@ -58,7 +58,7 @@ CSV HEADER;
CREATE UNIQUE INDEX payments_ukey_member_year ON sust.payments USING btree (member_id, payment_year)
select count(*) from payments_csv where pc_processed = true
select from payments_csv where pc_processed = false
select * from payments_csv pc where PC.pc_client like '%Anna%';
select * from members mem where mem.member_firstname like 'Anna%';
@@ -69,6 +69,8 @@ ALTER TABLE payments_csv ADD PRIMARY KEY (id);
select * from payments p where p.member_id in (205,206);
select * from payments p where date_part('year', p.payment_year) = 2025;
select * from payments_csv where pc_processed = false;
select pc.pc_reference from payments_csv pc where pc_processed =false order by pc_reference asc
select m.member_surname, m.member_firstname, m.member_firstname_spouse from members m
@@ -136,3 +138,14 @@ WHERE member_email is Null
(92), (93), (95), (96), (97), (98), (99), (203)
) AS extracted_members(member_id)
);
-- Members active in 2025 but haven't paid for 2025
SELECT m.*
FROM members_active m
WHERE m.id NOT IN (
SELECT mp.id
FROM members_paid mp
WHERE mp.payment_year = '2025-01-01'
AND mp.id IS NOT NULL
)
ORDER BY m.member_surname, m.member_firstname;
sustApp/import_transactions.py
+26 -20
View File
@@ -2,9 +2,10 @@
from numpy import genfromtxt
from time import time
from datetime import datetime
import models
import os
from database import engine, SessionLocal
from models import models
from models.database import engine, SessionLocal
import os, glob
from pytz import timezone
def Load_Data(file_name):
@@ -21,31 +22,36 @@ if __name__ == "__main__":
try:
home_directory = os.path.expanduser('~')
source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407_20241212_095142_mod_out.csv")
data = Load_Data(source_csv)
source_path = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "import_csv", "*_out.csv")
for source_csv in glob.glob(source_path):
#source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407_20250529_091945_mod_out.csv")
data = Load_Data(source_csv)
tz = timezone('UTC') # Replace with your desired timezone
for i in data:
dt = datetime.strptime(i[8], '%d.%m.%Y %H:%M:%S:%f')
tz = timezone('UTC') # Replace with your desired timezone
dt = datetime.strptime(data[8], '%d.%m.%Y %H:%M:%S:%f') # type: ignore
ts_timestamp = tz.localize(dt)
record = models.Payments_CSV(**{
'pc_valuta_date' : datetime.strptime(i[0], '%d.%m.%Y').date(),
'pc_client' : i[1],
'pc_reference' : i[2],
'pc_intended' : i[3],
'pc_iban' : i[4],
'pc_entry_date' : datetime.strptime(i[5], '%d.%m.%Y').date(),
'pc_amount' : float(i[6]),
'pc_currency' : i[7],
'pc_valuta_date' : datetime.strptime(data[0], '%d.%m.%Y').date(),
'pc_client' : data[1],
'pc_reference' : data[2],
'pc_intended' : data[3],
'pc_iban' : data[4],
'pc_entry_date' : datetime.strptime(data[5], '%d.%m.%Y').date(),
'pc_amount' : float(data[6]),
'pc_currency' : data[7],
'pc_timestamp' : ts_timestamp
})
db.add(record) #Add all the records
try:
db.commit() #Attempt to commit all the records
except Exception as e:
print(e)
try:
db.commit() #Attempt to commit all the records
#move the processed file to a backup directory
bak_dir = os.path.join(os.path.dirname(source_csv), "bak")
os.makedirs(bak_dir, exist_ok=True)
os.rename(source_csv, os.path.join(bak_dir, os.path.basename(source_csv)))
except Exception as e:
print(e)
except Exception as e:
print (e)
sustApp/mail_payment_slips.py
+2 -2
View File
@@ -1,6 +1,6 @@
from sqlalchemy import extract
from database import engine, SessionLocal
import models
from sustApp.models.database import engine, SessionLocal
import sustApp.models.models as models
import logging, sys
import smtplib
from email.mime.multipart import MIMEMultipart
sustApp/main.py
+30 -24
View File
@@ -3,8 +3,8 @@ from fastapi.security import OAuth2PasswordBearer
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from typing import List, Annotated, Optional
import models
from database import engine, SessionLocal
from sustApp.models.models import Base, Members, Payments
from sustApp.models.database import engine, SessionLocal
from sqlalchemy.orm import Session
from datetime import datetime
@@ -12,8 +12,14 @@ from datetime import datetime
#ngrok http http://localhost:8000
app = FastAPI()
@app.get("/")
def read_root():
return {"Hello": "World"}
#oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
models.Base.metadata.create_all(bind=engine)
Base.metadata.create_all(bind=engine)
origins = [
"http://localhost.tiangolo.com",
@@ -75,21 +81,21 @@ db_dependency = Annotated[Session, Depends(get_db)]
@app.get("/members/", response_model=List[MemberBase])
async def read_members(db: db_dependency, skip: int = 0, limit: int = 100 ):
members = db.query(models.Members).offset(skip).limit(limit).all()
members = db.query(Members).offset(skip).limit(limit).all()
if not members:
raise HTTPException(status_code=400, detail='No members')
return members
@app.get("/member/{member_id}")
async def read_member(member_id: int, db: db_dependency): #, token: Annotated[str, Depends(oauth2_scheme)]):
result = db.query(models.Members).filter(models.Members.id == member_id).first()
result = db.query(Members).filter(Members.id == member_id).first()
if not result:
raise HTTPException(status_code=400, detail='Member does not exist')
return result
@app.get("/payments/{member_id}")
async def read_payments(member_id: int, db: db_dependency):
result = db.query(models.Members).filter(models.Members.id == member_id).all()
result = db.query(Members).filter(Members.id == member_id).all()
if not result:
raise HTTPException(status_code=400, detail=f'No payments found for member {member_id}')
return result
@@ -97,34 +103,34 @@ async def read_payments(member_id: int, db: db_dependency):
@app.post("/members/")
async def create_member(member: MemberBase, db: db_dependency):
#db_member = models.Members(**member.model_dump())
db_member = models.Members(member_firstname=member.member_firstname,
member_surname=member.member_surname,
member_active=member.member_active)
db_member = Members(member_firstname=member.member_firstname,
member_surname=member.member_surname,
member_active=member.member_active)
db.add(db_member)
db.commit()
@app.post("/fullmembers/")
async def create_members(members: List[MemberBase], db: db_dependency):
for member in members:
db_member = models.Members(member_count=member.member_count,
member_firstname=member.member_firstname,
member_firstname_spouse=member.member_firstname_spouse,
member_surname=member.member_surname,
member_suffix=member.member_suffix,
member_street=member.member_street,
member_streetnumber=member.member_streetnumber,
member_email=member.member_email,
member_zip=member.member_zip,
member_city=member.member_city,
member_active=member.member_active,
member_gender=member.member_gender,
member_connected=member.member_connected)
db_member = Members(member_count=member.member_count,
member_firstname=member.member_firstname,
member_firstname_spouse=member.member_firstname_spouse,
member_surname=member.member_surname,
member_suffix=member.member_suffix,
member_street=member.member_street,
member_streetnumber=member.member_streetnumber,
member_email=member.member_email,
member_zip=member.member_zip,
member_city=member.member_city,
member_active=member.member_active,
member_gender=member.member_gender,
member_connected=member.member_connected)
db.add(db_member)
#db.commit()
#db.refresh(db_member)
if not member.payments is None:
for payment in member.payments:
db_payment = models.Payments(payment_date=payment.payment_date,
member_id=db_member.id)
db_payment = Payments(payment_date=payment.payment_date,
member_id=db_member.id)
db.add(db_payment)
db.commit()
sustApp/member_map.py
+19 -19
View File
@@ -1,28 +1,28 @@
# import the library and its Marker clusterization service
import folium
from database import SessionLocal
from models.database import SessionLocal
from folium.plugins import MarkerCluster
import models
from models.models import Base, Members, Payments, Deliveries
import logging
from geopy.geocoders import Bing
from geopy.geocoders import Nominatim
from sqlalchemy import *
import logging
import webbrowser
import re
from config_loader import load_config
def payment_slip_delivered(member: models.Members, year: int):
return db.query(models.Deliveries.delivery_date)\
.filter(models.Deliveries.member_id == member.id, extract('year', models.Deliveries.delivery_date) == year)\
def payment_slip_delivered(member: Members, year: int):
return db.query(Deliveries.delivery_date)\
.filter(Deliveries.member_id == member.id, extract('year', Deliveries.delivery_date) == year)\
.count()
def payment_made(member: models.Members, year: int):
return db.query(models.Payments.payment_year)\
.filter(models.Payments.member_id == member.id, extract('year', models.Payments.payment_year) == year)\
def payment_made(member: Members, year: int):
return db.query(Payments.payment_year)\
.filter(Payments.member_id == member.id, extract('year', Payments.payment_year) == year)\
.count()
def update_missing_locs():
members = db.query(models.Members).filter(models.Members.member_geo_latitude == None).all()
members = db.query(Members).filter(Members.member_geo_latitude == None).all()
for member in members:
address = f'{member.member_street} {member.member_streetnumber}, {member.member_zip} {member.member_city}, Austria'
@@ -57,8 +57,8 @@ year = cfg['year']
logger = logging.getLogger(__name__)
#geolocator = Nominatim(user_agent="sust", timeout=10)
geolocator = Bing(api_key='ApbTfEJ024nQCu6DC7TPIlqAcRMfZT8lPjAyMnVRcmo9rqxde5x-IS4fW-nGu0FI')
geolocator = Nominatim(user_agent="sust", timeout=10)
#geolocator = Bing(api_key='ApbTfEJ024nQCu6DC7TPIlqAcRMfZT8lPjAyMnVRcmo9rqxde5x-IS4fW-nGu0FI')
db = SessionLocal()
#models.Base.metadata.create_all(bind=engine)
@@ -73,13 +73,13 @@ for member in members:
for delivery in members.deliveries:
print(f'{delivery.delivery_date}') """
members = db.query(models.Members.member_geo_latitude,
models.Members.member_geo_longitude,
models.Members.member_firstname,
models.Members.member_surname,
models.Members.id,
models.Members.member_email)\
.filter(models.Members.member_geo_latitude != None, models.Members.member_count != 0)\
members = db.query(Members.member_geo_latitude,
Members.member_geo_longitude,
Members.member_firstname,
Members.member_surname,
Members.id,
Members.member_email)\
.filter(Members.member_geo_latitude != None, Members.member_count != 0)\
.all()
# .join(models.Members.member_deliveries)\
sustApp/models/__init__.py
View File
sustApp/database.py → sustApp/models/database.py
View File
sustApp/models.py → sustApp/models/models.py
+2 -1
View File
@@ -2,7 +2,7 @@ from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, DateTime, F
from sqlalchemy.types import TypeDecorator, VARCHAR, Integer
from sqlalchemy_utils import EmailType
from sqlalchemy.orm import relationship
from database import Base
from models.database import Base
import datetime
@@ -48,6 +48,7 @@ class Members(Base):
member_geo_latitude = Column(Float, nullable=True)
member_geo_longitude = Column(Float, nullable=True)
member_deliveries = relationship('Deliveries', backref='members')
member_payments = relationship('Payments', backref='members')
#member_deliveries: Mapped[List["Deliveries"]] = relationship()
def __repr__(self):
sustApp/print_payment_slips.py
+2 -2
View File
@@ -1,9 +1,9 @@
from reportlab.lib.pagesizes import A4
from create_payment_slip import create_payment_slip
from database import engine, SessionLocal
from sustApp.models.database import engine, SessionLocal
from pydantic import BaseModel
from sqlalchemy.orm import Session
import models
import sustApp.models.models as models
from reportlab.lib.utils import ImageReader
import logging, sys
from sqlalchemy import extract
sust.hcl → sustApp/sust.hcl
View File
sustApp/sust_schema.hcl
-273
View File
@@ -1,273 +0,0 @@
table "deliveries" {
schema = schema.sust
column "id" {
null = false
type = serial
}
column "delivery_date" {
null = true
type = timestamp
}
column "member_id" {
null = true
type = integer
}
column "delivered_by_id" {
null = true
type = integer
}
column "delivery_year" {
null = true
type = date
}
primary_key {
columns = [column.id]
}
foreign_key "deliveries_member_id_delivered_by_fkey" {
columns = [column.delivered_by_id]
ref_columns = [table.members.column.id]
on_update = NO_ACTION
on_delete = NO_ACTION
}
foreign_key "deliveries_member_id_fkey" {
columns = [column.member_id]
ref_columns = [table.members.column.id]
on_update = NO_ACTION
on_delete = NO_ACTION
}
index "ix_deliveries_delivery_date" {
columns = [column.delivery_date]
}
index "ix_deliveries_id" {
columns = [column.id]
}
unique "deliveries_ukey_member_year" {
columns = [column.member_id, column.delivery_year]
}
}
table "members" {
schema = schema.sust
column "id" {
null = false
type = serial
}
column "member_count" {
null = true
type = integer
default = 1
}
column "member_firstname" {
null = true
type = character_varying
}
column "member_firstname_spouse" {
null = true
type = character_varying
}
column "member_surname" {
null = true
type = character_varying
}
column "member_suffix" {
null = true
type = character_varying
}
column "member_street" {
null = true
type = character_varying
}
column "member_streetnumber" {
null = true
type = character_varying
}
column "member_email" {
null = true
type = character_varying(255)
}
column "member_zip" {
null = true
type = integer
}
column "member_city" {
null = true
type = character_varying
}
column "member_active" {
null = true
type = boolean
default = true
}
column "member_connected" {
null = true
type = integer
}
column "created_date" {
null = true
type = timestamp
default = sql("CURRENT_TIMESTAMP")
}
column "updated_date" {
null = true
type = timestamp
}
column "member_gender" {
null = true
type = enum.gender
}
column "member_died" {
null = true
type = timestamp
}
column "member_membership_start" {
null = true
type = timestamp
}
column "member_printed" {
null = true
type = boolean
}
column "member_coordinates" {
null = true
type = point
comment = "UNUSED"
}
column "member_geo_latitude" {
null = true
type = numeric
}
column "member_geo_longitude" {
null = true
type = numeric
}
primary_key {
columns = [column.id]
}
foreign_key "members_member_connected_fkey" {
columns = [column.member_connected]
ref_columns = [table.members.column.id]
on_update = NO_ACTION
on_delete = NO_ACTION
}
index "ix_members_id" {
columns = [column.id]
}
index "ix_members_member_city" {
columns = [column.member_city]
}
index "ix_members_member_email" {
columns = [column.member_email]
}
index "ix_members_member_firstname" {
columns = [column.member_firstname]
}
index "ix_members_member_street" {
columns = [column.member_street]
}
index "ix_members_member_surname" {
columns = [column.member_surname]
}
index "ix_members_member_zip" {
columns = [column.member_zip]
}
}
table "payments" {
schema = schema.sust
column "id" {
null = false
type = serial
}
column "payment_date" {
null = true
type = date
}
column "member_id" {
null = true
type = integer
}
column "payment_amount" {
null = true
type = numeric
}
column "payment_year" {
null = true
type = date
comment = "Year, for which the payment is intended for."
}
primary_key {
columns = [column.id]
}
foreign_key "payments_member_id_fkey" {
columns = [column.member_id]
ref_columns = [table.members.column.id]
on_update = NO_ACTION
on_delete = NO_ACTION
}
index "ix_payments_id" {
columns = [column.id]
}
index "ix_payments_payment_date" {
columns = [column.payment_date]
}
index "payments_ukey_member_year" {
unique = true
columns = [column.member_id, column.payment_year]
}
}
table "payments_csv" {
schema = schema.sust
column "id" {
null = false
type = serial
}
column "pc_valuta_date" {
null = true
type = date
}
column "pc_client" {
null = true
type = character_varying(50)
}
column "pc_reference" {
null = true
type = character_varying(50)
}
column "pc_intended" {
null = true
type = character_varying(100)
}
column "pc_iban" {
null = true
type = character_varying(50)
}
column "pc_entry_date" {
null = true
type = date
}
column "pc_amount" {
null = true
type = numeric
}
column "pc_currency" {
null = true
type = character_varying(3)
}
column "pc_timestamp" {
null = true
type = timestamp
}
column "pc_processed" {
null = true
type = boolean
default = false
}
primary_key {
columns = [column.id]
}
}
enum "gender" {
schema = schema.sust
values = ["male", "female", "divers"]
}
schema "sust" {
comment = "standard public schema"
}
sustApp/whatsapp_payment_slips.py
+2 -2
View File
@@ -3,8 +3,8 @@ from heyoo import WhatsApp
#import pywhatkit
from database import engine, SessionLocal
import models
from sustApp.models.database import engine, SessionLocal
import sustApp.models.models as models
import logging, sys
import smtplib
from email.mime.multipart import MIMEMultipart