zisco/SUST
1
1
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases Wiki Activity

Progress...

Browse Source
This commit is contained in:
Markus Zimmerberger
2024-03-13 11:31:10 +01:00
parent 8fb7af9205
commit 408e282fdc
10 changed files with 3576 additions and 3181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
assign_payments.py
+106
View File
@@ -0,0 +1,106 @@
# 3. assign manually
from database import Base, engine, SessionLocal
from time import time
from models import Payments_CSV, Payments, Members
import logging, sys
import datetime
def assign_by_reference(payments: Payments_CSV):
for payment in payments:
logging.debug(f'payment.pc_reference {payment.pc_reference}')
reference = payment.pc_reference.split('-')
if len(reference) == 1:
reference = payment.pc_reference.split(' ')
if len(reference) == 4:
addPayment(payment, reference[2], reference[0], payment.pc_amount)
else:
# more than one reference (e.g. 2024-4732-096-2,-097-1,-099-1)
referenceList = payment.pc_reference.split(',')
number_of_references = completeReference(referenceList=referenceList)
amount = payment.pc_amount # assume that the payment is for all references
for reference in referenceList:
referenceItem = reference.split('-')
addPayment(payment, referenceItem[2], referenceItem[0], amount/number_of_references*int(referenceItem[3]))
try:
db.commit()
except Exception as e:
print(e)
db.rollback()
def assign_by_name(payments: Payments_CSV):
year = 2024
removables = ('Dipl.', 'Ing.', 'Mag.')
for payment in payments:
logging.debug(f'payment.pc_reference {payment.pc_intended}')
name = payment.pc_client.split(' ')
for removeable in removables:
while removeable in name: name.remove(removeable)
# last part is usually surname
member = db.query(Members).filter(Members.member_surname.like(name[len(name)-1]), Members.member_firstname.like(name[0])).all()
if len(member) > 1:
logging.debug(f'More than one member with same name {payment.pc_client}')
elif len(member) == 0:
logging.debug(f'No member found with name {payment.pc_client}')
else:
# update Payments_CSV
payment.pc_reference = f'{year}-4732-{member[0].id}-{member[0].member_count}'
#addPayment(payment, referenceItem[2], referenceItem[0], amount/number_of_references*int(referenceItem[3]))
try:
db.commit()
except Exception as e:
print(e)
db.rollback()
def addPayment(payment, member_id, year, amount):
newPayment = Payments(payment_date = payment.pc_valuta_date,
member_id = member_id,
payment_year = datetime.date(int(year),1,1),
payment_amount = amount)
db.add(newPayment)
payment.pc_processed = True
def completeReference(referenceList: list) -> int:
number_of_references = 0
i=0
for reference in referenceList:
referenceItem = reference.split('-')
if referenceItem[0] == '':
referenceItem.insert(1, referenceList[0].split('-')[0])
referenceItem.insert(2, referenceList[0].split('-')[1])
referenceItem.pop(0)
reference = '-'.join(referenceItem)
referenceList[i]=reference
i+=1
number_of_references += int(referenceItem[3])
return number_of_references
if __name__ == "__main__":
t = time()
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
db = SessionLocal()
Base.metadata.create_all(bind=engine)
# 1. create missing reference by name
payments = db.query(Payments_CSV).filter(Payments_CSV.pc_intended!='', Payments_CSV.pc_reference=='', Payments_CSV.pc_processed!=True).all()
assign_by_name(payments)
# 2. assign payment by reference
payments = db.query(Payments_CSV).filter(Payments_CSV.pc_reference!='', Payments_CSV.pc_processed!=True).all()
assign_by_reference(payments)
logging.debug(f"Time elapsed: " + str(time() - t) + " s.") #0.091s
convert_payment_export_csv.py
+73
View File
@@ -0,0 +1,73 @@
# Importing libraries
from numpy import genfromtxt, savetxt
from time import time
from datetime import datetime
import models
import os, re
from database import engine, SessionLocal
from pytz import timezone
def Load_Data(file_name):
#data = genfromtxt(file_name, delimiter=';', skip_header=1, converters={0: lambda s: str(s)})
#str2date = lambda x: datetime.strptime(x, '%d-%m-%y')
data = genfromtxt(file_name, delimiter=';', dtype=None, skip_header=0, encoding='utf-8-sig')
return data.tolist()
def extract_values(text):
# Regular expressions to match the required fields
regex_patterns = {
'Auftraggeber': r'Auftraggeber: (.*?) (Verwendungszweck|Zahlungsreferenz|IBAN|BIC)',
'Verwendungszweck': r'Verwendungszweck: (.*?) (Zahlungsreferenz|IBAN|BIC)',
'Zahlungsreferenz': r'Zahlungsreferenz: (.*?) (Verwendungszweck|IBAN|BIC)',
'IBAN': r'IBAN Auftraggeber: (.*?) ',
'BIC': r'BIC Auftraggeber: (.*?)$'
}
# Initialize an empty list to store extracted information
extracted_info = []
# Initialize an empty dictionary to store information from this row
row_info = {}
# Extract information using regular expressions
for key, pattern in regex_patterns.items():
match = re.search(pattern, text)
if match:
row_info[key] = match.group(1)
else:
row_info[key] = None
# Append the extracted information from this row to the list
extracted_info.append(row_info)
return extracted_info
if __name__ == "__main__":
t = time()
db = SessionLocal()
models.Base.metadata.create_all(bind=engine)
try:
home_directory = os.path.expanduser('~')
source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407 (7)_org.csv")
target_csv = source_csv.replace(".csv", "_out.csv")
data = Load_Data(source_csv)
tz = timezone('UTC') # Replace with your desired timezone
outData = []
for i in data:
if ('Auftraggeber' in i[1]):
# extract key-value pairs Auftraggeber, Verwendungszweck, Zahlungsreferenz, IBAN
new_list = extract_values(i[1])
if len(new_list) > 0:
outData.append([i[0], new_list[0].get('Auftraggeber'), new_list[0].get('Zahlungsreferenz'), new_list[0].get('Verwendungszweck'), new_list[0].get('IBAN'), i[2], i[3].replace(',','.'), i[4], i[5]])
savetxt(target_csv, outData, delimiter=";", encoding='utf-8', header='valuta_date;client;reference;intended;IBAN;entry_date;amount;currency;timestamp', fmt='%s')
except Exception as e:
print (e)
finally:
print("Time elapsed: " + str(time() - t) + " s.") #0.091s
create_payment_slip.py
-1
View File
@@ -1,5 +1,4 @@
import models
import locale
from reportlab.lib.pagesizes import A4
from reportlab.pdfgen import canvas
epc_qr.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 339 B

After

Width:  |  Height:  |  Size: 331 B

helper_scripts.sql
+57 -1
View File
@@ -22,4 +22,60 @@ values (now(), 78, 186, '2024-01-01'),
(now(), 4, 186, '2024-01-01'),
(now(), 9, 186, '2024-01-01')
update deliveries set delivery_year = '2024-01-01'
insert into deliveries (delivery_date, member_id, delivered_by_id, delivery_year)
values (now(), 8, 186, '2024-02-21')
select count(*) from deliveries d
update deliveries set delivery_year = '2024-01-01'
create table sust.payments_csv
(id serial,
pc_valuta_date date,
pc_client varchar(50),
pc_reference varchar(50),
pc_intended varchar(50),
pc_iban varchar(50),
pc_entry_date date,
pc_amount money,
pc_currency varchar(3),
pc_timestamp timestamp)
COPY payments_csv(pc_valuta_date, pc_client, pc_reference, pc_intended, pc_iban, pc_entry_date, pc_amount, pc_currency, pc_timestamp)
FROM 'C:\Users\zisco\OneDrive\Dokumente\SUST\UmsatzListe_AT493443700000113407_v1.csv'
DELIMITER ';'
CSV HEADER;
CREATE UNIQUE INDEX payments_ukey_member_year ON sust.payments USING btree (member_id, payment_year)
select count(*) from payments_csv where pc_processed = true
select * from payments_csv pc where id = 168
ALTER TABLE payments_csv ADD PRIMARY KEY (id);
select * from payments p where p.member_id in (132)
select pc.pc_reference from payments_csv pc where pc_processed =false order by pc_reference asc
select m.member_surname, m.member_firstname, m.member_firstname_spouse from members m
except
select mp.member_surname, mp.member_firstname, mp.member_firstname_spouse from members_paid mp
select * from members_paid mp where id = 8
-- already paid?
select * from members_active m
join deliveries d on d.member_id = m.id
where m.id in (select id from members_delivered md
except
select id from members_paid mp)
order by delivery_date asc
insert into deliveries (delivery_date, member_id, delivery_year)
select now(), ss.id, '2024-01-01' from
(select id from members_active m
except
select member_id from deliveries d) ss
select count(*) from deliveries d
import_transactions.py
+56
View File
@@ -0,0 +1,56 @@
# Importing libraries
from numpy import genfromtxt
from time import time
from datetime import datetime
import models
import os
from database import engine, SessionLocal
from pytz import timezone
def Load_Data(file_name):
#data = genfromtxt(file_name, delimiter=';', skip_header=1, converters={0: lambda s: str(s)})
#str2date = lambda x: datetime.strptime(x, '%d-%m-%y')
data = genfromtxt(file_name, delimiter=';', dtype=None, skip_header=1, encoding='utf-8') #
return data.tolist()
if __name__ == "__main__":
t = time()
db = SessionLocal()
models.Base.metadata.create_all(bind=engine)
try:
home_directory = os.path.expanduser('~')
source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407_v7.csv")
data = Load_Data(source_csv)
tz = timezone('UTC') # Replace with your desired timezone
for i in data:
dt = datetime.strptime(i[8], '%d.%m.%Y %H:%M:%S:%f')
ts_timestamp = tz.localize(dt)
record = models.Payments_CSV(**{
'pc_valuta_date' : datetime.strptime(i[0], '%d.%m.%Y').date(),
'pc_client' : i[1],
'pc_reference' : i[2],
'pc_intended' : i[3],
'pc_iban' : i[4],
'pc_entry_date' : datetime.strptime(i[5], '%d.%m.%Y').date(),
'pc_amount' : float(i[6]),
'pc_currency' : i[7],
'pc_timestamp' : ts_timestamp
})
db.add(record) #Add all the records
try:
db.commit() #Attempt to commit all the records
except Exception as e:
print(e)
except Exception as e:
print (e)
db.rollback() #Rollback the changes on error
finally:
db.close() #Close the connection
print("Time elapsed: " + str(time() - t) + " s.") #0.091s
mail_payment_slips.py
+7 -3
View File
@@ -28,7 +28,8 @@ def send_mail(member: models.Members):
smtp_server = 'mail.gmx.com'
smtp_port = 587
smtp_username = 'zisco@gmx.at'
smtp_password = '57spunk07'
#TODO Secure
smtp_password = '57spunk07'
from_email = 'sv.stthomas@gmx.at'
to_email = member.member_email
@@ -70,7 +71,10 @@ models.Base.metadata.create_all(bind=engine)
#members = db.query(models.Members).filter(models.Members.member_email != None, ~models.Members.id.in_([183,51,200,49])).all()
#members = db.query(models.Members).filter(models.Members.member_active == True).all()
#members = db.query(models.Members).filter(models.Members.id.in_([2,3,9,199]), models.Members.member_active == True, models.Members.member_printed == False).all()
members = db.query(models.Members).filter(models.Members.id.in_([73,142])).all()
members = db.query(models.Members).filter(models.Members.id.in_([8])).all()
for member in members:
send_mail(member=member)
if member.member_email:
send_mail(member=member)
else:
logging.debug(f'Member {member.member_surname} has no email associated!')
map.html
+3241 -3163
View File
File diff suppressed because it is too large Load Diff
member_map.py
+15 -7
View File
@@ -16,6 +16,11 @@ def payment_slip_delivered(member: models.Members, year: int):
.filter(models.Deliveries.member_id == member.id, extract('year', models.Deliveries.delivery_date) == year)\
.count()
def payment_made(member: models.Members, year: int):
return db.query(models.Payments.payment_date)\
.filter(models.Payments.member_id == member.id, extract('year', models.Deliveries.delivery_date) == year)\
.count()
def update_missing_locs():
members = db.query(models.Members).filter(models.Members.member_geo_latitude == None).all()
@@ -90,23 +95,26 @@ marker_cluster = MarkerCluster().add_to(map)
for member in members:
if member.member_geo_latitude != None and member.member_geo_latitude != None:
delivered = payment_slip_delivered(member, 2024)
paid = payment_made(member, 2024)
iconColor = "blue"
hasMail = checkEmail(member.member_email)
if hasMail:
iconColor = "orange"
if not paid:
iconColor = "red"
elif delivered:
iconColor = "green"
elif hasMail:
iconColor = "orange"
if hasMail and delivered:
iconColor = "darkgreen"
# if hasMail and delivered:
# iconColor = "darkgreen"
#delivered = member.member_deliveries
location = (member.member_geo_latitude, member.member_geo_longitude)
folium.Marker(location=location,
popup=f'{member.member_firstname} {member.member_surname}',
tooltip=f'\nZugestellt' if delivered else 'Nicht zugestellt',
icon=folium.Icon(color=iconColor))\
popup=f'{member.member_firstname} {member.member_surname}',
tooltip=f'\nZugestellt & bezahlt' if (delivered and paid) else 'Nicht bezahlt' if not paid else 'Nicht zugestellt',
icon=folium.Icon(color=iconColor))\
.add_to(marker_cluster)
#Display the map
models.py
+21 -6
View File
@@ -1,11 +1,9 @@
from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, DateTime, Float
from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, DateTime, Float, Numeric
from sqlalchemy.types import TypeDecorator, VARCHAR, Integer
from sqlalchemy_utils import EmailType
from sqlalchemy.orm import relationship, Mapped
from geoalchemy2 import Geometry
from sqlalchemy.orm import relationship
from database import Base
import datetime
from coordinates_class import EasyGeometry
class NullCatchingString(TypeDecorator):
impl = VARCHAR
@@ -57,8 +55,10 @@ class Payments(Base):
__tablename__ = 'payments'
id = Column(Integer, primary_key=True, index=True)
payment_date = Column(String, index=True)
payment_date = Column(DateTime, index=True)
member_id = Column(Integer, ForeignKey("members.id"))
payment_amount = Column(Numeric)
payment_year = Column(DateTime, index=True)
def __repr__(self):
return f'<Comment "{self.content[:20]}...">'
@@ -77,4 +77,19 @@ class Deliveries(Base):
""" delivered_by_id = Column(Integer, ForeignKey("members.id" ))
member = relationship("Members", foreign_keys=[member_id])
delivering_member = relationship("Members", foreign_keys=[member_id]) """
delivering_member = relationship("Members", foreign_keys=[member_id]) """
class Payments_CSV(Base):
__tablename__ = 'payments_csv'
id = Column(Integer, primary_key=True, index=True)
pc_valuta_date = Column(DateTime)
pc_client = Column(String)
pc_reference = Column(String)
pc_intended = Column(String)
pc_iban = Column(String)
pc_entry_date = Column(DateTime)
pc_amount = Column(Numeric)
pc_currency = Column(String)
pc_timestamp = Column(DateTime)
pc_processed = Column(Boolean, default=False)