Progress...

assign_payments.py

@@ -0,0 +1,106 @@
+# 3. assign manually
+
+from database import Base, engine, SessionLocal
+from time import time
+from models import Payments_CSV, Payments, Members
+import logging, sys
+import datetime
+
+def assign_by_reference(payments: Payments_CSV):
+    for payment in payments:
+        logging.debug(f'payment.pc_reference {payment.pc_reference}')
+        reference = payment.pc_reference.split('-')
+        if len(reference) == 1:
+            reference = payment.pc_reference.split(' ')
+
+        if len(reference) == 4:
+            addPayment(payment, reference[2], reference[0], payment.pc_amount)
+        else:
+            # more than one reference (e.g. 2024-4732-096-2,-097-1,-099-1)
+            referenceList = payment.pc_reference.split(',')
+            number_of_references = completeReference(referenceList=referenceList)
+            amount = payment.pc_amount  # assume that the payment is for all references
+
+            for reference in referenceList:
+                referenceItem = reference.split('-')
+                addPayment(payment, referenceItem[2], referenceItem[0], amount/number_of_references*int(referenceItem[3]))
+
+    try:
+        db.commit()
+    except Exception as e:
+        print(e)
+        db.rollback()
+
+def assign_by_name(payments: Payments_CSV):
+    year = 2024
+    removables = ('Dipl.', 'Ing.', 'Mag.')
+    for payment in payments:
+        logging.debug(f'payment.pc_reference {payment.pc_intended}')
+        name = payment.pc_client.split(' ')
+        for removeable in removables:
+            while removeable in name: name.remove(removeable)
+        
+        # last part is usually surname
+        member = db.query(Members).filter(Members.member_surname.like(name[len(name)-1]), Members.member_firstname.like(name[0])).all()
+        if len(member) > 1:
+            logging.debug(f'More than one member with same name {payment.pc_client}')
+        elif len(member) == 0:
+            logging.debug(f'No member found with name {payment.pc_client}')
+        else:
+            # update Payments_CSV
+            payment.pc_reference = f'{year}-4732-{member[0].id}-{member[0].member_count}'
+
+        #addPayment(payment, referenceItem[2], referenceItem[0], amount/number_of_references*int(referenceItem[3]))
+
+    try:
+        db.commit()
+    except Exception as e:
+        print(e)
+        db.rollback()
+
+
+def addPayment(payment, member_id, year, amount):
+    newPayment = Payments(payment_date = payment.pc_valuta_date,
+                                  member_id = member_id,
+                                  payment_year = datetime.date(int(year),1,1),
+                                  payment_amount = amount)
+    
+    db.add(newPayment)
+    payment.pc_processed = True
+
+def completeReference(referenceList: list) -> int:
+    number_of_references = 0
+    i=0
+    for reference in referenceList:
+        referenceItem = reference.split('-')
+        if referenceItem[0] == '':
+            referenceItem.insert(1, referenceList[0].split('-')[0])
+            referenceItem.insert(2, referenceList[0].split('-')[1])
+            referenceItem.pop(0)
+
+            reference = '-'.join(referenceItem)
+            referenceList[i]=reference
+        i+=1
+
+        number_of_references += int(referenceItem[3])
+    
+    return number_of_references
+
+if __name__ == "__main__":
+    
+    t = time()
+    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+    logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+
+    db = SessionLocal()
+    Base.metadata.create_all(bind=engine)
+
+    # 1. create missing reference by name
+    payments = db.query(Payments_CSV).filter(Payments_CSV.pc_intended!='', Payments_CSV.pc_reference=='', Payments_CSV.pc_processed!=True).all()
+    assign_by_name(payments)
+
+    # 2. assign payment by reference
+    payments = db.query(Payments_CSV).filter(Payments_CSV.pc_reference!='', Payments_CSV.pc_processed!=True).all()
+    assign_by_reference(payments)
+
+    logging.debug(f"Time elapsed: " + str(time() - t) + " s.") #0.091s

convert_payment_export_csv.py

@@ -0,0 +1,73 @@
+# Importing libraries
+from numpy import genfromtxt, savetxt
+from time import time
+from datetime import datetime
+import models
+import os, re
+from database import engine, SessionLocal
+from pytz import timezone
+
+def Load_Data(file_name):
+    #data = genfromtxt(file_name, delimiter=';', skip_header=1, converters={0: lambda s: str(s)})
+    #str2date = lambda x: datetime.strptime(x, '%d-%m-%y')
+    data = genfromtxt(file_name, delimiter=';', dtype=None, skip_header=0, encoding='utf-8-sig')
+
+    return data.tolist()
+
+def extract_values(text):
+    # Regular expressions to match the required fields
+    regex_patterns = {
+        'Auftraggeber': r'Auftraggeber: (.*?) (Verwendungszweck|Zahlungsreferenz|IBAN|BIC)',
+        'Verwendungszweck': r'Verwendungszweck: (.*?) (Zahlungsreferenz|IBAN|BIC)',
+        'Zahlungsreferenz': r'Zahlungsreferenz: (.*?) (Verwendungszweck|IBAN|BIC)',
+        'IBAN': r'IBAN Auftraggeber: (.*?) ',
+        'BIC': r'BIC Auftraggeber: (.*?)$'
+    }
+    
+    # Initialize an empty list to store extracted information
+    extracted_info = []
+    
+    # Initialize an empty dictionary to store information from this row
+    row_info = {}
+    
+    # Extract information using regular expressions
+    for key, pattern in regex_patterns.items():
+        match = re.search(pattern, text)
+        if match:
+            row_info[key] = match.group(1)
+        else:
+            row_info[key] = None
+    
+    # Append the extracted information from this row to the list
+    extracted_info.append(row_info)
+    
+    return extracted_info
+
+if __name__ == "__main__":
+    t = time()
+
+    db = SessionLocal()
+    models.Base.metadata.create_all(bind=engine)
+
+    try:
+        home_directory = os.path.expanduser('~')
+        source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407 (7)_org.csv")
+        target_csv = source_csv.replace(".csv", "_out.csv")
+        data = Load_Data(source_csv) 
+
+        tz = timezone('UTC')  # Replace with your desired timezone
+        outData = []
+        for i in data:
+            if ('Auftraggeber' in i[1]):
+                # extract key-value pairs Auftraggeber, Verwendungszweck, Zahlungsreferenz, IBAN
+                new_list = extract_values(i[1])
+
+                if len(new_list) > 0:
+                    outData.append([i[0], new_list[0].get('Auftraggeber'), new_list[0].get('Zahlungsreferenz'), new_list[0].get('Verwendungszweck'), new_list[0].get('IBAN'), i[2], i[3].replace(',','.'), i[4], i[5]])
+        
+        savetxt(target_csv, outData, delimiter=";", encoding='utf-8', header='valuta_date;client;reference;intended;IBAN;entry_date;amount;currency;timestamp', fmt='%s')
+
+    except Exception as e:
+        print (e)
+    finally:
+        print("Time elapsed: " + str(time() - t) + " s.") #0.091s

create_payment_slip.py

@@ -1,5 +1,4 @@
 import models
-import locale
 
 from reportlab.lib.pagesizes import A4
 from reportlab.pdfgen import canvas

helper_scripts.sql

@@ -22,4 +22,60 @@ values (now(), 78, 186, '2024-01-01'),
 	   (now(),  4, 186, '2024-01-01'),
 	   (now(),  9, 186, '2024-01-01')
 	   
-update deliveries set delivery_year = '2024-01-01'
+insert into deliveries (delivery_date, member_id, delivered_by_id, delivery_year)
+values (now(), 8, 186, '2024-02-21')
+
+select count(*) from deliveries d
+
+update deliveries set delivery_year = '2024-01-01'
+
+create table sust.payments_csv
+(id serial, 
+ pc_valuta_date date,
+ pc_client varchar(50),
+ pc_reference varchar(50), 
+ pc_intended varchar(50),
+ pc_iban varchar(50), 
+ pc_entry_date date, 
+ pc_amount money, 
+ pc_currency varchar(3),
+ pc_timestamp timestamp)
+ 
+COPY payments_csv(pc_valuta_date, pc_client, pc_reference, pc_intended, pc_iban, pc_entry_date, pc_amount, pc_currency, pc_timestamp)
+FROM 'C:\Users\zisco\OneDrive\Dokumente\SUST\UmsatzListe_AT493443700000113407_v1.csv'
+DELIMITER ';'
+CSV HEADER;
+ 
+CREATE UNIQUE INDEX payments_ukey_member_year ON sust.payments USING btree (member_id, payment_year)
+
+select count(*) from payments_csv where pc_processed = true
+
+
+select * from payments_csv pc where id = 168
+
+ALTER TABLE payments_csv ADD PRIMARY KEY (id);
+
+select * from payments p where p.member_id in (132)
+select pc.pc_reference from payments_csv pc where pc_processed =false order by pc_reference asc 
+
+select m.member_surname, m.member_firstname, m.member_firstname_spouse from members m
+except
+select mp.member_surname, mp.member_firstname, mp.member_firstname_spouse from members_paid mp
+
+select * from members_paid mp where id = 8
+
+-- already paid?
+select * from members_active m 
+join deliveries d on d.member_id = m.id 
+where m.id in  (select id from members_delivered md
+except
+select id from members_paid mp)
+order by delivery_date asc
+
+insert into deliveries (delivery_date, member_id, delivery_year)
+select now(), ss.id, '2024-01-01' from
+(select id from members_active m 
+except
+select member_id from deliveries d) ss
+
+select count(*) from deliveries d

import_transactions.py

@@ -0,0 +1,56 @@
+# Importing libraries
+from numpy import genfromtxt
+from time import time
+from datetime import datetime
+import models
+import os
+from database import engine, SessionLocal
+from pytz import timezone
+
+def Load_Data(file_name):
+    #data = genfromtxt(file_name, delimiter=';', skip_header=1, converters={0: lambda s: str(s)})
+    #str2date = lambda x: datetime.strptime(x, '%d-%m-%y')
+    data = genfromtxt(file_name, delimiter=';', dtype=None, skip_header=1, encoding='utf-8') #
+    return data.tolist()
+
+if __name__ == "__main__":
+    t = time()
+
+    db = SessionLocal()
+    models.Base.metadata.create_all(bind=engine)
+
+    try:
+        home_directory = os.path.expanduser('~')
+        source_csv = os.path.join(home_directory, "OneDrive", "Dokumente", "SUST", "UmsatzListe_AT493443700000113407_v7.csv")
+        data = Load_Data(source_csv) 
+
+        tz = timezone('UTC')  # Replace with your desired timezone
+        for i in data:
+            dt = datetime.strptime(i[8], '%d.%m.%Y %H:%M:%S:%f')
+            ts_timestamp = tz.localize(dt)
+
+            record = models.Payments_CSV(**{
+                'pc_valuta_date' : datetime.strptime(i[0], '%d.%m.%Y').date(),
+                'pc_client' : i[1],
+                'pc_reference' : i[2],
+                'pc_intended' : i[3],
+                'pc_iban' : i[4],
+                'pc_entry_date' : datetime.strptime(i[5], '%d.%m.%Y').date(),
+                'pc_amount' : float(i[6]),
+                'pc_currency' : i[7],
+                'pc_timestamp' : ts_timestamp
+            })
+            db.add(record) #Add all the records
+
+        try:
+            db.commit() #Attempt to commit all the records
+        except Exception as e:
+            print(e)
+    
+    except Exception as e:
+        print (e)
+        db.rollback() #Rollback the changes on error
+    finally:
+        db.close() #Close the connection
+    
+    print("Time elapsed: " + str(time() - t) + " s.") #0.091s

mail_payment_slips.py

@@ -28,7 +28,8 @@ def send_mail(member: models.Members):
     smtp_server = 'mail.gmx.com'
     smtp_port = 587
     smtp_username = 'zisco@gmx.at'
-    smtp_password = '57spunk07'
+    #TODO Secure
+    smtp_password = '57spunk07' 
 
     from_email = 'sv.stthomas@gmx.at'
     to_email = member.member_email
@@ -70,7 +71,10 @@ models.Base.metadata.create_all(bind=engine)
 #members = db.query(models.Members).filter(models.Members.member_email != None, ~models.Members.id.in_([183,51,200,49])).all()
 #members = db.query(models.Members).filter(models.Members.member_active == True).all()
 #members = db.query(models.Members).filter(models.Members.id.in_([2,3,9,199]), models.Members.member_active == True, models.Members.member_printed == False).all()
-members = db.query(models.Members).filter(models.Members.id.in_([73,142])).all()
+members = db.query(models.Members).filter(models.Members.id.in_([8])).all()
 
 for member in members:
-    send_mail(member=member)
+    if member.member_email:
+        send_mail(member=member)
+    else:
+        logging.debug(f'Member {member.member_surname} has no email associated!')

member_map.py

@@ -16,6 +16,11 @@ def payment_slip_delivered(member: models.Members, year: int):
              .filter(models.Deliveries.member_id == member.id, extract('year', models.Deliveries.delivery_date) == year)\
              .count()
 
+def payment_made(member: models.Members, year: int):
+    return db.query(models.Payments.payment_date)\
+             .filter(models.Payments.member_id == member.id, extract('year', models.Deliveries.delivery_date) == year)\
+             .count()
+
 def update_missing_locs():
     members = db.query(models.Members).filter(models.Members.member_geo_latitude == None).all()
 
@@ -90,23 +95,26 @@ marker_cluster = MarkerCluster().add_to(map)
 for member in members:
     if member.member_geo_latitude != None and member.member_geo_latitude != None:
         delivered = payment_slip_delivered(member, 2024)
+        paid = payment_made(member, 2024)
         
         iconColor = "blue"
         hasMail = checkEmail(member.member_email)
-        if hasMail:
+        if not paid:
-            iconColor = "orange"
+            iconColor = "red"
         elif delivered:
             iconColor = "green"
+        elif hasMail:
+            iconColor = "orange"
 
-        if hasMail and delivered:
+        # if hasMail and delivered:
-            iconColor = "darkgreen"
+        #     iconColor = "darkgreen"
 
         #delivered = member.member_deliveries
         location = (member.member_geo_latitude, member.member_geo_longitude)
         folium.Marker(location=location,
-                        popup=f'{member.member_firstname} {member.member_surname}',
+                      popup=f'{member.member_firstname} {member.member_surname}',
-                        tooltip=f'\nZugestellt' if delivered else 'Nicht zugestellt',
+                      tooltip=f'\nZugestellt & bezahlt' if (delivered and paid) else 'Nicht bezahlt' if not paid else 'Nicht zugestellt',
-                        icon=folium.Icon(color=iconColor))\
+                      icon=folium.Icon(color=iconColor))\
         .add_to(marker_cluster)
 
 #Display the map

models.py

@@ -1,11 +1,9 @@
-from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, DateTime, Float
+from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, DateTime, Float, Numeric
 from sqlalchemy.types import TypeDecorator, VARCHAR, Integer
 from sqlalchemy_utils import EmailType
-from sqlalchemy.orm import relationship, Mapped
+from sqlalchemy.orm import relationship
-from geoalchemy2 import Geometry
 from database import Base
 import datetime
-from coordinates_class import EasyGeometry
 
 class NullCatchingString(TypeDecorator):
     impl = VARCHAR
@@ -57,8 +55,10 @@ class Payments(Base):
     __tablename__ = 'payments'
 
     id = Column(Integer, primary_key=True, index=True)
-    payment_date = Column(String, index=True)
+    payment_date = Column(DateTime, index=True)
     member_id = Column(Integer, ForeignKey("members.id"))
+    payment_amount = Column(Numeric)
+    payment_year = Column(DateTime, index=True)
 
     def __repr__(self):
         return f'<Comment "{self.content[:20]}...">'
@@ -77,4 +77,19 @@ class Deliveries(Base):
     
 """     delivered_by_id = Column(Integer, ForeignKey("members.id" ))
     member = relationship("Members", foreign_keys=[member_id])
-    delivering_member = relationship("Members", foreign_keys=[member_id]) """
+    delivering_member = relationship("Members", foreign_keys=[member_id]) """
+
+class Payments_CSV(Base):
+    __tablename__ = 'payments_csv'
+
+    id = Column(Integer, primary_key=True, index=True)
+    pc_valuta_date = Column(DateTime)
+    pc_client = Column(String)
+    pc_reference = Column(String)
+    pc_intended = Column(String)
+    pc_iban = Column(String) 
+    pc_entry_date = Column(DateTime)
+    pc_amount = Column(Numeric)
+    pc_currency = Column(String) 
+    pc_timestamp = Column(DateTime)
+    pc_processed = Column(Boolean, default=False)