From 9936df10912299b06562e382ecf80347be96d74f Mon Sep 17 00:00:00 2001 From: Markus Zimmerberger Date: Sat, 22 Feb 2025 18:00:16 +0100 Subject: [PATCH] Intermediate commit --- ...tattung_aichinger_enhanced.cpython-312.pyc | Bin 0 -> 6621 bytes ...r_bestattung_wels_enhanced.cpython-312.pyc | Bin 0 -> 7683 bytes crawler_bestattung_aichinger_enhanced.py | 145 ++++++++++++++ crawler_bestattung_wels_enhanced.py | 187 ++++++++++++++++++ crawler_bestatung-aichinger_enhanced.py | 132 ------------- crawler_bestatung-wels_enhanced.py | 125 ------------ main.py | 6 + schema.hcl | 38 ++++ 8 files changed, 376 insertions(+), 257 deletions(-) create mode 100644 __pycache__/crawler_bestattung_aichinger_enhanced.cpython-312.pyc create mode 100644 __pycache__/crawler_bestattung_wels_enhanced.cpython-312.pyc create mode 100644 crawler_bestattung_aichinger_enhanced.py create mode 100644 crawler_bestattung_wels_enhanced.py delete mode 100644 crawler_bestatung-aichinger_enhanced.py delete mode 100644 crawler_bestatung-wels_enhanced.py create mode 100644 schema.hcl diff --git a/__pycache__/crawler_bestattung_aichinger_enhanced.cpython-312.pyc b/__pycache__/crawler_bestattung_aichinger_enhanced.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa946bae475809639ab42e70d38dcdfb4e6272e3 GIT binary patch literal 6621 zcmbVQTTmNUnm#SHq}Ck}7fJXc8xvaw1lU|`oH*g;8#ayuc5JQVo<`k(!m@21M6QmIh8`@oc4wLHv&A3d3^Em!SJD`E%eur;39*@wIlJ5|~8yzGBk zYC(wCN$mlhK9~Re=kovO|G)G9yVGeykPQF*n~8t(AoO3Pq9=Q)@ZyvKp$CXZJSCzq z`5HtjZU`HwVw*8+)cWW!4PT>ZikrjcxFu|fGhrrf4O`>3uq|#6+ckPxbi|!uXWSKb zX?2t6j=RI|60eGIg~qEgT&aDl!d1LE>=8VlBHr?uAzW=hLJiOG)`zerk~&ei7TV}o zs94{g3$+iaVm<8TDe|X}qLAR_Z9eD>`*^$H;~k_7b2y>wDwW+(uGrRISt?gSyQf(G z%og^;?9PX!X~XpfbQvt$Pe&)_+1}S;YQ&Ne)r8o2;okc^Po@> zd-sPNb$)#P%>kgS0m|U3DfxP`Z=F6t@%7=uK;0-D*4rpFL(NbPs2xev6llmbM8V(P z$J54ZXo?E%{hCZa9HdldpTMQ%=y+NjNv2alv+9gsfs+MRNK8Z%f@3Gy)D#sA^;6t9VY_QT^+$dZz?4U?+ar?u({5o-Jzph zfkQ)QMlYOi4v5j4Lg2IzxtR0@2&|3e_%Adn-+k9Ge*AYGA?8wCA8h*$^Wm|pxnWZ>2* zx!fBoIaTN=C%I%zw#t;cjT|fS>dxWI6PVss4EULHvIFpN9&yZJS1e+nhW7tr=B#?`7{~7y6g#R^08m zGaGjAf_K#y%=>~%Zx!sv75cb#Pb1rpn%E^xB)1`dfU+o7776r}65tmw@lom#tsmB` zA#0EvG8w5Q-lQ9E@WN&|YGT@UB;KOa%o?-C0W?p2@Ye{<(pmbIa{|%=mr+{ltSM{4 zhqI=b?uxeYj*&g=NW7hQ=woAL71$!uQ4`<@G;7Y9@d4SX_r_d$Y3rLc^UgBA%B^<2 z^gVu-|8qyKZ98JJ431@)7+~L>iFZG$D4&%zYmL?HqyWWrmnm#=o!+0d>8*UFhNwJQ z8?T|qA3X*8E4fsaxl|GF@uOTE2|(i0)U17WyLNc87C^YwSv%x4S;wd7Gqb)<3NCDn zu-=hm;t92+Q^t1dEnDBLC2P+*0L_*(F}8QBPcP}RnRRBJxOSV)*1c|3^VHo-GV#G$ z+Bwgx3s${P@7?-lUHWP%v~4EukIeKpG@9`R{llsuEvhC!l9HqbC&AZgW*`lw0}M|T z3Y_51mpCVpOH26qiuzE4$}OrW!!+`t#(1dl+Ups}utxDEWdj~o2^J54IEgX2!(S7#lBpY< z23Q-B1wH_Cae?%#ryUGyDtdW}}ScR+_0pF=6u=co!vQ+DMR0LtMV>7C0Jc>)^0cka7^1%(MdVr(1;|p z^(>OI}T&u2oaOVD*g)2()#rZRZ>UZa?>!&U*_&(qB z$(~hTYu?woY+A+zU(c%VoxJaz6Dymv!kW6F)2N|pF~M;uVXu7k_AC+C#w z6D#$TbLZC_o_j-gh87$vjz(?l;iaCX=Hs^JT4w?-ND;*)((&z8Ad(2M zC3mWLmt2}HDi6RZ=;h!ioq;TeHXN>dXYZVy7Z-;Lj<($3y2Y{KtzQ`YeCU&*CF;}R zg12dj%X?ezIG=fabIy&rz4I|8(EsGZled-Xk=%tfyH}yT#U~os|Mws{!Bd*x9)mcy z3$E-nDJzo8i1Agp4un}!1%e9k#VvrIEsJf|D3?)M)|f3v!z3!vy&a-&ae6%-lgk(^ zRs}oQkqBPZ6-?&{IKzOcyB9&hUV*0c-D+s$6*yz7e+D%mxmKU>0K$i6X}OGFcw?3Z zIx!=y@`N; zaS{iT5lu6BaHMY4F;Waj5$`Ih4p3cobnwEv=X*y7Rr7cfgWhTn4b7NqItZ?j_&$>) z3PK9E0&mppq?DCUv!zNv2@@~}bU#$)1YlBON z3$>kd_H~zU!_xp)!_zi5u;JOeNH0#TctUFxH5>jzOZ5eR=iCrjJ!j>;b9c^teCxsO z`?nXlrOHBG(~6_{xf{9to7Ko#anE(fwZeEe9i;5MDM zmDZwtHi+zTB8gnhU}8_UqDjU0krqxWhMl%3Lm)d$;_lc^G7aSzr5Op6$et!*e3O)9 z$W%I+0(b+kNfZFiapMBS(>R`pp2QphpNx?gDk(Y!dPjS8i>T6l5P;MIPO7;WQpFPC z$HWDh)9{V06mSLnh1sK{fpIW&ic90dpGXfSQscc_vKD~2dwvln|6BUd-9Gw zIqO>Go`snue|~R=vZr(T((=p`f4=*?(skkKrKdB>H8y|oXUZ_AaAQhDRN_)TA}eEQ zC4FlX!DFe{0G~d?IpWQysUcFEqRx{~#RY@59>!a+_;U2Ow3XU7f8bjLpKrX#RyUt5 zSP$gP>+br++7yd)kHU>n6o0u0#3iM^PI9UDk8;KXPqQ5tmv5_*u^jRCEg zC{x}VQsb#W<_Dx^QX!kI38}bh_%XH1&X3(pCbQ+L{Ub$WfR4AuGrNEC58J8Atf#hW z&+S%Am^j(gUf}e7oO{lFp7Y(qzgsLu1nD=E|1|lVa)kbq6qIDjXCBzp2(2O(u~ZZV z$X6AmVyb|OlIzp~wbVujX!xq5nwU1Ajp+iqm_DG783KlwF<^|D0w#%{j+$eZfF)KE zD3QvVs5NE{So5-M0h=VNG*Bvi%K~MrHel!MKS!+YZB@XbLR>kkXAK{!$ri~a?bT3tQ$#z8sO1#&cbMFp83r1pIw#b*3U!TK6I(Ogj1YbkMsS3)MFzhv zUy;t%PWJ()_3>P6Em?s$#nuImeFs_flB{yzZ2ks$^+cYc8|2moyxcL?eVxK&g`Z$O zN*jfsY>Pbq^^vm&Rt9()xMND6zzHZfawj-%VJ%v?mdJ}=m!a$=g%+p6)hpmF#;Fl;*;SxCu(~W z(b-sB)IA*+xJeF++7SVV|4ueuX3w?!fj7!!}3BnBD?f1{k&H2VTUSI@aHa^)h$3=M`BlwfY z{GnNlxwz26!my_J^E1pO_tf7m9+ir{iP#JiUx0Gb6iUP=!jrxQCKeSbEa_l~_M~Md z!3&d^<6nt_B2hofb2!e#xQA4?sGnhYelCI8ht%JQ^b|0P^h^Q^4=KN>Wyc9y=-@hQ z^Mwc3A+w4&6rflXP_a}%4Vy%>Dpt+Xtmc;XLt5r!b*w&2O{fAIXxDN&*1#HW84H(7 zFVz|fTWyrerb5|#sBBpEPx`7g=la;DCDS*ytq<%PN|#S%^sar4xC zzNbe<+p=zbbjz`AIG4V#XDwf5f09aH+ADD^KmU_>`r=+$)mp{Re6L->PbezLdX9(V ztXGr7Ly2TK8fE;QzE*eh3s9MusdEcbak9{&v$ir-90liQSJq9{0hvg z^W-Vt$u4*El}n>TgD2e4@HFSX$c3g8UU%;lPQ*BW=V@Q7?__KH8DCplt9yi*U~pLO zmK^@BI?uZC_3q9NU#nLZ?r(2xYlYu7_dpnP6N!00X-vNKUFCh}IrOuWoWHHh*Vfg3 z=1g+-5&1(*2<%9LpE~O%HYe(a3imT3?rX4Q!M2mZLo3zQ+UaZawRJvrt)AyFu{-`Y z;84~`>5vRRqV`%_+gbNuI6gb?oqF(G-cjvh3@aXDy0 zZXm!SDxfs02C89cpxOlu7-iE7aGZPPQy$8L2k4E!n;=9S^@B1Xt0+aKP!Mu*T0kFA zl8u)wL`XK4aH(mYi%#G+;Npn@w;Laa54p_YcvuJqla2%Wd`fL6;U|bIjdp9FSfW4E zJEbH4$bg91lLquAWB_+jf-+weIXG}=B}vGe$)fiRQ;RBrTze@+h$%RfV3Mj^D#aHr zss#n>p_DpeKEw<~R;}=kATK?fG+qS<(9$yrCJZjBZi-`AI3`Kcln{&hW*7{uUY)4L z4BXY2s0~deh?zBA9qIpGu;*g`@aTxB#~gS)Uf@ObBqv}pVNnMTNFXw4+)m2sSscY= z%?LL?10Is&$!y(_1^G^avqr2v+UmSg><*k-mrkt~5 zU#+qmmvmohk)vky$(1M9)f+Et+dG%^cdRvQ*LG@Kvb8N6i#h948U0i8j*$T?iP&Rl z$LfHRBNQp@n8E^UNDy?}`UP+=6#wKCuYjm7F4B(VS96$P^DHf%sM_DJvYQRceaglrd#W znQvJYuo9^+wkf%SCQHhK%MY9eTgnJ8lG2n3Xj#hqbM&@W(M6$~|058eE~Ud-7E55v z9;LPTO_db;AJAj}Ha&oN+iA4m2muBTsTOr}s1XUoQc9@_VLOEm$g-piFiurb5A3LM z4Bh;A4&`xk`LL*wu#Bhyh|7b~kW6cfR0qaH1Mn40N%?TXEj75M2KPi_HqQD6M3sQy zkNtvNuvv0*{+wPHQBr?|^&N@%j*KPCpN?N=qG8re05WvLF-GWiV`4bHY8ep~z)-TT zZ+0dc23W_r!R+7#S&h5JO^|uFJE=e69tH#kIr9nkU>^TLsp4rcMOfe@Elh!b)LSaj z697;E{Q<%kL_H}4nP^m`1#VstH30F$GolW2Gf^hQVPa}UJygyRC@<1v%%bsmCOXUY zW1PT}gAt7(W+p7aq>`Md0i%g!C?*;v!cl=>wgpi$5yk+#)eOset+Mf?d3cS8y8OEV z5G(<_GK?hei$Wm?N*Pg&6LTUB1ne;vg`tEJQC=(|1IpuE7^P?|mT{dl_5vuja`Rzc zfM-n}5Q}D6M;=sLft#4^14*{vW>^=@gam0A$*P0a`(Vp?=+FNWb{#xtg>yB$5?b7b#{)@}uT*aj&)1K9N$6T5z^KP4;*tJ*PbsgPt`LiznCY^J2X7p8e?Trsmg@L1% zdUx#=l98u#_9IJuyR996uy=0QmwQ()tz23iUK#$((Ubw>-v>mr+IKHrx!v>e+`40( zdAIWY`giL$%ePcH&)FSMPuA13?di>oURy5NSJ9{RCZ4>Jt|xrSH2La1>-bx)#z;~OujmaTjChC7bxOilMz_tuGz zrfv(jCo-ciXP9v2QUaiT$WxU;&d1+$1adLTZ*Wp^dvT|kJyzOX_`nGR& zZ=Tqi`bhX_;`U%>jLBRZ&p1PWaIpI=WX1Mdkn~p#QB}izq$)is^=#Yp zW!ic&&I{X)-d)d;H!tsNpy?q`o(aFTdHRoQKlU{Dx2k^MT0LOa{F{ja`Uu(~K2GwT z{|K2?^zB%OrBa0$&Y~)%!gWHyM}o~OU>*>aPi(;!DK?2!C82Fm9S3g=$ftmg9_P7I zYVa73!9$AgAwF0}MCKpJ&+6tMu*bc)A2fU+nH-&{!Vm)BIR=NOL^UxAnn_GT|u5cUxK6_SFL9}ri4;FUnL)C~wW$nS&&wO1o*Ur1mGtAxloR1)beol7@l4nGT0 zDtIIi7bK+xgj*p4uL52E87&$i41?u;0NoT&;JISoRN-y{5`Mfk;x-N?xRV7Nz*F&y-_Y|hsPKS3 zm0(xIJ`NEI1we!qg0D!R zMoNIMnipxIfErnKihdMdfjp1GUHlg1jX{ilk^Uil^C|f|M=40G@&6o)!U-=v*`g8i zlC<~4-5dcY?nFp3P;9r`TPB)ENl_dIP8=bGoOt={lgIL*D-W=U;2sSh<+vH# z1q`ANf@{PU!mBY6i-iTz&_5sI03Rjd_!42z<3uz%&V;53fK2dQ-dbW35XRGlTqSH2 zci0-L@oYS4%-=M~6Pv^T8AuxaS6SW9d`r50y39~>$9Z(e*^+g(Y_Pvde2~}@{x$W_ zsh#s<+4EyLXJF~dURmvKrE4{{lG>^4$X0f2em_^)1O8m8Yt2@+ZaOySwkyx?I;wZ; z8s3S%72T;jldU_mbv#!$uxtWz#9No!jtiJePC%w{5NWt;k-t??4vE z8^f;;Z<}iXFF=0i^`W0W^OrA8$W{%G*HBMjj^o${wa#bCj&GY!?3(R6=9;XzX2)EY zHP`(WT(F`3ulvSA--kT0Pd6`g^fsX1Q8q#w41IR>?~d2?m8d^4X!W;Mh;q-}=s8?&ks2dJOhZ!wgsrEO?FC3rP^&v1gz>o{2;n|PdLDDbX*^c! zam?rS<1<8zdYtc&Vhfm)^hlv?BZRS;C<)UpU>d56p;I|nO3Yq9##-d`!I{Lw_+b+v z5^`7b3p}1DMd|43FZ7J`2f?jN$6urqGhCd!2LMhRcxHlw2r|Qxb4{Y`qJbSJK?{;K z_w|hSD4j+60)*V9IIF0Y|Am0(K!>Q0aRMWObz}Y?4DhhPB%~p`N@?-QPKj!W6pJ*( zg|AC^0#6bi=^{!uhz6(*Z!E&8g}^;{KAa{Tk8b8e{gZMu`W#+^R^oH`e*wC$rYP!j zWc?3R`#Cavj!b_w=`&wkNFL%R1P8)Vh>(&-FoZ0%0OkL;Z zbDIkvIkTq*GhIWspS!(~84G5gc{wx8WSH?xD4L1!*^rPKpUuo(-$!Vk>QTYxg6cBC z%k$I~QktU%$;URNlE$O2%IT`p^p~`O@+>!fiQsdu3K=VxQ#nIZTDxa;tygSYo6}|( z6Q%jF&h=R&OKseXZ82N}Im2bx^hU5Rk8;l#8mlhk)#NBig)_{8U}N*LZZD ze}Den`S+gMZ2PpaLw;neJ@@97t)uCoy(7&V_Fq+fQ1#x6o6miEq-({#JhFOi<=Psv z^8M{n&sIw1r@`9tYa{?=meKdb)zS 0 else None + title_right = title_parts[1].strip() if len(title_parts) > 1 else None + else: + title_left = title_right = None + + print(f"Extracted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") + + # Check for duplicates before inserting + existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=title_left, dn_location=title_right).first() + if existing_notice: + print(f"Duplicate entry found - Date: {date_death}, Name: {title_left}, Location: {title_right}") + else: + # Insert data into the database using SQLAlchemy + death_notice = DeathNotice(dn_death=date_death, dn_name=title_left, dn_location=title_right) + session.add(death_notice) + print(f"Inserted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") + + return True + + def find_next_page(current_url): + match = re.search(r'\?p=(\d+)', current_url) + if match: + next_page_num = int(match.group(1)) + 1 + else: + next_page_num = 2 + next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}" + return next_page_url + +class Worker: + def __init__(self, crawler, start_page=1): + self.crawler = crawler + self.start_page = start_page + + def run(self): + # Create a new session + session = SessionLocal() + + try: + current_url = URL_TEMPLATE.format(start_page) + while current_url: + soup = self.crawler.get_page_content(current_url) + if not self.crawler.extract_data(soup, session): + break + current_url = self.crawler.find_next_page(current_url) + if not current_url: + break + + + # Random delay between requests + time.sleep(random.uniform(1, 5)) + + # Commit the transaction after processing all rows on the page + session.commit() + except Exception as e: + print(f"An error occurred: {e}") + session.rollback() + finally: + # Close the session + session.close() + +if __name__ == "__main__": + start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1 + crawler = AichingerCrawler() + worker = Worker(crawler, start_page) + worker.run() \ No newline at end of file diff --git a/crawler_bestattung_wels_enhanced.py b/crawler_bestattung_wels_enhanced.py new file mode 100644 index 0000000..37bc170 --- /dev/null +++ b/crawler_bestattung_wels_enhanced.py @@ -0,0 +1,187 @@ +import requests +from bs4 import BeautifulSoup +import re +import yaml +from sqlalchemy import create_engine, Column, Integer, String, Date, LargeBinary, UniqueConstraint +from sqlalchemy.orm import declarative_base, sessionmaker +from datetime import datetime +import sys +import time +import random + +BASE_URL = "https://www.bestattung-wels.at/" +URL_TEMPLATE = f"{BASE_URL}/current-deaths?page={{}}&deathCompany={{}}" +URL_TEMPLATE_NO_COMPANY = f"{BASE_URL}/current-deaths?page={{}}" + +# Load database credentials from config.yaml +with open("config.yaml", "r") as file: + config = yaml.safe_load(file) + +db_config = config['database'] +DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}" + +# Define the SQLAlchemy base and model +Base = declarative_base() + +class DeathNotice(Base): + __tablename__ = 'death_notices' + __table_args__ = ( + UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'), + {'schema': 'pomos'} + ) + id = Column(Integer, primary_key=True, autoincrement=True) + dn_death = Column(Date) + dn_name = Column(String) + dn_location = Column(String) + dn_picture = Column(LargeBinary) + dn_age = Column(Integer) # New column for age + +# Create the SQLAlchemy engine and session +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(bind=engine) + +# Create the schema and tables +Base.metadata.create_all(engine) + +# List of User-Agent strings to rotate +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", +] + +class WelsCrawler: + def __init__(self, death_company): + self.death_company = death_company + + def get_page_content(self, url): + headers = { + "User-Agent": random.choice(USER_AGENTS) + } + response = requests.get(url, headers=headers) + return BeautifulSoup(response.content, "html.parser") + + def download_picture(self, url): + headers = { + "User-Agent": random.choice(USER_AGENTS) + } + response = requests.get(url, headers=headers) + return response.content if response.status_code == 200 else None + + def extract_data(self, soup, session): + death_table = soup.find("table", attrs={"class": "death-table"}) + if not death_table: + print("No death table found.") + return False + + death_table_data = death_table.find_all("tr") + + for row in death_table_data: + rows = row.find_all('td') + cols = [ele.text.strip() for ele in rows] + if cols: + date_death_str = cols[1].replace('†', '').strip() if cols[1] else None + try: + date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None + except ValueError: + print(f"Invalid date format: {date_death_str}") + date_death = None + continue + + # full-name is available as well + name = row.find('span', class_='full-name').text.strip() + + # name = cols[0] + location = self.death_company.capitalize() + + # Extract the picture URL + picture_tag = row.find('img', class_='profile-picture') + picture_url = picture_tag['src'] if picture_tag else None + picture_data = self.download_picture(picture_url) if picture_url else None + + # Extract the age + age_str = row.find('span', class_='deceased-age').text.strip() if row.find('span', class_='deceased-age') else None + age_match = re.search(r'(\d+)', age_str) if age_str else None + age = int(age_match.group(1)) if age_match else None + + # Check for duplicates before inserting + #existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=name, dn_location=location).first() + existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=name).first() + if existing_notice: + # update picture and age + existing_notice.dn_picture = picture_data + existing_notice.dn_age = age + existing_notice.dn_name = name + print(f"Duplicate entry found - Date: {date_death}, Name: {name}, Location: {location}") + else: + # Insert data into the database using SQLAlchemy + death_notice = DeathNotice(dn_death=date_death, dn_name=name, dn_location=location, dn_picture=picture_data, dn_age=age) + session.add(death_notice) + print(f"Inserted data - Date: {date_death}, Name: {name}, Location: {location}, Age: {age}, Picture URL: {picture_url}") + + return True + + def find_next_page(self, soup, current_url): + next_button_disabled = soup.find("a", class_="pagination__next-btn pagination__next-btn--disabled") + if next_button_disabled: + return None + + # next_button = soup.find("a", class_="pagination__next-btn") + # if next_button: + match = re.search(r'\?page=(\d+)', current_url) #next_button['href']) + if match: + next_page_num = int(match.group(1)) + 1 + if self.death_company == "unbekannt": + next_page_url = URL_TEMPLATE_NO_COMPANY.format(next_page_num) + else: + next_page_url = URL_TEMPLATE.format(next_page_num, self.death_company) + return next_page_url + return None + +class Worker: + def __init__(self, crawler, start_page=1): + self.crawler = crawler + self.start_page = start_page + + def run(self): + # Create a new session + session = SessionLocal() + + try: + if self.crawler.death_company == "unbekannt": + current_url = URL_TEMPLATE_NO_COMPANY.format(self.start_page) + else: + current_url = URL_TEMPLATE.format(self.start_page, self.crawler.death_company) + while current_url: + soup = self.crawler.get_page_content(current_url) + if not self.crawler.extract_data(soup, session): + break + current_url = self.crawler.find_next_page(soup, current_url) + if not current_url: + break + + # Random delay between requests + time.sleep(random.uniform(1, 5)) + + # Commit the transaction after processing all rows on the page + session.commit() + except Exception as e: + print(f"An error occurred: {e}") + session.rollback() + finally: + # Close the session + session.close() + +if __name__ == "__main__": + death_company = sys.argv[1] if len(sys.argv) > 1 else "unbekannt" + start_page = int(sys.argv[2]) if len(sys.argv) > 2 else 1 + + if death_company not in ["wels", "marchtrenk", "unbekannt"]: + print("Invalid death company. Use 'wels' or 'marchtrenk'.") + sys.exit(1) + + crawler = WelsCrawler(death_company) + worker = Worker(crawler, start_page) + worker.run() \ No newline at end of file diff --git a/crawler_bestatung-aichinger_enhanced.py b/crawler_bestatung-aichinger_enhanced.py deleted file mode 100644 index 1f91e62..0000000 --- a/crawler_bestatung-aichinger_enhanced.py +++ /dev/null @@ -1,132 +0,0 @@ -import requests -from bs4 import BeautifulSoup -import re -import yaml -from sqlalchemy import create_engine, Column, Integer, String, Date, UniqueConstraint -from sqlalchemy.orm import declarative_base, sessionmaker -from datetime import datetime -import sys -import time -import random - -BASE_URL = "https://www.bestattung-aichinger.at" -URL_TEMPLATE = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={{}}" - -# Load database credentials from config.yaml -with open("config.yaml", "r") as file: - config = yaml.safe_load(file) - -db_config = config['database'] -DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}" - -# Define the SQLAlchemy base and model -Base = declarative_base() - -class DeathNotice(Base): - __tablename__ = 'death_notices' - __table_args__ = ( - UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'), - {'schema': 'pomos'} - ) - id = Column(Integer, primary_key=True, autoincrement=True) - dn_death = Column(Date) - dn_name = Column(String) - dn_location = Column(String) - -# Create the SQLAlchemy engine and session -engine = create_engine(DATABASE_URL) -SessionLocal = sessionmaker(bind=engine) - -# Create the schema and tables -Base.metadata.create_all(engine) - -# List of User-Agent strings to rotate -USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", -] - -def get_page_content(url): - headers = { - "User-Agent": random.choice(USER_AGENTS) - } - response = requests.get(url, headers=headers) - return BeautifulSoup(response.content, "html.parser") - -def extract_data(soup, session): - death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"}) - if not death_table: - print("No death table found.") - return False - - death_table_data = death_table.find_all("li") - for row in death_table_data: - date_span = row.find('span', class_='homepage_unterseiten_layout_datum') - date_death_str = date_span.text.replace('†', '').strip() if date_span else None - - # Parse the date string to a datetime.date object - date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None - - title_span = row.find('span', class_='homepage_unterseiten_layout_titel') - - if title_span: - title_text = title_span.get_text(separator='\n') - title_parts = title_text.split('\n') - title_left = title_parts[0].strip() if len(title_parts) > 0 else None - title_right = title_parts[1].strip() if len(title_parts) > 1 else None - else: - title_left = title_right = None - - print(f"Extracted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") - - # Check for duplicates before inserting - existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=title_left, dn_location=title_right).first() - if existing_notice: - print(f"Duplicate entry found - Date: {date_death}, Name: {title_left}, Location: {title_right}") - else: - # Insert data into the database using SQLAlchemy - death_notice = DeathNotice(dn_death=date_death, dn_name=title_left, dn_location=title_right) - session.add(death_notice) - print(f"Inserted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") - - return True - -def find_next_page(current_url): - match = re.search(r'\?p=(\d+)', current_url) - if match: - next_page_num = int(match.group(1)) + 1 - else: - next_page_num = 2 - next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}" - return next_page_url - -def main(start_page=1): - # Create a new session - session = SessionLocal() - - try: - current_url = URL_TEMPLATE.format(start_page) - while current_url: - soup = get_page_content(current_url) - if not extract_data(soup, session): - break - current_url = find_next_page(current_url) - - # Random delay between requests - time.sleep(random.uniform(1, 5)) - - # Commit the transaction after processing all rows on the page - session.commit() - except Exception as e: - print(f"An error occurred: {e}") - session.rollback() - finally: - # Close the session - session.close() - -if __name__ == "__main__": - start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1 - main(start_page) \ No newline at end of file diff --git a/crawler_bestatung-wels_enhanced.py b/crawler_bestatung-wels_enhanced.py deleted file mode 100644 index 6f58efa..0000000 --- a/crawler_bestatung-wels_enhanced.py +++ /dev/null @@ -1,125 +0,0 @@ -import requests -from bs4 import BeautifulSoup -import re -import yaml -from sqlalchemy import create_engine, Column, Integer, String, Date, UniqueConstraint -from sqlalchemy.orm import declarative_base, sessionmaker -from datetime import datetime -import sys -import time -import random - -BASE_URL = "https://www.bestattung-wels.at/" -URL_TEMPLATE = f"{BASE_URL}/current-deaths?page={{}}&deathCompany={{}}" - -# Load database credentials from config.yaml -with open("config.yaml", "r") as file: - config = yaml.safe_load(file) - -db_config = config['database'] -DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}" - -# Define the SQLAlchemy base and model -Base = declarative_base() - -class DeathNotice(Base): - __tablename__ = 'death_notices' - __table_args__ = ( - UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'), - {'schema': 'pomos'} - ) - id = Column(Integer, primary_key=True, autoincrement=True) - dn_death = Column(Date) - dn_name = Column(String) - dn_location = Column(String) - -# Create the SQLAlchemy engine and session -engine = create_engine(DATABASE_URL) -SessionLocal = sessionmaker(bind=engine) - -# Create the schema and tables -Base.metadata.create_all(engine) - -# List of User-Agent strings to rotate -USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", -] - -def get_page_content(url): - headers = { - "User-Agent": random.choice(USER_AGENTS) - } - response = requests.get(url, headers=headers) - return BeautifulSoup(response.content, "html.parser") - -def extract_data(soup, session): - - death_table = soup.find("table", attrs={"class": "death-table"}) - if not death_table: - print("No death table found.") - return False - - death_table_data = death_table.find_all("tr") - - #pagination__next-btn__content - for row in death_table_data: - rows = row.find_all('td') - cols = [ele.text.strip() for ele in rows] - if cols: - date_death_str = cols[1].replace('†', '').strip() if cols[1] else None - date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None - - name = cols[0] - location = "Wels" - # Check for duplicates before inserting - existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=name, dn_location=location).first() - if existing_notice: - print(f"Duplicate entry found - Date: {date_death}, Name: {name}, Location: {location}") - else: - # Insert data into the database using SQLAlchemy - death_notice = DeathNotice(dn_death=date_death, dn_name=name, dn_location=location) - session.add(death_notice) - print(f"Inserted data - Date: {date_death}, Name: {name}, Location: {location}") - - return True - -def find_next_page(current_url): - match = re.search(r'\?p=(\d+)', current_url) - if match: - next_page_num = int(match.group(1)) + 1 - else: - next_page_num = 2 - next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}" - return next_page_url - -def main(start_page=1): - # Create a new session - session = SessionLocal() - - try: - current_url = URL_TEMPLATE.format(start_page) - while current_url: - soup = get_page_content(current_url) - if not extract_data(soup, session): - break - current_url = find_next_page(current_url) - - # Random delay between requests - time.sleep(random.uniform(1, 5)) - - # Commit the transaction after processing all rows on the page - session.commit() - except Exception as e: - print(f"An error occurred: {e}") - session.rollback() - finally: - # Close the session - session.close() - -if __name__ == "__main__": - start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1 - main(start_page) \ No newline at end of file diff --git a/main.py b/main.py index e69de29..a66d389 100644 --- a/main.py +++ b/main.py @@ -0,0 +1,6 @@ +from crawler_bestattung_wels_enhanced import WelsCrawler, Worker +from crawler_bestattung_aichinger_enhanced import AichingerCrawler, Worker + +Worker(WelsCrawler("wels"), 1).run() +Worker(WelsCrawler("marchtrenk"), 1).run() +Worker(AichingerCrawler(1)).run() \ No newline at end of file diff --git a/schema.hcl b/schema.hcl new file mode 100644 index 0000000..614c48a --- /dev/null +++ b/schema.hcl @@ -0,0 +1,38 @@ +table "death_notices" { + schema = schema.pomos + column "id" { + null = false + type = serial + } + column "dn_death" { + null = true + type = date + } + column "dn_name" { + null = true + type = character_varying + } + column "dn_location" { + null = true + type = character_varying + } + column "dn_picture" { + null = true + type = bytea + } + column "dn_age" { + null = true + type = smallint + } + primary_key { + columns = [column.id] + } + unique "unique_death_notice" { + columns = [column.dn_death, column.dn_name, column.dn_location] + } +} +schema "pomos" { +} +schema "public" { + comment = "standard public schema" +}