From 74449bf304be8b7905e946469bf6b2c4a2716930 Mon Sep 17 00:00:00 2001 From: Markus Zimmerberger Date: Thu, 20 Feb 2025 17:55:05 +0100 Subject: [PATCH] Enhancements --- config.yaml | 6 ++ crawler_bestattung-wels.py | 3 + crawler_bestatung-aichinger.py | 61 ++++++----- crawler_bestatung-aichinger_enhanced.py | 132 ++++++++++++++++++++++++ crawler_bestatung-wels_enhanced.py | 125 ++++++++++++++++++++++ 5 files changed, 301 insertions(+), 26 deletions(-) create mode 100644 config.yaml create mode 100644 crawler_bestatung-aichinger_enhanced.py create mode 100644 crawler_bestatung-wels_enhanced.py diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..b8c6642 --- /dev/null +++ b/config.yaml @@ -0,0 +1,6 @@ +database: + username: "pomos" + password: "$W$$q$T%+1y=*/cbFl2M" + host: "postgresql.home" + port: 5432 + dbname: "pomos" \ No newline at end of file diff --git a/crawler_bestattung-wels.py b/crawler_bestattung-wels.py index cdc837e..960a6ff 100644 --- a/crawler_bestattung-wels.py +++ b/crawler_bestattung-wels.py @@ -13,6 +13,9 @@ for row in death_table_data: cols = [ele.text.strip() for ele in cols] print(cols) # This will print each row as a list + +#pagination__next-btn__content + #for death in death_table_data: # print(death.full-name.text) diff --git a/crawler_bestatung-aichinger.py b/crawler_bestatung-aichinger.py index b0aa257..fc2e305 100644 --- a/crawler_bestatung-aichinger.py +++ b/crawler_bestatung-aichinger.py @@ -1,35 +1,44 @@ import requests from bs4 import BeautifulSoup +import re -URL = "https://www.bestattung-aichinger.at/traueranzeigen--5683197-de.html" -page = requests.get(URL) +BASE_URL = "https://www.bestattung-aichinger.at" +URL = f"{BASE_URL}/traueranzeigen--5683197-de.html" -soup = BeautifulSoup(page.content, "html.parser") -death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"}) +def get_page_content(url): + page = requests.get(url) + return BeautifulSoup(page.content, "html.parser") -death_table_data = death_table.find_all("li") -for row in death_table_data: - date_span = row.find('span', class_='homepage_unterseiten_layout_datum') - date_death = date_span.text.replace('†', '').strip() if date_span else None +def extract_data(soup): + death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"}) + death_table_data = death_table.find_all("li") + for row in death_table_data: + date_span = row.find('span', class_='homepage_unterseiten_layout_datum') + date_death = date_span.text.replace('†', '').strip() if date_span else None - title_span = row.find('span', class_='homepage_unterseiten_layout_titel') + title_span = row.find('span', class_='homepage_unterseiten_layout_titel') - if title_span: - title_text = title_span.get_text(separator='\n') - title_parts = title_text.split('\n') - title_left = title_parts[0].strip() if len(title_parts) > 0 else None - title_right = title_parts[1].strip() if len(title_parts) > 1 else None + if title_span: + title_text = title_span.get_text(separator='\n') + title_parts = title_text.split('\n') + title_left = title_parts[0].strip() if len(title_parts) > 0 else None + title_right = title_parts[1].strip() if len(title_parts) > 1 else None + else: + title_left = title_right = None + + print(date_death, title_left, title_right) + +def find_next_page(current_url): + match = re.search(r'\?p=(\d+)', current_url) + if match: + next_page_num = int(match.group(1)) + 1 else: - title_left = title_right = None + next_page_num = 2 + next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}" + return next_page_url - print(date_death, title_left, title_right) - - # cols = row.find_all('homepage_unterseiten_layout_datum') - # cols = [ele.text.strip() for ele in cols] - # print(cols) # This will print each row as a list - -#for death in death_table_data: -# print(death.full-name.text) - -#results = soup.findAll({"id" : lambda L: L and L.startswith('death-row-')}) -results = soup.find("death-table-body") \ No newline at end of file +current_url = URL +while current_url: + soup = get_page_content(current_url) + extract_data(soup) + current_url = find_next_page(current_url) \ No newline at end of file diff --git a/crawler_bestatung-aichinger_enhanced.py b/crawler_bestatung-aichinger_enhanced.py new file mode 100644 index 0000000..1f91e62 --- /dev/null +++ b/crawler_bestatung-aichinger_enhanced.py @@ -0,0 +1,132 @@ +import requests +from bs4 import BeautifulSoup +import re +import yaml +from sqlalchemy import create_engine, Column, Integer, String, Date, UniqueConstraint +from sqlalchemy.orm import declarative_base, sessionmaker +from datetime import datetime +import sys +import time +import random + +BASE_URL = "https://www.bestattung-aichinger.at" +URL_TEMPLATE = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={{}}" + +# Load database credentials from config.yaml +with open("config.yaml", "r") as file: + config = yaml.safe_load(file) + +db_config = config['database'] +DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}" + +# Define the SQLAlchemy base and model +Base = declarative_base() + +class DeathNotice(Base): + __tablename__ = 'death_notices' + __table_args__ = ( + UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'), + {'schema': 'pomos'} + ) + id = Column(Integer, primary_key=True, autoincrement=True) + dn_death = Column(Date) + dn_name = Column(String) + dn_location = Column(String) + +# Create the SQLAlchemy engine and session +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(bind=engine) + +# Create the schema and tables +Base.metadata.create_all(engine) + +# List of User-Agent strings to rotate +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", +] + +def get_page_content(url): + headers = { + "User-Agent": random.choice(USER_AGENTS) + } + response = requests.get(url, headers=headers) + return BeautifulSoup(response.content, "html.parser") + +def extract_data(soup, session): + death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"}) + if not death_table: + print("No death table found.") + return False + + death_table_data = death_table.find_all("li") + for row in death_table_data: + date_span = row.find('span', class_='homepage_unterseiten_layout_datum') + date_death_str = date_span.text.replace('†', '').strip() if date_span else None + + # Parse the date string to a datetime.date object + date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None + + title_span = row.find('span', class_='homepage_unterseiten_layout_titel') + + if title_span: + title_text = title_span.get_text(separator='\n') + title_parts = title_text.split('\n') + title_left = title_parts[0].strip() if len(title_parts) > 0 else None + title_right = title_parts[1].strip() if len(title_parts) > 1 else None + else: + title_left = title_right = None + + print(f"Extracted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") + + # Check for duplicates before inserting + existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=title_left, dn_location=title_right).first() + if existing_notice: + print(f"Duplicate entry found - Date: {date_death}, Name: {title_left}, Location: {title_right}") + else: + # Insert data into the database using SQLAlchemy + death_notice = DeathNotice(dn_death=date_death, dn_name=title_left, dn_location=title_right) + session.add(death_notice) + print(f"Inserted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") + + return True + +def find_next_page(current_url): + match = re.search(r'\?p=(\d+)', current_url) + if match: + next_page_num = int(match.group(1)) + 1 + else: + next_page_num = 2 + next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}" + return next_page_url + +def main(start_page=1): + # Create a new session + session = SessionLocal() + + try: + current_url = URL_TEMPLATE.format(start_page) + while current_url: + soup = get_page_content(current_url) + if not extract_data(soup, session): + break + current_url = find_next_page(current_url) + + # Random delay between requests + time.sleep(random.uniform(1, 5)) + + # Commit the transaction after processing all rows on the page + session.commit() + except Exception as e: + print(f"An error occurred: {e}") + session.rollback() + finally: + # Close the session + session.close() + +if __name__ == "__main__": + start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1 + main(start_page) \ No newline at end of file diff --git a/crawler_bestatung-wels_enhanced.py b/crawler_bestatung-wels_enhanced.py new file mode 100644 index 0000000..6f58efa --- /dev/null +++ b/crawler_bestatung-wels_enhanced.py @@ -0,0 +1,125 @@ +import requests +from bs4 import BeautifulSoup +import re +import yaml +from sqlalchemy import create_engine, Column, Integer, String, Date, UniqueConstraint +from sqlalchemy.orm import declarative_base, sessionmaker +from datetime import datetime +import sys +import time +import random + +BASE_URL = "https://www.bestattung-wels.at/" +URL_TEMPLATE = f"{BASE_URL}/current-deaths?page={{}}&deathCompany={{}}" + +# Load database credentials from config.yaml +with open("config.yaml", "r") as file: + config = yaml.safe_load(file) + +db_config = config['database'] +DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}" + +# Define the SQLAlchemy base and model +Base = declarative_base() + +class DeathNotice(Base): + __tablename__ = 'death_notices' + __table_args__ = ( + UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'), + {'schema': 'pomos'} + ) + id = Column(Integer, primary_key=True, autoincrement=True) + dn_death = Column(Date) + dn_name = Column(String) + dn_location = Column(String) + +# Create the SQLAlchemy engine and session +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(bind=engine) + +# Create the schema and tables +Base.metadata.create_all(engine) + +# List of User-Agent strings to rotate +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", +] + +def get_page_content(url): + headers = { + "User-Agent": random.choice(USER_AGENTS) + } + response = requests.get(url, headers=headers) + return BeautifulSoup(response.content, "html.parser") + +def extract_data(soup, session): + + death_table = soup.find("table", attrs={"class": "death-table"}) + if not death_table: + print("No death table found.") + return False + + death_table_data = death_table.find_all("tr") + + #pagination__next-btn__content + for row in death_table_data: + rows = row.find_all('td') + cols = [ele.text.strip() for ele in rows] + if cols: + date_death_str = cols[1].replace('†', '').strip() if cols[1] else None + date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None + + name = cols[0] + location = "Wels" + # Check for duplicates before inserting + existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=name, dn_location=location).first() + if existing_notice: + print(f"Duplicate entry found - Date: {date_death}, Name: {name}, Location: {location}") + else: + # Insert data into the database using SQLAlchemy + death_notice = DeathNotice(dn_death=date_death, dn_name=name, dn_location=location) + session.add(death_notice) + print(f"Inserted data - Date: {date_death}, Name: {name}, Location: {location}") + + return True + +def find_next_page(current_url): + match = re.search(r'\?p=(\d+)', current_url) + if match: + next_page_num = int(match.group(1)) + 1 + else: + next_page_num = 2 + next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}" + return next_page_url + +def main(start_page=1): + # Create a new session + session = SessionLocal() + + try: + current_url = URL_TEMPLATE.format(start_page) + while current_url: + soup = get_page_content(current_url) + if not extract_data(soup, session): + break + current_url = find_next_page(current_url) + + # Random delay between requests + time.sleep(random.uniform(1, 5)) + + # Commit the transaction after processing all rows on the page + session.commit() + except Exception as e: + print(f"An error occurred: {e}") + session.rollback() + finally: + # Close the session + session.close() + +if __name__ == "__main__": + start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1 + main(start_page) \ No newline at end of file