diff --git a/__pycache__/crawler_bestattung_aichinger_enhanced.cpython-312.pyc b/__pycache__/crawler_bestattung_aichinger_enhanced.cpython-312.pyc new file mode 100644 index 0000000..fa946ba Binary files /dev/null and b/__pycache__/crawler_bestattung_aichinger_enhanced.cpython-312.pyc differ diff --git a/__pycache__/crawler_bestattung_wels_enhanced.cpython-312.pyc b/__pycache__/crawler_bestattung_wels_enhanced.cpython-312.pyc new file mode 100644 index 0000000..e4adc11 Binary files /dev/null and b/__pycache__/crawler_bestattung_wels_enhanced.cpython-312.pyc differ diff --git a/crawler_bestattung_aichinger_enhanced.py b/crawler_bestattung_aichinger_enhanced.py new file mode 100644 index 0000000..7fd2f62 --- /dev/null +++ b/crawler_bestattung_aichinger_enhanced.py @@ -0,0 +1,145 @@ +import requests +from bs4 import BeautifulSoup +import re +import yaml +from sqlalchemy import create_engine, Column, Integer, String, Date, UniqueConstraint +from sqlalchemy.orm import declarative_base, sessionmaker +from datetime import datetime +import sys +import time +import random + +BASE_URL = "https://www.bestattung-aichinger.at" +URL_TEMPLATE = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={{}}" + +# Load database credentials from config.yaml +with open("config.yaml", "r") as file: + config = yaml.safe_load(file) + +db_config = config['database'] +DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}" + +# Define the SQLAlchemy base and model +Base = declarative_base() + +class DeathNotice(Base): + __tablename__ = 'death_notices' + __table_args__ = ( + UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'), + {'schema': 'pomos'} + ) + id = Column(Integer, primary_key=True, autoincrement=True) + dn_death = Column(Date) + dn_name = Column(String) + dn_location = Column(String) + +# Create the SQLAlchemy engine and session +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(bind=engine) + +# Create the schema and tables +Base.metadata.create_all(engine) + +# List of User-Agent strings to rotate +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", +] + +class AichingerCrawler: + # def __init__(self, death_company): + # self.death_company = death_company + def get_page_content(url): + headers = { + "User-Agent": random.choice(USER_AGENTS) + } + response = requests.get(url, headers=headers) + return BeautifulSoup(response.content, "html.parser") + + def extract_data(soup, session): + death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"}) + if not death_table: + print("No death table found.") + return False + + death_table_data = death_table.find_all("li") + for row in death_table_data: + date_span = row.find('span', class_='homepage_unterseiten_layout_datum') + date_death_str = date_span.text.replace('†', '').strip() if date_span else None + + # Parse the date string to a datetime.date object + date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None + + title_span = row.find('span', class_='homepage_unterseiten_layout_titel') + + if title_span: + title_text = title_span.get_text(separator='\n') + title_parts = title_text.split('\n') + title_left = title_parts[0].strip() if len(title_parts) > 0 else None + title_right = title_parts[1].strip() if len(title_parts) > 1 else None + else: + title_left = title_right = None + + print(f"Extracted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") + + # Check for duplicates before inserting + existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=title_left, dn_location=title_right).first() + if existing_notice: + print(f"Duplicate entry found - Date: {date_death}, Name: {title_left}, Location: {title_right}") + else: + # Insert data into the database using SQLAlchemy + death_notice = DeathNotice(dn_death=date_death, dn_name=title_left, dn_location=title_right) + session.add(death_notice) + print(f"Inserted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") + + return True + + def find_next_page(current_url): + match = re.search(r'\?p=(\d+)', current_url) + if match: + next_page_num = int(match.group(1)) + 1 + else: + next_page_num = 2 + next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}" + return next_page_url + +class Worker: + def __init__(self, crawler, start_page=1): + self.crawler = crawler + self.start_page = start_page + + def run(self): + # Create a new session + session = SessionLocal() + + try: + current_url = URL_TEMPLATE.format(start_page) + while current_url: + soup = self.crawler.get_page_content(current_url) + if not self.crawler.extract_data(soup, session): + break + current_url = self.crawler.find_next_page(current_url) + if not current_url: + break + + + # Random delay between requests + time.sleep(random.uniform(1, 5)) + + # Commit the transaction after processing all rows on the page + session.commit() + except Exception as e: + print(f"An error occurred: {e}") + session.rollback() + finally: + # Close the session + session.close() + +if __name__ == "__main__": + start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1 + crawler = AichingerCrawler() + worker = Worker(crawler, start_page) + worker.run() \ No newline at end of file diff --git a/crawler_bestattung_wels_enhanced.py b/crawler_bestattung_wels_enhanced.py new file mode 100644 index 0000000..37bc170 --- /dev/null +++ b/crawler_bestattung_wels_enhanced.py @@ -0,0 +1,187 @@ +import requests +from bs4 import BeautifulSoup +import re +import yaml +from sqlalchemy import create_engine, Column, Integer, String, Date, LargeBinary, UniqueConstraint +from sqlalchemy.orm import declarative_base, sessionmaker +from datetime import datetime +import sys +import time +import random + +BASE_URL = "https://www.bestattung-wels.at/" +URL_TEMPLATE = f"{BASE_URL}/current-deaths?page={{}}&deathCompany={{}}" +URL_TEMPLATE_NO_COMPANY = f"{BASE_URL}/current-deaths?page={{}}" + +# Load database credentials from config.yaml +with open("config.yaml", "r") as file: + config = yaml.safe_load(file) + +db_config = config['database'] +DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}" + +# Define the SQLAlchemy base and model +Base = declarative_base() + +class DeathNotice(Base): + __tablename__ = 'death_notices' + __table_args__ = ( + UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'), + {'schema': 'pomos'} + ) + id = Column(Integer, primary_key=True, autoincrement=True) + dn_death = Column(Date) + dn_name = Column(String) + dn_location = Column(String) + dn_picture = Column(LargeBinary) + dn_age = Column(Integer) # New column for age + +# Create the SQLAlchemy engine and session +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(bind=engine) + +# Create the schema and tables +Base.metadata.create_all(engine) + +# List of User-Agent strings to rotate +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", +] + +class WelsCrawler: + def __init__(self, death_company): + self.death_company = death_company + + def get_page_content(self, url): + headers = { + "User-Agent": random.choice(USER_AGENTS) + } + response = requests.get(url, headers=headers) + return BeautifulSoup(response.content, "html.parser") + + def download_picture(self, url): + headers = { + "User-Agent": random.choice(USER_AGENTS) + } + response = requests.get(url, headers=headers) + return response.content if response.status_code == 200 else None + + def extract_data(self, soup, session): + death_table = soup.find("table", attrs={"class": "death-table"}) + if not death_table: + print("No death table found.") + return False + + death_table_data = death_table.find_all("tr") + + for row in death_table_data: + rows = row.find_all('td') + cols = [ele.text.strip() for ele in rows] + if cols: + date_death_str = cols[1].replace('†', '').strip() if cols[1] else None + try: + date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None + except ValueError: + print(f"Invalid date format: {date_death_str}") + date_death = None + continue + + # full-name is available as well + name = row.find('span', class_='full-name').text.strip() + + # name = cols[0] + location = self.death_company.capitalize() + + # Extract the picture URL + picture_tag = row.find('img', class_='profile-picture') + picture_url = picture_tag['src'] if picture_tag else None + picture_data = self.download_picture(picture_url) if picture_url else None + + # Extract the age + age_str = row.find('span', class_='deceased-age').text.strip() if row.find('span', class_='deceased-age') else None + age_match = re.search(r'(\d+)', age_str) if age_str else None + age = int(age_match.group(1)) if age_match else None + + # Check for duplicates before inserting + #existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=name, dn_location=location).first() + existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=name).first() + if existing_notice: + # update picture and age + existing_notice.dn_picture = picture_data + existing_notice.dn_age = age + existing_notice.dn_name = name + print(f"Duplicate entry found - Date: {date_death}, Name: {name}, Location: {location}") + else: + # Insert data into the database using SQLAlchemy + death_notice = DeathNotice(dn_death=date_death, dn_name=name, dn_location=location, dn_picture=picture_data, dn_age=age) + session.add(death_notice) + print(f"Inserted data - Date: {date_death}, Name: {name}, Location: {location}, Age: {age}, Picture URL: {picture_url}") + + return True + + def find_next_page(self, soup, current_url): + next_button_disabled = soup.find("a", class_="pagination__next-btn pagination__next-btn--disabled") + if next_button_disabled: + return None + + # next_button = soup.find("a", class_="pagination__next-btn") + # if next_button: + match = re.search(r'\?page=(\d+)', current_url) #next_button['href']) + if match: + next_page_num = int(match.group(1)) + 1 + if self.death_company == "unbekannt": + next_page_url = URL_TEMPLATE_NO_COMPANY.format(next_page_num) + else: + next_page_url = URL_TEMPLATE.format(next_page_num, self.death_company) + return next_page_url + return None + +class Worker: + def __init__(self, crawler, start_page=1): + self.crawler = crawler + self.start_page = start_page + + def run(self): + # Create a new session + session = SessionLocal() + + try: + if self.crawler.death_company == "unbekannt": + current_url = URL_TEMPLATE_NO_COMPANY.format(self.start_page) + else: + current_url = URL_TEMPLATE.format(self.start_page, self.crawler.death_company) + while current_url: + soup = self.crawler.get_page_content(current_url) + if not self.crawler.extract_data(soup, session): + break + current_url = self.crawler.find_next_page(soup, current_url) + if not current_url: + break + + # Random delay between requests + time.sleep(random.uniform(1, 5)) + + # Commit the transaction after processing all rows on the page + session.commit() + except Exception as e: + print(f"An error occurred: {e}") + session.rollback() + finally: + # Close the session + session.close() + +if __name__ == "__main__": + death_company = sys.argv[1] if len(sys.argv) > 1 else "unbekannt" + start_page = int(sys.argv[2]) if len(sys.argv) > 2 else 1 + + if death_company not in ["wels", "marchtrenk", "unbekannt"]: + print("Invalid death company. Use 'wels' or 'marchtrenk'.") + sys.exit(1) + + crawler = WelsCrawler(death_company) + worker = Worker(crawler, start_page) + worker.run() \ No newline at end of file diff --git a/crawler_bestatung-aichinger_enhanced.py b/crawler_bestatung-aichinger_enhanced.py deleted file mode 100644 index 1f91e62..0000000 --- a/crawler_bestatung-aichinger_enhanced.py +++ /dev/null @@ -1,132 +0,0 @@ -import requests -from bs4 import BeautifulSoup -import re -import yaml -from sqlalchemy import create_engine, Column, Integer, String, Date, UniqueConstraint -from sqlalchemy.orm import declarative_base, sessionmaker -from datetime import datetime -import sys -import time -import random - -BASE_URL = "https://www.bestattung-aichinger.at" -URL_TEMPLATE = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={{}}" - -# Load database credentials from config.yaml -with open("config.yaml", "r") as file: - config = yaml.safe_load(file) - -db_config = config['database'] -DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}" - -# Define the SQLAlchemy base and model -Base = declarative_base() - -class DeathNotice(Base): - __tablename__ = 'death_notices' - __table_args__ = ( - UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'), - {'schema': 'pomos'} - ) - id = Column(Integer, primary_key=True, autoincrement=True) - dn_death = Column(Date) - dn_name = Column(String) - dn_location = Column(String) - -# Create the SQLAlchemy engine and session -engine = create_engine(DATABASE_URL) -SessionLocal = sessionmaker(bind=engine) - -# Create the schema and tables -Base.metadata.create_all(engine) - -# List of User-Agent strings to rotate -USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", -] - -def get_page_content(url): - headers = { - "User-Agent": random.choice(USER_AGENTS) - } - response = requests.get(url, headers=headers) - return BeautifulSoup(response.content, "html.parser") - -def extract_data(soup, session): - death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"}) - if not death_table: - print("No death table found.") - return False - - death_table_data = death_table.find_all("li") - for row in death_table_data: - date_span = row.find('span', class_='homepage_unterseiten_layout_datum') - date_death_str = date_span.text.replace('†', '').strip() if date_span else None - - # Parse the date string to a datetime.date object - date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None - - title_span = row.find('span', class_='homepage_unterseiten_layout_titel') - - if title_span: - title_text = title_span.get_text(separator='\n') - title_parts = title_text.split('\n') - title_left = title_parts[0].strip() if len(title_parts) > 0 else None - title_right = title_parts[1].strip() if len(title_parts) > 1 else None - else: - title_left = title_right = None - - print(f"Extracted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") - - # Check for duplicates before inserting - existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=title_left, dn_location=title_right).first() - if existing_notice: - print(f"Duplicate entry found - Date: {date_death}, Name: {title_left}, Location: {title_right}") - else: - # Insert data into the database using SQLAlchemy - death_notice = DeathNotice(dn_death=date_death, dn_name=title_left, dn_location=title_right) - session.add(death_notice) - print(f"Inserted data - Date: {date_death}, Name: {title_left}, Location: {title_right}") - - return True - -def find_next_page(current_url): - match = re.search(r'\?p=(\d+)', current_url) - if match: - next_page_num = int(match.group(1)) + 1 - else: - next_page_num = 2 - next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}" - return next_page_url - -def main(start_page=1): - # Create a new session - session = SessionLocal() - - try: - current_url = URL_TEMPLATE.format(start_page) - while current_url: - soup = get_page_content(current_url) - if not extract_data(soup, session): - break - current_url = find_next_page(current_url) - - # Random delay between requests - time.sleep(random.uniform(1, 5)) - - # Commit the transaction after processing all rows on the page - session.commit() - except Exception as e: - print(f"An error occurred: {e}") - session.rollback() - finally: - # Close the session - session.close() - -if __name__ == "__main__": - start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1 - main(start_page) \ No newline at end of file diff --git a/crawler_bestatung-wels_enhanced.py b/crawler_bestatung-wels_enhanced.py deleted file mode 100644 index 6f58efa..0000000 --- a/crawler_bestatung-wels_enhanced.py +++ /dev/null @@ -1,125 +0,0 @@ -import requests -from bs4 import BeautifulSoup -import re -import yaml -from sqlalchemy import create_engine, Column, Integer, String, Date, UniqueConstraint -from sqlalchemy.orm import declarative_base, sessionmaker -from datetime import datetime -import sys -import time -import random - -BASE_URL = "https://www.bestattung-wels.at/" -URL_TEMPLATE = f"{BASE_URL}/current-deaths?page={{}}&deathCompany={{}}" - -# Load database credentials from config.yaml -with open("config.yaml", "r") as file: - config = yaml.safe_load(file) - -db_config = config['database'] -DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}" - -# Define the SQLAlchemy base and model -Base = declarative_base() - -class DeathNotice(Base): - __tablename__ = 'death_notices' - __table_args__ = ( - UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'), - {'schema': 'pomos'} - ) - id = Column(Integer, primary_key=True, autoincrement=True) - dn_death = Column(Date) - dn_name = Column(String) - dn_location = Column(String) - -# Create the SQLAlchemy engine and session -engine = create_engine(DATABASE_URL) -SessionLocal = sessionmaker(bind=engine) - -# Create the schema and tables -Base.metadata.create_all(engine) - -# List of User-Agent strings to rotate -USER_AGENTS = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36", -] - -def get_page_content(url): - headers = { - "User-Agent": random.choice(USER_AGENTS) - } - response = requests.get(url, headers=headers) - return BeautifulSoup(response.content, "html.parser") - -def extract_data(soup, session): - - death_table = soup.find("table", attrs={"class": "death-table"}) - if not death_table: - print("No death table found.") - return False - - death_table_data = death_table.find_all("tr") - - #pagination__next-btn__content - for row in death_table_data: - rows = row.find_all('td') - cols = [ele.text.strip() for ele in rows] - if cols: - date_death_str = cols[1].replace('†', '').strip() if cols[1] else None - date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None - - name = cols[0] - location = "Wels" - # Check for duplicates before inserting - existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=name, dn_location=location).first() - if existing_notice: - print(f"Duplicate entry found - Date: {date_death}, Name: {name}, Location: {location}") - else: - # Insert data into the database using SQLAlchemy - death_notice = DeathNotice(dn_death=date_death, dn_name=name, dn_location=location) - session.add(death_notice) - print(f"Inserted data - Date: {date_death}, Name: {name}, Location: {location}") - - return True - -def find_next_page(current_url): - match = re.search(r'\?p=(\d+)', current_url) - if match: - next_page_num = int(match.group(1)) + 1 - else: - next_page_num = 2 - next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}" - return next_page_url - -def main(start_page=1): - # Create a new session - session = SessionLocal() - - try: - current_url = URL_TEMPLATE.format(start_page) - while current_url: - soup = get_page_content(current_url) - if not extract_data(soup, session): - break - current_url = find_next_page(current_url) - - # Random delay between requests - time.sleep(random.uniform(1, 5)) - - # Commit the transaction after processing all rows on the page - session.commit() - except Exception as e: - print(f"An error occurred: {e}") - session.rollback() - finally: - # Close the session - session.close() - -if __name__ == "__main__": - start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1 - main(start_page) \ No newline at end of file diff --git a/main.py b/main.py index e69de29..a66d389 100644 --- a/main.py +++ b/main.py @@ -0,0 +1,6 @@ +from crawler_bestattung_wels_enhanced import WelsCrawler, Worker +from crawler_bestattung_aichinger_enhanced import AichingerCrawler, Worker + +Worker(WelsCrawler("wels"), 1).run() +Worker(WelsCrawler("marchtrenk"), 1).run() +Worker(AichingerCrawler(1)).run() \ No newline at end of file diff --git a/schema.hcl b/schema.hcl new file mode 100644 index 0000000..614c48a --- /dev/null +++ b/schema.hcl @@ -0,0 +1,38 @@ +table "death_notices" { + schema = schema.pomos + column "id" { + null = false + type = serial + } + column "dn_death" { + null = true + type = date + } + column "dn_name" { + null = true + type = character_varying + } + column "dn_location" { + null = true + type = character_varying + } + column "dn_picture" { + null = true + type = bytea + } + column "dn_age" { + null = true + type = smallint + } + primary_key { + columns = [column.id] + } + unique "unique_death_notice" { + columns = [column.dn_death, column.dn_name, column.dn_location] + } +} +schema "pomos" { +} +schema "public" { + comment = "standard public schema" +}