Enhancements
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
database:
|
||||
username: "pomos"
|
||||
password: "$W$$q$T%+1y=*/cbFl2M"
|
||||
host: "postgresql.home"
|
||||
port: 5432
|
||||
dbname: "pomos"
|
||||
@@ -13,6 +13,9 @@ for row in death_table_data:
|
||||
cols = [ele.text.strip() for ele in cols]
|
||||
print(cols) # This will print each row as a list
|
||||
|
||||
|
||||
#pagination__next-btn__content
|
||||
|
||||
#for death in death_table_data:
|
||||
# print(death.full-name.text)
|
||||
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import re
|
||||
|
||||
URL = "https://www.bestattung-aichinger.at/traueranzeigen--5683197-de.html"
|
||||
page = requests.get(URL)
|
||||
BASE_URL = "https://www.bestattung-aichinger.at"
|
||||
URL = f"{BASE_URL}/traueranzeigen--5683197-de.html"
|
||||
|
||||
soup = BeautifulSoup(page.content, "html.parser")
|
||||
def get_page_content(url):
|
||||
page = requests.get(url)
|
||||
return BeautifulSoup(page.content, "html.parser")
|
||||
|
||||
def extract_data(soup):
|
||||
death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"})
|
||||
|
||||
death_table_data = death_table.find_all("li")
|
||||
for row in death_table_data:
|
||||
date_span = row.find('span', class_='homepage_unterseiten_layout_datum')
|
||||
@@ -24,12 +28,17 @@ for row in death_table_data:
|
||||
|
||||
print(date_death, title_left, title_right)
|
||||
|
||||
# cols = row.find_all('homepage_unterseiten_layout_datum')
|
||||
# cols = [ele.text.strip() for ele in cols]
|
||||
# print(cols) # This will print each row as a list
|
||||
def find_next_page(current_url):
|
||||
match = re.search(r'\?p=(\d+)', current_url)
|
||||
if match:
|
||||
next_page_num = int(match.group(1)) + 1
|
||||
else:
|
||||
next_page_num = 2
|
||||
next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}"
|
||||
return next_page_url
|
||||
|
||||
#for death in death_table_data:
|
||||
# print(death.full-name.text)
|
||||
|
||||
#results = soup.findAll({"id" : lambda L: L and L.startswith('death-row-')})
|
||||
results = soup.find("death-table-body")
|
||||
current_url = URL
|
||||
while current_url:
|
||||
soup = get_page_content(current_url)
|
||||
extract_data(soup)
|
||||
current_url = find_next_page(current_url)
|
||||
@@ -0,0 +1,132 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import re
|
||||
import yaml
|
||||
from sqlalchemy import create_engine, Column, Integer, String, Date, UniqueConstraint
|
||||
from sqlalchemy.orm import declarative_base, sessionmaker
|
||||
from datetime import datetime
|
||||
import sys
|
||||
import time
|
||||
import random
|
||||
|
||||
BASE_URL = "https://www.bestattung-aichinger.at"
|
||||
URL_TEMPLATE = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={{}}"
|
||||
|
||||
# Load database credentials from config.yaml
|
||||
with open("config.yaml", "r") as file:
|
||||
config = yaml.safe_load(file)
|
||||
|
||||
db_config = config['database']
|
||||
DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}"
|
||||
|
||||
# Define the SQLAlchemy base and model
|
||||
Base = declarative_base()
|
||||
|
||||
class DeathNotice(Base):
|
||||
__tablename__ = 'death_notices'
|
||||
__table_args__ = (
|
||||
UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'),
|
||||
{'schema': 'pomos'}
|
||||
)
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
dn_death = Column(Date)
|
||||
dn_name = Column(String)
|
||||
dn_location = Column(String)
|
||||
|
||||
# Create the SQLAlchemy engine and session
|
||||
engine = create_engine(DATABASE_URL)
|
||||
SessionLocal = sessionmaker(bind=engine)
|
||||
|
||||
# Create the schema and tables
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
# List of User-Agent strings to rotate
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36",
|
||||
]
|
||||
|
||||
def get_page_content(url):
|
||||
headers = {
|
||||
"User-Agent": random.choice(USER_AGENTS)
|
||||
}
|
||||
response = requests.get(url, headers=headers)
|
||||
return BeautifulSoup(response.content, "html.parser")
|
||||
|
||||
def extract_data(soup, session):
|
||||
death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"})
|
||||
if not death_table:
|
||||
print("No death table found.")
|
||||
return False
|
||||
|
||||
death_table_data = death_table.find_all("li")
|
||||
for row in death_table_data:
|
||||
date_span = row.find('span', class_='homepage_unterseiten_layout_datum')
|
||||
date_death_str = date_span.text.replace('†', '').strip() if date_span else None
|
||||
|
||||
# Parse the date string to a datetime.date object
|
||||
date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None
|
||||
|
||||
title_span = row.find('span', class_='homepage_unterseiten_layout_titel')
|
||||
|
||||
if title_span:
|
||||
title_text = title_span.get_text(separator='\n')
|
||||
title_parts = title_text.split('\n')
|
||||
title_left = title_parts[0].strip() if len(title_parts) > 0 else None
|
||||
title_right = title_parts[1].strip() if len(title_parts) > 1 else None
|
||||
else:
|
||||
title_left = title_right = None
|
||||
|
||||
print(f"Extracted data - Date: {date_death}, Name: {title_left}, Location: {title_right}")
|
||||
|
||||
# Check for duplicates before inserting
|
||||
existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=title_left, dn_location=title_right).first()
|
||||
if existing_notice:
|
||||
print(f"Duplicate entry found - Date: {date_death}, Name: {title_left}, Location: {title_right}")
|
||||
else:
|
||||
# Insert data into the database using SQLAlchemy
|
||||
death_notice = DeathNotice(dn_death=date_death, dn_name=title_left, dn_location=title_right)
|
||||
session.add(death_notice)
|
||||
print(f"Inserted data - Date: {date_death}, Name: {title_left}, Location: {title_right}")
|
||||
|
||||
return True
|
||||
|
||||
def find_next_page(current_url):
|
||||
match = re.search(r'\?p=(\d+)', current_url)
|
||||
if match:
|
||||
next_page_num = int(match.group(1)) + 1
|
||||
else:
|
||||
next_page_num = 2
|
||||
next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}"
|
||||
return next_page_url
|
||||
|
||||
def main(start_page=1):
|
||||
# Create a new session
|
||||
session = SessionLocal()
|
||||
|
||||
try:
|
||||
current_url = URL_TEMPLATE.format(start_page)
|
||||
while current_url:
|
||||
soup = get_page_content(current_url)
|
||||
if not extract_data(soup, session):
|
||||
break
|
||||
current_url = find_next_page(current_url)
|
||||
|
||||
# Random delay between requests
|
||||
time.sleep(random.uniform(1, 5))
|
||||
|
||||
# Commit the transaction after processing all rows on the page
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
print(f"An error occurred: {e}")
|
||||
session.rollback()
|
||||
finally:
|
||||
# Close the session
|
||||
session.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1
|
||||
main(start_page)
|
||||
@@ -0,0 +1,125 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import re
|
||||
import yaml
|
||||
from sqlalchemy import create_engine, Column, Integer, String, Date, UniqueConstraint
|
||||
from sqlalchemy.orm import declarative_base, sessionmaker
|
||||
from datetime import datetime
|
||||
import sys
|
||||
import time
|
||||
import random
|
||||
|
||||
BASE_URL = "https://www.bestattung-wels.at/"
|
||||
URL_TEMPLATE = f"{BASE_URL}/current-deaths?page={{}}&deathCompany={{}}"
|
||||
|
||||
# Load database credentials from config.yaml
|
||||
with open("config.yaml", "r") as file:
|
||||
config = yaml.safe_load(file)
|
||||
|
||||
db_config = config['database']
|
||||
DATABASE_URL = f"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['dbname']}"
|
||||
|
||||
# Define the SQLAlchemy base and model
|
||||
Base = declarative_base()
|
||||
|
||||
class DeathNotice(Base):
|
||||
__tablename__ = 'death_notices'
|
||||
__table_args__ = (
|
||||
UniqueConstraint('dn_death', 'dn_name', 'dn_location', name='unique_death_notice'),
|
||||
{'schema': 'pomos'}
|
||||
)
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
dn_death = Column(Date)
|
||||
dn_name = Column(String)
|
||||
dn_location = Column(String)
|
||||
|
||||
# Create the SQLAlchemy engine and session
|
||||
engine = create_engine(DATABASE_URL)
|
||||
SessionLocal = sessionmaker(bind=engine)
|
||||
|
||||
# Create the schema and tables
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
# List of User-Agent strings to rotate
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/16.16299",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36",
|
||||
]
|
||||
|
||||
def get_page_content(url):
|
||||
headers = {
|
||||
"User-Agent": random.choice(USER_AGENTS)
|
||||
}
|
||||
response = requests.get(url, headers=headers)
|
||||
return BeautifulSoup(response.content, "html.parser")
|
||||
|
||||
def extract_data(soup, session):
|
||||
|
||||
death_table = soup.find("table", attrs={"class": "death-table"})
|
||||
if not death_table:
|
||||
print("No death table found.")
|
||||
return False
|
||||
|
||||
death_table_data = death_table.find_all("tr")
|
||||
|
||||
#pagination__next-btn__content
|
||||
for row in death_table_data:
|
||||
rows = row.find_all('td')
|
||||
cols = [ele.text.strip() for ele in rows]
|
||||
if cols:
|
||||
date_death_str = cols[1].replace('†', '').strip() if cols[1] else None
|
||||
date_death = datetime.strptime(date_death_str, "%d.%m.%Y").date() if date_death_str else None
|
||||
|
||||
name = cols[0]
|
||||
location = "Wels"
|
||||
# Check for duplicates before inserting
|
||||
existing_notice = session.query(DeathNotice).filter_by(dn_death=date_death, dn_name=name, dn_location=location).first()
|
||||
if existing_notice:
|
||||
print(f"Duplicate entry found - Date: {date_death}, Name: {name}, Location: {location}")
|
||||
else:
|
||||
# Insert data into the database using SQLAlchemy
|
||||
death_notice = DeathNotice(dn_death=date_death, dn_name=name, dn_location=location)
|
||||
session.add(death_notice)
|
||||
print(f"Inserted data - Date: {date_death}, Name: {name}, Location: {location}")
|
||||
|
||||
return True
|
||||
|
||||
def find_next_page(current_url):
|
||||
match = re.search(r'\?p=(\d+)', current_url)
|
||||
if match:
|
||||
next_page_num = int(match.group(1)) + 1
|
||||
else:
|
||||
next_page_num = 2
|
||||
next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}"
|
||||
return next_page_url
|
||||
|
||||
def main(start_page=1):
|
||||
# Create a new session
|
||||
session = SessionLocal()
|
||||
|
||||
try:
|
||||
current_url = URL_TEMPLATE.format(start_page)
|
||||
while current_url:
|
||||
soup = get_page_content(current_url)
|
||||
if not extract_data(soup, session):
|
||||
break
|
||||
current_url = find_next_page(current_url)
|
||||
|
||||
# Random delay between requests
|
||||
time.sleep(random.uniform(1, 5))
|
||||
|
||||
# Commit the transaction after processing all rows on the page
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
print(f"An error occurred: {e}")
|
||||
session.rollback()
|
||||
finally:
|
||||
# Close the session
|
||||
session.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_page = int(sys.argv[1]) if len(sys.argv) > 1 else 1
|
||||
main(start_page)
|
||||
Reference in New Issue
Block a user