Enhancements

crawler_bestatung-aichinger.py

@@ -1,35 +1,44 @@
 import requests
 from bs4 import BeautifulSoup
+import re
 
-URL = "https://www.bestattung-aichinger.at/traueranzeigen--5683197-de.html"
-page = requests.get(URL)
+BASE_URL = "https://www.bestattung-aichinger.at"
+URL = f"{BASE_URL}/traueranzeigen--5683197-de.html"
 
-soup = BeautifulSoup(page.content, "html.parser")
-death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"})
+def get_page_content(url):
+    page = requests.get(url)
+    return BeautifulSoup(page.content, "html.parser")
 
-death_table_data = death_table.find_all("li")
-for row in death_table_data:
-    date_span = row.find('span', class_='homepage_unterseiten_layout_datum')
-    date_death = date_span.text.replace('†', '').strip() if date_span else None
+def extract_data(soup):
+    death_table = soup.find("ul", attrs={"class": "homepage_unterseiten_layout_todesanzeigen"})
+    death_table_data = death_table.find_all("li")
+    for row in death_table_data:
+        date_span = row.find('span', class_='homepage_unterseiten_layout_datum')
+        date_death = date_span.text.replace('†', '').strip() if date_span else None
 
-    title_span = row.find('span', class_='homepage_unterseiten_layout_titel')
+        title_span = row.find('span', class_='homepage_unterseiten_layout_titel')
 
-    if title_span:
-        title_text = title_span.get_text(separator='\n')
-        title_parts = title_text.split('\n')
-        title_left = title_parts[0].strip() if len(title_parts) > 0 else None
-        title_right = title_parts[1].strip() if len(title_parts) > 1 else None
+        if title_span:
+            title_text = title_span.get_text(separator='\n')
+            title_parts = title_text.split('\n')
+            title_left = title_parts[0].strip() if len(title_parts) > 0 else None
+            title_right = title_parts[1].strip() if len(title_parts) > 1 else None
+        else:
+            title_left = title_right = None
+
+        print(date_death, title_left, title_right)
+
+def find_next_page(current_url):
+    match = re.search(r'\?p=(\d+)', current_url)
+    if match:
+        next_page_num = int(match.group(1)) + 1
     else:
-        title_left = title_right = None
+        next_page_num = 2
+    next_page_url = f"{BASE_URL}/traueranzeigen--5683197-de.html?p={next_page_num}"
+    return next_page_url
 
-    print(date_death, title_left, title_right)
-    
-    # cols = row.find_all('homepage_unterseiten_layout_datum')
-    # cols = [ele.text.strip() for ele in cols]
-    # print(cols)  # This will print each row as a list
-
-#for death in death_table_data:
-#    print(death.full-name.text)
-
-#results = soup.findAll({"id" : lambda L: L and L.startswith('death-row-')})
-results = soup.find("death-table-body")
+current_url = URL
+while current_url:
+    soup = get_page_content(current_url)
+    extract_data(soup)
+    current_url = find_next_page(current_url)