zisco/pbt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extend stops to DACH-region
Deploy PBT / deploy (push) Failing after 2m53s
Details

Browse Source
This commit is contained in:
zisco
2026-09-12 07:06:29 +02:00
parent 039e2905dd
commit a52e233c31
9 changed files with 204 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/fetch_stops.py
+162 -121
View File
@@ -1,59 +1,51 @@
#!/usr/bin/env python3
"""Fetch all Austrian public-transport stops (and the lines serving them) from
OpenStreetMap via the Overpass API and write a deduplicated snapshot to
data/stops_at.csv.gz.
"""Fetch all public-transport stops (and the lines serving them) in Austria,
Germany and Switzerland (DACH) from OpenStreetMap and write a deduplicated
snapshot to data/stops_dach.csv.gz.
Data © OpenStreetMap contributors, licensed under the ODbL.
Run this only when you want to refresh the snapshot; the import into the
database reads the checked-in CSV and needs no network access. The route pass
downloads a few hundred MB in ~25 tiles and needs roughly 1 GB of RAM.
At this scale the public Overpass API can't be used directly - even a plain
count query for one of these countries times out on it. Instead this
downloads the official Geofabrik .osm.pbf extract per country (cached in
data/.osm-cache/, never committed) and processes it locally with pyosmium.
No load is put on any shared API.
Run this only when you want to refresh the snapshot; the import into the
database (`flask import-stops`) reads the checked-in CSV and needs no network
access. Refreshing needs: `pip install osmium`, ~6 GB of disk for the
extracts, a few GB of RAM, and maybe 15-30 minutes depending on disk speed
(most of it is Germany).
pip install osmium
python scripts/fetch_stops.py
"""
from __future__ import annotations
import csv
import gzip
import json
import math
import re
import sys
import time
import unicodedata
import urllib.parse
import urllib.request
from pathlib import Path
OUT_PATH = Path(__file__).resolve().parent.parent / "data" / "stops_at.csv.gz"
import osmium
OVERPASS_ENDPOINTS = [
"https://overpass-api.de/api/interpreter",
"https://overpass.kumi.systems/api/interpreter",
"https://overpass.private.coffee/api/interpreter",
]
OUT_PATH = Path(__file__).resolve().parent.parent / "data" / "stops_dach.csv.gz"
CACHE_DIR = Path(__file__).resolve().parent.parent / "data" / ".osm-cache"
# Named nodes in Austria that represent a boardable stop ("Haltestelle").
STOPS_QUERY = """
[out:json][timeout:600];
area["ISO3166-1"="AT"][admin_level=2]->.at;
(
node(area.at)["highway"="bus_stop"]["name"];
node(area.at)["railway"="tram_stop"]["name"];
node(area.at)["railway"="station"]["name"];
node(area.at)["railway"="halt"]["name"];
node(area.at)["public_transport"="station"]["name"];
node(area.at)["amenity"="bus_station"]["name"];
);
out body;
"""
# ISO 3166-1 alpha-2 -> Geofabrik region file basename.
GEOFABRIK_REGIONS = {
"AT": "austria",
"CH": "switzerland",
"DE": "germany",
}
ROUTE_MODES = "bus|trolleybus|tram|light_rail|subway|train|monorail|share_taxi"
# Austria bbox, tiled so each Overpass response stays small enough to parse.
BBOX = (46.35, 9.50, 49.05, 17.20) # south, west, north, east
LAT_STEP = 0.9
LON_STEP = 1.0
ROUTE_MODES = {
"bus", "trolleybus", "tram", "light_rail", "subway", "train",
"monorail", "share_taxi",
}
STOP_ROLES = {
"stop", "platform",
@@ -62,7 +54,7 @@ STOP_ROLES = {
}
CSV_FIELDS = [
"osm_type", "osm_id", "name", "stop_type",
"osm_type", "osm_id", "name", "stop_type", "country",
"municipality", "latitude", "longitude", "lines",
]
@@ -72,6 +64,8 @@ TYPE_RANK = {"subway": 5, "train": 4, "bus_station": 3, "tram": 2, "bus": 1, "ot
# nodes with the same name closer than this are treated as one stop.
MERGE_METERS = 250.0
MIN_STOPS = 100_000 # DACH sanity floor; refuse to overwrite a good snapshot with junk
def normalize_name(value: str) -> str:
"""Lowercase, fold accents and ß so search is diacritic-insensitive.
@@ -112,46 +106,76 @@ def municipality_of(tags: dict) -> str:
return ""
def overpass(query: str, *, tries: int = 4) -> dict:
body = urllib.parse.urlencode({"data": query}).encode()
last_error: Exception | None = None
for attempt in range(tries):
endpoint = OVERPASS_ENDPOINTS[attempt % len(OVERPASS_ENDPOINTS)]
try:
req = urllib.request.Request(
endpoint, data=body, headers={"User-Agent": "pbt-stop-import/1.0"}
)
with urllib.request.urlopen(req, timeout=600) as resp:
return json.load(resp)
except Exception as exc: # noqa: BLE001 - retry on another mirror
last_error = exc
print(f" {endpoint} failed ({exc}); retrying", file=sys.stderr)
time.sleep(10 * (attempt + 1))
raise SystemExit(f"Overpass failed after {tries} tries; last error: {last_error}")
# --------------------------------------------------------------------------- #
# Geofabrik download
# --------------------------------------------------------------------------- #
def download_extract(iso: str) -> Path:
region = GEOFABRIK_REGIONS[iso]
CACHE_DIR.mkdir(parents=True, exist_ok=True)
dest = CACHE_DIR / f"{region}-latest.osm.pbf"
if dest.exists() and dest.stat().st_size > 0:
print(f" using cached {dest.name} ({dest.stat().st_size / 1e6:.0f} MB)", file=sys.stderr)
return dest
url = f"https://download.geofabrik.de/europe/{region}-latest.osm.pbf"
tmp = dest.with_suffix(".pbf.part")
print(f" downloading {url} ...", file=sys.stderr)
with urllib.request.urlopen(url, timeout=1800) as resp, open(tmp, "wb") as fh:
while True:
chunk = resp.read(1 << 20)
if not chunk:
break
fh.write(chunk)
tmp.rename(dest)
print(f" downloaded {dest.name} ({dest.stat().st_size / 1e6:.0f} MB)", file=sys.stderr)
return dest
# --------------------------------------------------------------------------- #
# Stops
#
# All three passes below use osmium.FileProcessor with a native (C++-side)
# filter rather than SimpleHandler.apply_file(): filtering there means most
# objects never cross into Python at all. A plain SimpleHandler callback -
# even one that returns immediately - has to be invoked for every single
# node in the file, and that per-call overhead alone made a whole-country
# extract (tens of millions of nodes) impractically slow in testing.
# --------------------------------------------------------------------------- #
def build_stop_rows(payload: dict) -> list[dict]:
by_name: dict[str, list[dict]] = {}
for element in payload.get("elements", []):
if element.get("type") != "node":
def collect_stop_nodes(pbf_path: Path, iso: str) -> list[dict]:
nodes: list[dict] = []
fp = osmium.FileProcessor(str(pbf_path)).with_filter(osmium.filter.EmptyTagFilter())
for obj in fp:
if not obj.is_node() or not obj.location.valid():
continue
tags = obj.tags
if not (
tags.get("highway") == "bus_stop"
or tags.get("railway") in ("tram_stop", "station", "halt")
or tags.get("public_transport") == "station"
or tags.get("amenity") == "bus_station"
):
continue
tags = element.get("tags", {})
name = (tags.get("name") or "").strip()
lat, lon = element.get("lat"), element.get("lon")
if not name or lat is None or lon is None:
if not name:
continue
node = {
"osm_id": int(element["id"]),
tags_dict = {t.k: t.v for t in tags}
nodes.append({
"osm_id": int(obj.id),
"name": name,
"stop_type": classify(tags),
"municipality": municipality_of(tags),
"latitude": round(float(lat), 6),
"longitude": round(float(lon), 6),
}
by_name.setdefault(normalize_name(name), []).append(node)
"stop_type": classify(tags_dict),
"country": iso,
"municipality": municipality_of(tags_dict),
"latitude": round(obj.location.lat, 6),
"longitude": round(obj.location.lon, 6),
})
print(f" {iso}: {len(nodes)} raw stop nodes", file=sys.stderr)
return nodes
def build_stop_rows(nodes: list[dict]) -> list[dict]:
by_name: dict[str, list[dict]] = {}
for node in nodes:
by_name.setdefault(normalize_name(node["name"]), []).append(node)
rows: list[dict] = []
for group in by_name.values():
@@ -180,6 +204,7 @@ def build_stop_rows(payload: dict) -> list[dict]:
"osm_id": min(n["osm_id"] for n in cluster),
"name": lead["name"],
"stop_type": lead["stop_type"],
"country": lead["country"],
"municipality": lead["municipality"]
or next((n["municipality"] for n in cluster if n["municipality"]), ""),
"latitude": round(sum(n["latitude"] for n in cluster) / len(cluster), 6),
@@ -194,54 +219,52 @@ def build_stop_rows(payload: dict) -> list[dict]:
# --------------------------------------------------------------------------- #
# Lines (route relations -> stops)
# --------------------------------------------------------------------------- #
def fetch_route_members() -> tuple[list[tuple[str, int]], dict[int, tuple[str, float, float]]]:
"""Return (ref, node_id) pairs plus node_id -> (name, lat, lon)."""
seen_rel: set[int] = set()
ref_members: list[tuple[str, int]] = []
def collect_route_members(pbf_path: Path) -> tuple[list[tuple[str, int]], dict[int, tuple[str, float, float]]]:
"""Pass 1: which route relations do we care about, and which node ids do
their stop/platform members point at? Pass 2: resolve name/coordinates
for exactly those node ids (they're often bare stop_position nodes with
no tags at all, so this can't be narrowed with a tag filter like the
stops pass - it has to look at every node, hence EntityFilter rather
than EmptyTagFilter to at least skip all ways in the same pass)."""
relations: list[tuple[list[str], list[int]]] = []
wanted_node_ids: set[int] = set()
fp1 = osmium.FileProcessor(str(pbf_path)).with_filter(osmium.filter.EntityFilter(osmium.osm.RELATION))
for obj in fp1:
tags = obj.tags
if tags.get("route") not in ROUTE_MODES:
continue
raw_ref = tags.get("ref")
if not raw_ref:
continue
# A few relations pack several numbers into one ref ("407, 413").
refs = [part.strip() for part in raw_ref.replace(",", ";").split(";") if part.strip()]
if not refs:
continue
member_ids = [m.ref for m in obj.members if m.type == "n" and m.role in STOP_ROLES]
if not member_ids:
continue
relations.append((refs, member_ids))
wanted_node_ids.update(member_ids)
print(
f" {len(relations)} route relations, "
f"{len(wanted_node_ids)} distinct stop-member nodes to resolve",
file=sys.stderr,
)
node_pos: dict[int, tuple[str, float, float]] = {}
fp2 = osmium.FileProcessor(str(pbf_path)).with_filter(osmium.filter.EntityFilter(osmium.osm.NODE))
for obj in fp2:
if obj.id not in wanted_node_ids or not obj.location.valid():
continue
node_pos[obj.id] = (obj.tags.get("name", ""), obj.location.lat, obj.location.lon)
south, west, north, east = BBOX
lat = south
tiles: list[tuple[float, float, float, float]] = []
while lat < north:
lon = west
while lon < east:
tiles.append((lat, lon, min(lat + LAT_STEP, north), min(lon + LON_STEP, east)))
lon += LON_STEP
lat += LAT_STEP
for i, (s, w, n, e) in enumerate(tiles, 1):
print(f" route tile {i}/{len(tiles)} ({s:.1f},{w:.1f})", file=sys.stderr)
query = (
f"[out:json][timeout:400];"
f'relation["type"="route"]["route"~"^({ROUTE_MODES})$"]["ref"]'
f"({s},{w},{n},{e})->.r;"
f".r out body;"
f"node(r.r);out body;"
)
payload = overpass(query)
for el in payload.get("elements", []):
if el["type"] == "node":
tags = el.get("tags", {})
node_pos[el["id"]] = (tags.get("name", ""), el["lat"], el["lon"])
elif el["type"] == "relation":
if el["id"] in seen_rel:
continue
seen_rel.add(el["id"])
# A few relations pack several numbers into one ref ("407, 413").
refs = [r.strip() for r in re.split(r"[;,]", el["tags"]["ref"]) if r.strip()]
if not refs:
continue
members = [
m["ref"] for m in el.get("members", [])
if m["type"] == "node" and m["role"] in STOP_ROLES
]
for ref in refs:
for node_id in members:
ref_members.append((ref, node_id))
del payload
print(f" {len(seen_rel)} routes, {len(ref_members)} stop memberships", file=sys.stderr)
ref_members: list[tuple[str, int]] = []
for refs, member_ids in relations:
for ref in refs:
for node_id in member_ids:
ref_members.append((ref, node_id))
return ref_members, node_pos
@@ -302,16 +325,31 @@ def line_sort_key(ref: str):
# --------------------------------------------------------------------------- #
def main() -> None:
print("Fetching stops ...", file=sys.stderr)
rows = build_stop_rows(overpass(STOPS_QUERY))
if len(rows) < 10_000:
all_nodes: list[dict] = []
all_ref_members: list[tuple[str, int]] = []
all_node_pos: dict[int, tuple[str, float, float]] = {}
for iso in GEOFABRIK_REGIONS:
print(f"=== {iso} ===", file=sys.stderr)
pbf_path = download_extract(iso)
print(" collecting stops ...", file=sys.stderr)
all_nodes.extend(collect_stop_nodes(pbf_path, iso))
print(" collecting routes ...", file=sys.stderr)
ref_members, node_pos = collect_route_members(pbf_path)
all_ref_members.extend(ref_members)
all_node_pos.update(node_pos)
print("Clustering stops ...", file=sys.stderr)
rows = build_stop_rows(all_nodes)
if len(rows) < MIN_STOPS:
raise SystemExit(
f"Only {len(rows)} stops parsed - refusing to overwrite the snapshot."
f"Only {len(rows)} stops parsed (< {MIN_STOPS}) - refusing to overwrite the snapshot."
)
print("Fetching routes ...", file=sys.stderr)
ref_members, node_pos = fetch_route_members()
matched, unmatched = assign_lines(rows, ref_members, node_pos)
print("Matching lines to stops ...", file=sys.stderr)
matched, unmatched = assign_lines(rows, all_ref_members, all_node_pos)
OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
with gzip.open(OUT_PATH, "wt", newline="", encoding="utf-8") as fh:
@@ -322,10 +360,13 @@ def main() -> None:
writer.writerow(row)
by_type: dict[str, int] = {}
by_country: dict[str, int] = {}
for row in rows:
by_type[row["stop_type"]] = by_type.get(row["stop_type"], 0) + 1
by_country[row["country"]] = by_country.get(row["country"], 0) + 1
with_lines = sum(1 for r in rows if r["lines"])
print(f"Wrote {len(rows)} stops to {OUT_PATH}", file=sys.stderr)
print(f" by country: {by_country}", file=sys.stderr)
print(f" by type: {by_type}", file=sys.stderr)
print(
f" lines: {with_lines} stops have >=1 line "