Cleanup of station catalogue

FL550 · Oct 6, 2023 · 1660228 · 1660228
1 parent 5fdc7a1
commit 1660228
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 67 deletions.
diff --git a/development/bundeslaender.json b/development/bundeslaender.json
diff --git a/development/generate_bundesland.py b/development/generate_bundesland.py
@@ -0,0 +1,11 @@
+import json
+
+with open("stations_old.json", "r", encoding="utf-8") as f:
+    file = json.load(f)
+
+new_list = {}
+for item in file:
+    new_list[f"{file[item]['lat']};{file[item]['lon']}"] = file[item]["bundesland"]
+
+with open("development/bundeslaender.json", "w", encoding="utf-8") as f:
+    json.dump(new_list, f, ensure_ascii=False)
diff --git a/development/generate_stations.py b/development/generate_stations.py
@@ -4,6 +4,7 @@
 from bs4 import BeautifulSoup
 import json
 import re
+import pandas as pd
 
 success = False
 print("Retrieving MOSMIX stations catalogue...")
@@ -45,101 +46,95 @@
 print("Parsing...")
 soup = BeautifulSoup(request.text, "html.parser")
 print("Done.")
-stations_list = {}
+stations_list = []
 rows_html = soup.table.find_all("tr")
+
 for row in rows_html:
     cols = row.find_all("td")
     if len(cols) > 5 and "." in cols[10].text:
         end_year = int(cols[10].text.split(".")[2])
         if end_year >= datetime.datetime.now().year:
-            stations_list[cols[3].text.strip()] = {
-                "name": cols[0].text.strip(),
-                "lat": cols[4].text.strip(),
-                "lon": cols[5].text.strip(),
-                "elev": cols[6].text.strip(),
-                "bundesland": cols[8].text.strip(),
-                "report_available": 1 if cols[3].text.strip() in poi_links else 0,
-            }
-print(f"Found {len(stations_list.keys())} active stations.")
+            stations_list.append(
+                {
+                    "id": cols[1].text.strip(),
+                    "kennung": cols[3].text.strip(),
+                    "name": cols[0].text.strip(),
+                    "lat": cols[4].text.strip(),
+                    "lon": cols[5].text.strip(),
+                    "elev": cols[6].text.strip(),
+                    "bundesland": cols[8].text.strip(),
+                    "report_available": 1 if cols[3].text.strip() in poi_links else 0,
+                    "date": datetime.datetime.strptime(
+                        cols[10].text.strip(), "%d.%m.%Y"
+                    ),
+                }
+            )
+
+stations_list = pd.DataFrame(
+    stations_list,
+    columns=[
+        "id",
+        "kennung",
+        "name",
+        "lat",
+        "lon",
+        "elev",
+        "bundesland",
+        "report_available",
+        "date",
+    ],
+)
+stations_list = stations_list[
+    stations_list["date"] == stations_list.groupby("id")["date"].transform("max")
+]
+stations_list = stations_list.drop_duplicates()
+print(f"Found {len(stations_list)} active stations.")
 range_iter = iter(range(len(mosmix_data) - 1))
 first_run = True
 stations = {}
+with open("development/bundeslaender.json", "r", encoding="utf-8") as f:
+    bundeslaender = json.load(f)
+
 for i in range_iter:
     if first_run:
         mosmix_data[i] += f" POI REGION"
         next(range_iter)
         first_run = False
         continue
-    print(mosmix_data[i])
+    # print(mosmix_data[i])
     groups = re.search(
         "^(?P<id>\S*)\s*(?P<icao>\S*)\s+(?P<name>.+?)\s+(?P<lat>-?\d{1,2}\.\d{2})\s*(?P<lon>-?\d{1,3}\.\d{2})\s+(?P<elev>-*\d+)$",
         mosmix_data[i],
     )
-    is_in_stations_list = groups.group("id") in stations_list.keys()
+    is_in_stations_list = groups.group("id") in stations_list["kennung"].values
     region = ""
     if is_in_stations_list:
-        region = stations_list[groups.group("id")]["bundesland"]
-    else:
-        _lat = groups.group("lat").split(".")
-        _lat = round(float(_lat[0]) + float(_lat[1]) / 60, 2)
-        _lon = groups.group("lon").split(".")
-        _lon = round(float(_lon[0]) + float(_lon[1]) / 60, 2)
-        url = f"https://maps.googleapis.com/maps/api/geocode/json?latlng={_lat},{_lon}&location_type=APPROXIMATE&result_type=administrative_area_level_1&language=de&key={API_KEY}"
-        try:
-# not allowed
-            #request = requests.get(url, timeout=10)
-        except Timeout:
-            print("Timeout")
-        else:
-            region = ""
-            request = request.json()
-            if request["status"] == "OK":
-                region = request["results"][0]["address_components"][0]["long_name"]
-                if region == "Hessen":
-                    region = "HE"
-                elif region == "Nordrhein-Westfalen":
-                    region = "NW"
-                elif region == "Rheinland-Pfalz":
-                    region = "RP"
-                elif region == "Saarland":
-                    region = "SL"
-                elif region == "Baden-Württemberg":
-                    region = "BW"
-                elif region == "Bayern":
-                    region = "BY"
-                elif region == "Berlin":
-                    region = "BE"
-                elif region == "Brandenburg":
-                    region = "BB"
-                elif region == "Mecklenburg-Vorpommern":
-                    region = "MV"
-                elif region == "Sachsen":
-                    region = "SN"
-                elif region == "Sachsen-Anhalt":
-                    region = "ST"
-                elif region == "Thüringen":
-                    region = "TH"
-                elif region == "Hamburg":
-                    region = "HH"
-                elif region == "Bremen":
-                    region = "HB"
-                elif region == "Schleswig-Holstein":
-                    region = "SH"
-                elif region == "Niedersachsen":
-                    region = "NI"
+        region = stations_list[stations_list["kennung"] == groups.group("id")][
+            "bundesland"
+        ].values[0]
+    elif f'{groups.group("lat")};{groups.group("lon")}' in bundeslaender:
+        region = bundeslaender[f'{groups.group("lat")};{groups.group("lon")}']
 
     stations[groups.group("id")] = {
-        "name": stations_list[groups.group("id")]["name"]
+        "name": stations_list[stations_list["kennung"] == groups.group("id")][
+            "name"
+        ].values[0]
         if is_in_stations_list
         else groups.group("name").title(),
         "icao": groups.group("icao"),
-        "lat": stations_list[groups.group("id")]["lat"]
+        "lat": stations_list[stations_list["kennung"] == groups.group("id")][
+            "lat"
+        ].values[0]
         if is_in_stations_list
         else groups.group("lat"),
-        "lon": stations_list[groups.group("id")]["lon"]
+        "lon": stations_list[stations_list["kennung"] == groups.group("id")][
+            "lon"
+        ].values[0]
         if is_in_stations_list
         else groups.group("lon"),
-        "elev": stations_list[groups.group("id")]["elev"]
+        "elev": stations_list[stations_list["kennung"] == groups.group("id")][
+            "elev"
+        ].values[0]
         if is_in_stations_list
         else groups.group("elev"),
         "bundesland": region,

diff --git a/development/stations.json b/development/stations.json
diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="simple_dwd_weatherforecast",
-    version="2.0.14",
+    version="2.0.15",
     author="Max Fermor",
     description="A simple tool to retrieve a weather forecast from DWD OpenData",
     long_description=long_description,

diff --git a/stations.json b/stations.json