From 5cacaf42fb88f1960196c37638ee8b2f32e7df28 Mon Sep 17 00:00:00 2001 From: FL550 Date: Wed, 18 Sep 2024 11:32:24 +0000 Subject: [PATCH] Implement etag for hourly update --- simple_dwd_weatherforecast/dwdforecast.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/simple_dwd_weatherforecast/dwdforecast.py b/simple_dwd_weatherforecast/dwdforecast.py index 36aa72e..890526a 100644 --- a/simple_dwd_weatherforecast/dwdforecast.py +++ b/simple_dwd_weatherforecast/dwdforecast.py @@ -951,21 +951,31 @@ def download_small_kml(self, stationid) -> bytes | None: except Exception as error: print(f"Error in download_latest_kml: {type(error)} args: {error.args}") - def get_chunks(self): - def zipped_chunks(): + def get_chunks(self, url): + def zipped_chunks(url): # Iterable that yields the bytes of a zip file with httpx.stream( "GET", - "https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_S/all_stations/kml/MOSMIX_S_LATEST_240.kmz", + url, ) as r: + self.etags[url] = r.headers["etag"] # type: ignore yield from r.iter_bytes(chunk_size=171072) - return stream_unzip(zipped_chunks()) + return stream_unzip(zipped_chunks(url)) def download_large_kml(self, stationid): placemark = b"" + url = "https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_S/all_stations/kml/MOSMIX_S_LATEST_240.kmz" + # Check if content has updated + headers = {"If-None-Match": self.etags[url] if url in self.etags else ""} # type: ignore + r = httpx.head( + "https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_S/all_stations/kml/MOSMIX_S_LATEST_240.kmz", + headers=headers, + ) + if r.status_code == 304: + return - for file_name, file_size, unzipped_chunks in self.get_chunks(): + for file_name, file_size, unzipped_chunks in self.get_chunks(url): chunk1 = b"" chunk2 = b"" first_chunk = None