diff --git a/.gitignore b/.gitignore index a3937d5..c499546 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc *.csv -.vscode/ \ No newline at end of file +.vscode/ +.vs/ \ No newline at end of file diff --git a/.vs/PythonSettings.json b/.vs/PythonSettings.json new file mode 100644 index 0000000..a229f65 --- /dev/null +++ b/.vs/PythonSettings.json @@ -0,0 +1,3 @@ +{ + "SuppressEnvironmentCreationPrompt": true +} \ No newline at end of file diff --git a/.vs/VSWorkspaceState.json b/.vs/VSWorkspaceState.json new file mode 100644 index 0000000..87537d8 --- /dev/null +++ b/.vs/VSWorkspaceState.json @@ -0,0 +1,8 @@ +{ + "ExpandedNodes": [ + "", + "\\util" + ], + "SelectedNode": "\\weather_scraper.py", + "PreviewInSolutionExplorer": false +} \ No newline at end of file diff --git a/.vs/slnx.sqlite b/.vs/slnx.sqlite new file mode 100644 index 0000000..9ea3d60 Binary files /dev/null and b/.vs/slnx.sqlite differ diff --git a/.vs/the-weather-scraper/v17/.suo b/.vs/the-weather-scraper/v17/.suo new file mode 100644 index 0000000..514bf32 Binary files /dev/null and b/.vs/the-weather-scraper/v17/.suo differ diff --git a/README.md b/README.md index 8c4e514..c818d4d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ python weather_scraper.py ``` ### How to run TWS? -First, find the weather stations you are looking for. +First, find the weather stations you are looking for. Then you just have to update 2 config files before running TWS. 1. Go to https://www.wunderground.com/wundermap and zoom in to your location @@ -44,12 +44,12 @@ UNIT_SYSTEM = "metric" FIND_FIRST_DATE = False ``` -Now you are read to run your downloads: +Now you are ready to run your downloads: ```sh $ python weather_scraper.py ``` Wait until TWS finishes writing your data to files with this naming pattern ***station_name.csv***! -You resulting CSV file will look something like this (if you give it a nice format) +You resulting CSV file will look something like this (if you format it nicely) ![CSV example](https://raw.githubusercontent.com/Karlheinzniebuhr/the-weather-scraper/master/resources/csv.JPG) diff --git a/TODO b/TODO index 500ece3..d5720bd 100644 --- a/TODO +++ b/TODO @@ -10,4 +10,4 @@ Todo: ✔ add error handling to UnitConverter @done(20-07-03 15:11) ✔ Write Dates in YYYY MM DD format @done(20-07-04 11:54) ☐ See if there is a way to start downloading from first available date - ☐ Write one more column with date_time \ No newline at end of file + ✔ Write one more column with date_time \ No newline at end of file diff --git a/util/Parser.py b/util/Parser.py index 8eaea3c..e922597 100644 --- a/util/Parser.py +++ b/util/Parser.py @@ -28,8 +28,10 @@ def parse_html_table(date_string: str, history_table: list) -> dict: if i == 0: date = datetime.strptime(date_string, "%Y-%m-%d") time = datetime.strptime(td_content, "%I:%M %p") + blank = datetime.strptime('','') row_dict['Date'] = date.strftime('%Y/%m/%d') row_dict['Time'] = time.strftime('%I:%M %p') + row_dict['Date_time'] = ((time-blank)+date).strftime('%Y/%m/%d %I:%M:%S') else: row_dict[Parser.format_key(headers_list[i])] = td_content diff --git a/util/UnitConverter.py b/util/UnitConverter.py index 5d47195..e7cc12b 100644 --- a/util/UnitConverter.py +++ b/util/UnitConverter.py @@ -24,16 +24,7 @@ def temperature(self, temp_string: str): except Exception as e: print(f'{e}! probably caused by an empty row in the data') return 'NA' - - def dew_point(self, dew_point_string: str): - try: - fahrenheit = float(re.findall(self.extract_numbers_pattern, dew_point_string)[0]) if dew_point_string else 'NA' - if self.system == "metric": - celsius = (fahrenheit - 32) * 5/9 - return round(celsius, self.round_to_decimals) - else: - return fahrenheit - + except Exception as e: print(f'{e}! probably caused by an empty row in the data') return 'NA' @@ -111,29 +102,31 @@ def clean_and_convert(self, dict_list: list): for key, value in dict.items(): if key == 'Date': converted_dict['Date'] = value - if key == 'Time': + elif key == 'Time': converted_dict['Time'] = value - if key == 'Temperature': + elif key == 'Date_time': + converted_dict['Date_time'] = value + elif key == 'Temperature': converted_dict['Temperature'] = self.temperature(value) - if key == 'Dew_Point': - converted_dict['Dew_Point'] = self.dew_point(value) - if key == 'Humidity': + elif key == 'Dew_Point': + converted_dict['Dew_Point'] = self.temperature(value) + elif key == 'Humidity': converted_dict['Humidity'] = self.humidity(value) - if key == 'Wind': + elif key == 'Wind': converted_dict['Wind'] = value - if key == 'Speed': + elif key == 'Speed': converted_dict['Speed'] = self.speed(value) - if key == 'Gust': + elif key == 'Gust': converted_dict['Gust'] = self.speed(value) - if key == 'Pressure': + elif key == 'Pressure': converted_dict['Pressure'] = self.pressure(value) - if key == 'Precip_Rate': + elif key == 'Precip_Rate': converted_dict['Precip_Rate'] = self.precipitation(value) - if key == 'Precip_Accum': + elif key == 'Precip_Accum': converted_dict['Precip_Accum'] = self.precipitation(value) - if key == 'UV': + elif key == 'UV': converted_dict['UV'] = self.uv(value) - if key == 'Solar': + elif key == 'Solar': converted_dict['Solar'] = self.solar(value) converted_dict_list.append(converted_dict) diff --git a/weather_scraper.py b/weather_scraper.py index bc71c0d..a2ab790 100644 --- a/weather_scraper.py +++ b/weather_scraper.py @@ -24,7 +24,7 @@ FIND_FIRST_DATE = config.FIND_FIRST_DATE -def scrap_station(weather_station_url): +def scrape_station(weather_station_url): session = requests.Session() timeout = 5 @@ -87,4 +87,4 @@ def scrap_station(weather_station_url): for url in URLS: url = url.strip() print(url) - scrap_station(url) \ No newline at end of file + scrape_station(url) \ No newline at end of file