Skip to content

Commit

Permalink
Ingest air pollutants dataset (#406)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Corgam
Corgam authored Jul 14, 2024
2 parents 5c27d61 + aee863b commit fc53eae
Show file tree
Hide file tree
Showing 5 changed files with 129 additions and 72 deletions.
4 changes: 4 additions & 0 deletions backend/metadata-database/init-db.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,10 @@ const datasets = [
{ displayName: "Nitrogen Dioxide (NO2)", value: "no2" },
{ displayName: "Particulate Matter (PM10)", value: "pm10" },
{ displayName: "Particulate Matter (PM2'5)", value: "pm2_5" },
{ displayName: "Ozone (O3)", value: "o3" },
{ displayName: "Carbon monoxide (CO)", value: "co" },
{ displayName: "Date", value: "date" },
{ displayName: "Time", value: "time" },
],
PolygonColoring: null,
Tables: [],
Expand Down
12 changes: 8 additions & 4 deletions backend/src/BIE.Core/BIE.Core.API/DatasetHandlers/CsvDatasetHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public string GetDataInsideArea(BoundingBox boundingBox)
var query = "";
if (mMetadata.basicData.DatasetId == "air_pollutants")
{
query = "SELECT top 1000 station_name,pm10,pm2_5,no2,Location.AsTextZM() AS Location" +
query = "SELECT top 1000 station_name,pm10,pm2_5,no2,co,o3,unit,time,date,Location.AsTextZM() AS Location" +
ApiHelper.FromTableWhereIntersectsPolygon(tableName, polygon);
}
else if (mMetadata.basicData.DatasetId == "EV_charging_stations")
Expand Down Expand Up @@ -72,9 +72,13 @@ public string GetDataInsideArea(BoundingBox boundingBox)
if (mMetadata.basicData.DatasetId == "air_pollutants")
{
properties["station_name"] = row["station_name"];
properties["pm10"] = row["pm10"] + " µg/m3^3";
properties["pm2_5"] = row["pm2_5"] + " µg/m3^3";
properties["no2"] = row["no2"] + " µg/m3^3";
properties["pm10"] = row["pm10"] + row["unit"];
properties["pm2_5"] = row["pm2_5"] + row["unit"];
properties["no2"] = row["no2"] + row["unit"];
properties["o3"] = row["o3"] + row["unit"];
properties["co"] = row["co"] + row["unit"];
properties["date"] = row["date"];
properties["time"] = row["time"];
}
else if (mMetadata.basicData.DatasetId == "EV_charging_stations")
{
Expand Down
64 changes: 50 additions & 14 deletions backend/src/BIE.Core/BIE.Core.API/LoadLocationHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public class LoadLocationHelper
private static readonly GeometryFactory GeometryFactory = new(new PrecisionModel(), 4326);

private static readonly HashSet<string> cityGmlColumns = new() { "Id", "GroundArea", "BuildingWallHeight", "LivingArea", "RoofArea", "BuildingVolume", "RoofArea", "SolarPotential" };
private static readonly HashSet<string> airPollutionColumns = new() { "station_name", "pm10", "pm2_5", "no2" };
private static readonly HashSet<string> airPollutionColumns = new() { "station_name", "pm10", "pm2_5", "no2", "o3", "co" };
private static readonly HashSet<string> chargingStationsColumns = new() { "operator", "Id", "rated_power_kw", "charging_equipment_type" };
private static readonly HashSet<string> actualUseColumns = new() { "nutzart" };
private static readonly HashSet<string> houseFootprintsColumns = new() { "Id" };
Expand Down Expand Up @@ -212,39 +212,61 @@ private static void AggregateDataForAirPollutant(List<DatasetItem> generalData,
List<double> avgPm10 = new();
List<double> avgPm25 = new();
List<double> avgNo2 = new();
List<double> avgO3 = new();
List<double> avgCo = new();

foreach (var h in airPollutants)
{
Point location = GeoReader.Read(h["Location"]) as Point;
double pm10 = double.Parse(h["pm10"]);
double pm25 = double.Parse(h["pm2_5"]);
double no2 = double.Parse(h["no2"]);

List<SubdataItem> subdataItems = new();
if (pm10 > 0)

if (h.TryGetValue("pm10", out var pm10Value) && double.TryParse(pm10Value, out var pm10) && pm10 > 0)
{
avgPm10.Add(pm10);
subdataItems.Add(new()
{
Key = "Particulate matter PM₁₀",
Value = h["pm10"] + "μg/m³"
Value = pm10 + "μg/m³"
});
}
if (pm25 > 0)

if (h.TryGetValue("pm2_5", out var pm25Value) && double.TryParse(pm25Value, out var pm25) && pm25 > 0)
{
avgPm25.Add(pm25);
subdataItems.Add(new()
{
Key = "Fine particulate matter PM₂,₅",
Value = h["pm2_5"] + "μg/m³"
Value = pm25 + "μg/m³"
});
}
if (no2 > 0)

if (h.TryGetValue("no2", out var no2Value) && double.TryParse(no2Value, out var no2) && no2 > 0)
{
avgNo2.Add(no2);
subdataItems.Add(new()
{
Key = "Nitrogen oxide NO₂",
Value = h["no2"] + "μg/m³"
Value = no2 + "μg/m³"
});
}

if (h.TryGetValue("o3", out var o3Value) && double.TryParse(o3Value, out var o3) && o3 > 0)
{
avgO3.Add(o3);
subdataItems.Add(new()
{
Key = "Ozone O₃",
Value = o3 + "μg/m³"
});
}

if (h.TryGetValue("co", out var coValue) && double.TryParse(coValue, out var co) && co > 0)
{
avgCo.Add(co);
subdataItems.Add(new()
{
Key = "Carbon monoxide CO",
Value = co + "μg/m³"
});
}

Expand All @@ -260,13 +282,16 @@ private static void AggregateDataForAirPollutant(List<DatasetItem> generalData,

airPollutionItems.Add(item);
}

individualData.AddRange(airPollutionItems);

generalData.Add(new DatasetItem
{
DatasetId = "air_pollutants",
DisplayName = "Total number of air pollution measuring stations",
Value = airPollutionItems.Count.ToString()
});

generalData.Add(new DatasetItem
{
DatasetId = "air_pollutants",
Expand All @@ -276,22 +301,33 @@ private static void AggregateDataForAirPollutant(List<DatasetItem> generalData,
new()
{
Key = "Average Particulate matter PM₁₀",
Value = avgPm10.Average().ToString("N2") + "μg/m³"
Value = avgPm10.Count > 0 ? avgPm10.Average().ToString("N2") + "μg/m³" : "-"
},
new()
{
Key = "Average Fine particulate matter PM₂.₅",
Value = avgPm25.Average().ToString("N2") + "μg/m³"
Value = avgPm25.Count > 0 ? avgPm25.Average().ToString("N2") + "μg/m³" : "-"
},
new()
{
Key = "Average Nitrogen oxide NO₂",
Value = avgNo2.Average().ToString("N2") + "μg/m³"
Value = avgNo2.Count > 0 ? avgNo2.Average().ToString("N2") + "μg/m³" : "-"
},
new()
{
Key = "Average Ozone O₃",
Value = avgO3.Count > 0 ? avgO3.Average().ToString("N2") + "μg/m³" : "-"
},
new()
{
Key = "Average Carbon monoxide CO",
Value = avgCo.Count > 0 ? avgCo.Average().ToString("N2") + "μg/m³" : "-"
}
}
});
}


private static void AggregateDataForActualUse(List<DatasetItem> generalData, List<DatasetItem> individualData, double totalAreaSearchPolygon, List<Dictionary<string, string>> dataEntries)
{
long count = dataEntries.Count;
Expand Down
107 changes: 56 additions & 51 deletions backend/src/BIE.DataPipeline/yaml-data/air_pollutants.csv
View file
Original file line number Diff line number Diff line change
@@ -1,51 +1,56 @@
Bundesland / Messnetz,Stationscode,Stationsname,Breitengrad,Längengrad,PM10,PM2'5,NO2
Bayern,DEBY001,Ansbach/Residenzstraße,49.30,10.58,11,7,12
Bayern,DEBY002,Arzberg/Egerstraße,50.06,12.19,5,0,0
Bayern,DEBY004,Kleinwallstadt/Hofstetter Straße,49.87,9.17,8,0,6
Bayern,DEBY005,Aschaffenburg/Bussardweg,49.99,9.12,6,0,5
Bayern,DEBY006,Augsburg/Königsplatz,48.37,10.89,12,0,7
Bayern,DEBY007,Augsburg/Bourges-Platz,48.38,10.89,11,6,6
Bayern,DEBY009,Bamberg/Löwenbrücke,49.90,10.89,12,7,6
Bayern,DEBY012,Burghausen/Marktler Straße,48.17,12.83,11,9,4
Bayern,DEBY013,Mehring/Sportplatz,49.79,6.83,0,5,3
Bayern,DEBY014,Coburg/Lossaustraße,50.26,10.96,0,12,0
Bayern,DEBY020,Hof/LfU,50.32,11.90,0,0,3
Bayern,DEBY030,Saal a.d. Donau/Regensburger Straße,48.90,11.94,0,0,9
Bayern,DEBY031,Kempten (Allgäu)/Westendstraße,47.73,10.31,0,6,5
Bayern,DEBY032,Kulmbach/Konrad-Adenauer-Straße,50.10,11.44,8,0,3
Bayern,DEBY033,Landshut/Podewilsstraße,48.54,12.16,10,0,10
Bayern,DEBY035,Lindau (Bodensee)/Friedrichshafener Straße,47.56,9.68,8,6,12
Bayern,DEBY037,München/Stachus,48.14,11.57,13,6,13
Bayern,DEBY039,München/Lothstraße,48.15,11.55,11,6,7
Bayern,DEBY049,Neustadt a.d. Donau/Eining,48.86,11.77,0,5,3
Bayern,DEBY052,Neu-Ulm/Gabelsbergerstraße,48.40,10.01,12,8,12
Bayern,DEBY053,Nürnberg/Bahnhof,49.45,11.08,0,9,5
Bayern,DEBY056,Fürth/Theresienstraße,49.47,10.98,8,0,7
Bayern,DEBY058,Nürnberg/Muggenhof,49.47,11.02,0,6,14
Bayern,DEBY063,Regensburg/Rathaus,49.02,12.09,6,0,4
Bayern,DEBY067,Schwandorf/Wackersdorfer Straße,49.32,12.13,0,0,9
Bayern,DEBY068,Schweinfurt/Obertor,50.05,10.23,8,0,3
Bayern,DEBY072,Tiefenbach/Altenschneeberg,49.44,12.56,5,4,7
Bayern,DEBY075,Weiden i.d.OPf./Nikolaistraße,49.68,12.16,0,8,3
Bayern,DEBY077,Würzburg/Kopfklinik,49.80,9.96,13,7,3
Bayern,DEBY088,Trostberg/Schwimmbadstraße,48.02,12.54,11,6,3
Bayern,DEBY089,München/Johanneskirchen,48.17,11.64,18,7,0
Bayern,DEBY093,Sulzbach-Rosenberg/Lohe,48.17,11.64,8,0,2
Bayern,DEBY099,Augsburg/LfU,48.33,10.90,8,6,2
Bayern,DEBY109,Andechs/Rothenfeld,47.97,11.22,8,5,19
Bayern,DEBY110,Augsburg/Karlstraße,48.37,10.90,12,0,5
Bayern,DEBY111,Bayreuth/Hohenzollernring,49.95,11.58,9,0,3
Bayern,DEBY115,München/Landshuter Allee,48.17,11.54,10,6,37
Bayern,DEBY118,Passau/Stelzhamerstraße,48.57,13.42,15,10,15
Bayern,DEBY119,Würzburg/Stadtring Süd,49.79,9.94,11,0,27
Bayern,DEBY120,Nürnberg/Von-der-Tann-Straße,49.44,11.04,14,0,10
Bayern,DEBY121,Oberaudorf/Inntal-Autobahn,49.02,12.06,11,7,22
Bayern,DEBY122,Bad Hindelang/Unterjoch,47.54,10.43,0,0,2
Bayern,DEBY124,Burgbernheim/Grüne Au,49.44,10.32,12,12,2
Bayern,DEBY187,Oettingen/Goethestraße,48.95,10.60,0,3,5
Bayern,DEBY188,Schwabach/Angerstraße,49.32,11.03,8,8,5
Bayern,DEBY189,München/Allach,48.19,11.47,0,0,2
Bayern,DEBY196,Garmisch-Partenkirchen/Wasserwerk,48.25,11.37,0,0,1
Bayern,DEBY198,Ingolstadt/Münchener Str.,48.74,11.44,13,7,18
Bayern,DEBY199,Neustadt a.d. Aisch/An der Weißen Marter,49.58,10.62,0,0,4
Bayern,DEBY200,Passau/Angerstr.,48.58,13.46,15,9,37
Bundesland / Messnetz,Stationscode,Stationsname,Breitengrad,Längengrad,PM10,PM2'5,NO2,O3,CO,Unit,Time,Date
Bayern,DEBY001,Ansbach/Residenzstraße,49.30,10.58,11,7,12,-,"0,2062",µg/m³,12:00,05.07.2024
Bayern,DEBY002,Arzberg/Egerstraße,50.06,12.19,5,-,-,92,-,µg/m³,12:00,05.07.2024
Bayern,DEBY004,Kleinwallstadt/Hofstetter Straße,49.87,9.17,8,-,6,90,-,µg/m³,12:00,05.07.2024
Bayern,DEBY005,Aschaffenburg/Bussardweg,49.99,9.12,6,-,5,88,-,µg/m³,12:00,05.07.2024
Bayern,DEBY006,Augsburg/Königsplatz,48.37,10.89,12,-,7,-,"0,1687",µg/m³,12:00,05.07.2024
Bayern,DEBY007,Augsburg/Bourges-Platz,48.38,10.89,11,6,6,77,-,µg/m³,12:00,05.07.2024
Bayern,DEBY009,Bamberg/Löwenbrücke,49.90,10.89,12,7,6,-,-,µg/m³,12:00,05.07.2024
Bayern,DEBY012,Burghausen/Marktler Straße,48.17,12.83,11,9,4,83,-,µg/m³,12:00,05.07.2024
Bayern,DEBY013,Mehring/Sportplatz,49.79,6.83,-,5,3,96,-,µg/m³,12:00,05.07.2024
Bayern,DEBY014,Coburg/Lossaustraße,50.26,10.96,-,12,-,-,"0,1762",µg/m³,12:00,05.07.2024
Bayern,DEBY020,Hof/LfU,50.32,11.90,-,-,3,95,-,µg/m³,12:00,05.07.2024
Bayern,DEBY026,Vohburg a.d. Donau/Alter Wöhrer Weg,48.76,11.63,-,-,-,81,-,µg/m³,12:00,05.07.2024
Bayern,DEBY030,Saal a.d. Donau/Regensburger Straße,48.90,11.94,-,-,9,76,-,µg/m³,12:00,05.07.2024
Bayern,DEBY031,Kempten (Allgäu)/Westendstraße,47.73,10.31,-,6,5,83,-,µg/m³,12:00,05.07.2024
Bayern,DEBY032,Kulmbach/Konrad-Adenauer-Straße,50.10,11.44,8,-,3,-,-,µg/m³,12:00,05.07.2024
Bayern,DEBY033,Landshut/Podewilsstraße,48.54,12.16,1-,-,1-,-,-,µg/m³,12:00,05.07.2024
Bayern,DEBY035,Lindau (Bodensee)/Friedrichshafener Straße,47.56,9.68,8,6,12,-,"0,1975",µg/m³,12:00,05.07.2024
Bayern,DEBY037,München/Stachus,48.14,11.57,13,6,13,-,"0,1912",µg/m³,12:00,05.07.2024
Bayern,DEBY039,München/Lothstraße,48.15,11.55,11,6,7,79,-,µg/m³,12:00,05.07.2024
Bayern,DEBY047,Naila/Selbitzer Berg,50.32,11.72,-,-,-,93,-,µg/m³,12:00,05.07.2024
Bayern,DEBY049,Neustadt a.d. Donau/Eining,48.86,11.77,-,5,3,83,-,µg/m³,12:00,05.07.2024
Bayern,DEBY052,Neu-Ulm/Gabelsbergerstraße,48.40,10.01,12,8,12,74,-,µg/m³,12:00,05.07.2024
Bayern,DEBY053,Nürnberg/Bahnhof,49.45,11.08,-,9,5,-,-,µg/m³,12:00,05.07.2024
Bayern,DEBY056,Fürth/Theresienstraße,49.47,10.98,8,-,7,-,-,µg/m³,12:00,05.07.2024
Bayern,DEBY058,Nürnberg/Muggenhof,49.47,11.02,-,6,14,92,-,µg/m³,12:00,05.07.2024
Bayern,DEBY062,Regen/Bodenmaiser Straße,48.98,13.13,-,-,-,75,-,µg/m³,12:00,05.07.2024
Bayern,DEBY063,Regensburg/Rathaus,49.02,12.09,6,-,4,-,"0,1912",µg/m³,12:00,05.07.2024
Bayern,DEBY067,Schwandorf/Wackersdorfer Straße,49.32,12.13,-,-,9,88,-,µg/m³,12:00,05.07.2024
Bayern,DEBY068,Schweinfurt/Obertor,50.05,10.23,8,-,3,88,-,µg/m³,12:00,05.07.2024
Bayern,DEBY072,Tiefenbach/Altenschneeberg,49.44,12.56,5,4,7,91,-,µg/m³,12:00,05.07.2024
Bayern,DEBY075,Weiden i.d.OPf./Nikolaistraße,49.68,12.16,-,8,3,90,-,µg/m³,12:00,05.07.2024
Bayern,DEBY077,Würzburg/Kopfklinik,49.80,9.96,13,7,3,95,-,µg/m³,12:00,05.07.2024
Bayern,DEBY079,Bad Reichenhall/Kirchholzstraße,47.73,12.90,-,-,-,88,-,µg/m³,12:00,05.07.2024
Bayern,DEBY088,Trostberg/Schwimmbadstraße,48.02,12.54,11,6,3,91,-,µg/m³,12:00,05.07.2024
Bayern,DEBY089,München/Johanneskirchen,48.17,11.64,18,7,-,78,-,µg/m³,12:00,05.07.2024
Bayern,DEBY093,Sulzbach-Rosenberg/Lohe,48.17,11.64,8,-,2,92,-,µg/m³,12:00,05.07.2024
Bayern,DEBY099,Augsburg/LfU,48.33,10.90,8,6,2,83,"0,1475",µg/m³,12:00,05.07.2024
Bayern,DEBY109,Andechs/Rothenfeld,47.97,11.22,8,5,19,86,-,µg/m³,12:00,05.07.2024
Bayern,DEBY110,Augsburg/Karlstraße,48.37,10.90,12,-,5,-,"0,2225",µg/m³,12:00,05.07.2024
Bayern,DEBY111,Bayreuth/Hohenzollernring,49.95,11.58,9,-,3,-,-,µg/m³,12:00,05.07.2024
Bayern,DEBY113,Erlangen/Kraepelinstraße,49.61,10.96,-,-,-,92,-,µg/m³,12:00,05.07.2024
Bayern,DEBY115,München/Landshuter Allee,48.17,11.54,1-,6,37,-,"0,36",µg/m³,12:00,05.07.2024
Bayern,DEBY118,Passau/Stelzhamerstraße,48.57,13.42,15,1-,15,-,-,µg/m³,12:00,05.07.2024
Bayern,DEBY119,Würzburg/Stadtring Süd,49.79,9.94,11,-,27,-,"0,1413",µg/m³,12:00,05.07.2024
Bayern,DEBY120,Nürnberg/Von-der-Tann-Straße,49.44,11.04,14,-,1-,-,"0,1425",µg/m³,12:00,05.07.2024
Bayern,DEBY121,Oberaudorf/Inntal-Autobahn,49.02,12.06,11,7,22,-,-,µg/m³,12:00,05.07.2024
Bayern,DEBY122,Bad Hindelang/Unterjoch,47.54,10.43,-,-,2,101,-,µg/m³,12:00,05.07.2024
Bayern,DEBY124,Burgbernheim/Grüne Au,49.44,10.32,12,12,2,96,-,µg/m³,12:00,05.07.2024
Bayern,DEBY187,Oettingen/Goethestraße,48.95,10.60,-,3,5,84,-,µg/m³,12:00,05.07.2024
Bayern,DEBY188,Schwabach/Angerstraße,49.32,11.03,8,8,5,89,-,µg/m³,12:00,05.07.2024
Bayern,DEBY189,München/Allach,48.19,11.47,-,-,2,73,-,µg/m³,12:00,05.07.2024
Bayern,DEBY196,Garmisch-Partenkirchen/Wasserwerk,48.25,11.37,-,-,1,76,-,µg/m³,12:00,05.07.2024
Bayern,DEBY198,Ingolstadt/Münchener Str.,48.74,11.44,13,7,18,-,"0,2062",µg/m³,12:00,05.07.2024
Bayern,DEBY199,Neustadt a.d. Aisch/An der Weißen Marter,49.58,10.62,-,-,4,95,-,µg/m³,12:00,05.07.2024
Bayern,DEBY200,Passau/Angerstr.,48.58,13.46,15,9,37,-,"0,275",µg/m³,12:00,05.07.2024
14 changes: 11 additions & 3 deletions backend/src/BIE.DataPipeline/yaml/common/air_pollutants.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,18 @@ table_cols:
type: DECIMAL(9,6)
- name: PM10
name_in_table: pm10
type: INT
- name: PM2'5
name_in_table: pm2_5
type: INT
- name: NO2
name_in_table: no2
type: INT
- name: O3
name_in_table: o3
- name: CO
name_in_table: co
- name: Unit
name_in_table: unit
- name: Time
name_in_table: time
- name: Date
name_in_table: date

0 comments on commit fc53eae

Please sign in to comment.