Better showcases (#1)

* Use country specific showcases * Update deprecated calls * Update deprecated calls run ruff on code
OCHA-DAP · Jul 8, 2024 · 0d261e2 · 0d261e2
1 parent 8bc67a4
commit 0d261e2
Show file tree

Hide file tree

Showing 9 changed files with 143 additions and 74 deletions.
diff --git a/analysis_dates.txt b/analysis_dates.txt
@@ -1 +1 @@
-DEFAULT=2017-01-01,AFG=2024-03-01,START_DATE=2017-01-01,END_DATE=2025-02-28,AGO=2021-06-01,BDI=2024-05-01,BEN=2024-03-01,BFA=2024-03-01,CAF=2024-04-01,CIV=2024-03-01,CMR=2024-03-01,COD=2023-11-01,CPV=2024-03-01,DJI=2024-04-01,DOM=2023-09-01,ETH=2021-05-01,GHA=2024-03-01,GIN=2024-03-01,GMB=2024-03-01,GNB=2024-03-01,GTM=2023-05-01,HND=2024-02-01,HTI=2024-03-01,KEN=2024-02-01,LBN=2024-03-01,LBR=2023-03-01,LSO=2023-07-01,MDG=2023-12-01,MLI=2024-03-01,MOZ=2023-07-01,MRT=2024-03-01,MWI=2023-07-01,NAM=2023-07-01,NER=2024-03-01,NGA=2024-03-01,PAK=2024-03-01,SDN=2024-04-01,SEN=2024-03-01,SLE=2024-03-01,SLV=2021-08-01,SOM=2024-01-01,SSD=2023-09-01,SWZ=2023-06-01,TCD=2024-03-01,TGO=2024-03-01,TLS=2023-11-01,TZA=2023-11-01,UGA=2024-04-01,YEM=2023-10-01,ZAF=2020-10-01,ZMB=2023-09-01,ZWE=2020-10-01,BGD=2024-02-01,MMR=2023-09-01,PSE=2024-05-01
+DEFAULT=2017-01-01
diff --git a/config/ch_countries.csv b/config/ch_countries.csv
@@ -0,0 +1,19 @@
+Country_name,ISO_3,ISO_2
+Benin,BEN,BJ
+Burkina Faso,BFA,BF
+Cabo Verde,CPV,CV
+Cameroun,CMR,CM
+Cote D’Ivoire,CIV,CI
+Gambia (the),GMB,GM
+Ghana,GHA,GH
+Guinea Bissau,GNB,GW
+Guinee,GIN,GN
+Liberia,LBR,LR
+Mali,MLI,ML
+Mauritanie,MRT,MR
+Niger (the),NER,NE
+Nigeria,NGA,NG
+Senegal,SEN,SE
+Sierra Leone,SLE,SL
+Chad,TCD,TD
+Togo,TGO,TG
diff --git a/config/hdx_dataset_static.yml → config/hdx_dataset_static.yaml b/config/hdx_dataset_static.yml → config/hdx_dataset_static.yaml
diff --git a/config/project_configuration.yml → config/project_configuration.yaml b/config/project_configuration.yml → config/project_configuration.yaml
@@ -1,5 +1,7 @@
 # Collector specific configuration
 base_url: "https://api.ipcinfo.org"
+showcase_url: "https://www.ipcinfo.org/ipc-country-analysis/en/?country="
+ch_showcase_url: "https://www.ipcinfo.org/ch/en/"
 long_hxltags:
   "Date of analysis": "#date+analysis"
   "Country": "#country+code"

diff --git a/ipc.py b/ipc.py
@@ -24,7 +24,7 @@
 
 
 class IPC:
-    def __init__(self, configuration, retriever, state):
+    def __init__(self, configuration, retriever, state, ch_countries):
         self.configuration = configuration
         self.retriever = retriever
         self.state = state
@@ -62,6 +62,7 @@ def __init__(self, configuration, retriever, state):
         name, title = self.get_dataset_title_name("Global")
         temp_dataset = Dataset({"name": name, "title": title})
         self.global_dataset_url = temp_dataset.get_hdx_url()
+        self.ch_countries = ch_countries
 
     def get_dataset_title_name(self, countryname):
         title = f"{countryname}: Acute Food Insecurity Country Data"
@@ -95,32 +96,33 @@ def parse_date(datestring):
             return date.replace(tzinfo=timezone.utc)
 
         analysis_date = parse_date(most_recent_analysis["analysis_date"])
-        if analysis_date <= self.state.get(countryiso3, self.default_start_date):
+        if analysis_date <= self.state.get(countryiso3,
+                                           self.default_start_date):
             update = False
         else:
             update = True
         self.state[countryiso3] = analysis_date
-        reference_period = {"start_date": default_enddate, "end_date": default_date}
+        time_period = {"start_date": default_enddate, "end_date": default_date}
 
         def parse_date_range(date_range):
             start, end = date_range.split(" - ")
             startdate = parse_date(start)
-            if startdate < reference_period["start_date"]:
-                reference_period["start_date"] = startdate
+            if startdate < time_period["start_date"]:
+                time_period["start_date"] = startdate
             enddate = parse_date(end)
             enddate = enddate + relativedelta(months=1, days=-1)
-            if enddate > reference_period["end_date"]:
-                reference_period["end_date"] = enddate
+            if enddate > time_period["end_date"]:
+                time_period["end_date"] = enddate
             startdatestr = startdate.date().isoformat()
             enddatestr = enddate.date().isoformat()
             return startdatestr, enddatestr
 
         def add_country_subnational_rows(
-            base_row,
-            location,
-            rows,
-            rows_wide,
-            analysis=None,
+                base_row,
+                location,
+                rows,
+                rows_wide,
+                analysis=None,
         ):
             if analysis is None:
                 analysis = location
@@ -156,7 +158,8 @@ def add_country_subnational_rows(
                     else:
                         colname = f"Phase {phase} number {projection_name_l}"
                     row_wide[colname] = affected
-                    percentage = location.get(f"{prefix}_percentage{projection_suffix}")
+                    percentage = location.get(
+                        f"{prefix}_percentage{projection_suffix}")
                     row["Percentage"] = percentage
                     if prefix != "estimated":
                         row_wide[
@@ -184,7 +187,8 @@ def add_country_rows(analysis, rows, rows_wide):
             )
 
         def add_subnational_rows(
-            analysis, group_rows, group_rows_wide, area_rows, area_rows_wide
+                analysis, group_rows, group_rows_wide, area_rows,
+                area_rows_wide
         ):
             def process_areas(adm_row, adm):
                 if adm["areas"] is None:
@@ -266,8 +270,8 @@ def process_areas(adm_row, adm):
         self.output["group_rows_wide"].extend(group_rows_wide)
         self.output["area_rows"].extend(area_rows)
         self.output["area_rows_wide"].extend(area_rows_wide)
-        start_date = reference_period["start_date"]
-        end_date = reference_period["end_date"]
+        start_date = time_period["start_date"]
+        end_date = time_period["end_date"]
         output["start_date"] = start_date
         output["end_date"] = end_date
         if start_date < self.output["start_date"]:
@@ -320,18 +324,18 @@ def generate_dataset_and_showcase(self, folder, output):
             "integrated food security phase classification-ipc",
         )
         dataset.add_tags(tags)
-        dataset.set_reference_period(output["start_date"], output["end_date"])
+        dataset.set_time_period(output["start_date"], output["end_date"])
 
         filename = f"ipc_{countryiso3lower}_national_long_latest.csv"
         resourcedata = {
             "name": filename,
-            "description": f"Latest IPC national data in long form with HXL tags",
+            "description": "Latest IPC national data in long form with HXL tags",
         }
         country_rows = output["country_rows_latest"]
         if not country_rows:
             logger.warning(f"{filename} has no data!")
             return None, None
-        success, results = dataset.generate_resource_from_iterator(
+        success, results = dataset.generate_resource_from_iterable(
             list(country_rows[0].keys()),
             country_rows,
             self.configuration["long_hxltags"],
@@ -349,9 +353,9 @@ def generate_dataset_and_showcase(self, folder, output):
             filename = f"ipc_{countryiso3lower}_national_wide_latest.csv"
             resourcedata = {
                 "name": filename,
-                "description": f"Latest IPC national data in wide form with HXL tags",
+                "description": "Latest IPC national data in wide form with HXL tags",
             }
-            success, results = dataset.generate_resource_from_iterator(
+            success, results = dataset.generate_resource_from_iterable(
                 list(country_rows_wide[0].keys()),
                 country_rows_wide,
                 self.configuration["wide_hxltags"],
@@ -360,12 +364,19 @@ def generate_dataset_and_showcase(self, folder, output):
                 resourcedata,
             )
 
+        if countryiso3lower == "global":
+            showcase_url = "https://www.ipcinfo.org/ipcinfo-website/ipc-dashboard/en/"
+        elif countryiso3 in self.ch_countries:
+            showcase_url = self.configuration["ch_showcase_url"]
+        else:
+            showcase_url = self.configuration["showcase_url"]
+            showcase_url = f"{showcase_url}{countryiso3}"
         showcase = Showcase(
             {
                 "name": f"{name}-showcase",
                 "title": f"{title} showcase",
-                "notes": f"IPC-CH Dashboard",
-                "url": "https://www.ipcinfo.org/ipcinfo-website/ipc-dashboard/en/",
+                "notes": "IPC-CH Dashboard",
+                "url": showcase_url,
                 "image_url": "https://www.ipcinfo.org/fileadmin/user_upload/ipcinfo/img/dashboard_thumbnail.jpg",
             }
         )
@@ -375,9 +386,9 @@ def generate_dataset_and_showcase(self, folder, output):
             filename = f"ipc_{countryiso3lower}_level1_long_latest.csv"
             resourcedata = {
                 "name": filename,
-                "description": f"Latest IPC level 1 data in long form with HXL tags",
+                "description": "Latest IPC level 1 data in long form with HXL tags",
             }
-            success, results = dataset.generate_resource_from_iterator(
+            success, results = dataset.generate_resource_from_iterable(
                 list(group_rows[0].keys()),
                 group_rows,
                 self.configuration["long_hxltags"],
@@ -391,9 +402,9 @@ def generate_dataset_and_showcase(self, folder, output):
             filename = f"ipc_{countryiso3lower}_level1_wide_latest.csv"
             resourcedata = {
                 "name": filename,
-                "description": f"Latest IPC level 1 data in wide form with HXL tags",
+                "description": "Latest IPC level 1 data in wide form with HXL tags",
             }
-            success, results = dataset.generate_resource_from_iterator(
+            success, results = dataset.generate_resource_from_iterable(
                 list(group_rows_wide[0].keys()),
                 group_rows_wide,
                 self.configuration["wide_hxltags"],
@@ -406,9 +417,9 @@ def generate_dataset_and_showcase(self, folder, output):
         filename = f"ipc_{countryiso3lower}_area_long_latest.csv"
         resourcedata = {
             "name": filename,
-            "description": f"Latest IPC area data in long form with HXL tags",
+            "description": "Latest IPC area data in long form with HXL tags",
         }
-        success, results = dataset.generate_resource_from_iterator(
+        success, results = dataset.generate_resource_from_iterable(
             list(area_rows[0].keys()),
             area_rows,
             self.configuration["long_hxltags"],
@@ -421,9 +432,9 @@ def generate_dataset_and_showcase(self, folder, output):
         filename = f"ipc_{countryiso3lower}_area_wide_latest.csv"
         resourcedata = {
             "name": filename,
-            "description": f"Latest IPC area data in wide form with HXL tags",
+            "description": "Latest IPC area data in wide form with HXL tags",
         }
-        success, results = dataset.generate_resource_from_iterator(
+        success, results = dataset.generate_resource_from_iterable(
             list(area_rows_wide[0].keys()),
             area_rows_wide,
             self.configuration["wide_hxltags"],
@@ -440,9 +451,9 @@ def generate_dataset_and_showcase(self, folder, output):
         filename = f"ipc_{countryiso3lower}_national_long.csv"
         resourcedata = {
             "name": filename,
-            "description": f"All IPC national data in long form with HXL tags",
+            "description": "All IPC national data in long form with HXL tags",
         }
-        success, results = dataset.generate_resource_from_iterator(
+        success, results = dataset.generate_resource_from_iterable(
             list(country_rows[0].keys()),
             country_rows,
             self.configuration["long_hxltags"],
@@ -454,9 +465,9 @@ def generate_dataset_and_showcase(self, folder, output):
         filename = f"ipc_{countryiso3lower}_national_wide.csv"
         resourcedata = {
             "name": filename,
-            "description": f"All IPC national data in wide form with HXL tags",
+            "description": "All IPC national data in wide form with HXL tags",
         }
-        success, results = dataset.generate_resource_from_iterator(
+        success, results = dataset.generate_resource_from_iterable(
             list(country_rows_wide[0].keys()),
             country_rows_wide,
             self.configuration["wide_hxltags"],
@@ -470,9 +481,9 @@ def generate_dataset_and_showcase(self, folder, output):
             filename = f"ipc_{countryiso3lower}_level1_long.csv"
             resourcedata = {
                 "name": filename,
-                "description": f"All IPC level 1 data in long form with HXL tags",
+                "description": "All IPC level 1 data in long form with HXL tags",
             }
-            success, results = dataset.generate_resource_from_iterator(
+            success, results = dataset.generate_resource_from_iterable(
                 list(group_rows[0].keys()),
                 group_rows,
                 self.configuration["long_hxltags"],
@@ -486,9 +497,9 @@ def generate_dataset_and_showcase(self, folder, output):
             filename = f"ipc_{countryiso3lower}_level1_wide.csv"
             resourcedata = {
                 "name": filename,
-                "description": f"All IPC level 1 data in wide form with HXL tags",
+                "description": "All IPC level 1 data in wide form with HXL tags",
             }
-            success, results = dataset.generate_resource_from_iterator(
+            success, results = dataset.generate_resource_from_iterable(
                 list(group_rows_wide[0].keys()),
                 group_rows_wide,
                 self.configuration["wide_hxltags"],
@@ -501,9 +512,9 @@ def generate_dataset_and_showcase(self, folder, output):
         filename = f"ipc_{countryiso3lower}_area_long.csv"
         resourcedata = {
             "name": filename,
-            "description": f"All IPC area data in long form with HXL tags",
+            "description": "All IPC area data in long form with HXL tags",
         }
-        success, results = dataset.generate_resource_from_iterator(
+        success, results = dataset.generate_resource_from_iterable(
             list(area_rows[0].keys()),
             area_rows,
             self.configuration["long_hxltags"],
@@ -516,9 +527,9 @@ def generate_dataset_and_showcase(self, folder, output):
         filename = f"ipc_{countryiso3lower}_area_wide.csv"
         resourcedata = {
             "name": filename,
-            "description": f"All IPC area data in wide form with HXL tags",
+            "description": "All IPC area data in wide form with HXL tags",
         }
-        success, results = dataset.generate_resource_from_iterator(
+        success, results = dataset.generate_resource_from_iterable(
             list(area_rows_wide[0].keys()),
             area_rows_wide,
             self.configuration["wide_hxltags"],

diff --git a/requirements.txt b/requirements.txt
@@ -1,2 +1,2 @@
-hdx-python-api==6.0.8
-python-slugify==8.0.1
+hdx-python-api==6.3.1
+python-slugify==8.0.4
diff --git a/run.py b/run.py
@@ -10,7 +10,8 @@
 from hdx.api.configuration import Configuration
 from hdx.facades.infer_arguments import facade
 from hdx.utilities.downloader import Download
-from hdx.utilities.path import progress_storing_folder, wheretostart_tempdir_batch
+from hdx.utilities.path import progress_storing_folder, \
+    wheretostart_tempdir_batch
 from hdx.utilities.retriever import Retrieve
 from hdx.utilities.state import State
 from ipc import IPC
@@ -34,27 +35,31 @@ def main(save: bool = False, use_saved: bool = False) -> None:
 
     configuration = Configuration.read()
     with State(
-        "analysis_dates.txt",
-        State.dates_str_to_country_date_dict,
-        State.country_date_dict_to_dates_str,
+            "analysis_dates.txt",
+            State.dates_str_to_country_date_dict,
+            State.country_date_dict_to_dates_str,
     ) as state:
         state_dict = deepcopy(state.get())
         with wheretostart_tempdir_batch(lookup) as info:
             folder = info["folder"]
             with Download(
-                extra_params_yaml=join(expanduser("~"), ".extraparams.yml"),
-                extra_params_lookup=lookup,
+                    extra_params_yaml=join(expanduser("~"),
+                                           ".extraparams.yaml"),
+                    extra_params_lookup=lookup,
             ) as downloader:
+                _, iterator = downloader.get_tabular_rows(
+                    join("config", "ch_countries.csv"), dict_form=True)
+                ch_countries = [row["ISO_3"] for row in iterator]
                 retriever = Retrieve(
                     downloader, folder, "saved_data", folder, save, use_saved
                 )
-                ipc = IPC(configuration, retriever, state_dict)
+                ipc = IPC(configuration, retriever, state_dict, ch_countries)
                 countries = ipc.get_countries()
                 logger.info(f"Number of countries: {len(countries)}")
 
                 def create_dataset(
-                    dataset,
-                    showcase,
+                        dataset,
+                        showcase,
                 ):
                     if not dataset:
                         return
@@ -79,7 +84,8 @@ def create_dataset(
                         showcase.add_dataset(dataset)
 
                 country_data_updated = False
-                for _, country in progress_storing_folder(info, countries, "iso3"):
+                for _, country in progress_storing_folder(info, countries,
+                                                          "iso3"):
                     countryiso = country["iso3"]
                     output = ipc.get_country_data(countryiso)
                     if output:
@@ -108,7 +114,7 @@ def create_dataset(
 if __name__ == "__main__":
     facade(
         main,
-        user_agent_config_yaml=join(expanduser("~"), ".useragents.yml"),
+        user_agent_config_yaml=join(expanduser("~"), ".useragents.yaml"),
         user_agent_lookup=lookup,
-        project_config_yaml=join("config", "project_configuration.yml"),
+        project_config_yaml=join("config", "project_configuration.yaml"),
     )
diff --git a/test-requirements.txt b/test-requirements.txt
@@ -1,3 +1,3 @@
-pytest==7.4.0
-pytest-cov==4.1.0
+pytest==8.2.2
+pytest-cov==5.0.0
 -r requirements.txt
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		DEFAULT=2017-01-01,AFG=2024-03-01,START_DATE=2017-01-01,END_DATE=2025-02-28,AGO=2021-06-01,BDI=2024-05-01,BEN=2024-03-01,BFA=2024-03-01,CAF=2024-04-01,CIV=2024-03-01,CMR=2024-03-01,COD=2023-11-01,CPV=2024-03-01,DJI=2024-04-01,DOM=2023-09-01,ETH=2021-05-01,GHA=2024-03-01,GIN=2024-03-01,GMB=2024-03-01,GNB=2024-03-01,GTM=2023-05-01,HND=2024-02-01,HTI=2024-03-01,KEN=2024-02-01,LBN=2024-03-01,LBR=2023-03-01,LSO=2023-07-01,MDG=2023-12-01,MLI=2024-03-01,MOZ=2023-07-01,MRT=2024-03-01,MWI=2023-07-01,NAM=2023-07-01,NER=2024-03-01,NGA=2024-03-01,PAK=2024-03-01,SDN=2024-04-01,SEN=2024-03-01,SLE=2024-03-01,SLV=2021-08-01,SOM=2024-01-01,SSD=2023-09-01,SWZ=2023-06-01,TCD=2024-03-01,TGO=2024-03-01,TLS=2023-11-01,TZA=2023-11-01,UGA=2024-04-01,YEM=2023-10-01,ZAF=2020-10-01,ZMB=2023-09-01,ZWE=2020-10-01,BGD=2024-02-01,MMR=2023-09-01,PSE=2024-05-01
		DEFAULT=2017-01-01