📊 vaccine safety (#3904)

* 📊 vaccine safety

* adding steps

* dag

* clean data to remove NR and ND

* clean nr and nd

* adding metadata

* add saloni's titles

dag/health.yml

@@ -944,3 +944,13 @@ steps:
     - data://meadow/who/2025-01-17/mortality_database_vaccine_preventable
   data://grapher/who/2025-01-17/mortality_database_vaccine_preventable:
     - data://garden/who/2025-01-17/mortality_database_vaccine_preventable
+
+  #
+  # WHO Vaccine Safety
+  #
+  data://meadow/who/2025-01-28/vaccine_safety:
+    - snapshot://who/2025-01-28/vaccine_safety.xlsx
+  data://garden/who/2025-01-28/vaccine_safety:
+    - data://meadow/who/2025-01-28/vaccine_safety
+  data://grapher/who/2025-01-28/vaccine_safety:
+    - data://garden/who/2025-01-28/vaccine_safety

etl/steps/data/garden/who/2025-01-28/vaccine_safety.countries.json

@@ -0,0 +1,224 @@
+{
+  "Afghanistan": "Afghanistan",
+  "Albania": "Albania",
+  "Algeria": "Algeria",
+  "American Samoa": "American Samoa",
+  "Andorra": "Andorra",
+  "Angola": "Angola",
+  "Anguilla": "Anguilla",
+  "Antigua and Barbuda": "Antigua and Barbuda",
+  "Argentina": "Argentina",
+  "Armenia": "Armenia",
+  "Aruba": "Aruba",
+  "Australia": "Australia",
+  "Austria": "Austria",
+  "Azerbaijan": "Azerbaijan",
+  "Bahamas": "Bahamas",
+  "Bahrain": "Bahrain",
+  "Bangladesh": "Bangladesh",
+  "Barbados": "Barbados",
+  "Belarus": "Belarus",
+  "Belgium": "Belgium",
+  "Belize": "Belize",
+  "Benin": "Benin",
+  "Bermuda": "Bermuda",
+  "Bhutan": "Bhutan",
+  "Bolivia (Plurinational State of)": "Bolivia",
+  "Bosnia and Herzegovina": "Bosnia and Herzegovina",
+  "Botswana": "Botswana",
+  "Brazil": "Brazil",
+  "British Virgin Islands": "British Virgin Islands",
+  "Brunei Darussalam": "Brunei",
+  "Bulgaria": "Bulgaria",
+  "Burkina Faso": "Burkina Faso",
+  "Burundi": "Burundi",
+  "Cabo Verde": "Cape Verde",
+  "Cambodia": "Cambodia",
+  "Cameroon": "Cameroon",
+  "Canada": "Canada",
+  "Cayman Islands": "Cayman Islands",
+  "Central African Republic": "Central African Republic",
+  "Chad": "Chad",
+  "Chile": "Chile",
+  "China": "China",
+  "China, Hong Kong SAR": "Hong Kong",
+  "China, Macao SAR": "Macao",
+  "Colombia": "Colombia",
+  "Comoros": "Comoros",
+  "Congo": "Congo",
+  "Cook Islands": "Cook Islands",
+  "Costa Rica": "Costa Rica",
+  "Croatia": "Croatia",
+  "Cuba": "Cuba",
+  "Cura\u00e7ao": "Curacao",
+  "Cyprus": "Cyprus",
+  "Czechia": "Czechia",
+  "C\u00f4te d'Ivoire": "Cote d'Ivoire",
+  "Democratic People's Republic of Korea": "North Korea",
+  "Democratic Republic of the Congo": "Democratic Republic of Congo",
+  "Denmark": "Denmark",
+  "Djibouti": "Djibouti",
+  "Dominica": "Dominica",
+  "Dominican Republic": "Dominican Republic",
+  "Ecuador": "Ecuador",
+  "Egypt": "Egypt",
+  "El Salvador": "El Salvador",
+  "Equatorial Guinea": "Equatorial Guinea",
+  "Eritrea": "Eritrea",
+  "Estonia": "Estonia",
+  "Eswatini": "Eswatini",
+  "Ethiopia": "Ethiopia",
+  "Fiji": "Fiji",
+  "Finland": "Finland",
+  "France": "France",
+  "French Guiana": "French Guiana",
+  "French Polynesia": "French Polynesia",
+  "Gabon": "Gabon",
+  "Gambia": "Gambia",
+  "Georgia": "Georgia",
+  "Germany": "Germany",
+  "Ghana": "Ghana",
+  "Greece": "Greece",
+  "Grenada": "Grenada",
+  "Guadeloupe": "Guadeloupe",
+  "Guam": "Guam",
+  "Guatemala": "Guatemala",
+  "Guinea": "Guinea",
+  "Guinea-Bissau": "Guinea-Bissau",
+  "Guyana": "Guyana",
+  "Haiti": "Haiti",
+  "Honduras": "Honduras",
+  "Hungary": "Hungary",
+  "Iceland": "Iceland",
+  "India": "India",
+  "Indonesia": "Indonesia",
+  "Iran (Islamic Republic of)": "Iran",
+  "Iraq": "Iraq",
+  "Ireland": "Ireland",
+  "Israel": "Israel",
+  "Italy": "Italy",
+  "Jamaica": "Jamaica",
+  "Japan": "Japan",
+  "Jordan": "Jordan",
+  "Kazakhstan": "Kazakhstan",
+  "Kenya": "Kenya",
+  "Kiribati": "Kiribati",
+  "Kuwait": "Kuwait",
+  "Kyrgyzstan": "Kyrgyzstan",
+  "Lao People's Democratic Republic": "Laos",
+  "Latvia": "Latvia",
+  "Lebanon": "Lebanon",
+  "Lesotho": "Lesotho",
+  "Liberia": "Liberia",
+  "Libya": "Libya",
+  "Lithuania": "Lithuania",
+  "Luxembourg": "Luxembourg",
+  "Madagascar": "Madagascar",
+  "Malawi": "Malawi",
+  "Malaysia": "Malaysia",
+  "Maldives": "Maldives",
+  "Mali": "Mali",
+  "Malta": "Malta",
+  "Marshall Islands": "Marshall Islands",
+  "Martinique": "Martinique",
+  "Mauritania": "Mauritania",
+  "Mauritius": "Mauritius",
+  "Mexico": "Mexico",
+  "Micronesia (Federated States of)": "Micronesia (country)",
+  "Monaco": "Monaco",
+  "Mongolia": "Mongolia",
+  "Montenegro": "Montenegro",
+  "Montserrat": "Montserrat",
+  "Morocco": "Morocco",
+  "Mozambique": "Mozambique",
+  "Myanmar": "Myanmar",
+  "Namibia": "Namibia",
+  "Nauru": "Nauru",
+  "Nepal": "Nepal",
+  "Netherlands Antilles": "Netherlands Antilles",
+  "New Caledonia": "New Caledonia",
+  "New Zealand": "New Zealand",
+  "Nicaragua": "Nicaragua",
+  "Niger": "Niger",
+  "Nigeria": "Nigeria",
+  "Niue": "Niue",
+  "North Macedonia": "North Macedonia",
+  "Northern Mariana Islands": "Northern Mariana Islands",
+  "Norway": "Norway",
+  "Oman": "Oman",
+  "Pakistan": "Pakistan",
+  "Palau": "Palau",
+  "Panama": "Panama",
+  "Papua New Guinea": "Papua New Guinea",
+  "Paraguay": "Paraguay",
+  "Peru": "Peru",
+  "Philippines": "Philippines",
+  "Poland": "Poland",
+  "Portugal": "Portugal",
+  "Puerto Rico": "Puerto Rico",
+  "Qatar": "Qatar",
+  "Republic of Korea": "South Korea",
+  "Republic of Moldova": "Moldova",
+  "Romania": "Romania",
+  "Russian Federation": "Russia",
+  "Rwanda": "Rwanda",
+  "R\u00e9union": "Reunion",
+  "Saint Helena": "Saint Helena",
+  "Saint Kitts and Nevis": "Saint Kitts and Nevis",
+  "Saint Lucia": "Saint Lucia",
+  "Saint Vincent and the Grenadines": "Saint Vincent and the Grenadines",
+  "Samoa": "Samoa",
+  "San Marino": "San Marino",
+  "Sao Tome and Principe": "Sao Tome and Principe",
+  "Saudi Arabia": "Saudi Arabia",
+  "Senegal": "Senegal",
+  "Serbia": "Serbia",
+  "Seychelles": "Seychelles",
+  "Sierra Leone": "Sierra Leone",
+  "Singapore": "Singapore",
+  "Sint Maarten (Dutch part)": "Sint Maarten (Dutch part)",
+  "Slovakia": "Slovakia",
+  "Slovenia": "Slovenia",
+  "Solomon Islands": "Solomon Islands",
+  "Somalia": "Somalia",
+  "South Africa": "South Africa",
+  "South Sudan": "South Sudan",
+  "Spain": "Spain",
+  "Sri Lanka": "Sri Lanka",
+  "Sudan": "Sudan",
+  "Suriname": "Suriname",
+  "Sweden": "Sweden",
+  "Switzerland": "Switzerland",
+  "Syrian Arab Republic": "Syria",
+  "Tajikistan": "Tajikistan",
+  "Thailand": "Thailand",
+  "Timor-Leste": "East Timor",
+  "Togo": "Togo",
+  "Tokelau": "Tokelau",
+  "Tonga": "Tonga",
+  "Trinidad and Tobago": "Trinidad and Tobago",
+  "Tunisia": "Tunisia",
+  "Turkmenistan": "Turkmenistan",
+  "Turks and Caicos Islands": "Turks and Caicos Islands",
+  "Tuvalu": "Tuvalu",
+  "Uganda": "Uganda",
+  "Ukraine": "Ukraine",
+  "United Arab Emirates": "United Arab Emirates",
+  "United Kingdom of Great Britain and Northern Ireland": "United Kingdom",
+  "United Republic of Tanzania": "Tanzania",
+  "United States Virgin Islands": "United States Virgin Islands",
+  "United States of America": "United States",
+  "Uruguay": "Uruguay",
+  "Uzbekistan": "Uzbekistan",
+  "Vanuatu": "Vanuatu",
+  "Venezuela (Bolivarian Republic of)": "Venezuela",
+  "Viet Nam": "Vietnam",
+  "Wallis and Futuna": "Wallis and Futuna",
+  "Yemen": "Yemen",
+  "Zambia": "Zambia",
+  "Zimbabwe": "Zimbabwe",
+  "Kosovo (in accordance with UN Security Council resolution 1244 (1999))": "Kosovo",
+  "Netherlands (Kingdom of the)": "Netherlands",
+  "T\u00fcrkiye": "Turkey",
+  "occupied Palestinian territory, including east Jerusalem": "Palestine"
+}

etl/steps/data/garden/who/2025-01-28/vaccine_safety.excluded_countries.json

@@ -0,0 +1,2 @@
+[
+]

etl/steps/data/garden/who/2025-01-28/vaccine_safety.meta.yml

@@ -0,0 +1,67 @@
+# NOTE: To learn more about the fields, hover over their names.
+definitions:
+  common:
+    presentation:
+      topic_tags:
+        - Vaccination
+    unit: ""
+    display:
+      numDecimalPlaces: 0
+
+# Learn more about the available fields:
+# http://docs.owid.io/projects/etl/architecture/metadata/reference/
+dataset:
+  update_period_days: 365
+
+tables:
+  vaccine_safety:
+    variables:
+      burial_recommended:
+        title: Is burial a recommended practice for disposal of immunization waste?
+        description_short: Burial is the burying of waste materials in a secured and fenced-off burial site.
+      burning_recommended:
+        title: Is burning in open containers a recommended practice for disposal of immunization waste?
+        description_short: Burning of waste in open containers is not encouraged and should be considered as a last resort to dispose packaging, paper, cardboard. It is not an acceptable practice for PVC, plastics or pharmaceutical products.
+      encapsulation_recommended:
+        title: Is encapsulation a recommended practice for disposal of immunization waste?
+        description_short: Encapsulation involves filling a plastic or steel drum with solid and semi-solid wastes up to 75% capacity and pouring in impervious material (i.e. cement or cement/lime mixture, plastic foam or bituminous sand) to immobilize the wastes.
+      engineered_sanitary_landfill_recommended:
+        title: Is engineered sanitary landfill a recommended practice for disposal of immunization waste?
+        description_short: Engineered sanitary landfill involves safely disposing wastes into a properly constructed and operated landfill designed to protect the aquifer. An appropriate landfill consists of an evacuated pit isolated from watercourses and above the water table.
+      incineration_recommended:
+        title: Is incineration a recommended practice for disposal of immunization waste?
+        description_short: Incineration is the controlled burning of vials in a furnace/ incinerator at temperatures >800C.
+      inertization_recommended:
+        title: Is inertization a recommended practice for disposal of immunization waste?
+        description_short: Inertization is the removal of the packaging materials, paper, cardboard and plastic, from the pharmaceutical products, then grounding and mixing water, lime and cement to form a paste. The paste is transported and poured into the normal urban waste.
+      national_policy_waste:
+        title: Does the country have a national policy for the disposal of immunization waste?
+      national_system_monitor_adverse_events:
+        title: National system to monitor vaccine adverse events
+        description_from_producer: |-
+          A national system must include ALL of the following: 1) written guidelines on monitoring and investigation of reported adverse events; 2) a written list of events to monitor; 3) an established mechanism to communicate data for regulatory action; and 4) implementation of points 1, 2 and 3. If any of the four conditions are not met, countries are instructed to report 'No'. How many total adverse events, including suspected or confirmed, were reported to the national level? Countries are instructed to include ALL cases that are documented from ALL sources, i.e. cases that have at least a reporting form and/or are line listed from routine, campaigns, immunization weeks, school health programs, etc.
+      other_recommended_practices:
+        title: Are there other recommended practices for disposal of immunization waste?
+        description_short: Other refers to any waste-disposal policy or practice that is not listed elsewhere.
+      recycling_recommended:
+        title: Is recycling a recommended practice for disposal of immunization waste?
+        description_short: "If a recycling programme exists, items can be reused (i.e.: glassware can be given to laboratories, mechanical items given to scrap dealers) or recycled (if facilities are available). Depending on the type of material and its proposed reuse, appropriate treatment, such as cleaning or disinfection, may be needed."
+      serious_adverse_events:
+        title: Serious adverse events reported
+        description_short: "An adverse event from immunization is serious if it results in death, is life-threatening, requires in-patient hospitalization or prolongation of existing hospitalization, results in persistent or significant incapacity, or causes a congenital anomaly/birth defect."
+        unit: adverse events
+        description_from_producer: |-
+          An adverse event from immunization will be considered serious if it results in death, is life-threatening, requires in-patient hospitalization or prolongation of existing hospitalization, results in persistent or significant disability/incapacity, or is a congenital anomaly/birth defect. Other situations, such as other important medical events that may jeopardize the patient or may require intervention to prevent one of the outcomes above, should also be considered serious after applying medical and scientific judgment.
+      source_data_serious_adverse_events:
+        title: What is the source of data for the total number of serious adverse events reported?
+        description_from_producer: |-
+          Where the reports were sourced from for AEFI cases, the "EPI program" or the 'National Regulatory Authority'? Countries are instructed to select 'Both EPI and NRA jointly' if the data from both sources have been pooled together. If it is obtained from another source (such as a National Pharmacovigilance center or manufacturer), instructions state to select 'Other'.
+      source_data_total_adverse_events:
+        title: What is the source of data for the total number of adverse events reported?
+        description_from_producer: |-
+          Where the reports were sourced from for AEFI cases, the "EPI program" or the 'National Regulatory Authority'? Countries are instructed to select 'Both EPI and NRA jointly' if the data from both sources have been pooled together. If it is obtained from another source (such as a National Pharmacovigilance center or manufacturer), instructions state to select 'Other'.
+      total_adverse_events_reported:
+        title: Adverse events reported
+      vaccine_adverse_events_review_committee:
+        title: Vaccine adverse events review committee'
+        description_from_producer: Adverse events review committee is an independent committee of recognized experts that provides technical advice and recommendations to the government regarding vaccine safety issues. The adverse events review committee is a tool that enables the government to assess vaccine safety issues through a transparent, systematic process. The adverse events review committees are composed of recognized national experts, independent from the immunization program and the national regulatory authority, and their primary function should focus on offering technical recommendations.

etl/steps/data/garden/who/2025-01-28/vaccine_safety.py

@@ -0,0 +1,76 @@
+"""Load a meadow dataset and create a garden dataset."""
+
+import pandas as pd
+from owid.catalog import Table
+
+from etl.data_helpers import geo
+from etl.helpers import PathFinder, create_dataset
+
+# Get paths and naming conventions for current step.
+paths = PathFinder(__file__)
+
+# A dictionary to map the original variable names to the new column names.
+DESCRIPTION_SHORT = {
+    "Does the country have a national system to monitor adverse events following immunization?": "national_system_monitor_adverse_events",
+    "How many total adverse events, including suspected or confirmed, were reported to the national level?": "total_adverse_events_reported",
+    "Is incineration a recommended practice for disposal of immunization waste?": "incineration_recommended",
+    "Are there other recommended practices for disposal of immunization waste?": "other_recommended_practices",
+    "Is burial a recommended practice for disposal of immunization waste?": "burial_recommended",
+    "Is burning in open containers a recommended practice for disposal of immunization waste?": "burning_recommended",
+    'Of the total adverse events reported, how many were "serious"?': "serious_adverse_events",
+    "Does the country have a national policy for waste from immunization activities?": "national_policy_waste",
+    "Does the country have a vaccine adverse events review committee?": "vaccine_adverse_events_review_committee",
+    "What is the source of data for the total number of adverse events reported?": "source_data_total_adverse_events",
+    "What is the source of data for the total number of serious adverse events reported?": "source_data_serious_adverse_events",
+    "Is encapsulation a recommended practice for disposal of immunization waste?": "encapsulation_recommended",
+    "Is engineered sanitary landfill a recommended practice for disposal of immunization waste?": "engineered_sanitary_landfill_recommended",
+    "Is inertization a recommended practice for disposal of immunization waste?": "inertization_recommended",
+    "Is recycling a recommended practice for disposal of immunization waste?": "recycling_recommended",
+}
+
+
+def run(dest_dir: str) -> None:
+    #
+    # Load inputs.
+    #
+    # Load meadow dataset.
+    ds_meadow = paths.load_dataset("vaccine_safety")
+
+    # Read table from meadow dataset.
+    tb = ds_meadow.read("vaccine_safety")
+
+    #
+    # Process data.
+    tb = tb.rename(columns={"countryname": "country"})
+    tb = geo.harmonize_countries(
+        df=tb, countries_file=paths.country_mapping_path, excluded_countries_file=paths.excluded_countries_path
+    )
+    tb = clean_data(tb)
+    tb["description"] = tb["description"].replace(DESCRIPTION_SHORT)
+    tb = tb.pivot(index=["country", "year"], columns="description", values="value").reset_index()
+
+    tb = tb.format(["country", "year"])
+
+    #
+    # Save outputs.
+    #
+    # Create a new garden dataset with the same metadata as the meadow dataset.
+    ds_garden = create_dataset(
+        dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata
+    )
+
+    # Save changes in the new garden dataset.
+    ds_garden.save()
+
+
+def clean_data(tb: Table) -> Table:
+    """
+    - Drop extraneous columns
+    - Replace 'ND' and 'NR' with NA
+    - There are two variables for the yellow fever vaccine, which do not overlap in time-coverage.
+    I believe they are just two spellings of the same vaccine, so I will merge them.
+    """
+    tb = tb.drop(columns=["iso_3_code", "who_region", "indcode", "indcatcode", "indcat_description", "indsort"])
+    tb = tb.replace({"value": {"ND": pd.NA, "NR": pd.NA}})
+
+    return tb