Merge pull request #35 from blw-ofag-ufag/refactor-schema-with-data

Refactor schema with data

.github/workflows/combine-datasets.yml

@@ -0,0 +1,41 @@
+name: Combine Datasets
+
+on:
+  push:
+    branches:
+      - refactor-schema-with-data  # Runs only on pushes to the 'refactor-schema-with-data' branch; to be changed!
+  pull_request:
+    branches:
+      - refactor-schema-with-data  # Runs on PRs targeting the 'refactor-schema-with-data' branch; to be changed!
+
+jobs:
+  process_datasets:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Set Up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          # Install any dependencies if required
+          # pip install -r requirements.txt
+
+      - name: Run Combine Datasets Script
+        run: python automation/combine-datasets.py
+
+      - name: Commit and Push Changes
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git add docs/assets/datasets.json
+          git commit -m "Updated combined datasets.json" || echo "No changes to commit"
+          git push
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1 +1,2 @@
-.DS_Store
+.DS_Store
+*.R

automation/combine-datasets.py

@@ -0,0 +1,56 @@
+import os
+import json
+
+# Define input and output directories
+DATASET_DIR = os.path.expanduser("data/datasets")
+OUTPUT_FILE = os.path.expanduser("docs/assets/datasets.json")
+
+# Attributes to extract
+ATTRIBUTES = [
+    "dcterms:identifier",
+    "dcterms:title",
+    "dcterms:description",
+    "dcterms:issued",
+    "dcat:keyword",
+    "schema:OrganizationRole",
+    "schema:image"
+]
+
+def extract_relevant_data(file_path):
+    """Extract relevant attributes from a JSON file."""
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+
+        extracted_data = {key: data[key] for key in ATTRIBUTES if key in data}
+
+        # Filter only the data owner from schema:OrganizationRole
+        if "schema:OrganizationRole" in extracted_data:
+            extracted_data["schema:OrganizationRole"] = [
+                person for person in extracted_data["schema:OrganizationRole"] 
+                if person.get("role") == "dataOwner"
+            ]
+
+        return extracted_data
+    except Exception as e:
+        print(f"Error processing {file_path}: {e}")
+        return None
+
+def process_all_files():
+    """Process all JSON files in the dataset directory and write into one output file."""
+    combined_data = []
+
+    for filename in os.listdir(DATASET_DIR):
+        if filename.endswith(".json"):
+            input_path = os.path.join(DATASET_DIR, filename)
+            extracted_data = extract_relevant_data(input_path)
+            if extracted_data:
+                combined_data.append(extracted_data)
+
+    with open(OUTPUT_FILE, "w", encoding="utf-8") as out_f:
+        json.dump(combined_data, out_f, ensure_ascii=False, indent=4)
+    print("All data combined into", OUTPUT_FILE)
+
+if __name__ == "__main__":
+    process_all_files()
+    print("Processing complete.")

data/datasets/D00001.json

@@ -0,0 +1,67 @@
+{
+  "schema:image": "https://raw.githubusercontent.com/blw-ofag-ufag/data-catalog/refs/heads/main/docs/assets/images/wetland-001.jpg",
+  "dcterms:identifier": "D00001",
+  "dcterms:title": {
+    "de": "Massnahmenmonitoring Klimastrategie Landwirtschaft und Ernährung",
+    "fr": "Suivi des mesures Stratégie Climat pour l'agriculture et l'alimentation",
+    "it": "Monitoraggio delle misure Strategia Climatica per l'Agricoltura e l'Alimentazione",
+    "en": "Monitoring Measures Climate Strategy for Agriculture and Food"
+  },
+  "dcterms:description": {
+    "de": "Daten zur Massnahmenumsetzung geben einen Überblick zum Umsetzungsstand der 42 Massnahmen bezüglich Klimastrategie Landwirtschaft und Ernährung. Dieser Massnahmenplan läuft bis 2030.",
+    "fr": "Les données relatives à la mise en œuvre des mesures donnent un aperçu de l'état de mise en œuvre des 42 mesures relatives à la stratégie climat pour l'agriculture et l'alimentation. Ce plan de mesures court jusqu'en 2030.",
+    "it": "I dati sull'attuazione delle misure forniscono una panoramica dello stato di attuazione delle 42 misure relative alla Strategia Climatica per l'Agricoltura e l'Alimentazione. Questo piano d'azione è valido fino al 2030.",
+    "en": "Data on the implementation of measures provide an overview of the implementation status of the 42 measures related to the Climate Strategy for Agriculture and Food. This action plan runs until 2030."
+  },
+  "dcterms:accessRights": "NON_PUBLIC",
+  "dcterms:issued": "2024-12-04",
+  "dcat:keyword": [
+    "environment",
+    "sustainability",
+    "climate",
+    "monitoring",
+    "food"
+  ],
+  "dcterms:accrualPeriodicity": "ANNUAL",
+  "dcterms:modified": "2025-01-22",
+  "adms:status": "workInProgress",
+  "bv:personalData": "none",
+  "dcat:themeTaxonomy": [
+    "environment",
+    "agricultureFisheriesForestryFood"
+  ],
+  "dcat:distribution": [
+    {
+      "dcterms:title": {
+        "de": "Tabelle Massnahmenmonitoring",
+        "fr": "Tableau de suivi des mesures",
+        "it": "Tabella monitoraggio delle misure",
+        "en": "Monitoring Measures Table"
+      },
+      "dcterms:description": {
+        "de": "Excel-Tabelle mit den eigentlichen Daten zum Umsetzungsstand der 42 Massnahmen bezüglich Klimastrategie Landwirtschaft und Ernährung.",
+        "fr": "Tableau Excel contenant les données réelles sur l'état d'avancement des 42 mesures relatives à la stratégie climat pour l'agriculture et l'alimentation.",
+        "it": "Tabella Excel con i dati effettivi sullo stato di attuazione delle 42 misure relative alla Strategia Climatica per l'Agricoltura e l'Alimentazione.",
+        "en": "Excel table with the actual data on the implementation status of the 42 measures related to the Climate Strategy for Agriculture and Food."
+      },
+      "dcat:accessURL": "https://object.gever.admin.ch:443/web/?ObjectToOpenID=%24ActaNovaDocument%7C574FDB97-8106-4294-94C3-8A121A593B9E&TenantID=123",
+      "dcterms:format": "XSLX"
+    },
+    {
+      "dcterms:title": {
+        "de": "PowerBI Report für das Massnahmenmonitoring",
+        "fr": "Rapport PowerBI pour le suivi des mesures",
+        "it": "Report PowerBI per il monitoraggio delle misure",
+        "en": "PowerBI Report for Monitoring Measures"
+      },
+      "dcterms:description": {
+        "de": "Interaktiver PowerBI-Report, welcher einen Überblick zum Umsetzungsstand der 42 Massnahmen bezüglich Klimastrategie Landwirtschaft und Ernährung gibt.",
+        "fr": "Rapport PowerBI interactif donnant un aperçu de l'état d'avancement des 42 mesures relatives à la stratégie climat pour l'agriculture et l'alimentation.",
+        "it": "Report PowerBI interattivo che fornisce una panoramica dello stato di attuazione delle 42 misure relative alla Strategia Climatica per l'Agricoltura e l'Alimentazione.",
+        "en": "Interactive PowerBI report providing an overview of the implementation status of the 42 measures related to the Climate Strategy for Agriculture and Food."
+      },
+      "dcat:accessURL": "https://www.blw.admin.ch/de/klimastrategie-landwirtschaft-und-ernaehrung-2050#%C3%9Cberpr%C3%BCfung-der-Massnahmen-und-Zielerreichung",
+      "dcterms:format": "PowerBI Report"
+    }
+  ]
+}