Processing fixes

.github/workflows/process_data.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import unicodedata
 from urllib.parse import urljoin, urlparse
 
 import pandas as pd
@@ -145,7 +146,8 @@ def extract_item_data(item):
     """Extracts relevant data from an item and downloads its thumbnail if available."""
     local_image_path = (
         download_thumbnail(item.get("thumbnail_display_urls", {}).get("large", ""))
-        or "assets/img/no-image.svg"
+        if item.get("o:is_public", False)
+        else "assets/img/placeholder.svg"
     )
 
     return {
@@ -184,8 +186,9 @@ def extract_media_data(media, item_dc_identifier):
     if "platzhalter" in media.get("o:source", ""):
         local_image_path = "assets/img/placeholder.svg"
     else:
-        local_image_path = download_thumbnail(
-            media.get("thumbnail_display_urls", {}).get("large", "")
+        local_image_path = (
+            download_thumbnail(media.get("thumbnail_display_urls", {}).get("large", ""))
+            or "assets/img/no-image.svg"
         )
 
     # Extract media data
@@ -223,23 +226,36 @@ def extract_media_data(media, item_dc_identifier):
     }
 
 
+def normalize_record(record):
+    """Normalizes all string fields in a record to Unicode NFC form."""
+    return {
+        key: unicodedata.normalize("NFC", value) if isinstance(value, str) else value
+        for key, value in record.items()
+    }
+
+
 # --- Main Processing Function ---
 def main():
     # Fetch item data
     items_data = get_items_from_collection(ITEM_SET_ID)
 
     # Process each item and associated media
-    item_records, media_records = [], []
+    items_processed = []
     for item in items_data:
         item_record = extract_item_data(item)
-        item_records.append(item_record)
+        items_processed.append(item_record)
         media_data = get_media(item.get("o:id", ""))
         if media_data:
             for media in media_data:
-                media_records.append(extract_media_data(media, item_record["objectid"]))
+                items_processed.append(
+                    extract_media_data(media, item_record["objectid"])
+                )
+
+    # Normalize all string fields in the records to avoid decomposed Unicode form Umlaute ¨ + o -> ö
+    items_normalized = [normalize_record(record) for record in items_processed]
 
     # Save data to CSV and JSON formats
-    save_to_files(item_records + media_records, CSV_PATH, JSON_PATH)
+    save_to_files(items_normalized, CSV_PATH, JSON_PATH)
 
 
 def save_to_files(records, csv_path, json_path):