From c81945e3e8d77de640094f3cfaf521622842187f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexandre=20Bult=C3=A9?= <alexandre.bulte@data.gouv.fr>
Date: Fri, 16 Oct 2020 16:21:04 +0200
Subject: [PATCH] Migrate ODS datasets urls (#2559)

---
 CHANGELOG.md                                  |  1 +
 .../2020-10-16-migrate-ods-resources.py       | 38 +++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 udata/migrations/2020-10-16-migrate-ods-resources.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b628aa848b..da26da2384 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@
   - Datetime fields `password_rotation_demanded` and `password_rotation_performed` added to user model.
   - Override Flask-Security's login and reset password forms to implement the password rotation checks.
 - Password complexity settings hardening [#2554](https://github.com/opendatateam/udata/pull/2554)
+- Migrate ODS datasets urls [#2559](https://github.com/opendatateam/udata/pull/2559)
 
 ## 2.3.0 (2020-09-29)
 
diff --git a/udata/migrations/2020-10-16-migrate-ods-resources.py b/udata/migrations/2020-10-16-migrate-ods-resources.py
new file mode 100644
index 0000000000..f764e4bb7f
--- /dev/null
+++ b/udata/migrations/2020-10-16-migrate-ods-resources.py
@@ -0,0 +1,38 @@
+'''
+Migrate ODS harvested resources URLS from
+use_labels_for_header=true to use_labels_for_header=false
+cf https://github.com/opendatateam/udata-ods/pull/169
+'''
+import logging
+
+from mongoengine.errors import ValidationError
+from udata.models import Dataset
+
+log = logging.getLogger(__name__)
+
+MIX_MATCH = [
+    'use_labels_for_header=true',
+    'use_labels_for_header=false'
+]
+
+
+def migrate(db):
+    log.info('Migrating ODS harvest datasets...')
+
+    count = 0
+    # datasets from ODS
+    datasets = Dataset.objects.filter(**{'extras__ods:url__ne': None})
+    for d in datasets:
+        touched = False
+        for r in d.resources:
+            if r.url.endswith(MIX_MATCH[0]):
+                r.url = r.url.replace(MIX_MATCH[0], MIX_MATCH[1])
+                touched = True
+                count += 1
+        if touched:
+            try:
+                d.save()
+            except ValidationError as e:
+                log.warning(f'Error while saving dataset {d.id}: {str(e)}')
+
+    log.info(f'Completed, {count} resources migrated.')