⚡ sync: add data files support

sync/doc/changelog.rst

@@ -2,6 +2,7 @@
 -------
 
 - **New:** Use prime numbers for major releases ;-)
+- **New:** Support data files
 - **Improvement:** make links dependent on project
 
 `7.0.0`

sync/models/__init__.py

@@ -10,6 +10,7 @@
 from . import sync_project_demo
 from . import sync_task
 from . import sync_job
+from . import sync_data
 from . import ir_logging
 from . import ir_actions
 from . import ir_attachment

sync/models/sync_data.py

@@ -0,0 +1,45 @@
+# Copyright 2024 Ivan Yelizariev <https://twitter.com/yelizariev>
+import base64
+import csv
+import json
+from io import StringIO
+
+import yaml
+
+from odoo import fields, models
+
+
+class SyncData(models.Model):
+    _name = "sync.data"
+    _description = "Sync Data File"
+
+    name = fields.Char("Technical name")
+    project_id = fields.Many2one("sync.project", ondelete="cascade")
+    file_name = fields.Char("File Name")
+    file_content = fields.Binary("File Content")
+
+    def csv(self, *args, **kwargs):
+        """Parse CSV file from binary field."""
+        if self.file_content:
+            file_content = base64.b64decode(self.file_content)
+            file_content = file_content.decode("utf-8")
+            file_like_object = StringIO(file_content)
+            reader = csv.DictReader(file_like_object, *args, **kwargs)
+            return [row for row in reader]
+        return []
+
+    def json(self):
+        """Parse JSON file from binary field."""
+        if self.file_content:
+            file_content = base64.b64decode(self.file_content)
+            file_content = file_content.decode("utf-8")
+            return json.loads(file_content)
+        return {}
+
+    def yaml(self):
+        """Parse YAML file from binary field."""
+        if self.file_content:
+            file_content = base64.b64decode(self.file_content)
+            file_content = file_content.decode("utf-8")
+            return yaml.safe_load(file_content)
+        return None

sync/models/sync_project.py

@@ -5,8 +5,10 @@
 
 import base64
 import logging
+import os
 from datetime import datetime
 
+import urllib3
 from pytz import timezone
 
 from odoo import api, fields, models
@@ -107,6 +109,7 @@ class SyncProject(models.Model):
     log_count = fields.Integer(compute="_compute_log_count")
     link_ids = fields.One2many("sync.link", "project_id")
     link_count = fields.Integer(compute="_compute_link_count")
+    data_ids = fields.One2many("sync.data", "project_id")
 
     def copy(self, default=None):
         default = dict(default or {})
@@ -319,6 +322,10 @@ def record2image(record, fname="image_1920"):
         for w in self.task_ids.mapped("webhook_ids"):
             WEBHOOKS[w.trigger_name] = w.website_url
 
+        DATA = AttrDict()
+        for d in self.data_ids:
+            DATA[d.name] = d
+
         core_eval_context = {
             "MAGIC": MAGIC,
             "SECRETS": SECRETS,
@@ -330,6 +337,7 @@ def record2image(record, fname="image_1920"):
             "CORE": CORE,
             "PARAMS": PARAMS,
             "WEBHOOKS": WEBHOOKS,
+            "DATA": DATA,
         }
         LIB = eval_export(safe_eval, self.common_code, lib_eval_context)
 
@@ -499,6 +507,38 @@ def magic_upgrade(self):
             if gist_files.get(file_name):
                 vals[field_name] = gist_files[file_name]
 
+        # [DATA]
+        http = urllib3.PoolManager()
+        for file_info in gist_content["files"].values():
+            # e.g. "data.emoji.csv"
+            file_name = file_info["filename"]
+            if not file_name.startswith("data."):
+                continue
+            raw_url = file_info["raw_url"]
+            response = http.request("GET", raw_url)
+            if response.status == 200:
+                file_content = response.data
+                file_content = base64.b64encode(file_content)
+            else:
+                raise Exception(
+                    f"Failed to fetch raw content from {raw_url}. Status code: {response.status}"
+                )
+
+            technical_name = file_name
+            technical_name = technical_name[len("data.") :]
+            technical_name = os.path.splitext(technical_name)[0]
+            technical_name = technical_name.replace(".", "_")
+
+            data_vals = {
+                "name": technical_name,
+                "project_id": self.id,
+                "file_name": file_name,
+                "file_content": file_content,
+            }
+            self.env["sync.data"]._create_or_update_by_xmlid(
+                data_vals, file_name, namespace=gist_id
+            )
+
         # Tasks 🦋
         for file_name in gist_files:
             # e.g. "task.setup.py"

sync/security/ir.model.access.csv

@@ -5,6 +5,9 @@ access_sync_project_manager,sync.project manager,model_sync_project,sync_group_m
 access_sync_task_user,sync.task user,model_sync_task,sync_group_user,1,0,0,0
 access_sync_task_dev,sync.task dev,model_sync_task,sync_group_dev,1,1,1,1
 access_sync_task_manager,sync.task manager,model_sync_task,sync_group_manager,1,1,1,1
+access_sync_data_user,sync.data user,model_sync_data,sync_group_user,1,0,0,0
+access_sync_data_dev,sync.data dev,model_sync_data,sync_group_dev,1,1,1,1
+access_sync_data_manager,sync.data manager,model_sync_data,sync_group_manager,1,1,1,1
 access_sync_trigger_automation_user,sync.trigger.automation user,model_sync_trigger_automation,sync_group_user,1,0,0,0
 access_sync_trigger_automation_dev,sync.trigger.automation dev,model_sync_trigger_automation,sync_group_dev,1,1,1,1
 access_sync_trigger_automation_manager,sync.trigger.automation manager,model_sync_trigger_automation,sync_group_manager,1,1,1,1

sync/views/sync_project_views.xml

@@ -214,28 +214,6 @@
                                 </em>
                             </p>
                         </page>
-                        <!--
-                        <page name="buttons" string="Manual Triggers">
-                            <p class="oe_edit_only">
-                                <br />
-                                Hint: to add manual triggers navigate to corresponding
-                                Task via "Available Tasks" tab
-                            </p>
-                            <field name="trigger_button_ids">
-                                <tree create="0">
-                                    <field name="sync_task_id" />
-                                    <field name="trigger_name" />
-                                    <button
-                                        name="start_button"
-                                        string="Run Now"
-                                        class="oe_highlight"
-                                        type="object"
-                                    />
-                                    <field name="name" />
-                                </tree>
-                            </field>
-                        </page>
-                        -->
                         <page name="params" string="PARAMS.🌹">
                             <notebook>
                                 <page string=" ✏️ Templates 🌹">
@@ -275,6 +253,26 @@
                                 </page>
                             </notebook>
                         </page>
+                        <page string="DATA.🐫">
+                            <field name="data_ids">
+                                <tree editable="bottom">
+                                    <field name="name" />
+                                    <field name="file_name" invisible="1" />
+                                    <field name="file_content" filename="file_name" />
+                                </tree>
+                            </field>
+                            <p>
+                                <em>
+                                    Hint:
+                                    <a
+                                        target="_blank"
+                                        href="https://gist.github.com/yelizariev/e0585a0817c4d87b65b8a3d945da7ca2#file-api-data-markdown"
+                                    >
+                                        Documentation
+                                    </a>
+                                </em>
+                            </p>
+                        </page>
                         <page string="SECRETS.🔐">
                             <p class="oe_read_only">
                                 <em>

sync/views/sync_task_views.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<!-- Copyright 2020-2021 Ivan Yelizariev <https://twitter.com/yelizariev>
+<!-- Copyright 2020-2021,2024 Ivan Yelizariev <https://twitter.com/yelizariev>
      License MIT (https://opensource.org/licenses/MIT). -->
 <odoo>
     <record id="sync_task_view_tree" model="ir.ui.view">
@@ -68,6 +68,13 @@
                                 <em>
                                     Hint: Updating this code won't change the triggers.
                                     Instead, update the gist file.
+                                    <br />
+                                    <a
+                                        target="_blank"
+                                        href="https://gist.github.com/yelizariev/e0585a0817c4d87b65b8a3d945da7ca2#file-api-tasks-markdown"
+                                    >
+                                        Documentation
+                                    </a>
                                 </em>
                             </p>
                         </page>