Merge pull request #32 from MerginMaps/ci_tests

Ci tests + black style

.github/workflows/code_style.yaml

@@ -0,0 +1,14 @@
+name: Code Layout
+
+on: push
+
+jobs:
+  code_style_python:
+    name: Python code convention check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: psf/black@stable
+        with:
+          options: "--check --verbose --diff"

.github/workflows/tests_mergin_media_sync.yaml

@@ -0,0 +1,70 @@
+name: Tests for Mergin Maps Media Sync
+
+on:
+  push:
+    paths:
+    - "test/**"
+    - "**.py"
+    - ".github/workflows/tests_mergin_media_sync.yaml"
+
+env:
+  TEST_MERGIN_URL: https://test.dev.merginmaps.com
+  TEST_API_USERNAME: test_media_sync
+  TEST_API_PASSWORD: ${{ secrets.TEST_API_PASSWORD }}
+  TEST_API_WORKSPACE: test-media-sync
+  TEST_MINIO_URL: 127.0.0.1:9000
+  TEST_MINIO_ACCESS_KEY: minioaccesskey
+  TEST_MINIO_SECRET_KEY: miniosecretkey
+
+jobs:
+
+  Tests-for-Mergin-Maps-Media-Sync:
+
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Install Spatialite
+        run: sudo apt-get install -y libsqlite3-mod-spatialite
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Python dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install pipenv
+          pipenv install
+          pipenv install --dev
+
+      - name: Run MinIO Docker
+        run: |
+          docker run -d \
+          -p 127.0.0.1:9000:9000 \
+          -p 127.0.0.1:9001:9001 \
+          -e MINIO_ROOT_USER=$TEST_MINIO_ACCESS_KEY \
+          -e MINIO_ROOT_PASSWORD=$TEST_MINIO_SECRET_KEY \
+          --entrypoint /bin/bash \
+          minio/minio:latest -c "minio server /data --console-address :9001 --address :9000"
+
+      - name: Test MinIO and Mergin Client
+        shell: pipenv run python {0}
+        run: |
+          import os
+          from minio import Minio
+          from mergin import MerginClient
+
+          client = Minio(
+              endpoint="127.0.0.1:9000",
+              access_key=os.environ.get('TEST_MINIO_ACCESS_KEY'),
+              secret_key=os.environ.get('TEST_MINIO_SECRET_KEY'),
+              secure=False,
+          )
+
+          mc = MerginClient(os.environ.get('TEST_MERGIN_URL'), 
+                            login=os.environ.get('TEST_API_USERNAME'), 
+                            password=os.environ.get('TEST_API_PASSWORD'))
+
+      - name: Run tests
+        run: |
+          pipenv run pytest test --cov=. --cov-report=term-missing:skip-covered -vv

Pipfile

@@ -9,7 +9,7 @@ pytest-cov = "~=3.0"
 
 [packages]
 minio = "~=7.1"
-mergin-client = "==0.8.3"
+mergin-client = "==0.9.0"
 dynaconf = {extras = ["ini"],version = "~=3.1"}
 
 [requires]

config.py

@@ -12,7 +12,7 @@
 
 config = Dynaconf(
     envvar_prefix=False,
-    settings_files=['config.yaml'],
+    settings_files=["config.yaml"],
 )
 
 
@@ -21,9 +21,11 @@ class ConfigError(Exception):
 
 
 def validate_config(config):
-    """ Validate config - make sure values are consistent """
+    """Validate config - make sure values are consistent"""
 
-    if not (config.mergin.username and config.mergin.password and config.mergin.project_name):
+    if not (
+        config.mergin.username and config.mergin.password and config.mergin.project_name
+    ):
         raise ConfigError("Config error: Incorrect mergin settings")
 
     if config.driver not in ["local", "minio"]:
@@ -32,10 +34,15 @@ def validate_config(config):
     if config.operation_mode not in ["move", "copy"]:
         raise ConfigError("Config error: Unsupported operation mode")
 
-    if config.driver == 'local' and not config.local.dest:
+    if config.driver == "local" and not config.local.dest:
         raise ConfigError("Config error: Incorrect Local driver settings")
 
-    if config.driver == 'minio' and not (config.minio.endpoint and config.minio.access_key and config.minio.secret_key and config.minio.bucket):
+    if config.driver == "minio" and not (
+        config.minio.endpoint
+        and config.minio.access_key
+        and config.minio.secret_key
+        and config.minio.bucket
+    ):
         raise ConfigError("Config error: Incorrect MinIO driver settings")
 
     if not (config.allowed_extensions and len(config.allowed_extensions)):
@@ -45,7 +52,10 @@ def validate_config(config):
         raise ConfigError("Config error: References list can not be empty")
 
     for ref in config.references:
-        if not all(hasattr(ref, attr) for attr in ['file', 'table', 'local_path_column', 'driver_path_column']):
+        if not all(
+            hasattr(ref, attr)
+            for attr in ["file", "table", "local_path_column", "driver_path_column"]
+        ):
             raise ConfigError("Config error: Incorrect media reference settings")
 
 

drivers.py

@@ -22,12 +22,12 @@ def __init__(self, config):
         self.config = config
 
     def upload_file(self, src, obj_path):
-        """ Copy object to destination and return path """
+        """Copy object to destination and return path"""
         raise NotImplementedError
 
 
 class LocalDriver(Driver):
-    """ Driver to work with local drive, for testing purpose mainly """
+    """Driver to work with local drive, for testing purpose mainly"""
 
     def __init__(self, config):
         super(LocalDriver, self).__init__(config)
@@ -52,7 +52,7 @@ def upload_file(self, src, obj_path):
 
 
 class MinioDriver(Driver):
-    """ Driver to handle connection to minio-like server """
+    """Driver to handle connection to minio-like server"""
 
     def __init__(self, config):
         super(MinioDriver, self).__init__(config)
@@ -63,7 +63,7 @@ def __init__(self, config):
                 access_key=config.minio.access_key,
                 secret_key=config.minio.secret_key,
                 secure=config.as_bool("minio.secure"),
-                region=config.minio.region
+                region=config.minio.region,
             )
             self.bucket = config.minio.bucket
             bucket_found = self.client.bucket_exists(self.bucket)
@@ -77,7 +77,7 @@ def __init__(self, config):
 
             # construct base url for bucket
             scheme = "https://" if config.as_bool("minio.secure") else "http://"
-            self.base_url = scheme + config.minio.endpoint + '/' + self.bucket
+            self.base_url = scheme + config.minio.endpoint + "/" + self.bucket
         except S3Error as e:
             raise DriverError("MinIO driver init error: " + str(e))
 
@@ -86,14 +86,14 @@ def upload_file(self, src, obj_path):
             obj_path = f"{self.bucket_subpath}/{obj_path}"
         try:
             res = self.client.fput_object(self.bucket, obj_path, src)
-            dest = self.base_url + '/' + res.object_name
+            dest = self.base_url + "/" + res.object_name
         except S3Error as e:
             raise DriverError("MinIO driver error: " + str(e))
         return dest
 
 
 def create_driver(config):
-    """ Create driver object based on type defined in config """
+    """Create driver object based on type defined in config"""
     driver = None
     if config.driver == "local":
         driver = LocalDriver(config)