Feature/SK-289 [studio-models] | Add code-checks

.github/workflows/code-checks.yaml

@@ -0,0 +1,52 @@
+name: Code checks
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+  pull_request:
+    branches:
+      - main
+      - develop
+  release:
+    types: [published]
+
+jobs:
+  check-code:
+
+    runs-on: ubuntu-20.04
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          #Python version should be the same as the base image in Dockerfile
+          python-version: "3.8.10"
+      - name: Install dependencies
+
+        run: |
+          python -m pip install --upgrade pip
+          pip install black isort flake8
+          if [ -f requirements.txt ]; then pip install --no-cache-dir -r requirements.txt; fi
+      - name: Check Python imports
+        run: |
+          isort . --check --diff --skip migrations --profile black -p studio --line-length 79
+      - name: Check Python formatting
+        run: |
+          black . --check --diff --line-length 79 --exclude migrations
+      - name: Check Python linting
+        run: |
+          flake8 .  --exclude migrations

models/apps.py

@@ -2,6 +2,6 @@
 
 
 class ModelsConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'models'
-    verbose_name = 'Studio Models'
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "models"
+    verbose_name = "Studio Models"

models/forms.py

@@ -6,35 +6,57 @@
 class ModelForm(forms.ModelForm):
     class Meta:
         model = Model
-        fields = ('name', 'description', 'release_type',
-                  'version', 'access', 'path')
+        fields = (
+            "name",
+            "description",
+            "release_type",
+            "version",
+            "access",
+            "path",
+        )
         labels = {
-            'path': ('Current folder name of your trained model*'),
+            "path": ("Current folder name of your trained model*"),
         }
 
 
 class ModelLogForm(forms.ModelForm):
     class Meta:
         model = ModelLog
         fields = (
-            'run_id', 'trained_model', 'project', 'training_started_at', 'execution_time', 'code_version',
-            'current_git_repo', 'latest_git_commit', 'system_details', 'cpu_details', 'training_status')
+            "run_id",
+            "trained_model",
+            "project",
+            "training_started_at",
+            "execution_time",
+            "code_version",
+            "current_git_repo",
+            "latest_git_commit",
+            "system_details",
+            "cpu_details",
+            "training_status",
+        )
 
 
 class Metadata(forms.ModelForm):
     class Meta:
         model = Metadata
         fields = (
-            'run_id', 'trained_model', 'project', 'model_details', 'parameters', 'metrics')
+            "run_id",
+            "trained_model",
+            "project",
+            "model_details",
+            "parameters",
+            "metrics",
+        )
 
 
 class UploadModelCardHeadlineForm(forms.Form):
     file = forms.ImageField()
 
 
 class EnvironmentForm(forms.Form):
-    registry = forms.CharField(label='Registry DNS', max_length=100)
-    username = forms.CharField(label='Username', max_length=100)
-    repository = forms.CharField(label='Repository', max_length=100)
-    image = forms.CharField(label='Image', max_length=100)
-    tag = forms.CharField(label='Tag', max_length=100)
+    registry = forms.CharField(label="Registry DNS", max_length=100)
+    username = forms.CharField(label="Username", max_length=100)
+    repository = forms.CharField(label="Repository", max_length=100)
+    image = forms.CharField(label="Image", max_length=100)
+    tag = forms.CharField(label="Tag", max_length=100)

models/helpers.py

@@ -2,16 +2,14 @@
 import shutil
 import tarfile
 
+import s3fs
 from django.apps import apps
 from django.conf import settings
-
-import s3fs
 from minio import Minio
 
-
 from .models import Model
 
-ublicModelObject = apps.get_model(app_label=settings.PUBLICMODELOBJECT_MODEL)
+PublicModelObject = apps.get_model(app_label=settings.PUBLICMODELOBJECT_MODEL)
 
 
 def add_pmo_to_publish(mdl, pmodel):
@@ -22,38 +20,37 @@ def add_pmo_to_publish(mdl, pmodel):
     access_key = mdl.s3.access_key
     secret_key = mdl.s3.secret_key
     bucket = mdl.bucket
-    region = mdl.s3.region
     filename = mdl.uid
     path = mdl.path
-    #TODO: this is a quick bugfix, path for tensorflow models is "models", it should contain the model uid
+    # TODO: this is a quick bugfix, path for tensorflow models is "models",
+    # it should contain the model uid
     if path == "models":
         path = filename
     try:
         s3 = s3fs.S3FileSystem(
             key=access_key,
             secret=secret_key,
-            client_kwargs={
-                'endpoint_url': 'http://'+host
-            }
+            client_kwargs={"endpoint_url": "http://" + host},
         )
     except Exception as err:
         print(err)
     print("Created S3 fs")
     try:
         # if e.g tensorflow model, the object is already compressed
         if path == filename:
-            fobj = s3.open(bucket+'/'+path, 'rb')
-        #else if model is e.g mlflow, the model artifact is a folder and needs to be compressed
+            fobj = s3.open(bucket + "/" + path, "rb")
+        # else if model is e.g mlflow, the model artifact is a folder and
+        # needs to be compressed
         else:
-            #download files on folder inside bucket into tmp folder
-            s3.get(bucket+'/'+path, './tmp/', recursive=True)
-            #create tar file
+            # download files on folder inside bucket into tmp folder
+            s3.get(bucket + "/" + path, "./tmp/", recursive=True)
+            # create tar file
             with tarfile.open("model.tar", "w") as tar:
                 tar.add("./tmp/")
-            #remove tmp folder
+            # remove tmp folder
             shutil.rmtree("./tmp")
-            #open tar for later read 
-            fobj = open("./model.tar" ,"rb")
+            # open tar for later read
+            fobj = open("./model.tar", "rb")
 
         print("Opened s3 file.")
 
@@ -81,15 +78,15 @@ def get_download_url(model_id):
 
     download_url = ""
     path = ""
-    if model.object_type.slug == 'mlflow':
-        path = model.path 
+    if model.object_type.slug == "mlflow":
+        path = model.path
     try:
         client = Minio(
             minio_host,
             access_key=minio_access_key,
             secret_key=minio_secret_key,
             region=minio_region,
-            secure=False
+            secure=False,
         )
 
         download_url = client.presigned_get_object(bucket, path + uid)
@@ -99,69 +96,99 @@ def get_download_url(model_id):
     return download_url
 
 
-# This Method use Minio Python API to create a minio client and connect it to a minio server instance
+# This Method use Minio Python API to create a minio client and
+# connect it to a minio server instance
 def create_client(S3_storage, secure_mode=True):
     try:
         access_key = S3_storage.access_key
     except Exception:
-        print("No access key could be found with the current S3 storage instance: {}".format(
-            S3_storage))
+        print(
+            (
+                "No access key could be found with "
+                "the current S3 storage instance: {}"
+            ).format(S3_storage)
+        )
         return []
     try:
         secret_key = S3_storage.secret_key
     except Exception:
-        print("No secret key could be found with the current S3 storage instance: {}".format(
-            S3_storage))
+        print(
+            (
+                "No secret key could be found with "
+                "the current S3 storage instance: {}"
+            ).format(S3_storage)
+        )
         return []
 
     # API connection does not want scheme in the minio URL
-    # Yet we need the URL to have the scheme included when some app charts use the minio client mc
-    if 'http://' in S3_storage.host:
-        minio_url = S3_storage.host.replace('http://', '')
-    elif 'https://' in S3_storage.host:
-        minio_url = S3_storage.host.replace('https://', '')
+    # Yet we need the URL to have the scheme included when
+    # some app charts use the minio client mc
+    if "http://" in S3_storage.host:
+        minio_url = S3_storage.host.replace("http://", "")
+    elif "https://" in S3_storage.host:
+        minio_url = S3_storage.host.replace("https://", "")
     else:
         minio_url = S3_storage.host
 
     if not secure_mode:
-        client = Minio(minio_url, access_key=access_key,
-                       secret_key=secret_key, secure=secure_mode)
+        client = Minio(
+            minio_url,
+            access_key=access_key,
+            secret_key=secret_key,
+            secure=secure_mode,
+        )
     else:
         client = Minio(minio_url, access_key=access_key, secret_key=secret_key)
 
     return client
 
 
-# This Method use Minio Python API to save an artificat into a running minio server instance
-def set_artifact(artifact_name, artifact_file, bucket, S3_storage, is_file=False, secure_mode=True):
-    """ Instance must be a byte-like object. """
+# This Method use Minio Python API to save an artificat
+# into a running minio server instance
+def set_artifact(
+    artifact_name,
+    artifact_file,
+    bucket,
+    S3_storage,
+    is_file=False,
+    secure_mode=True,
+):
+    """Instance must be a byte-like object."""
     client = create_client(S3_storage, secure_mode)
 
     try:
         found = client.bucket_exists(bucket)
     except Exception as err:
-        print('EXCEPTION LOG: Client could not verify if bucket exists')
+        print(err)
+        print("EXCEPTION LOG: Client could not verify if bucket exists")
         return False
 
     if not found:
         try:
             client.make_bucket(bucket)
         except Exception as err:
-            print('Bucket does not exist, and failed to create bucket.')
+            print(err)
+            print("Bucket does not exist, and failed to create bucket.")
             return False
 
-    if is_file == True:
+    if is_file:
         try:
             client.fput_object(bucket, artifact_name, artifact_file)
-        except Exception as e:
-            print('Client method fput_object failed')
+        except Exception as err:
+            print(err)
+            print("Client method fput_object failed")
             return False
     else:
         try:
-            client.put_object(bucket, artifact_name, io.BytesIO(
-                artifact_file), len(artifact_file))
-        except Exception as e:
-            print('Client method put_object failed')
+            client.put_object(
+                bucket,
+                artifact_name,
+                io.BytesIO(artifact_file),
+                len(artifact_file),
+            )
+        except Exception as err:
+            print(err)
+            print("Client method put_object failed")
             return False
 
     return True