Merge pull request #128 from GSTT-CSC/110-update-server-configuration…

…-to-scenario-5

updates mlflow to used proxied artifact storage

.github/workflows/test_cli.yml

@@ -35,12 +35,16 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install requirements
+    - name: install sys requirements
       run: |
         sudo apt-get update
         sudo apt-get install -y nvidia-cuda-toolkit
-        python -m pip install --upgrade pip
-        python setup.py install
+
+    - name: Install python requirements
+      run: |
+        python -m pip install --upgrade pip build
+        python -m build 
+        pip install dist/*.whl
 
 #    - name: start MLOps server
 #      run: |
@@ -51,7 +55,7 @@ jobs:
     - name: Test CLI
       run: |
          mlops run -h
-#
+
 #    - name: Stop containers
 #      if: always()
 #      run: |

.gitignore

@@ -28,5 +28,4 @@ mlflow_server/pytest-coverage.txt
 /mlops/mlops.egg-info/
 /mlops/csc_mlops.egg-info/
 /csc_mlops.egg-info/
-assests/MOps_logo.svg
-*.cfg
+assests/MOps_logo.svg

mlflow_server/docker-compose.yml

@@ -96,8 +96,8 @@ services:
             - no_proxy=minio,db
         command: >
             bash -c "
-            mlflow db upgrade postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} ;
-            mlflow server --backend-store-uri postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} --default-artifact-root s3://mlflow/ --host 0.0.0.0
+            mlflow db upgrade postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} ;
+            mlflow server --backend-store-uri postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} --serve-artifacts --artifacts-destination s3://mlflow --host 0.0.0.0
             "
 
 # use these docker volumes for testings and development - use bind mounts in production

mlops/Experiment.py

@@ -21,8 +21,7 @@
 class Experiment:
 
     def __init__(self, script, config_path, project_path: str = '.',
-                 verbose: bool = True, ignore_git_check: bool = False,
-                 artifact_path: str = 's3://mlflow'):
+                 verbose: bool = True, ignore_git_check: bool = False):
         """
         The Experiment class is the interface through which all projects should be run.
         :param script: path to script to run
@@ -32,7 +31,6 @@ def __init__(self, script, config_path, project_path: str = '.',
         """
         self.script = script
         self.config = None
-        self.artifact_path = artifact_path
         self.experiment_name = None
         self.experiment_id = None
         self.config_path = config_path
@@ -119,7 +117,6 @@ def env_setup(self):
         :return:
         """
         os.environ['MLFLOW_TRACKING_URI'] = self.config['server']['MLFLOW_TRACKING_URI']
-        os.environ['MLFLOW_S3_ENDPOINT_URL'] = self.config['server']['MLFLOW_S3_ENDPOINT_URL']
 
     def init_experiment(self):
         """
@@ -134,7 +131,7 @@ def init_experiment(self):
         experiment = mlflow.get_experiment_by_name(self.experiment_name)
 
         if experiment is None:
-            exp_id = mlflow.create_experiment(self.experiment_name, artifact_location=self.artifact_path)
+            exp_id = mlflow.create_experiment(self.experiment_name)
             logger.info('Creating experiment: name: {0} *** ID: {1}'.format(self.experiment_name, exp_id))
         else:
             exp_id = experiment.experiment_id

mlops/_version.py

@@ -0,0 +1 @@
+__version__ = '0.9.16'

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"

requirements.txt

@@ -12,8 +12,7 @@ tqdm
 pandas
 matplotlib
 GitPython
-protobuf~=3.19.0 # temporary solution to avoid breaking changes made to 3rd party streamlit (necessary for mlflow) https://discuss.streamlit.io/t/typeerror-descriptors-cannot-not-be-created-directly/25639
 PyYAML
-requests~=2.27.1
+requests
 click
 setuptools

setup.cfg

@@ -0,0 +1,31 @@
+[metadata]
+name = csc-mlops
+version = attr: mlops._version.__version__
+url = https://github.com/GSTT-CSC/MLOps
+author = Laurence Jackson
+author_email = [email protected]
+description = An MLOps framework for development of clinical applications
+long_description = file:README.md
+long_description_content_type = text/markdown
+
+[options]
+packages = find:
+python_requires = <3.11
+install_requires =
+    mlflow==2.0.1
+    boto3
+    docker
+    minio
+    colorlog
+    monai
+    itk
+    pandas
+    matplotlib
+    xnat
+    GitPython
+    PyYAML
+    click
+
+[options.entry_points]
+console_scripts =
+    mlops = mlops.cli:cli

tests/data/test_config.cfg

@@ -1,7 +1,5 @@
 [server]
-MLFLOW_S3_ENDPOINT_URL = http://0.0.0.0:8002
 MLFLOW_TRACKING_URI = http://0.0.0.0:85
-ARTIFACT_PATH = s3://mlflow
 
 [project]
 NAME = test_project

tests/mlops/test_Experiment.py

@@ -28,12 +28,10 @@ def test_check_dirty(self):
     def test_config_setup(self):
         self.experiment.config_setup()
         assert self.experiment.experiment_name == 'test_project'
-        assert self.experiment.artifact_path == 's3://mlflow'
 
     def test_env_setup(self):
         self.experiment.env_setup()
         assert os.getenv('MLFLOW_TRACKING_URI') == self.experiment.config['server']['MLFLOW_TRACKING_URI']
-        assert os.getenv('MLFLOW_S3_ENDPOINT_URL') == self.experiment.config['server']['MLFLOW_S3_ENDPOINT_URL']
 
     def test_read_config(self):
         # Create config file and assert identical
@@ -52,7 +50,6 @@ def test_print_experiment_info(self, caplog):
         # Check correct information is printed to console
         self.experiment.print_experiment_info()  # Call function.
         assert 'Name: test_project' in caplog.text
-        assert 'Artifact Location: s3://mlflow' in caplog.text
 
     def test_build_project_file(self):
         if os.path.exists('MLproject'):