TMMA-338: Upgrade deployment scripts to support python3.8 and Fabric …

…1.15.0 (#245)

* TMMA-338: Upgrade Fabric in use and remove dependency for Python 2

* TMMA-338: Update comments

* TMMA-338: Bug fix installing fabric using python 3 for GitHub actions

* TMMA-338: Remove installation of Python 2 in GitHub actions and update comments

.github/workflows/check-python-dependcies.yml

@@ -17,15 +17,6 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
-    - name: Install Python 2 support
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y \
-          python2.7 python2.7-dev python2-pip-whl
-        sudo ln -sf python2.7 /usr/bin/python
-        export PYTHONPATH=`echo /usr/share/python-wheels/pip-*py2*.whl`
-        sudo --preserve-env=PYTHONPATH python -m pip install --upgrade pip setuptools wheel
-        sudo chown -R $USER /usr/local/lib/python2.7
     - uses: actions/setup-python@v4
       with:
         python-version: '3.8'

.github/workflows/docker.yml

@@ -1,7 +1,7 @@
 name: Docker build
 # TODO: Trigger a fresh build every Wednesday 
-# Triggered after changes to code or deployment config for model 1
-# Or can be triggered manualy
+# Triggered after changes to docker build related code
+# Or can be triggered manually
 on:
   workflow_dispatch:
   pull_request:

.github/workflows/run-django-test.yml

@@ -25,15 +25,6 @@ jobs:
           MYSQL_ROOT_PASSWORD: 'ci-db-pswd'
     steps:
     - uses: actions/checkout@v4
-    - name: Install Python 2 support
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y \
-          python2.7 python2.7-dev python2-pip-whl
-        sudo ln -sf python2.7 /usr/bin/python
-        export PYTHONPATH=`echo /usr/share/python-wheels/pip-*py2*.whl`
-        sudo --preserve-env=PYTHONPATH python -m pip install --upgrade pip setuptools wheel
-        sudo chown -R $USER /usr/local/lib/python2.7
     - uses: actions/setup-python@v4
       with:
         python-version: '3.8'

.github/workflows/run-fabric-test.yml

@@ -14,14 +14,5 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
-    - name: Install Python 2 support
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y \
-          python2.7 python2.7-dev python2-pip-whl
-        sudo ln -sf python2.7 /usr/bin/python
-        export PYTHONPATH=`echo /usr/share/python-wheels/pip-*py2*.whl`
-        sudo --preserve-env=PYTHONPATH python -m pip install --upgrade pip setuptools wheel
-        sudo chown -R $USER /usr/local/lib/python2.7
-    - name: Syntax check Python 2 Fabric deployment script
+    - name: Syntax check Python 3 Fabric deployment script
       run: bash tests/run-fabric-tests.sh

deploy/deploy-centos.sh

@@ -13,15 +13,6 @@ echo "Install Centos dev tools, like audit2allow"
 yum -y install policycoreutils-python
 yum -y install mlocate
 
-echo "# Install python 2 components for Fabric usage (and legacy Python 2 builds <= v4.4.0)"
-yum -y install python-devel
-yum -y install python-magic
-yum -y install python-pip
-yum -y install python-lxml
-
-pip2 install -U pip==19.3.1
-pip2 install Fabric==1.13.1 # NB: v1.15.0 supports Python 2, & 3.6, 3.7, & 3.8
-
 echo "###   Install dev tools"
 yum -y install git
 yum -y install nano
@@ -69,9 +60,13 @@ echo "export LD_LIBRARY_PATH=/usr/local/lib/" > ld_library.sh
 mv ld_library.sh /etc/profile.d/ld_library.sh
 export set LD_LIBRARY_PATH=/usr/local/lib/
 
-pip3.8 install mod_wsgi==4.9.0
+# Install symtem wide python requirements
+pip3.8 install -U pip==22.0.2 # As per CI server
+pip3.8 install Fabric==1.15.0 # NB: v1.15.0 supports Python 2, & 3.6, 3.7, & 3.8
+
+pip3.8 install mod_wsgi==4.7.1 # As per app servers
 ls /usr/local/lib64/python3.8/site-packages/mod_wsgi/server/
-pip3.8 install virtualenv==20.13.0
+pip3.8 install virtualenv==15.1.0 # As per app servers
 
 ln -s /usr/local/bin/virtualenv /usr/bin/virtualenv-3.8
 

deploy/fabfile.py

@@ -5,7 +5,7 @@
 
 from datetime import datetime
 import os
-import urllib2
+from urllib.request import urlopen
 
 from fabric.api import *
 from fabric.contrib import files
@@ -150,7 +150,7 @@ def make_virtualenv(env="dev", configure_apache=False, clone_repo=False, branch=
                 # ref: https://chromedriver.chromium.org/downloads/version-selection
                 google_chrome_version = caller('google-chrome --version').strip("Google Chrome ")
                 chrome_driver_version = google_chrome_version[:google_chrome_version.rindex(".")]
-                version = urllib2.urlopen('https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_'+chrome_driver_version).read()
+                version = urlopen('https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_'+chrome_driver_version).read()
                 if int(google_chrome_version[:google_chrome_version.index(".")]) < 115:
                     zip_name = 'chromedriver_linux64.zip'
                     caller('wget https://chromedriver.storage.googleapis.com/' + version + '/' + zip_name)

tests/build-test-env.sh

@@ -3,7 +3,7 @@
 set -e
 
 cd $GITHUB_WORKSPACE
-# Move source code to a directory structure the application expects.
+echo "Move source code to a directory structure the application expects"
 mkdir -p lib/test/src/temmpo
 shopt -s extglob
 mv !(lib) lib/test/src/temmpo/
@@ -21,25 +21,26 @@ mkdir -p var/results/testing/v4
 mkdir -p var/tmp
 
 
-echo "Check both versions of pip are installed"
-pip2 -V
+echo "Check the version of pip that is installed"
+pip3 -V
+echo "Ensure using a consistent version of pip as per on premises CI server"
+pip3 install -U pip==22.0.2
 pip3 -V
 
 echo "Install Fabric"
-pip2 install -U pip==20.3.4
-pip2 install Fabric==1.13.1 # NB: v1.15.0 supports Python 2, & 3.6, 3.7, & 3.8
+pip3 install fabric==1.15.0 # NB: v1.15.0 supports Python 2, & 3.6, 3.7, & 3.8
 
 echo "Install virtualenv"
-pip3 install virtualenv==20.13.0
+pip3 install virtualenv==15.1.0
 
 echo "Install wheel"
 sudo apt-get install -y python3-wheel-whl
 
-echo "Set up alias for python3.8"
+echo "Check aliases for python3.8"
 which python3
 which python3.8
 
-# Install Chrome
+echo "Install Chrome"
 wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
 sudo apt-get install -y ./google-chrome-stable_current_amd64.deb
 rm -f google-chrome-stable_current_amd64.deb

tests/run-fabric-tests.sh

@@ -3,5 +3,5 @@
 set -e
 
 cd $GITHUB_WORKSPACE
-pip install fabric==1.13.1
+pip3 install fabric==1.15.0
 fab -l -f deploy/fabfile.py