Faraz/testing ci checks

.ci/docker-compose-ci.yml

@@ -5,12 +5,18 @@ services:
   db:
     image: mysql:5.7
     container_name: db
+    platform: linux/amd64
     command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci
     environment:
       MYSQL_ROOT_PASSWORD: "password"
       MYSQL_DATABASE: "ecommerce"
   ecommerce:
-    image: edxops/ecommerce:latest
+    platform: linux/amd64
+    build:
+      context: ../.
+      target: dev
+      args:
+        PYTHON_VERSION: $PYTHON_VERSION
     container_name: ecommerce_testing
     volumes:
       - ..:/edx/app/ecommerce/ecommerce

.github/workflows/ci.yml

@@ -11,61 +11,65 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        include:
-          - django-env: django32
-            testname: quality-and-jobs
-            targets: PYTHON_ENV=py38 requirements.js check_translations_up_to_date validate_translations clean_static static quality validate_js check_keywords
-          - django-env: django32
-            testname: test-python
-            targets: PYTHON_ENV=py38 requirements.js clean_static static validate_python
-          - django-env: django32
-            testname: acceptance-python
-            targets: PYTHON_ENV=py38 requirements.js clean_static static acceptance
-
+        python-version: ['py38', 'py311', 'py312']
+        django-env: ['django32']
+        test: ['quality-and-jobs', 'test-python']
     steps:
       - uses: actions/checkout@v2
+      - name: Setup and Format Python Version
+        id: format_python_version
+        shell: bash
+        run: |
+          # Remove 'py' and insert a dot to format the version
+          FORMATTED_VERSION=${{ matrix.python-version }}   # e.g., py38
+          FORMATTED_VERSION=${FORMATTED_VERSION/py3/3.}    # becomes 3.8
+          # Set environment variables
+          echo "PYTHON_VERSION=$FORMATTED_VERSION" >> $GITHUB_ENV
       - name: Start container
         run: |
-          docker-compose -f ./.ci/docker-compose-ci.yml up -d
+          PYTHON_VERSION=$PYTHON_VERSION docker compose -f ./.ci/docker-compose-ci.yml up -d
       - name: Install dependencies
         run: |
           docker exec -t ecommerce_testing bash -c "
             cd /edx/app/ecommerce/ecommerce/ &&
-            python3 -m pip install tox
+            python$PYTHON_VERSION -m pip install tox
           "
       - name: Run tests
         run: |
           docker exec -t -e CI=1 ecommerce_testing bash -c "
             cd /edx/app/ecommerce/ecommerce/ &&
             PATH=\$PATH:/edx/app/ecommerce/nodeenvs/ecommerce/bin:/snap/bin
-            DJANGO_ENV=${{ matrix.django-env }} make ${{ matrix.targets }}
+            DJANGO_ENV=${{ matrix.django-env }} PYTHON_ENV=${{ matrix.python-version }} PYTHON_VERSION=$PYTHON_VERSION make ${{ matrix.test }}
           "
       - name: Run coverage
-        if: matrix.testname == 'test-python'
+        if: matrix.test == 'test-python'
         run: |
           docker exec ecommerce_testing /edx/app/ecommerce/ecommerce/.ci/run_coverage.sh
       - name: Setup Python
-        if: matrix.testname == 'test-python'
+        if: matrix.test == 'test-python' && matrix.python-version == 'py38'
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
           architecture: x64
       - name: Report coverage
-        if: matrix.testname == 'test-python'
+        if: matrix.test == 'test-python' && matrix.python-version == 'py38'
         uses: codecov/codecov-action@v3
         with:
           flags: unittests
           fail_ci_if_error: false
 
-  docs:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.8"
-          architecture: x64
-      - name: Install Dependencies
-        run: pip install -r requirements/docs.txt -r requirements/tox.txt
-      - name: Build the docs
-        run: make docs
+  # docs:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       python-version: ['3.8', '3.11', '3.12']
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - uses: actions/setup-python@v2
+  #       with:
+  #         python-version: ${{matrix.python-version}}
+  #         architecture: x64
+  #     - name: Install Dependencies
+  #       run: pip install -r requirements/docs.txt -r requirements/tox.txt
+  #     - name: Build the docs
+  #       run: make docs

Dockerfile

@@ -1,18 +1,23 @@
 FROM ubuntu:focal as app
 
+# Define 3.12 as default but it changes to the PYTHON_VERSION passed as argument
+ARG PYTHON_VERSION=3.12
+
 ENV DEBIAN_FRONTEND noninteractive
 # System requirements.
 RUN apt update && \
+  apt-get install -y software-properties-common && \
+  apt-add-repository -y ppa:deadsnakes/ppa && \
   apt-get install -qy \
   curl \
   gettext \
   git \
   language-pack-en \
   build-essential \
-  python3.8-dev \
-  python3-virtualenv \
-  python3.8-distutils \
+  python${PYTHON_VERSION}-dev \
+  python${PYTHON_VERSION}-distutils \
   libmysqlclient-dev \
+  pkg-config \
   libssl-dev \
   libcairo2-dev && \
   rm -rf /var/lib/apt/lists/*
@@ -34,11 +39,15 @@ ARG ECOMMERCE_NODEENV_DIR="${ECOMMERCE_APP_DIR}/nodeenvs/${SERVICE_NAME}"
 ENV ECOMMERCE_CFG "${COMMON_CFG_DIR}/ecommerce.yml"
 ENV ECOMMERCE_CODE_DIR "${ECOMMERCE_CODE_DIR}"
 ENV ECOMMERCE_APP_DIR "${ECOMMERCE_APP_DIR}"
+ENV PYTHON_VERSION "${PYTHON_VERSION}"
+
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION}
+RUN pip install virtualenv
 
 # Add virtual env and node env to PATH, in order to activate them
 ENV PATH "${ECOMMERCE_VENV_DIR}/bin:${ECOMMERCE_NODEENV_DIR}/bin:$PATH"
 
-RUN virtualenv -p python3.8 --always-copy ${ECOMMERCE_VENV_DIR}
+RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${ECOMMERCE_VENV_DIR}
 
 RUN pip install nodeenv
 

Makefile

@@ -1,7 +1,8 @@
 NODE_BIN=./node_modules/.bin
 DIFF_COVER_BASE_BRANCH=master
-PYTHON_ENV=py38
+PYTHON_ENV_VAR=$(if $(PYTHON_ENV),$(PYTHON_ENV),py312)
 DJANGO_ENV_VAR=$(if $(DJANGO_ENV),$(DJANGO_ENV),django32)
+PYTHON_VERSION_VAR=$(if $(PYTHON_VERSION),$(PYTHON_VERSION),3.12)
 
 help:
 	@echo ''
@@ -41,21 +42,21 @@ requirements.js:
 	$(NODE_BIN)/bower install --allow-root
 
 requirements: requirements.js
-	pip3 install -r requirements/pip_tools.txt
-	pip3 install -r requirements/dev.txt --exists-action w
+	pip install -r requirements/pip_tools.txt
+	pip install -r requirements/dev.txt --exists-action w
 
 requirements.tox:
-	pip3 install -U pip==20.0.2
-	pip3 install -r requirements/tox.txt --exists-action w
+	pip install -U pip
+	pip install -r requirements/tox.txt --exists-action w
 
 production-requirements: requirements.js
-	pip3 install -r requirements.txt --exists-action w
+	pip install -r requirements.txt --exists-action w
 
 migrate: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-migrate
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-migrate
 
 serve: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-serve
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-serve
 
 clean:
 	find . -name '*.pyc' -delete
@@ -65,18 +66,18 @@ clean_static:
 	rm -rf assets/* ecommerce/static/build/*
 
 run_check_isort: requirements.tox
-	tox -e $(PYTHON_ENV)-check_isort
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-check_isort
 
 run_isort: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-run_isort
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-run_isort
 
 run_pycodestyle: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-pycodestyle
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-pycodestyle
 
 run_pep8: run_pycodestyle
 
 run_pylint: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-pylint
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-pylint
 
 quality: run_check_isort run_pycodestyle run_pylint
 
@@ -86,42 +87,42 @@ validate_js:
 	$(NODE_BIN)/gulp lint
 
 validate_python: clean requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-tests
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-tests
 
 acceptance: clean requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-acceptance
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-acceptance
 
 fast_validate_python: clean requirements.tox
-	DISABLE_ACCEPTANCE_TESTS=True tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-tests
+	DISABLE_ACCEPTANCE_TESTS=True python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-tests
 
 validate: validate_python validate_js quality
 
 theme_static: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-theme_static
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-theme_static
 
 static: requirements.js theme_static requirements.tox
 	$(NODE_BIN)/r.js -o build.js
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-static
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-static
 
 html_coverage: requirements.tox
-	tox -e $(PYTHON_ENV)-coverage_html
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-coverage_html
 
 diff_coverage: validate fast_diff_coverage
 
 fast_diff_coverage: requirements.tox
-	tox -e $(PYTHON_ENV)-fast_diff_coverage
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-fast_diff_coverage
 
 e2e: requirements.tox
-	tox -e $(PYTHON_ENV)-e2e
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-e2e
 
 extract_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-extract_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-extract_translations
 
 dummy_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-dummy_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-dummy_translations
 
 compile_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-compile_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-compile_translations
 
 fake_translations: extract_translations dummy_translations compile_translations
 
@@ -134,18 +135,18 @@ update_translations: pull_translations fake_translations
 
 # extract_translations should be called before this command can detect changes
 detect_changed_source_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-detect_changed_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-detect_changed_translations
 
 # @FIXME: skip detect_changed_source_translations until git diff works again (REV-2737)
 check_translations_up_to_date: fake_translations # detect_changed_source_translations
 
 # Validate translations
 validate_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-validate_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-validate_translations
 
 # Scan the Django models in all installed apps in this project for restricted field names
 check_keywords: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-check_keywords
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-check_keywords
 
 COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
 .PHONY: $(COMMON_CONSTRAINTS_TXT)
@@ -173,6 +174,12 @@ upgrade: $(COMMON_CONSTRAINTS_TXT)
 docs:
 	tox -e docs
 
+quality-and-jobs: requirements.js check_translations_up_to_date validate_translations clean_static static quality validate_js check_keywords
+
+test-python: requirements.js clean_static static validate_python
+
+acceptance-python: requirements.js clean_static static acceptance
+
 # Targets in a Makefile which do not produce an output file with the same name as the target name
 .PHONY: help requirements migrate serve clean validate_python quality validate_js validate html_coverage e2e \
 	extract_translations dummy_translations compile_translations fake_translations pull_translations \