Merge pull request #102 from blaggacao/ci/adopt-best-practice

ci: adopt best practice

.github/helper/install.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+
+cd ~ || exit
+
+sudo apt update
+sudo apt remove mysql-server mysql-client
+sudo apt install libcups2-dev redis-server mariadb-client-10.6
+
+pip install frappe-bench
+
+githubbranch=${GITHUB_BASE_REF:-${GITHUB_REF##*/}}
+frappeuser=${FRAPPE_USER:-"frappe"}
+frappebranch=${FRAPPE_BRANCH:-$githubbranch}
+erpnextbranch=${ERPNEXT_BRANCH:-$githubbranch}
+
+git clone "https://github.com/${frappeuser}/frappe" --branch "${frappebranch}" --depth 1
+bench init --skip-assets --frappe-path ~/frappe --python "$(which python)" frappe-bench
+
+mkdir ~/frappe-bench/sites/test_site
+cp -r "${GITHUB_WORKSPACE}/.github/helper/site_config.json" ~/frappe-bench/sites/test_site/
+
+mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'"
+mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"
+
+mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'"
+mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "CREATE DATABASE test_frappe"
+mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost'"
+
+mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "FLUSH PRIVILEGES"
+
+install_whktml() {
+    wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz
+    tar -xf /tmp/wkhtmltox.tar.xz -C /tmp
+    sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf
+    sudo chmod o+x /usr/local/bin/wkhtmltopdf
+}
+install_whktml &
+
+cd ~/frappe-bench || exit
+
+sed -i 's/watch:/# watch:/g' Procfile
+sed -i 's/schedule:/# schedule:/g' Procfile
+sed -i 's/socketio:/# socketio:/g' Procfile
+sed -i 's/redis_socketio:/# redis_socketio:/g' Procfile
+
+bench get-app "https://github.com/${frappeuser}/erpnext" --branch "$erpnextbranch" --resolve-deps
+bench get-app payments "${GITHUB_WORKSPACE}"
+bench setup requirements --dev
+
+bench start &>> ~/frappe-bench/bench_start.log &
+CI=Yes bench build --app frappe &
+bench --site test_site reinstall --yes
+
+bench --verbose --site test_site install-app payments

.github/helper/site_config.json

@@ -0,0 +1,16 @@
+{
+    "db_host": "127.0.0.1",
+    "db_port": 3306,
+    "db_name": "test_frappe",
+    "db_password": "test_frappe",
+    "auto_email_id": "[email protected]",
+    "mail_server": "smtp.example.com",
+    "mail_login": "[email protected]",
+    "mail_password": "test",
+    "admin_password": "admin",
+    "root_login": "root",
+    "root_password": "root",
+    "host_name": "http://test_site:8000",
+    "install_apps": ["payments", "erpnext"],
+    "throttle_user_limit": 100
+}

.github/workflows/ci.yml

@@ -1,62 +1,96 @@
-
-name: Server
+name: CI
 
 on:
-  push:
-    branches:
-      - develop
   pull_request:
+    paths-ignore:
+      - "**.css"
+      - "**.js"
+      - "**.md"
+      - "**.html"
+      - "**.csv"
+  schedule:
+    # Run everday at midnight UTC / 5:30 IST
+    - cron: "0 0 * * *"
 
 concurrency:
-  group: develop-payments-${{ github.event.number }}
+  group: develop-${{ github.event.number }}
   cancel-in-progress: true
 
 jobs:
   tests:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      NODE_ENV: "production"
+      WITH_COVERAGE: ${{ github.event_name != 'pull_request' }}
+
     strategy:
       fail-fast: false
-    name: Server
+
+      matrix:
+        container: [1]
+
+    name: Python Unit Tests
 
     services:
-      mariadb:
+      mysql:
         image: mariadb:10.6
         env:
-          MYSQL_ROOT_PASSWORD: root
+          MARIADB_ROOT_PASSWORD: 'root'
         ports:
           - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+        options: --health-cmd="mariadb-admin ping" --health-interval=5s --health-timeout=2s --health-retries=3
 
     steps:
       - name: Clone
         uses: actions/checkout@v2
 
-      - name: Install MariaDB Client
-        run: sudo apt-get -y install mariadb-client-10.6
-
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: '3.10'
 
+      - name: Check for valid Python & Merge Conflicts
+        run: |
+          python -m compileall -f "${GITHUB_WORKSPACE}"
+          if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}"
+              then echo "Found merge conflicts"
+              exit 1
+          fi
+
       - name: Setup Node
         uses: actions/setup-node@v2
         with:
           node-version: 18
           check-latest: true
 
+      - name: Add to Hosts
+        run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+
       - name: Cache pip
         uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py', '**/setup.cfg') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml') }}
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
 
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
-        run: 'echo "::set-output name=dir::$(yarn cache dir)"'
+        run: echo "::set-output name=dir::$(yarn cache dir)"
 
       - uses: actions/cache@v2
         id: yarn-cache
@@ -66,30 +100,41 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
-      - name: Setup
-        run: |
-          pip install frappe-bench
-          bench init --skip-redis-config-generation --skip-assets --frappe-branch ${GITHUB_BASE_REF:-${GITHUB_REF##*/}} --python "$(which python)" ~/frappe-bench
-          mysql --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'"
-          mysql --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"
-
       - name: Install
-        working-directory: /home/runner/frappe-bench
         run: |
-          bench get-app https://github.com/frappe/erpnext --branch "develop" --resolve-deps
-          bench get-app payments $GITHUB_WORKSPACE
-          bench setup requirements --dev
-          bench new-site --db-root-password root --admin-password admin test_site
-          bench start &> bench_start.log &
-          bench --site test_site install-app erpnext payments
-          bench build
+          bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
-          CI: 'Yes'
+          FRAPPE_USER: ${{ github.event.inputs.user }}
+          FRAPPE_BRANCH: ${{ github.event.inputs.branch }}
 
       - name: Run Tests
-        working-directory: /home/runner/frappe-bench
-        run: |
-          bench --site test_site set-config allow_tests true
-          bench --site test_site run-tests --app payments
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --app payments --total-builds ${{ strategy.job-total }} --build-number ${{ matrix.container }}
         env:
           TYPE: server
+          CAPTURE_COVERAGE: ${{ github.event_name != 'pull_request' }}
+
+      - name: Upload coverage data
+        uses: actions/upload-artifact@v3
+        if: github.event_name != 'pull_request'
+        with:
+          name: coverage-${{ matrix.container }}
+          path: /home/runner/frappe-bench/sites/coverage.xml
+
+  coverage:
+    name: Coverage Wrap Up
+    needs: tests
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' }}
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Upload coverage data
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/labeller.yml

@@ -0,0 +1,12 @@
+name: "Pull Request Labeler"
+on:
+  pull_request_target:
+    types: [opened, reopened]
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/labeler@v4
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.gitignore

@@ -6,3 +6,5 @@ tags
 payments/docs/current
 node_modules/
 __pycache__/
+.aider*
+.helix