Merge branch 'ansible:devel' into convert-skip-to-xfail

ansible · Sep 17, 2024 · 5fb47a0 · 5fb47a0
2 parents 19ccfbb + eccc32c
commit 5fb47a0
Show file tree

Hide file tree

Showing 60 changed files with 294 additions and 1,248 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -0,0 +1,57 @@
+---
+
+codecov:
+  notify:
+    after_n_builds: 5  # Number of test matrix+lint jobs uploading coverage
+    wait_for_ci: false
+
+  require_ci_to_pass: false
+
+  token: >-  # repo-scoped, upload-only, needed for stability in PRs from forks
+    2b8c7a7a-7293-4a00-bf02-19bd55a1389b
+
+comment:
+  require_changes: true
+
+coverage:
+  range: 100..100
+  status:
+    patch:
+      default:
+        target: 100%
+      pytest:
+        target: 100%
+        flags:
+          - pytest
+      typing:
+        flags:
+          - MyPy
+    project:
+      default:
+        target: 100%
+      lib:
+        flags:
+          - pytest
+        paths:
+          - awx/
+        target: 100%
+      tests:
+        flags:
+          - pytest
+        paths:
+          - tests/
+          - >-
+            **/test/
+          - >-
+            **/tests/
+          - >-
+            **/test/**
+          - >-
+            **/tests/**
+        target: 100%
+      typing:
+        flags:
+          - MyPy
+        target: 100%
+
+...
diff --git a/.coveragerc b/.coveragerc
@@ -1,16 +1,6 @@
-[run]
-source = awx
-branch = True
-omit =
-    awx/main/migrations/*
-    awx/lib/site-packages/*
-
 [report]
 # Regexes for lines to exclude from consideration
-exclude_lines =
-    # Have to re-enable the standard pragma
-    pragma: no cover
-
+exclude_also =
     # Don't complain about missing debug-only code:
     def __repr__
     if self\.debug
@@ -23,7 +13,16 @@ exclude_lines =
     if 0:
     if __name__ == .__main__.:
 
-ignore_errors = True
+    ^\s*@pytest\.mark\.xfail
+
+[run]
+branch = True
+omit =
+    awx/main/migrations/*
+source =
+    .
+source_pkgs =
+    awx
 
 [xml]
 output = ./reports/coverage.xml
diff --git a/.github/actions/awx_devel_image/action.yml b/.github/actions/awx_devel_image/action.yml
@@ -24,11 +24,11 @@ runs:
 
     - name: Pre-pull latest devel image to warm cache
       shell: bash
-      run: docker pull -q ghcr.io/${OWNER_LC}/awx_devel:${{ github.base_ref }}
+      run: docker pull -q ghcr.io/${OWNER_LC}/awx_devel:${{ github.base_ref || github.ref_name }}
 
     - name: Build image for current source checkout
       shell: bash
       run: |
         DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} \
-        COMPOSE_TAG=${{ github.base_ref }} \
+        COMPOSE_TAG=${{ github.base_ref || github.ref_name }} \
         make docker-compose-build
diff --git a/.github/actions/run_awx_devel/action.yml b/.github/actions/run_awx_devel/action.yml
@@ -36,7 +36,7 @@ runs:
       shell: bash
       run: |
         DEV_DOCKER_OWNER=${{ github.repository_owner }} \
-        COMPOSE_TAG=${{ github.base_ref }} \
+        COMPOSE_TAG=${{ github.base_ref || github.ref_name }} \
         COMPOSE_UP_OPTS="-d" \
         make docker-compose
 

diff --git a/.github/triage_replies.md b/.github/triage_replies.md
@@ -1,5 +1,4 @@
 ## General
-- For the roundup of all the different mailing lists available from AWX, Ansible, and beyond visit: https://docs.ansible.com/ansible/latest/community/communication.html
 - Hello, we think your question is answered in our FAQ. Does this: https://www.ansible.com/products/awx-project/faq cover your question?
 - You can find the latest documentation here: https://ansible.readthedocs.io/projects/awx/en/latest/userguide/index.html
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,8 +5,12 @@ env:
   CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   DEV_DOCKER_OWNER: ${{ github.repository_owner }}
   COMPOSE_TAG: ${{ github.base_ref || 'devel' }}
+  UPSTREAM_REPOSITORY_ID: 91594105
 on:
   pull_request:
+  push:
+    branches:
+      - devel  # needed to publish code coverage post-merge
 jobs:
   common-tests:
     name: ${{ matrix.tests.name }}
@@ -20,7 +24,7 @@ jobs:
       matrix:
         tests:
           - name: api-test
-            command: /start_tests.sh
+            command: /start_tests.sh test_coverage
           - name: api-migrations
             command: /start_tests.sh test_migrations
           - name: api-lint
@@ -30,7 +34,10 @@ jobs:
           - name: awx-collection
             command: /start_tests.sh test_collection_all
           - name: api-schema
-            command: /start_tests.sh detect-schema-change SCHEMA_DIFF_BASE_BRANCH=${{ github.event.pull_request.base.ref }}
+            command: >-
+              /start_tests.sh detect-schema-change SCHEMA_DIFF_BASE_BRANCH=${{
+                github.event.pull_request.base.ref || github.ref_name
+              }}
 
     steps:
       - uses: actions/checkout@v4
@@ -43,7 +50,70 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run check ${{ matrix.tests.name }}
-        run: AWX_DOCKER_CMD='${{ matrix.tests.command }}' make docker-runner
+        id: make-run
+        run: >-
+          AWX_DOCKER_ARGS='-e GITHUB_ACTIONS -e GITHUB_OUTPUT -v "${GITHUB_OUTPUT}:${GITHUB_OUTPUT}:rw,Z"'
+          AWX_DOCKER_CMD='${{ matrix.tests.command }}'
+          make docker-runner
+
+      - name: Upload test coverage to Codecov
+        if: >-
+          !cancelled()
+          && steps.make-run.outputs.cov-report-files != ''
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: >-
+            ${{
+              toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+            }}
+          files: >-
+            ${{ steps.make-run.outputs.cov-report-files }}
+          flags: >-
+            CI-GHA,
+            pytest,
+            OS-${{
+              runner.os
+            }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+      - name: Upload test results to Codecov
+        if: >-
+          !cancelled()
+          && steps.make-run.outputs.test-result-files != ''
+        uses: codecov/test-results-action@v1
+        with:
+          fail_ci_if_error: >-
+            ${{
+              toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+            }}
+          files: >-
+            ${{ steps.make-run.outputs.test-result-files }}
+          flags: >-
+            CI-GHA,
+            pytest,
+            OS-${{
+              runner.os
+            }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Upload awx jUnit test reports
+        if: >-
+          !cancelled()
+          && steps.make-run.outputs.test-result-files != ''
+          && github.event_name == 'push'
+          && env.UPSTREAM_REPOSITORY_ID == github.repository_id
+          && github.ref_name == github.event.repository.default_branch
+        run: |
+          for junit_file in $(echo '${{ steps.make-run.outputs.test-result-files }}' | sed 's/,/ /')
+          do
+              curl \
+                -v \
+                --user "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}:${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}"
+                --form "xunit_xml=@${junit_file}" \
+                --form "component_name=awx"
+                --form "git_commit_sha=${{ github.sha }}"
+                --form "git_repository_url=https://github.com/${{ github.repository }}"
+                "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}/api/results/upload/"
+          done
 
   dev-env:
     runs-on: ubuntu-latest

diff --git a/Makefile b/Makefile
@@ -327,7 +327,12 @@ swagger: reports
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	(set -o pipefail && py.test $(PYTEST_ARGS) awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs | tee reports/$@.report)
+	(set -o pipefail && py.test --cov --cov-report=xml --junitxml=reports/junit.xml $(PYTEST_ARGS) awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs | tee reports/$@.report)
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo 'cov-report-files=reports/coverage.xml' >> "${GITHUB_OUTPUT}"; \
+	  echo 'test-result-files=reports/junit.xml' >> "${GITHUB_OUTPUT}"; \
+	fi
 
 check: black
 
@@ -351,15 +356,26 @@ test:
 	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3
 	awx-manage check_migrations --dry-run --check  -n 'missing_migration_file'
 
+## Run all API unit tests with coverage enabled.
+test_coverage:
+	$(MAKE) test PYTEST_ARGS="--create-db --cov --cov-report=xml --junitxml=reports/junit.xml"
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo 'cov-report-files=reports/coverage.xml' >> "${GITHUB_OUTPUT}"; \
+	  echo 'test-result-files=awxkit/report.xml,reports/junit.xml' >> "${GITHUB_OUTPUT}"; \
+	fi
+
 test_migrations:
-	if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider --migrations -m migration_test $(PYTEST_ARGS) $(TEST_DIRS)
+	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider --migrations -m migration_test --create-db --cov=awx --cov-report=xml --junitxml=reports/junit.xml $(PYTEST_ARGS) $(TEST_DIRS)
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo 'cov-report-files=reports/coverage.xml' >> "${GITHUB_OUTPUT}"; \
+	  echo 'test-result-files=reports/junit.xml' >> "${GITHUB_OUTPUT}"; \
+	fi
 
 ## Runs AWX_DOCKER_CMD inside a new docker container.
 docker-runner:
-	docker run -u $(shell id -u) --rm -v $(shell pwd):/awx_devel/:Z --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(AWX_DOCKER_CMD)
+	docker run -u $(shell id -u) --rm -v $(shell pwd):/awx_devel/:Z $(AWX_DOCKER_ARGS) --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(AWX_DOCKER_CMD)
 
 test_collection:
 	rm -f $(shell ls -d $(VENV_BASE)/awx/lib/python* | head -n 1)/no-global-site-packages.txt
@@ -368,7 +384,12 @@ test_collection:
 	fi && \
 	if ! [ -x "$(shell command -v ansible-playbook)" ]; then pip install ansible-core; fi
 	ansible --version
-	py.test $(COLLECTION_TEST_DIRS) -v
+	py.test $(COLLECTION_TEST_DIRS) --cov --cov-report=xml --junitxml=reports/junit.xml -v
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo 'cov-report-files=reports/coverage.xml' >> "${GITHUB_OUTPUT}"; \
+	  echo 'test-result-files=reports/junit.xml' >> "${GITHUB_OUTPUT}"; \
+	fi
 	# The python path needs to be modified so that the tests can find Ansible within the container
 	# First we will use anything expility set as PYTHONPATH
 	# Second we will load any libraries out of the virtualenv (if it's unspecified that should be ok because python should not load out of an empty directory)
@@ -414,13 +435,6 @@ test_unit:
 	fi; \
 	py.test awx/main/tests/unit awx/conf/tests/unit awx/sso/tests/unit
 
-## Run all API unit tests with coverage enabled.
-test_coverage:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	py.test --create-db --cov=awx --cov-report=xml --junitxml=./reports/junit.xml $(TEST_DIRS)
-
 ## Output test coverage as HTML (into htmlcov directory).
 coverage_html:
 	coverage html

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-[![CI](https://github.com/ansible/awx/actions/workflows/ci.yml/badge.svg?branch=devel)](https://github.com/ansible/awx/actions/workflows/ci.yml) [![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-Ansible-yellow.svg)](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html) [![Apache v2 License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](https://github.com/ansible/awx/blob/devel/LICENSE.md) [![AWX on the Ansible Forum](https://img.shields.io/badge/mailing%20list-AWX-orange.svg)](https://forum.ansible.com/tag/awx)
+[![CI](https://github.com/ansible/awx/actions/workflows/ci.yml/badge.svg?branch=devel)](https://github.com/ansible/awx/actions/workflows/ci.yml) [![codecov](https://codecov.io/github/ansible/awx/graph/badge.svg?token=4L4GSP9IAR)](https://codecov.io/github/ansible/awx) [![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-Ansible-yellow.svg)](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html) [![Apache v2 License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](https://github.com/ansible/awx/blob/devel/LICENSE.md) [![AWX on the Ansible Forum](https://img.shields.io/badge/mailing%20list-AWX-orange.svg)](https://forum.ansible.com/tag/awx)
 [![Ansible Matrix](https://img.shields.io/badge/matrix-Ansible%20Community-blueviolet.svg?logo=matrix)](https://chat.ansible.im/#/welcome) [![Ansible Discourse](https://img.shields.io/badge/discourse-Ansible%20Community-yellowgreen.svg?logo=discourse)](https://forum.ansible.com)
 
 <img src="https://raw.githubusercontent.com/ansible/awx-logos/master/awx/ui/client/assets/logo-login.svg?sanitize=true" width=200 alt="AWX" />

diff --git a/awx/__init__.py b/awx/__init__.py
@@ -5,6 +5,7 @@
 import os
 import sys
 import warnings
+from importlib.metadata import PackageNotFoundError, version as _get_version
 
 
 def get_version():
@@ -34,10 +35,8 @@ def version_file():
 
 
 try:
-    import pkg_resources
-
-    __version__ = pkg_resources.get_distribution('awx').version
-except pkg_resources.DistributionNotFound:
+    __version__ = _get_version('awx')
+except PackageNotFoundError:
     __version__ = get_version()
 
 __all__ = ['__version__']

diff --git a/awx/api/views/mixin.py b/awx/api/views/mixin.py
@@ -15,6 +15,7 @@
 from rest_framework import status
 
 from awx.main.constants import ACTIVE_STATES
+from awx.main.models import Organization
 from awx.main.utils import get_object_or_400
 from awx.main.models.ha import Instance, InstanceGroup, schedule_policy_task
 from awx.main.models.organization import Team
@@ -60,6 +61,21 @@ def destroy(self, request, *args, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
+class OrganizationInstanceGroupMembershipMixin(object):
+    """
+    This mixin overloads attach/detach so that it calls Organization.save(),
+    to ensure instance group updates are persisted
+    """
+
+    def unattach(self, request, *args, **kwargs):
+        with transaction.atomic():
+            organization_queryset = Organization.objects.select_for_update()
+            organization = organization_queryset.get(pk=self.get_parent_object().id)
+            response = super(OrganizationInstanceGroupMembershipMixin, self).unattach(request, *args, **kwargs)
+            organization.save()
+            return response
+
+
 class InstanceGroupMembershipMixin(object):
     """
     This mixin overloads attach/detach so that it calls InstanceGroup.save(),

diff --git a/awx/api/views/organization.py b/awx/api/views/organization.py
@@ -52,7 +52,7 @@
     WorkflowJobTemplateSerializer,
     CredentialSerializer,
 )
-from awx.api.views.mixin import RelatedJobsPreventDeleteMixin, OrganizationCountsMixin
+from awx.api.views.mixin import RelatedJobsPreventDeleteMixin, OrganizationCountsMixin, OrganizationInstanceGroupMembershipMixin
 from awx.api.views import immutablesharedfields
 
 logger = logging.getLogger('awx.api.views.organization')
@@ -202,7 +202,7 @@ class OrganizationNotificationTemplatesApprovalList(OrganizationNotificationTemp
     relationship = 'notification_templates_approvals'
 
 
-class OrganizationInstanceGroupsList(SubListAttachDetachAPIView):
+class OrganizationInstanceGroupsList(OrganizationInstanceGroupMembershipMixin, SubListAttachDetachAPIView):
     model = InstanceGroup
     serializer_class = InstanceGroupSerializer
     parent_model = Organization

diff --git a/awx/main/db/profiled_pg/base.py b/awx/main/db/profiled_pg/base.py
@@ -1,9 +1,9 @@
 import os
-import pkg_resources
 import sqlite3
 import sys
 import traceback
 import uuid
+from importlib.metadata import version as _get_version
 
 from django.core.cache import cache
 from django.core.cache.backends.locmem import LocMemCache
@@ -70,7 +70,7 @@ def write(self, query):
             else:
                 progname = os.path.basename(sys.argv[0])
             filepath = os.path.join(self.dest, '{}.sqlite'.format(progname))
-            version = pkg_resources.get_distribution('awx').version
+            version = _get_version('awx')
             log = sqlite3.connect(filepath, timeout=3)
             log.execute(
                 'CREATE TABLE IF NOT EXISTS queries ('