Merge pull request #5 from newrelic/add-python-ci-workflow-2

Add Python CI Workflow

.github/workflows/python.yml

@@ -0,0 +1,137 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+name: Python Agent CI
+
+on:
+  pull_request:
+  push:
+    # Run on push to main or PR. 
+    branches:
+      - main
+    # Do not run when a tag is created.
+    tags-ignore:
+      - "**"
+  release:
+    types:
+      - published
+
+env:
+  INITCONTAINER_LANGUAGE: python
+  K8S_OPERATOR_IMAGE_TAG: edge
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # 3.3.0
+
+      - name: Start minikube
+        uses: medyagh/setup-minikube@317d92317e473a10540357f1f4b2878b80ee7b95 # 0.0.16
+
+      - name: Deploy cert-manager to minikube
+        run: |
+          helm repo add jetstack https://charts.jetstack.io --force-update
+          helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.14.5 --set installCRDs=true
+          sleep 5
+          kubectl wait --for=condition=Ready -n cert-manager --all pods
+
+      - name: Deploy New Relic k8s-agents-operator to minikube
+        run: |
+          helm repo add k8s-agents-operator https://newrelic.github.io/k8s-agents-operator
+          helm upgrade --install k8s-agents-operator k8s-agents-operator/k8s-agents-operator \
+            --namespace=default \
+            --set=licenseKey=${{ secrets.NEW_RELIC_LICENSE_KEY }} \
+            --set=controllerManager.manager.image.tag=${{ env.K8S_OPERATOR_IMAGE_TAG }}
+          sleep 5
+          kubectl wait --for=condition=Ready -n default --all pods
+
+      - name: Build init container
+        run: |
+          minikube image build -t e2e/newrelic-${{ env.INITCONTAINER_LANGUAGE }}-init:e2e ${{ env.INITCONTAINER_LANGUAGE }}/
+
+      - name: Build test app container
+        run: |
+          minikube image build -t e2e/test-app-${{ env.INITCONTAINER_LANGUAGE }}:e2e tests/${{ env.INITCONTAINER_LANGUAGE }}/
+
+      - name: Run e2e-test
+        uses: newrelic/newrelic-integration-e2e-action@a97ced80a4841c8c6261d1f9dca6706b1d89acb1  # 1.11.0
+        with:
+          retry_seconds: 60
+          retry_attempts: 5
+          agent_enabled: false
+          spec_path: tests/${{ env.INITCONTAINER_LANGUAGE }}/test-specs.yml
+          account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
+          api_key: ${{ secrets.NEW_RELIC_API_KEY }}
+          license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+
+  publish:
+    if: github.event_name == 'release' && endsWith(github.ref, '_python')
+    runs-on: ubuntu-latest
+    needs:
+      - test
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Extract Agent Version
+        id: version
+        run: |
+          agent_version=${{ github.ref_name }}  # Use tag name
+          agent_version=${agent_version##v}  # Remove v prefix
+          agent_version=${agent_version%%_${{ env.INITCONTAINER_LANGUAGE }}}  # Remove language suffix
+          echo "agent_version=${agent_version}" | tee -a "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # 3.3.0
+
+      - name: Generate Docker metadata (tags and labels)
+        id: meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # 5.5.1
+        with:
+          images: newrelic/newrelic-${{ env.INITCONTAINER_LANGUAGE }}-init
+          tags: |
+            type=raw,value=${{ steps.version.outputs.agent_version }}
+            type=raw,value=latest
+
+      - name: Login to Docker Hub Container Registry
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # 3.1.0
+        with:
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and publish init container image
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # 5.3.0
+        with:
+          push: true
+          context: ${{ env.INITCONTAINER_LANGUAGE }}/
+          platforms: linux/amd64,linux/arm64,linux/arm
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            AGENT_VERSION=${{ steps.version.outputs.agent_version }}

python/Dockerfile

@@ -9,17 +9,39 @@
 # - Grant the necessary access to `/instrumentation` directory. `chmod -R go+r /instrumentation`
 
 FROM python:3.10-alpine AS build
+
 WORKDIR /operator-build
-ARG version
-ENV NEW_RELIC_EXTENSIONS = False
-# WARNING: Disabling optional C extension components of the Python agent 
-# will result in some non core features of the Python agent, such as
-# capacity analysis instance busy metrics, that will not be available.
-# Pure Python versions of code supporting some features, rather than the
-# optimised C versions, will also be used resulting in additional overheads.
-ADD requirements.txt .
-RUN mkdir workspace && pip install --target workspace newrelic==$version
 
+# Install dependencies
+COPY requirements-builder.txt .
+RUN pip install -r requirements-builder.txt
+
+# Download and prepare wheels
+ARG AGENT_VERSION
+ENV AGENT_VERSION=${AGENT_VERSION}
+COPY download_wheels.py .
+RUN python ./download_wheels.py
+
+# Install sdist without extensions to set up the directory
+ENV NEW_RELIC_EXTENSIONS False
+ENV WRAPT_DISABLE_EXTENSIONS True
+RUN pip install ./workspace/newrelic.tar.gz --target=./workspace/newrelic && \
+    rm ./workspace/newrelic.tar.gz
+
+# Install pip as a vendored package
+COPY requirements-vendor.txt .
+RUN mkdir -p ./workspace/vendor && \
+    pip install --target=./workspace/vendor -r requirements-vendor.txt
+
+# Install sitecustomize and newrelic_k8s_operator modules
+RUN cp ./workspace/newrelic/newrelic/bootstrap/sitecustomize.py ./workspace/sitecustomize.py
+COPY newrelic_k8s_operator.py ./workspace/
+
+# *** TODO: Remove this after sitecustomize changes have been released with the latest copy of the agent ***
+COPY sitecustomize.py ./workspace/sitecustomize.py
+
+# initcontainer
 FROM busybox
+
 COPY --from=build /operator-build/workspace /instrumentation
 RUN chmod -R go+r /instrumentation

python/download_wheels.py

@@ -0,0 +1,107 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import requests
+import os
+import shutil
+import subprocess
+import tempfile
+
+
+AGENT_VERSION = os.getenv("AGENT_VERSION", "").lstrip("v")
+FILE_DIR = os.path.dirname(__file__)
+WORKSPACE_DIR = os.path.join(FILE_DIR, "workspace")
+
+
+def main():
+    """
+    Download and unpack wheels and sdist for a given agent version as part of the build process for the Python agent init container.
+    """
+    with requests.session() as session:
+        # Fetch JSON list of all agent package artifacts
+        resp = session.get("https://pypi.org/pypi/newrelic/json")
+        resp.raise_for_status()
+        resp_dict = resp.json()
+
+        if AGENT_VERSION:
+            # Grab the supplied release version
+            release = resp_dict["releases"][AGENT_VERSION]
+        else:
+            # Grab latest release version
+            release = list(resp_dict["releases"].values())[-1]
+
+        # Filter artifacts for wheels and tarballs
+        wheel_urls = [
+            artifact["url"]
+            for artifact in release
+            if artifact["url"].endswith(".whl")
+        ]
+
+        tarball_url = [
+            artifact["url"]
+            for artifact in release
+            if artifact["url"].endswith(".tar.gz")
+        ][0]
+
+        # Make workspace directory
+        if not os.path.exists(WORKSPACE_DIR):
+            os.mkdir(WORKSPACE_DIR)
+
+        # Download tarball
+        resp = session.get(tarball_url)
+        resp.raise_for_status()
+
+        tarball_filepath = os.path.join(WORKSPACE_DIR, "newrelic.tar.gz")
+        with open(tarball_filepath, "wb") as file:
+            file.write(resp.content)
+
+        # Download and extract all wheels
+        for wheel_url in wheel_urls:
+            # Download wheel
+            resp = session.get(wheel_url)
+            resp.raise_for_status()
+
+            # Write wheel to file under a folder of the same name
+            with tempfile.TemporaryDirectory() as wheel_temp_dir:
+                # Compute paths
+                wheel_filename = wheel_url.split("/")[-1]
+                wheel_filepath = os.path.join(wheel_temp_dir, wheel_filename)
+
+                # Write wheel file to tempdir
+                with open(wheel_filepath, "wb") as file:
+                    file.write(resp.content)
+
+                # Unpack wheel using wheel command
+                subprocess.run(
+                    ["wheel", "unpack", str(wheel_filename)],
+                    cwd=wheel_temp_dir,
+                )
+
+                # Delete wheel file after unpacking
+                os.remove(wheel_filepath)
+
+                # Move unpacked module from tempdir to final destination
+                unpacked_module_folder = os.path.join(
+                    wheel_temp_dir, list(os.listdir(wheel_temp_dir))[0]
+                )
+
+                # Put wheel contents into folders named after the original wheel types
+                final_wheel_dest = os.path.join(
+                    WORKSPACE_DIR, wheel_filename.rstrip(".whl")
+                )
+                shutil.move(str(unpacked_module_folder), str(final_wheel_dest))
+
+
+if __name__ == "__main__":
+    main()

python/newrelic_k8s_operator.py

@@ -0,0 +1,77 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+INSTRUMENTATION_PATH = os.path.dirname(__file__)
+INSTRUMENTATION_VENDOR_PATH = os.path.join(INSTRUMENTATION_PATH, "vendor")
+SDIST_PATH = str(os.path.join(INSTRUMENTATION_PATH, "newrelic"))
+
+
+def get_supported_tags():
+    """
+    Load pip and run compatibility_tags.get_supported(). If pip is not present
+    or does not contain this function, load our vendored version of pip instead.
+    """
+    try:
+        # Attempt to use any existing installs of pip to find supported tags
+        from pip._internal.utils.compatibility_tags import get_supported
+
+        return get_supported()
+    except Exception:
+        pass
+
+    # Unable to find pip, or pip version does not have compatibility tags
+    if "pip" in sys.modules:
+        # Save original pip module and load our own copy
+        original_pip = sys.modules["pip"]
+        del sys.modules["pip"]
+    else:
+        original_pip = None
+
+    try:
+        # Insert our vendored version of pip into the path, and try to return
+        # the supported tags again.
+        sys.path.insert(0, INSTRUMENTATION_VENDOR_PATH)
+        from pip._internal.utils.compatibility_tags import get_supported
+
+        return get_supported()
+    finally:
+        # Remove our vendored version of pip from the path
+        if INSTRUMENTATION_VENDOR_PATH in sys.path:
+            del sys.path[sys.path.index(INSTRUMENTATION_VENDOR_PATH)]
+
+        # Replace original pip module for compatibility
+        if original_pip:
+            sys.modules["pip"] = original_pip
+        elif "pip" in sys.modules:
+            del sys.modules["pip"]
+
+
+def find_supported_newrelic_distribution():
+    """
+    Return the path to a supported wheel or sdist included under INSTRUMENTATION_PATH by the Python agent init container.
+    """
+    try:
+        wheels = list(os.listdir(INSTRUMENTATION_PATH))
+        for tag in get_supported_tags():
+            tag = str(tag)
+            for wheel in wheels:
+                if tag in wheel:
+                    return str(os.path.join(INSTRUMENTATION_PATH, wheel))
+    except Exception:
+        pass
+
+    return SDIST_PATH

python/requirements-builder.txt

@@ -0,0 +1,2 @@
+requests
+setuptools>=40.8.0

python/requirements-vendor.txt

@@ -0,0 +1 @@
+pip