diff --git a/.github/workflows/ci-docker-compose-integration.yml b/.github/workflows/ci-docker-compose-integration.yml new file mode 100644 index 0000000000..1bba4267c4 --- /dev/null +++ b/.github/workflows/ci-docker-compose-integration.yml @@ -0,0 +1,85 @@ +--- +name: Docker Compose Integration +on: + pull_request: + branches: + - '**' + push: + branches: + - '**' + workflow_dispatch: + +jobs: + integration: + strategy: + fail-fast: false + matrix: + env: + - { TEST_PROFILE: standalone, HUB_TEST_MARKS: "deployment_standalone or all" } + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: (Linux) Install docker compose + run: | + curl -L -o /tmp/docker-compose https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64 + install /tmp/docker-compose /usr/local/bin/ + + - name: sudo apt update + run: | + sudo apt update + + - name: Install python requirements and podman + run: | + sudo apt-get install -y libsasl2-dev python-dev-is-python3 libldap2-dev libssl-dev podman + + - name: Spin up dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml + run: | + docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml up --detach + + - name: Export environment variables to host + if: ${{ matrix.env.TEST_PROFILE != 'certified-sync' }} + run: | + docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV + + - name: Export environment variables to host (certified-sync) + if: ${{ matrix.env.TEST_PROFILE == 'certified-sync' }} + run: | + docker compose -f dev/compose/${{ matrix.env.TEST_PROFILE }}.yaml exec standalone-manager /bin/bash -c 'env | grep -v -w "HOME"' >> $GITHUB_ENV + + - run: cat $GITHUB_ENV + + - name: Wait for API online status + run: | + max_runs=10 + for i in $(seq 1 $max_runs); do + echo "$i: checking api status..." + RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -u "${{ env.DJANGO_SUPERUSER_USERNAME }}:${{ env.DJANGO_SUPERUSER_PASSWORD }}" -LI http://localhost:5001/api/galaxy/pulp/api/v3/status/) + if [ "$RESPONSE" -ne 200 ]; then + echo "API is down. Retrying in 10 seconds..." + sleep 10 + else + echo "API online." + exit 0 + fi + done + + - name: Install integration requirements + run: | + pip install -r integration_requirements.txt + + - name: Run integration tests + run: | + pytest -v -r sx --color=yes -m '${{ matrix.env.HUB_TEST_MARKS }}' galaxy_ng/tests/integration + env: + GALAXYKIT_SLEEP_SECONDS_POLLING: ".5" + GALAXYKIT_SLEEP_SECONDS_ONETIME: ".5" + GALAXYKIT_POLLING_MAX_ATTEMPTS: "50" + GALAXY_SLEEP_SECONDS_POLLING: ".5" + GALAXY_SLEEP_SECONDS_ONETIME: ".5" + GALAXY_POLLING_MAX_ATTEMPTS: "50" \ No newline at end of file diff --git a/.github/workflows/ci_oci-env-integration.yml b/.github/workflows/ci_oci-env-integration.yml index 045ae54fef..9030c72a90 100644 --- a/.github/workflows/ci_oci-env-integration.yml +++ b/.github/workflows/ci_oci-env-integration.yml @@ -17,7 +17,7 @@ jobs: env: - TEST_PROFILE: ldap - TEST_PROFILE: keycloak - - TEST_PROFILE: standalone + # - TEST_PROFILE: standalone - TEST_PROFILE: rbac - TEST_PROFILE: certified-sync - TEST_PROFILE: insights diff --git a/dev/compose/Dockerfile.dev b/dev/compose/Dockerfile.dev new file mode 100644 index 0000000000..5a54e8f018 --- /dev/null +++ b/dev/compose/Dockerfile.dev @@ -0,0 +1,24 @@ +# This Dockerfile is meant to extend the main Dockerfile.rhel8 +# Moving files to the required places +# Installing extra development libraries and tools +FROM localhost/galaxy_ng/galaxy_ng:base + +ARG GNUPGHOME +ENV GNUPGHOME $GNUPGHOME + +USER root + +RUN /app/dev/compose/signing/setup_gpg_workarounds.sh + +RUN set -ex; \ + install -Dm 0775 /app/dev/compose/database/database_fields.symmetric.key /etc/pulp/certs/database_fields.symmetric.key && \ + install -Dm 0775 /app/dev/compose/signing/signing-secret.key /etc/pulp/certs/signing-secret.key && \ + install -Dm 0775 /app/dev/compose/signing/signing-secret.key.password.txt /etc/pulp/certs/signing-secret.key.password.txt && \ + install -Dm 0775 /app/dev/compose/signing/signing-secret.key /etc/pulp/certs/signing-public.key && \ + install -Dm 0775 /app/dev/compose/signing/collection_sign.sh /var/lib/pulp/scripts/collection_sign.sh && \ + install -Dm 0775 /app/dev/compose/signing/container_sign.sh /var/lib/pulp/scripts/container_sign.sh && \ + chmod +x /var/lib/pulp/scripts/*_sign.sh + +RUN /app/dev/compose/signing/setup_gpg_keys.sh + +RUN ${VIRTUAL_ENV}/bin/python3 -m pip install ipython ipdb django-extensions diff --git a/dev/compose/README.md b/dev/compose/README.md new file mode 100644 index 0000000000..99dea4b24e --- /dev/null +++ b/dev/compose/README.md @@ -0,0 +1,263 @@ +# Galaxy Simplified Compose Stack + +Profiles: + +- `standalone` - Run galaxy_ng for integration with Ansible Automation Platform + +## Requirements + +- `docker compose` version `>=2` + +## Usage + +Pick a profile as needed and on the root of `galaxy_ng` repository. + +> Examples assumes `standalone` as the profile, change as needed. + +Build images +```bash +docker compose -f dev/compose/standalone.yaml build +``` + +Run the stack +```bash +docker compose -f dev/compose/standalone.yaml up +# optionally pass `-d` to release the terminal +``` + +Exec commands on the `manager` service + +Bash +```console +$ docker compose -f dev/compose/standalone.yaml exec manager /bin/bash +bash-4.4# +``` +Django Admin +```console +$ docker compose -f dev/compose/standalone.yaml exec manager pulpcore-manager +Type 'pulpcore-manager help ' for help on a specific subcommand. + +Available subcommands: + +[app] + add-signing-service + analyze-publication +... +``` + +Settings +```console +$ docker compose -f dev/compose/standalone.yaml exec manager dynaconf get DATABASES | python -m json.tool +{ + "default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": "postgres", + "NAME": "galaxy_ng", + "PASSWORD": "galaxy_ng", + "PORT": 5432, + "USER": "galaxy_ng" + } +} +``` +```console +$ docker compose -f dev/compose/standalone.yaml exec manager dynaconf list +CONTENT_ORIGIN 'https://localhost' +CACHE_ENABLED False +CACHE_SETTINGS {'EXPIRES_TTL': 600} +ALLOWED_CONTENT_CHECKSUMS ['sha224', 'sha256', 'sha384', 'sha512'] +... +``` + +Stopping +```console +$ docker compose -f dev/compose/standalone.yaml down +# add -v to stop and remove volumes +``` + +> [!TIP] +> Stop with Ctrl + C if running without `-d` and then execute the `down` command. + +## API Access + +Galaxy API is available on: + +[http://localhost:5001/api/galaxy/v3/swagger-ui/](http://localhost:5001/api/galaxy/v3/swagger-ui/) + +AAP UI and API will be available only if started separately on: + +[https://localhost](https://localhost) + + +## Running UI for standalone development + +Ansible Hub UI can be started separately as a standalone `npm` run. + +```console +# Assuming galaxy_ng is running on standalone compose. + +$ git clone https://github.com/ansible/ansible-hub-ui ~/projects/ansible-hub-ui +$ git clone https://github.com/ansible/galaxy_ng ~/projects/galaxy_ng +``` +Open 2 terminals: + +On the first terminal: + +```console +$ cd galaxy_ng +$ docker compose -f dev/compose/standalone.yaml up +``` + +On the second terminal: + +```console +cd ansible-hub-ui +npm install +npm run start-standalone +``` + +UI will be available on http://localhost:8002 and API on http://localhost:5001 + + +## Auto Reload and Local Checkouts + +To have the services `api`, `worker` and `content` to automatically reload when +source code changes it is required to set which paths the `reloader` must watch for changes. + +Set the variable `DEV_SOURCE_PATH` with the packages you want to add to the reloader list. + +Those repositories must be local checkouts located on the same level of the `galaxy_ng` repository. + +Example: + +Get the repositories locally in the same root path. +```console +$ git clone https://github.com/dynaconf/dynaconf ~/projects/dynaconf +$ git clone https://github.com/pulp/pulp_ansible ~/projects/pulp_ansible +$ git clone https://github.com/ansible/galaxy_ng ~/projects/galaxy_ng +``` + +> **IMPORTANT** Ensure all the repos are checked out to compatible branches. +> for example. you may be on galaxy_ng:master and reading `setup.py` you +> see that it requires `pulp_ansible>2.10,<3` then ensure you checkout `pulp_ansible` +> to a compatible branch. + +Start the compose setting the desired editable paths. + +```console +$ cd ~/projects/galaxy_ng +$ export DEV_SOURCE_PATH="dynaconf:pulp_ansible:galaxy_ng" +$ docker compose -f dev/compose/standalone.yaml up --build +``` + +Optionally it can be informed in a single line: + +```console +$ DEV_SOURCE_PATH="dynaconf:pulp_ansible:galaxy_ng" docker compose -f dev/compose/app.yaml up --build +``` + +> [!NOTE] +> if passing on the call line, remember to repass the same variable every time you interact with +>`docker compose` using the `run` command, usually `exec,logs,stats` doesn't require, but commands +> that starts the service container from scratch needs the variables. + +Now when changes are detected on `.py` and `.yaml` files on any of the `DEV_SOURCE_PATH` +directories it will trigger reload of `api`, `worker`, and `content` services. + + +## Troubleshooting + +### VersionConflict error + +Example: +```bash +api-1 | raise VersionConflict(dist, req).with_context(dependent_req) +api-1 | pkg_resources.VersionConflict: (pkg_foo 3.2.6 (/venv/lib/python3.11/site-packages), Requirement.parse('pkg_foo<3.1.13,>=3.1.12')) +``` + +Solution 1: +Clean up local build files: + +```bash +cd ~/projects/galaxy_ng +rm -rf .eggs +rm -rf build +rm -rf galaxy_ng.egg-info +``` + +Solution 2: + +- Ensure `LOCK_REQUIREMENTS` is set to `0` +- Ensure all your local checkouts are checked out to compatible branches + +### LLB definition error + +```bash +failed to solve: rpc error: code = Unknown desc = failed to solve with frontend dockerfile.v0: failed to create LLB definition: failed to do request: Head "http://localhost/v2/galaxy_ng/galaxy_ng/manifests/base": dial tcp [::1]:80: connect: connection refused +``` + +Solution + +```bash +export DOCKER_BUILDKIT=0 +export COMPOSE_DOCKER_CLI_BUILD=0 +``` + +## Tests + +### Unit tests +Run unit tests with docker compose [Running unit tests](../../galaxy_ng//tests/unit/README.md) + +### Integration tests +Run integration tests with docker compose, check the [Running integration tests](../../galaxy_ng/tests/integration/README.md) + +Run with legacy oci-env, check the [Running oci-env integration tests](../../docs/dev/integration_tests.md) + +## Tips and Tricks. + +--- + +### Debugging + +#### Step 1 - Add the breakpoint + +Edit the file you want to debug and add a breakpoint + +```python +__import__("ipdb").set_trace() +``` + +> [!TIP] +> Replace if you are using a different debugger, however the images has only **pdb** and **ipdb** installed. + +If you discover other ways of debugging, like connecting dap protocol or vscode debugger, please update this file! + +#### Step 2 - Now execute your stack or just the container you are trying to debug. + +Example: + +```bash +$ export DEV_SOURCE_PATH=galaxy_ng +$ docker compose -f dev/compose/standalone.yaml up migrations +# The container will keep running stopped on the breakpoint. +``` + +#### Step 3 - Attach + +```bash +$ docker compose -f dev/compose/standalone.yaml attach migrations +ipdb> +``` + +> [!IMPORTANT] +> To detach from the container DO NOT use Ctrl+c, +> instead, use Ctrl-p Ctrl-q + +--- + +**TBD** + +### Connecting to Database + +### Dumping and restoring the database + +### Populating testing data diff --git a/dev/compose/bin/README.md b/dev/compose/bin/README.md new file mode 100755 index 0000000000..b447b359d2 --- /dev/null +++ b/dev/compose/bin/README.md @@ -0,0 +1,48 @@ +# Dev Binary Tools + +## Reloader + +A Script that splits `DEV_SOURCE_PATH` variable and list all files on its directories, +then omit some unwanted files watching for `.py|yaml` only for changes. + +Then it calls `entr` to reload the passed command. + +## entr + +A tool to watch files for changes and reload +This file is compiled specifically for the container in use. + +LINK: https://eradman.com/entrproject/ + +In case recompilation is needed, run inside the container. + +```bash +# Install entr so we can have reload on worker processes +# the checkout is to fix a compilation error caused by this commit +# https://github.com/eradman/entr/commit/f9ac92d17e42236fe6b5e8492e087620173c7b24 +RUN git clone https://github.com/eradman/entr && \ + cd entr && \ + git checkout 0d2d92d6052624a1e03a2a654e98e1c49f9955d9 && \ + cp Makefile.linux Makefile && \ + make +``` + +Then copy the generated `entr` binary. + +## devinstall + +Install all paths listed on `DEV_SOURCE_PATH` as editable every time the compose starts. + +## wait + +Wait until a file is present on the shared volumes, + +```console +$ wait /etc/pulp/certs/filename; +Waiting for /etc/pulp/certs/filename ... +# sleep 2 secs +Waiting for /etc/pulp/certs/filename ... +# sleep 2 secs +Waiting for /etc/pulp/certs/filename ... +... +``` diff --git a/dev/compose/bin/devinstall b/dev/compose/bin/devinstall new file mode 100755 index 0000000000..79167d1eea --- /dev/null +++ b/dev/compose/bin/devinstall @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +if [ -z "${DEV_SOURCE_PATH}" ]; then + echo "DEV_SOURCE_PATH is not defined, skipping editable installs." + exit 0 +fi + +IFS=':' read -ra SRC_PATH_LIST <<< "$DEV_SOURCE_PATH" + +for item in "${SRC_PATH_LIST[@]}"; do + src_path="/src/${item}" + if [[ -d "$src_path" ]]; then + echo "Installing path ${item} in editable mode." + + if [[ "${LOCK_REQUIREMENTS}" -eq "1" ]]; then + pip3.11 install --no-cache-dir --no-deps --editable "$src_path" >/dev/null + else + pip3.11 install --no-cache-dir --editable "$src_path" >/dev/null + fi + + else + echo "WARNING: Source path ${item} is not a directory." + fi +done + diff --git a/dev/compose/bin/entr b/dev/compose/bin/entr new file mode 100755 index 0000000000..b9b3c55826 Binary files /dev/null and b/dev/compose/bin/entr differ diff --git a/dev/compose/bin/pulpcore-api b/dev/compose/bin/pulpcore-api new file mode 100755 index 0000000000..b9abda9963 --- /dev/null +++ b/dev/compose/bin/pulpcore-api @@ -0,0 +1,32 @@ +#!/bin/bash +set -o errexit +set -o nounset + + +readonly GUNICORN=${GUNICORN:-'/venv/bin/gunicorn'} +readonly GUNICORN_FORWARDED_ALLOW_IPS="${GUNICORN_FORWARDED_ALLOW_IPS:-}" +readonly GUNICORN_WORKERS="${GUNICORN_WORKERS:-4}" +readonly GUNICORN_LOGGER_CLASS="${GUNICORN_LOGGER_CLASS:-}" +readonly GUNICORN_TIMEOUT="${GUNICORN_TIMEOUT:-60}" + +readonly BIND_HOST='0.0.0.0' +readonly BIND_PORT=${GUNICORN_PORT:-24817} +readonly APP_MODULE='pulpcore.app.wsgi:application' + + +GUNICORN_OPTIONS=( + --bind "${BIND_HOST}:${BIND_PORT}" + --workers "${GUNICORN_WORKERS}" + --access-logfile - + --limit-request-field_size 32768 + --timeout "${GUNICORN_TIMEOUT}" +) + +if [[ -n "${GUNICORN_FORWARDED_ALLOW_IPS}" ]]; then + GUNICORN_OPTIONS+=(--forwarded-allow-ips "${GUNICORN_FORWARDED_ALLOW_IPS}") +fi +if [[ -n "${GUNICORN_LOGGER_CLASS}" ]]; then + GUNICORN_OPTIONS+=(--logger-class "${GUNICORN_LOGGER_CLASS}") +fi + +exec "${GUNICORN}" "${GUNICORN_OPTIONS[@]}" "${APP_MODULE}" \ No newline at end of file diff --git a/dev/compose/bin/pulpcore-content b/dev/compose/bin/pulpcore-content new file mode 100755 index 0000000000..f0d43f1784 --- /dev/null +++ b/dev/compose/bin/pulpcore-content @@ -0,0 +1,20 @@ +#!/bin/bash + +set -o errexit +set -o nounset + + +readonly GUNICORN_WORKERS=${GUNICORN_WORKERS:-4} + +readonly BIND_HOST='0.0.0.0' +readonly BIND_PORT="${GUNICORN_PORT:-24816}" +readonly WORKER_CLASS='aiohttp.GunicornWebWorker' +readonly APP_MODULE='pulpcore.content:server' + + +exec gunicorn \ + --bind "${BIND_HOST}:${BIND_PORT}" \ + --worker-class "${WORKER_CLASS}" \ + --workers "${GUNICORN_WORKERS}" \ + --access-logfile - \ + "${APP_MODULE}" diff --git a/dev/compose/bin/reloader b/dev/compose/bin/reloader new file mode 100755 index 0000000000..90723be0bd --- /dev/null +++ b/dev/compose/bin/reloader @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +if [ -z "${DEV_SOURCE_PATH}" ]; then + echo "DEV_SOURCE_PATH is not defined, running without reload." + $1 +else + # List all files in the DEV_SOURCE_PATH and watch for changes and use entr to SIGKILL + restart + echo "Watching file changes on $DEV_SOURCE_PATH" + find $(echo $DEV_SOURCE_PATH | tr ':' '\n' | while read item; do echo -n /src/$item\ ; done) \( -path /src/galaxy_ng/.venv -o -path /src/galaxy_ng/build -o -path /src/galaxy_ng/.eggs \) -prune -o -name '*.py' -o -name '*.yaml' | /src/galaxy_ng/dev/compose/bin/entr -n -r timeout -k 5 0 $1 +fi diff --git a/dev/compose/bin/wait b/dev/compose/bin/wait new file mode 100755 index 0000000000..65d08290e0 --- /dev/null +++ b/dev/compose/bin/wait @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +while [[ ! -f $1 ]]; do + echo "Waiting for $1 ..."; + sleep 2; +done; diff --git a/dev/compose/database/README.md b/dev/compose/database/README.md new file mode 100644 index 0000000000..d878677e75 --- /dev/null +++ b/dev/compose/database/README.md @@ -0,0 +1,21 @@ +# Database Field Encryption + +Pulp relies on the file `database_fields.symmetric.key` being on `/etc/pulp/certs` +For development purposes there is a hardcoded key on this folder. + +```bash +mkdir -p /etc/pulp/certs/; +cp database_fields.symmetric.key /etc/pulp/certs/database_fields.symmetric.key +``` + +> NOTE: For development it is better to use a persistent key, so database dumps +> can be easily restored across dev environments. + +## Generating a new key + +```bash +rpm -q openssl || dnf -y install openssl; +mkdir -p /etc/pulp/certs/; +openssl rand -base64 32 > /etc/pulp/certs/database_fields.symmetric.key; +chmod 640 /etc/pulp/certs/database_fields.symmetric.key; +``` diff --git a/dev/compose/database/database_fields.symmetric.key b/dev/compose/database/database_fields.symmetric.key new file mode 100644 index 0000000000..14697a0310 --- /dev/null +++ b/dev/compose/database/database_fields.symmetric.key @@ -0,0 +1 @@ +DNmNdwgyZugTax9S64J0FITTr9IHPxbuoF1F1CGPr68= diff --git a/dev/compose/signing/README.md b/dev/compose/signing/README.md new file mode 100644 index 0000000000..5341520bda --- /dev/null +++ b/dev/compose/signing/README.md @@ -0,0 +1,181 @@ +# Node Setup + +Requirements: + +- system has a running `gpg-agent` +- required files are in place + +```bash +# Keys +/etc/pulp/certs/signing-secret.key +/etc/pulp/certs/signing-secret.key.password.tx +/etc/pulp/certs/signing-public.key + +# Scripts +/var/lib/pulp/scripts/collection_sign.sh +/var/lib/pulp/scripts/container_sign.sh + +# Workarounds for Pulp envvar limitation. +# required only if GPGHOME differs from running user HOME +# ./setup_gpg_workarounds.sh +/etc/pulp/certs/HOME.workaround.txt +/etc/pulp/certs/GNUPGHOME.workaround.txt +``` + +## Workers + +```bash +./setup_gpg_keys.sh +``` + +## Run once, on any node, after workers are alive + +```bash +./setup_gpg_keys.sh +./setup_signing_services.sh +./setup_repo_keyring.sh +``` + +--- + +# How Signing Works + +## 1. GPG and keys + +System needs a GPGHOME and a KEYRING + +Usually: + +```bash +~/.gnupg/pubring.kbx +``` + +> NOTE: If `GNUPGHOME` differs from the current user `HOME`, then it is required to save the path to `/etc/pulp/certs/HOME.workaround.txt` and `/etc/pulp/certs/GNUPGHOME.workaround.txt` as a workaround due to a limitation on Pulp SigningService that cannot access external environment variables. + +The keyring must have the secret-key for signing and public key for verification. +The files are located on this directory and must be copied to `/etc/pulp/certs` + +> These keys are for development purposes only! NEVER USE THOSE IN PRODUCTION! + +- cp `signing-secret.key` -> `/etc/pulp/certs/signing-secret.key` +- cp `signing-public.key` -> `/etc/pulp/certs/signing-public.key` + +Key information: + +- fingerprint is `FB8B3F2D24BCAF7EFDF793A9F37575C52D4F16F3` +- short id `F37575C52D4F16F3` +- admin ID `galaxydev@ansible.com` +- passphrase `Galaxy2024` + +The passphrase must be added to a file named `/etc/pulp/certs/signing-secret.key.password.txt` + +```bash +echo "Galaxy2024" > /etc/pulp/certs/signing-secret.key.password.txt +``` + +To add both keys to the keyring start the agent and run: + +```bash +gpgconf --kill gpg-agent && gpg --batch --no-default-keyring --import /etc/pulp/certs/signing-secret.key; +``` + +it is also require to adjust the key trust level: + +```bash +(echo 5; echo y; echo save) | gpg --command-fd 0 --no-tty --no-greeting -q --edit-key 'FB8B3F2D24BCAF7EFDF793A9F37575C52D4F16F3' +``` + +Ensure key is added and trusted + +```bash +gpg --list-secret-keys +``` + +Must output a `[ultimate]` trust level key. + + +## 2. Signing Scripts and Signing Service + +> For collection the script is `collection_sign.sh` and for containers `container_sign.sh`, both +> located on this directory, the files must be copied to `/var/lib/pulp/scripts/` + +- cp `collection_sign.sh` -> `/var/lib/pulp/scripts/collection_sign.sh` +- cp `container_sign.sh` -> `/var/lib/pulp/scripts/container_sign.sh` + +> NOTE: The path for the signing service can be any path, `/var/lib/pulp/script` is just a convention. + +Use `pulpcore-manager add-signing-service` to register the signing service. + +- name: container-default | ansible-default +- script: path/to/executable that can access the GPG keyring +- key: fingerprint_of_gpg_key that lives on the keyring + +Examples: + +For collections: + +```bash +pulpcore-manager add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh F37575C52D4F16F3 +``` + +For containers: + +```bash +pulpcore-manager add-signing-service container-default /var/lib/pulp/scripts/container_sign.sh F37575C52D4F16F3 --class container:ManifestSigningService; +``` + +> NOTE: The command above will actually try to sign an arbitrary artifact in order to validate the key. + +## 3. Signing Collections and Containers + +Galaxy will call the `sign` endpoint passing the following information: + +- content unit id: +- signing_service: foo +- repository: + +Then: + +Signature content is created based on artifact and added to the same +repo as the content and contains a foreign_key to the content signed. + +## 5. Public keys and Validation + +### Collection + +Galaxy exposes collection signature on UI and API. + +### Container + +For Containers the signature is added to the registry extended API so usually +clients such as `podman` will automatically fetch and validate the signature +if configured with a proper `policies.json` file. + +### Public Key + +When validating a signature the client `ansible-galaxy` will needs to have +the public key on a local keyring, there are 2 ways to get the public key. + +- On Galaxy UI there is a menu on sidebar `Signature Keys` that exposes +the public keys and user can download it. + +- On API there is the `/signing-services` API to expose the same information. + +> NOTE: User needs to create the keyring locally and import the key manually. + +## 6. Signature Upload + +Galaxy can be alternatively configured to accept signature upload instead +of relying on a local GPG keyring and SigningService to generate it. + +In this case the artifact can be externally signed (e.g using a hardware token) +and then signature uploaded during the approval dashboard process. + +1. Repository needs a `gpgkey` field containing a public key. + * `pulpcore-manager set-repo-keyring --repository staging --publickeypath /etc/pulp/certs/signing-public.key -y;` + * OR + * `pulpcore-manager set-repo-keyring --repository staging --keyring ~/.gnupg/pubring.kbx -y;` +2. CollectionVersion is uploaded to the repo and ends in `pending` state on approval dashboard when system has `REQUIRE_SIGNATURE_FOR_APPROVAL` +3. Signature is uploaded to the same repo, file matches the collectionversion (e.g `namespace-collection-1.2.3.asc`) name and repo uses `gpgkey` to verify the signature is valid. + +In this case there is no signing service involved, no need for GPG keyrings. diff --git a/dev/compose/signing/collection_sign.sh b/dev/compose/signing/collection_sign.sh new file mode 100755 index 0000000000..685309ad50 --- /dev/null +++ b/dev/compose/signing/collection_sign.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +GNUPGHOME=$(cat /etc/pulp/certs/GNUPGHOME.workaround.txt) + +gpg --lock-never \ + --quiet \ + --batch \ + --pinentry-mode loopback \ + --yes \ + --passphrase $(cat /etc/pulp/certs/signing-secret.key.password.txt) \ + --homedir "$GNUPGHOME" \ + --detach-sign \ + --default-key $PULP_SIGNING_KEY_FINGERPRINT \ + --armor \ + --output $1.asc \ + $1 + +[ $? -eq 0 ] && echo {\"file\": \"$1\", \"signature\": \"$1.asc\"} || exit $? diff --git a/dev/compose/signing/container_sign.sh b/dev/compose/signing/container_sign.sh new file mode 100755 index 0000000000..160a1d6b2b --- /dev/null +++ b/dev/compose/signing/container_sign.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +MANIFEST_PATH=$1 +HOME=$(cat /etc/pulp/certs/HOME.workaround.txt) + +skopeo standalone-sign \ + --passphrase-file /etc/pulp/certs/signing-secret.key.password.txt \ + --output $SIG_PATH \ + $MANIFEST_PATH $REFERENCE $PULP_SIGNING_KEY_FINGERPRINT + +[ $? -eq 0 ] && echo {\"signature_path\": \"$SIG_PATH\"} || exit $? diff --git a/dev/compose/signing/setup_gpg_keys.sh b/dev/compose/signing/setup_gpg_keys.sh new file mode 100755 index 0000000000..d20d018301 --- /dev/null +++ b/dev/compose/signing/setup_gpg_keys.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# ensure agent is running +gpgconf --kill gpg-agent + +# Import the key +gpg --batch --no-default-keyring --import /etc/pulp/certs/signing-secret.key; + +# Set the key trust level +(echo 5; echo y; echo save) | gpg --command-fd 0 --no-tty --no-greeting -q --edit-key 'FB8B3F2D24BCAF7EFDF793A9F37575C52D4F16F3' trust; diff --git a/dev/compose/signing/setup_gpg_workarounds.sh b/dev/compose/signing/setup_gpg_workarounds.sh new file mode 100755 index 0000000000..59a068d824 --- /dev/null +++ b/dev/compose/signing/setup_gpg_workarounds.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +DEFAULT_GNUPGHOME="$HOME/.gnupg" +CURRENT_GNUPGHOME="${GNUPGHOME:-$DEFAULT_GNUPGHOME}" + +# Remove both `.gnupg` and potential trailing `/` after `.gnupg` +CURRENT_HOME="${CURRENT_GNUPGHOME%/.gnupg/}" +CURRENT_HOME="${CURRENT_HOME%/.gnupg}" + +echo "$CURRENT_GNUPGHOME" > /etc/pulp/certs/GNUPGHOME.workaround.txt +echo "$CURRENT_HOME" > /etc/pulp/certs/HOME.workaround.txt diff --git a/dev/compose/signing/setup_repo_keyring.sh b/dev/compose/signing/setup_repo_keyring.sh new file mode 100755 index 0000000000..7eefc10b46 --- /dev/null +++ b/dev/compose/signing/setup_repo_keyring.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +for repo in staging published; do + pulpcore-manager set-repo-keyring --repository $repo --keyring /root/.gnupg/pubring.kbx -y; +done + diff --git a/dev/compose/signing/setup_signing_services.sh b/dev/compose/signing/setup_signing_services.sh new file mode 100755 index 0000000000..9de430badf --- /dev/null +++ b/dev/compose/signing/setup_signing_services.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Collection +HAS_COLLECTION_SIGNING=$(pulpcore-manager shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="ansible-default").count())' 2>/dev/null || true) +if [[ "$HAS_COLLECTION_SIGNING" -eq "0" ]]; then + pulpcore-manager add-signing-service ansible-default /var/lib/pulp/scripts/collection_sign.sh F37575C52D4F16F3 +else + echo "Collection Signing Service Already exists" +fi + +# Container +HAS_CONTAINER_SIGNING=$(pulpcore-manager shell -c 'from pulpcore.app.models import SigningService;print(SigningService.objects.filter(name="container-default").count())' 2>/dev/null || true) +if [[ "$HAS_CONTAINER_SIGNING" -eq "0" ]]; then + pulpcore-manager add-signing-service container-default /var/lib/pulp/scripts/container_sign.sh F37575C52D4F16F3 --class container:ManifestSigningService +else + echo "Container Signing Service Already exists" +fi diff --git a/dev/compose/signing/signing-public.key b/dev/compose/signing/signing-public.key new file mode 100644 index 0000000000..5a27c5c58d --- /dev/null +++ b/dev/compose/signing/signing-public.key @@ -0,0 +1,18 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBGcBkDMBCADcXzLFpIHjlPSNtNsQBvtnRCcqBUKUk7Xz8eZIHVIOt6lF3TPN +ZMfyyPhaihLlKzJQdhzDoc3/hdTKRxfBltrff9nYx5dHWegomVL1i/S0HAB7MOqn +Sj1dErkQEPhYbno1VI5ml71LMeu/anDmEkEhtvkVcqWnXs6DiiudU0Tmc4W/+lVH +Cluk+ISkg1/3pxLAQjSiPpWg/YB/cNDcuuHxueC0KqIHgGIrx0iDI8Tq7KlNbcOV +Cyj0Xpcv653dDlJZAA9O/ljfleNa08NxspRBEoMXhIKuOL/owVY65ZDCBBgg5GGH +uKGnFBITmKd9dWUVRezj7sKu5p4ErATgRi7lABEBAAG0JkdhbGF4eSBEZXYgS2V5 +IDxnYWxheHlkZXZAYW5zaWJsZS5jb20+iQFRBBMBCAA7FiEE+4s/LSS8r37995Op +83V1xS1PFvMFAmcBkDMCGwMFCwkIBwICIgIGFQoJCAsCBBYCAwECHgcCF4AACgkQ +83V1xS1PFvPDmAgAwJJ/TPDrW9clGbObG6d3RTzM08R1Hvx1xRhpeWGB3uPag+zS +LmmgSHUskWWChRr1mcdIfH60HgdhR1PIk9sjKsTWorzS0Polit7Dd7pS+e1sFdGe +MFGX0ljDYF558yaX5NCVBstIaqxTuHc+SGOYT8/7w5P7/y7Li9OBO4vpPx6gq/RV +J1x6GnDELphGeyzAlPzevRwVr7h27kfIidI8vD2LYs3qV7scnOcW+hFcJ5utLee4 +4ia1BDCYDRgOgz4kMoz/rGQ4rR15aTclihc0+puZJrvsUQMPS1+q9XoJPJs+s+Gr +kY24YID/BNUXy0j/BR5vQQvCSCaWdBDGyFvV7A== +=0hZK +-----END PGP PUBLIC KEY BLOCK----- diff --git a/dev/compose/signing/signing-secret.key b/dev/compose/signing/signing-secret.key new file mode 100644 index 0000000000..3408182cc6 --- /dev/null +++ b/dev/compose/signing/signing-secret.key @@ -0,0 +1,33 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQPGBGcBkDMBCADcXzLFpIHjlPSNtNsQBvtnRCcqBUKUk7Xz8eZIHVIOt6lF3TPN +ZMfyyPhaihLlKzJQdhzDoc3/hdTKRxfBltrff9nYx5dHWegomVL1i/S0HAB7MOqn +Sj1dErkQEPhYbno1VI5ml71LMeu/anDmEkEhtvkVcqWnXs6DiiudU0Tmc4W/+lVH +Cluk+ISkg1/3pxLAQjSiPpWg/YB/cNDcuuHxueC0KqIHgGIrx0iDI8Tq7KlNbcOV +Cyj0Xpcv653dDlJZAA9O/ljfleNa08NxspRBEoMXhIKuOL/owVY65ZDCBBgg5GGH +uKGnFBITmKd9dWUVRezj7sKu5p4ErATgRi7lABEBAAH+BwMCHPHopLXNy/H5x9sv +w+Ca3lAqKjqFX0MTJElj5dOndRgdUh3oUAoYfSB6/F3OHGIl3zlKhh5QrP8UUFjb +PFZjJYVoEU+ymBNucXMLBAymRAb8FxsDD94G75TYZxg+PXjQCnUbBxeeFoYl5vA2 +35KMZpGAIzhhNAjZlyFoX7GiMqdWxaKuUkR+NAi7cEWjhrgTE8gTtPfDl8aWN0if +2b2Jl8YXXlG32P+YN4qn0P91Z/nRAHPnrm0wGbDtcFCPGLGl2wXtoY2eb/2gwruU +r0uh3LEXq/27G6EfgX5ADikmXI4W6VdyCkkIRjbAv7jfou3jEseiy1jI2oGCvWVn +k0y79g04oTSjDGadW0gvZbGA87HndnS7p7S0oVmPVuhzKot894VH3TCCr1V3nO+Z +y7FfjdCDkN9bZ4ijoI0aXdOGZD1lbRHlq2UCaCe9ZKy27xFpqnkqlfuk59wjmcVJ +xv+w11kPh1QW51xY5quv+CXKFd3DF1L9auRvRBJP+UZa6Qq/2w5YZVPKLj9VF5oS +4iQON3l/4casojT/s3F7N1+HfsSZH6y3+WFr9up806GUte8eQUiqHxAnKfKdoUC9 +6izFeGWi6AL57vEiLp939RWh5AERnLHHAp4KpR4fDbwoBREbgqtBe2FHCJCnhqVl +/E/1Ydle1bXV/swpPI5lsKqjf2EhKK2hFUTmBlDkQjfN4tVU1BvC6s7HGD/F0OQu +loZghUisN66t9cA0rZHYzvd0yIknglDXfjgzj+rdXaMoaqOKV8IgEa/SwdNjLw5A +m/dFXCfz3AuezjHrdAw8y1+F/wRlezySFRmd6REStMgHtWJeRgJiv42ZgktBJziL +a671wp6c8bV0Ataa43WIBu7KZbq6NuiJnkZHNI7BILpENA98yV6WNetCb7SsZ6OQ +lPdzW0wp9BILtCZHYWxheHkgRGV2IEtleSA8Z2FsYXh5ZGV2QGFuc2libGUuY29t +PokBUQQTAQgAOxYhBPuLPy0kvK9+/feTqfN1dcUtTxbzBQJnAZAzAhsDBQsJCAcC +AiICBhUKCQgLAgQWAgMBAh4HAheAAAoJEPN1dcUtTxbzw5gIAMCSf0zw61vXJRmz +mxund0U8zNPEdR78dcUYaXlhgd7j2oPs0i5poEh1LJFlgoUa9ZnHSHx+tB4HYUdT +yJPbIyrE1qK80tD6JYrew3e6UvntbBXRnjBRl9JYw2BeefMml+TQlQbLSGqsU7h3 +PkhjmE/P+8OT+/8uy4vTgTuL6T8eoKv0VSdcehpwxC6YRnsswJT83r0cFa+4du5H +yInSPLw9i2LN6le7HJznFvoRXCebrS3nuOImtQQwmA0YDoM+JDKM/6xkOK0deWk3 +JYoXNPqbmSa77FEDD0tfqvV6CTybPrPhq5GNuGCA/wTVF8tI/wUeb0ELwkgmlnQQ +xshb1ew= +=hw1Q +-----END PGP PRIVATE KEY BLOCK----- diff --git a/dev/compose/signing/signing-secret.key.password.txt b/dev/compose/signing/signing-secret.key.password.txt new file mode 100644 index 0000000000..7e0c21b8b5 --- /dev/null +++ b/dev/compose/signing/signing-secret.key.password.txt @@ -0,0 +1 @@ +Galaxy2024 diff --git a/dev/compose/standalone.yaml b/dev/compose/standalone.yaml new file mode 100644 index 0000000000..d1b5685268 --- /dev/null +++ b/dev/compose/standalone.yaml @@ -0,0 +1,309 @@ +x-common-env: &common-env + + GNUPGHOME: /root/.gnupg/ + + DJANGO_SUPERUSER_USERNAME: admin + DJANGO_SUPERUSER_EMAIL: admin@example.com + DJANGO_SUPERUSER_PASSWORD: admin + + POSTGRES_USER: galaxy_ng + POSTGRES_PASSWORD: galaxy_ng + POSTGRES_DB: galaxy_ng + + # no spying + PULP_ANALYTICS: 'false' + + # normally goes into settings.py ... + PULP_DATABASES__default__ENGINE: django.db.backends.postgresql + PULP_DATABASES__default__NAME: galaxy_ng + PULP_DATABASES__default__USER: galaxy_ng + PULP_DATABASES__default__PASSWORD: galaxy_ng + PULP_DATABASES__default__HOST: postgres + PULP_DATABASES__default__PORT: 5432 + + PULP_DEBUG: 1 + PULP_GALAXY_DEPLOYMENT_MODE: 'standalone' + PULP_DEFAULT_FILE_STORAGE: "pulpcore.app.models.storage.FileSystem" + PULP_REDIRECT_TO_OBJECT_STORAGE: 'false' + + # Hostname and prefix has to be correct + PULP_GALAXY_API_PATH_PREFIX: '/api/galaxy/' + PULP_CONTENT_PATH_PREFIX: '/pulp/content/' + PULP_ANSIBLE_API_HOSTNAME: 'http://localhost:5001' + PULP_ANSIBLE_CONTENT_HOSTNAME: "http://localhost:5001" + PULP_CONTENT_ORIGIN: "http://localhost:5001" + PULP_CSRF_TRUSTED_ORIGINS: "['http://localhost']" + + # signing ... + PULP_GALAXY_AUTO_SIGN_COLLECTIONS: 'false' + PULP_GALAXY_REQUIRE_CONTENT_APPROVAL: 'true' + PULP_GALAXY_REQUIRE_SIGNATURE_FOR_APPROVAL: 'false' + PULP_GALAXY_COLLECTION_SIGNING_SERVICE: 'ansible-default' + PULP_GALAXY_CONTAINER_SIGNING_SERVICE: 'container-default' + + # pulp container ... + PULP_TOKEN_AUTH_DISABLED: 'false' + PULP_TOKEN_SERVER: 'http://localhost:5001/token/' + PULP_TOKEN_SIGNATURE_ALGORITHM: 'ES256' + PULP_PUBLIC_KEY_PATH: '/src/galaxy_ng/dev/common/container_auth_public_key.pem' + PULP_PRIVATE_KEY_PATH: '/src/galaxy_ng/dev/common/container_auth_private_key.pem' + + # auth ... + PULP_GALAXY_AUTHENTICATION_CLASSES: "['rest_framework.authentication.SessionAuthentication', 'rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.BasicAuthentication']" + PULP_ANSIBLE_BASE_JWT_VALIDATE_CERT: 'false' + PULP_ANSIBLE_BASE_JWT_KEY: 'https://localhost' + PULP_GALAXY_FEATURE_FLAGS__external_authentication: 'true' + + # disable user/group modifications + PULP_ALLOW_LOCAL_RESOURCE_MANAGEMENT: 'true' + + # role content workaround .. + PULP_ANSIBLE_BASE_ROLES_REQUIRE_VIEW: 'false' + + # Resource server + # This disables the attempt for resource syncing + PULP_RESOURCE_SERVER_SYNC_ENABLED: 'false' + # Set above to 'true' if/when RESOURCE_SERVER is configured + # The next variables must be configured to enable resource sync + # PULP_RESOURCE_SERVER__URL='https://localhost' + # PULP_RESOURCE_SERVER__SECRET_KEY='?' + # PULP_RESOURCE_SERVER__VALIDATE_HTTPS='false' + + # Integration test settings + HUB_API_ROOT: 'http://localhost:5001/api/galaxy/' + HUB_TEST_MARKS: deployment_standalone or all + HUB_USE_MOVE_ENDPOINT: 'true' + CONTAINER_REGISTRY: 'localhost:5001' + + # Unpin dependencies on setup.py if set to 0 + LOCK_REQUIREMENTS: 0 + + # DEV EDITABLE STUFF + # To enable editable installs of local checkouts set DEV_SOURCE_PATH keeping the ordering as follows: + # "dynaconf:pulpcore:galaxy_importer:pulp_ansible:pulp_container:galaxy_ng:django-ansible-base" + # This can be done as part of the `docker compose` call: + # $ DEV_SOURCE_PATH="pulp_container:galaxy_ng" docker compose -f dev/compose/standalone.yaml up + DEV_SOURCE_PATH: + +# allow attaching to the running container +x-debugging: &debugging + stdin_open: true + tty: true + +services: + base_img: + build: + context: ../../ + dockerfile: Dockerfile + image: "localhost/galaxy_ng/galaxy_ng:base" + + base_img_dev: # Extends base_img with extra files and dev tools + depends_on: + - base_img + build: + context: . + dockerfile: Dockerfile.dev + args: + <<: *common-env + image: "localhost/galaxy_ng/galaxy_ng:dev" + + redis: + image: "redis:5" + + postgres: + image: "postgres:13" + ports: + - '5433:5432' + environment: + <<: *common-env + healthcheck: + test: ["CMD", "pg_isready", "-U", "galaxy_ng"] + interval: 10s + retries: 5 + # Uncomment below to spam out every DB statement to the service stderr + # WARNING: enabling log_statement=all makes database slower + # command: ["postgres", "-c", "log_statement=ddl", "-c", "log_destination=stderr"] + + migrations: + image: "localhost/galaxy_ng/galaxy_ng:dev" + depends_on: + - base_img_dev + - postgres + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp_standalone:/var/lib/pulp" + - "../../../:/src" + - "../../:/app" + environment: + <<: *common-env + user: root + <<: *debugging + command: | + bash -c " + set -e; + rm -rf /var/lib/pulp/.migrated; + /src/galaxy_ng/dev/compose/bin/devinstall; + + pulpcore-manager check --database default; + pulpcore-manager migrate; + pulpcore-manager shell < /src/galaxy_ng/dev/common/setup_test_data.py; + pulpcore-manager createsuperuser --noinput || true; + + touch /var/lib/pulp/.migrated; + " + + api: + image: "localhost/galaxy_ng/galaxy_ng:dev" + depends_on: + - base_img_dev + - postgres + - migrations + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp_standalone:/var/lib/pulp" + - "../../../:/src" + - "../../:/app" + environment: + <<: *common-env + extra_hosts: + localhost: "host-gateway" + networks: + - default + - service-mesh + user: root + <<: *debugging + command: | + bash -c " + /src/galaxy_ng/dev/compose/bin/devinstall; + /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated; + + /src/galaxy_ng/dev/compose/bin/reloader /src/galaxy_ng/dev/compose/bin/pulpcore-api + " + + content: + image: "localhost/galaxy_ng/galaxy_ng:dev" + depends_on: + - base_img_dev + - postgres + - migrations + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp_standalone:/var/lib/pulp" + - "../../../:/src" + - "../../:/app" + environment: + <<: *common-env + extra_hosts: + localhost: "host-gateway" + networks: + - default + - service-mesh + user: root + <<: *debugging + command: | + bash -c " + /src/galaxy_ng/dev/compose/bin/devinstall; + /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated; + + /src/galaxy_ng/dev/compose/bin/reloader /src/galaxy_ng/dev/compose/bin/pulpcore-content + " + + worker: + image: "localhost/galaxy_ng/galaxy_ng:dev" + depends_on: + - base_img_dev + - postgres + - migrations + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp_standalone:/var/lib/pulp" + - "../../../:/src" + - "../../:/app" + environment: + <<: *common-env + user: root + <<: *debugging + command: | + bash -c " + /src/galaxy_ng/dev/compose/bin/devinstall; + /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated; + + # Worker needs gpg in order to consume signing tasks; + gpg --list-secret-keys; + + /src/galaxy_ng/dev/compose/bin/reloader /venv/bin/pulpcore-worker + " + + manager: + image: "localhost/galaxy_ng/galaxy_ng:dev" + depends_on: + - base_img_dev + - postgres + - migrations + - worker + volumes: + - "etc_pulp_certs:/etc/pulp/certs" + - "var_lib_pulp_standalone:/var/lib/pulp" + - "../../../:/src" + - "../../:/app" + environment: + <<: *common-env + user: root + <<: *debugging + command: | + bash -c " + /src/galaxy_ng/dev/compose/bin/devinstall; + /src/galaxy_ng/dev/compose/bin/wait /var/lib/pulp/.migrated; + + # Give some time for API to start; + sleep 5; + + # Setup signing services; + gpg --list-secret-keys; + /src/galaxy_ng/dev/compose/signing/setup_signing_services.sh; + echo 'Signing Services'; + curl -s -u $$DJANGO_SUPERUSER_USERNAME:$$DJANGO_SUPERUSER_PASSWORD http://api:24817/api/galaxy/pulp/api/v3/signing-services/?fields=name,script,pubkey_fingerprint | python -m json.tool; + + # Setup repository gpgkey for upload verification; + /src/galaxy_ng/dev/compose/signing/setup_repo_keyring.sh; + + echo ' '; + echo '###################### API ROOT ##############################'; + curl -s -u $$DJANGO_SUPERUSER_USERNAME:$$DJANGO_SUPERUSER_PASSWORD http://api:24817/api/galaxy/ | python -m json.tool; + echo '################### DEV_SOURCE_PATH ##########################'; + echo $$DEV_SOURCE_PATH; + echo ' '; + echo '######################## READY ###############################'; + echo ' '; + echo 'Credentials: ' $$DJANGO_SUPERUSER_USERNAME:$$DJANGO_SUPERUSER_PASSWORD; + echo 'API Spec: http://localhost:5001/api/galaxy/v3/swagger-ui/'; + echo 'Django Admin: docker compose -f dev/compose/standalone.yaml exec manager pulpcore-manager'; + echo 'Settings list: docker compose -f dev/compose/standalone.yaml exec manager dynaconf list'; + echo 'Docs: https://github.com/ansible/galaxy_ng/blob/master/dev/compose/README.md'; + echo '##############################################################'; + + # Keep it running indefinitely to enable `docker compose -f ... exec manager /bin/bash`; + tail -f /dev/null + " + + nginx: + image: "nginx:latest" + depends_on: + - postgres + - migrations + - api + - content + ports: + - '5001:5001' + volumes: + - '../nginx/nginx.conf:/etc/nginx/nginx.conf:ro' + +volumes: + var_lib_pulp_standalone: + name: var_lib_pulp_standalone + etc_pulp_certs: + name: etc_pulp_certs + +networks: + service-mesh: + name: service-mesh diff --git a/dev/nginx/nginx.conf b/dev/nginx/nginx.conf new file mode 100644 index 0000000000..73f91b11d4 --- /dev/null +++ b/dev/nginx/nginx.conf @@ -0,0 +1,92 @@ +# TODO: Support IPv6. +# TODO: Maybe serve multiple `location`s, not just one. + +# The "nginx" package on fedora creates this user and group. +# user nginx nginx; + +# Gunicorn docs suggest this value. +worker_processes 1; +# daemon off; +events { + worker_connections 1024; # increase if you have lots of clients + accept_mutex off; # set to 'on' if nginx worker_processes > 1 +} + +http { + include mime.types; + # fallback in case we can't determine a type + default_type application/octet-stream; + sendfile on; + + # If left at the default of 1024, nginx emits a warning about being unable + # to build optimal hash types. + types_hash_max_size 4096; + + upstream pulp-content { + server content:24816; + } + + upstream pulp-api { + server api:24817; + } + + server { + # Gunicorn docs suggest the use of the "deferred" directive on Linux. + listen 5001 default_server deferred; + server_name $hostname; + + # The default client_max_body_size is 1m. Clients uploading + # files larger than this will need to chunk said files. + client_max_body_size 10m; + + # Gunicorn docs suggest this value. + keepalive_timeout 5; + + location /pulp/content/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-content; + } + + location /api/galaxy/api/v3/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + client_max_body_size 0; + } + + location /auth/login/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + } + + # include pulp/*.conf; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + # most pulp static files are served through whitenoise + # http://whitenoise.evans.io/en/stable/ + } + + } +} +