From becd1449a272d93f977e80f6576780e5699bccd7 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Wed, 8 May 2024 02:14:39 -0700 Subject: [PATCH] feat: options to configure local registry Signed-off-by: Thuan Vo --- .github/workflows/test.yaml | 88 +++++++++++++++++++ README.md | 41 +++++++++ action.yml | 20 +++++ cleanup.sh | 12 ++- kind.sh | 34 ++++++++ main.sh | 28 +++++- registry.sh | 170 ++++++++++++++++++++++++++++++++++++ 7 files changed, 385 insertions(+), 8 deletions(-) create mode 100755 registry.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a961b73..5fcae98 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -151,3 +151,91 @@ jobs: run: | kubectl cluster-info kubectl get nodes + + test-without-registry: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + + - name: Create kind cluster without registry + uses: ./ + with: + registry: false + + - name: Test + run: | + kubectl cluster-info + kubectl get storageclass standard + + if [[ -n "$(docker ps --filter "name=kind-registry" --format "{{.ID}}")" ]]; then + echo "Registry is present" + exit 1 + fi + + test-with-registry: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + + - name: Create kind cluster with registry + uses: ./ + with: + registry: true + registry_name: custom-registry + registry_port: 5001 + + - name: Test + run: | + kubectl cluster-info + kubectl get storageclass standard + + if [[ -z "$(docker ps --filter "name=custom-registry" --format "{{.ID}}")" ]]; then + echo "Registry is not present" + exit 1 + fi + + local_reg=custom-registry:5001 + + docker pull busybox + docker tag busybox $local_reg/localbusybox + docker push $local_reg/localbusybox + + kubectl create job test --image=$local_reg/localbusybox + kubectl wait --for=condition=complete --timeout=30s job/test + + test-with-registry-and-delete-enabled: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + + - name: Create kind cluster with registry and delete enabled + uses: ./ + with: + registry: true + registry_name: custom-registry + registry_port: 5001 + registry_enable_delete: true + + - name: Test + run: | + kubectl cluster-info + kubectl get storageclass standard + + if [[ -z "$(docker ps --filter "name=custom-registry" --format "{{.ID}}")" ]]; then + echo "Registry is not present" + exit 1 + fi + + local_reg=custom-registry:5001 + + docker pull busybox + docker tag busybox $local_reg/localbusybox + + DIGEST=$(docker push $local_reg/localbusybox | grep -oE 'sha256:\w+') + + curl -X DELETE custom-registry:5001/v2/localbusybox/manifests/$DIGEST + [[ "$(curl -Ls custom-registry:5001/v2/localbusybox/tags/list | jq .tags)" == null ]] + diff --git a/README.md b/README.md index 1309d66..981af4b 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,50 @@ For more information on inputs, see the [API Documentation](https://developer.gi - `wait`: The duration to wait for the control plane to become ready (default: `60s`) - `verbosity`: info log verbosity, higher value produces more output - `kubectl_version`: The kubectl version to use (default: v1.28.6) +- `registry`: Whether to configure an insecure local registry (default: false) +- `registry_image`: The registry image to use (default: registry:2) +- `registry_name`: The registry name to use (default: kind-registry) +- `registry_port`: The local port used to bind the registry (default: 5000) +- `registry_enable_delete`: Enable delete operations on the registry (default: false) - `install_only`: Skips cluster creation, only install kind (default: false) - `ignore_failed_clean`: Whether to ignore the post delete cluster action failing (default: false) +## Configuring Local Registry + +Create a workflow (eg: .github/workflows/create-cluster-with-registry.yml): + + +```yaml +name: Create Cluster with Registry + +on: pull_request + +jobs: + create-cluster-with-registry: + runs-on: ubuntu-latest + steps: + - name: Kubernetes KinD Cluster + uses: helm/kind-action@v1 + with: + registry: true + registry_name: my-registry + registry_port: 5001 + registry_enable_delete: true +``` + +This will configure the cluster with an insecure local registry at `my-registry:5001` on both the host and within cluster. + +**Note**: If `config` option is used, you must manually configure the cluster with registry config dir enabled at `/etc/containerd/certs.d`. For example: + +```yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +containerdConfigPatches: +- |- + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +``` + ### Example Workflow Create a workflow (eg: `.github/workflows/create-cluster.yml`): diff --git a/action.yml b/action.yml index 4a77a84..9901bc8 100644 --- a/action.yml +++ b/action.yml @@ -31,6 +31,26 @@ inputs: description: "The kubectl version to use (default: v1.29.3)" required: false default: "v1.29.3" + registry: + description: "Whether to configure an insecure local registry (default: false)" + required: false + default: "false" + registry_image: + description: "The registry image to use (default: registry:2)" + required: false + default: "registry:2" + registry_name: + description: "The registry name to use (default: kind-registry)" + required: false + default: "kind-registry" + registry_port: + description: "The local port used to bind the registry (default: 5000)" + required: false + default: "5000" + registry_enable_delete: + description: "Enable delete operations on the registry (default: false)" + required: false + default: "false" install_only: description: "Skips cluster creation, only install kind (default: false)" required: false diff --git a/cleanup.sh b/cleanup.sh index bfc800e..f746734 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -19,16 +19,14 @@ set -o nounset set -o pipefail DEFAULT_CLUSTER_NAME=chart-testing +DEFAULT_REGISTRY_NAME=kind-registry main() { - args=() - - if [[ -n "${INPUT_CLUSTER_NAME:-}" ]]; then - args+=(--name "${INPUT_CLUSTER_NAME}") - else - args+=(--name "${DEFAULT_CLUSTER_NAME}") - fi + args=(--name "${INPUT_CLUSTER_NAME:-$DEFAULT_CLUSTER_NAME}") + registry_args=("${INPUT_REGISTRY_NAME:-$DEFAULT_REGISTRY_NAME}") + docker rm -f "${registry_args[@]}" || "${INPUT_IGNORE_FAILED_CLEAN}" + kind delete cluster "${args[@]}" || "${INPUT_IGNORE_FAILED_CLEAN}" } diff --git a/kind.sh b/kind.sh index 54460b4..c280797 100755 --- a/kind.sh +++ b/kind.sh @@ -35,6 +35,7 @@ Usage: $(basename "$0") -l, --verbosity info log verbosity, higher value produces more output -k, --kubectl-version The kubectl version to use (default: $DEFAULT_KUBECTL_VERSION) -o, --install-only Skips cluster creation, only install kind (default: false) + --with-registry Enables registry config dir for the cluster (default: false) EOF } @@ -48,6 +49,8 @@ main() { local verbosity= local kubectl_version="${DEFAULT_KUBECTL_VERSION}" local install_only=false + local with_registry=false + local config_with_registry_path="/etc/kind-registry/config.yaml" parse_command_line "$@" @@ -175,6 +178,14 @@ parse_command_line() { install_only=true fi ;; + --with-registry) + if [[ -n "${2:-}" ]]; then + with_registry="$2" + shift + else + with_registry=true + fi + ;; *) break ;; @@ -208,6 +219,20 @@ install_kubectl() { chmod +x "${kubectl_dir}/kubectl" } +create_config_with_registry() { + sudo mkdir -p $(dirname "$config_with_registry_path") + cat < + + -h, --help Display help + -i, --registry-image The registry image to use (default: $DEFAULT_REGISTRY_IMAGE) + -n, --registry-name The registry name to use (default: $DEFAULT_REGISTRY_NAME) + -p, --registry-port The local port used to bind the registry (default: $DEFAULT_REGISTRY_PORT) + -d, --enable-delete Enable delete operations on the registry (default: false) + --cluster-name The name of the cluster to configure with the registry (default: $DEFAULT_CLUSTER_NAME) + +EOF +} + +main() { + local registry_name="$DEFAULT_REGISTRY_NAME" + local registry_image="$DEFAULT_REGISTRY_IMAGE" + local registry_port="$DEFAULT_REGISTRY_PORT" + local enable_delete=false + local cluster_name="${DEFAULT_CLUSTER_NAME}" + + parse_command_line "$@" + + create_registry + connect_registry + config_registry_for_nodes + document_registry +} + +parse_command_line() { + while :; do + case "${1:-}" in + -h|--help) + show_help + exit + ;; + -n|--registry-name) + if [[ -n "${2:-}" ]]; then + registry_name="$2" + shift + else + echo "ERROR: '-n|--registry-name' cannot be empty." >&2 + show_help + exit 1 + fi + ;; + -i|--registry-image) + if [[ -n "${2:-}" ]]; then + registry_image="$2" + shift + else + echo "ERROR: '-i|--registry-image' cannot be empty." >&2 + show_help + exit 1 + fi + ;; + -p|--registry-port) + if [[ -n "${2:-}" ]]; then + registry_port="$2" + shift + else + echo "ERROR: '-p|--registry-port' cannot be empty." >&2 + show_help + exit 1 + fi + ;; + -d|--enable-delete) + if [[ -n "${2:-}" ]]; then + enable_delete="$2" + shift + else + enable_delete=true + fi + ;; + --cluster-name) + if [[ -n "${2:-}" ]]; then + cluster_name="$2" + shift + else + echo "ERROR: '--cluster-name' cannot be empty." >&2 + show_help + exit 1 + fi + ;; + *) + break + ;; + esac + + shift + done +} + +create_registry() { + echo "Creating local registry..." + + docker run -d --restart=always \ + --name "${registry_name}" \ + --network bridge \ + -p "${registry_port}:5000" \ + -e REGISTRY_STORAGE_DELETE_ENABLED="$enable_delete" \ + $registry_image + + # Local registry is available at $registry_name:$registry_port + echo "127.0.0.1 $registry_name" | sudo tee -a /etc/hosts +} + +connect_registry() { + echo "Connecting local registry to cluster network..." + + docker network connect kind "$registry_name" +} + +config_registry_for_nodes() { + # Reference: https://github.com/containerd/containerd/blob/main/docs/hosts.md + REGISTRY_DIR="/etc/containerd/certs.d/${registry_name}:${registry_port}" + + for node in $(kind get nodes -n "${cluster_name}"); do + docker exec "${node}" mkdir -p "${REGISTRY_DIR}" + cat <