diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e432bba..20c5a1a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,6 +1,7 @@ name: build on: + workflow_dispatch: push: branches: - '*' @@ -18,60 +19,101 @@ on: jobs: - docker-build: + validate: runs-on: ubuntu-latest if: ${{ !contains(github.event.head_commit.message,'[skip ci]') }} + container: golang:1.21-alpine + steps: + - name: checkout + uses: actions/checkout@v4 + - name: test + shell: sh + env: + CGO_ENABLED: 0 + run: | + apk --update add ca-certificates tzdata make git bash + make lint + make test-json + + - name: upload test results + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: test-reports + if-no-files-found: ignore + path: | + test-report.out + coverage.out + golangci-lint.out + + - name: SonarCloud scan + uses: SonarSource/sonarcloud-github-action@master + if: ${{ always() }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + + docker-build: + + runs-on: ubuntu-latest + needs: validate + # build only on master branch and tags + if: ${{ !contains(github.event.head_commit.message,'[skip ci]') && (github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/'))) }} steps: - - uses: actions/checkout@v3 + - name: checkout + uses: actions/checkout@v4 + + - name: get short sha + id: short_sha + run: echo ::set-output name=sha::$(git rev-parse --short HEAD) + + - name: get version + id: version + run: echo ::set-output name=version::$([[ -z "${{ github.event.pull_request.number }}" ]] && echo "sha-${{ steps.short_sha.outputs.sha }}" || echo "pr-${{ github.event.pull_request.number }}") + + - name: set up QEMU + uses: docker/setup-qemu-action@v3 - name: set up Docker buildx id: buildx uses: docker/setup-buildx-action@v3 - with: - version: latest - - - name: decide on tag - id: prep - run: | - DOCKER_IMAGE=${{ github.repository }} - VERSION=noop - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then - VERSION=edge - fi - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then - MINOR=${VERSION%.*} - MAJOR=${MINOR%.*} - TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR},${DOCKER_IMAGE}:${MAJOR},${DOCKER_IMAGE}:latest" - elif [ "${{ github.event_name }}" = "push" ]; then - TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" - fi - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - name: login to DockerHub + if: ${{ github.event_name != 'pull_request' }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: prepare meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ github.repository }}-agent + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha + labels: | + github.run.id=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + org.opencontainers.image.title=kubeip-agent + org.opencontainers.image.description=kubeip agent + org.opencontainers.image.vendor=DoiT International + - name: build and push - id: docker_build uses: docker/build-push-action@v5 with: + build-args: | + VERSION=${{ steps.version.outputs.version }} + COMMIT=${{ steps.short_sha.outputs.sha }} + BRANCH=${{ github.ref_name }} push: true - platforms: | - linux/amd64 - linux/arm64 - tags: ${{ steps.prep.outputs.tags }} - - - name: image digest - run: echo ${{ steps.docker_build.outputs.digest }} + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 1c9b6ff..4961c4a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,50 @@ -# code coverage -.cover +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +.bin + +# Test binary, built with `go test -c` +*.test +.run + +# Local environment +.env +.in +.out + +# Local .terraform directories +**/.terraform/* + +# Sonar Scanner +.scannerwork/ -# git repo -.git +# .tfstate files +*.tfstate +*.tfstate.* -# IDE customization + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# IDE support files .idea .vscode -# binaries -.bin +# MacOS file system metadata +.DS_Store -# env customization -.env -.in +# local credentials +.credentials -# Test binary, build with `go test -c` -*.test +# Keep out some temporary example code and temprary file +example_code/ +older/ +temp/ + +# Dependency directories (remove the comment below to include it) +vendor/ + +tmp/ \ No newline at end of file diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..7602ea2 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,146 @@ +run: + # which dirs to skip + skip-dirs: + - mocks + # Timeout for analysis, e.g. 30s, 5m. + # Default: 1m + timeout: 5m + # Exit code when at least one issue was found. + # Default: 1 + issues-exit-code: 2 + # Include test files or not. + # Default: true + tests: false + # allow parallel run + allow-parallel-runners: true + +linters-settings: + govet: + check-shadowing: true + gocyclo: + min-complexity: 15 + maligned: + suggest-new: true + dupl: + threshold: 100 + goconst: + min-len: 2 + min-occurrences: 2 + misspell: + locale: US + ignore-words: + - "cancelled" + goimports: + local-prefixes: github.com/golangci/golangci-lint + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + disabled-checks: + - dupImport # https://github.com/go-critic/go-critic/issues/845 + - ifElseChain + - octalLiteral + - rangeValCopy + - unnamedResult + - whyNoLint + - wrapperFunc + funlen: + lines: 105 + statements: 50 + +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + disable-all: true + enable: + - asciicheck + - bidichk + - bodyclose + # - containedctx + # - contextcheck disabled because of generics + - dupword + - decorder + # - depguard + - dogsled + - dupl + - durationcheck + - errcheck + - errchkjson + - errname + - errorlint + - exhaustive + # - exhaustivestruct TODO: check how to fix it + - exportloopref + # - forbidigo TODO: configure forbidden code patterns + # - forcetypeassert + - funlen + - gci + # - gochecknoglobals TODO: remove globals from code + # - gochecknoinits TODO: remove main.init + - gocognit + - goconst + - gocritic + - gocyclo + # - godox + - goerr113 + - gofmt + - goimports + - gomnd + # - gomoddirectives + - gosec + - gosimple + - govet + - goprintffuncname + - grouper + - importas + # - ireturn TODO: not sure if it is a good linter + - ineffassign + - interfacebloat + - loggercheck + - maintidx + - makezero + - misspell + - nakedret + # - nestif + - nilerr + - nilnil + # - noctx + - nolintlint + - prealloc + - predeclared + - promlinter + - reassign + - revive + # - rowserrcheck disabled because of generics + # - staticcheck doesn't work with go1.19 + # - structcheck disabled because of generics + - stylecheck + - tenv + - testableexamples + - typecheck + - unconvert + - unparam + - unused + # - varnamelen TODO: review naming + # - varcheck depricated 1.49 + # - wastedassign disabled because of generics + - whitespace + - wrapcheck + # - wsl + +issues: + exclude-rules: + - path: _test\.go + linters: + - funlen + - bodyclose + - gosec + - dupl + - gocognit + - goconst + - gocyclo + exclude: + - Using the variable on range scope `tt` in function literal diff --git a/Dockerfile b/Dockerfile index 0baab36..9ecbf4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,40 +1,38 @@ -FROM --platform=$BUILDPLATFORM golang:1.20-alpine AS builder -# curl git bash -RUN apk add --no-cache curl git bash make +FROM --platform=$BUILDPLATFORM golang:1.21-alpine AS builder -# ----- build and test ----- -FROM builder as build +# add CA certificates and TZ for local time +RUN apk --update add ca-certificates tzdata make git -# set working directorydoc -RUN mkdir -p /go/src/app -WORKDIR /go/src/app +# create a working directory +WORKDIR /app -# load dependency -COPY Makefile . -COPY go.mod . -COPY go.sum . -RUN --mount=type=cache,target=/root/.cache/go-build go mod download +# Retrieve application dependencies. +# This allows the container build to reuse cached dependencies. +# Expecting to copy go.mod and if present go.sum. +COPY go.* ./ +RUN go mod download -# copy sources -COPY . . +# Copy local code to the container image. +COPY . ./ -# build arguments (passed from buildx) +# get version, commit and branch from build args +ARG VERSION +ARG COMMIT +ARG BRANCH ARG TARGETOS ARG TARGETARCH -# build -RUN --mount=type=cache,target=/root/.cache/go-build make binary TARGETOS=${TARGETOS} TARGETARCH=${TARGETARCH} +# Build the binary with make (using the version, commit and branch) +RUN make build VERSION=${VERSION} COMMIT=${COMMIT} BRANCH=${BRANCH} TARGETOS=${TARGETOS} TARGETARCH=${TARGETARCH} -# -# ------ release Docker image ------ -# +# final image FROM scratch # copy CA certificates COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -# this is the last command since it's never cached -COPY --from=build /go/src/app/.bin/github.com/doitintl/kubeip /kubeip +# copy timezone settings +COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +# copy the binary to the production image from the builder stage +COPY --from=builder /app/.bin/kubeip-agent /kubeip-agent -ENTRYPOINT ["/kubeip"] - - -# RUN cd /go/src/github.com/doitintl/kubeip && GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a --installsuffix cgo --ldflags="-s" -ldflags "-X main.version=$(git log | head -n 1 | cut -f 2 -d ' ') -X main.buildDate=$(date +%Y-%m-%d\-%H:%M)" -o /kubeip +ENTRYPOINT ["/kubeip-agent"] +CMD ["run"] \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index d3bb2b9..0000000 --- a/Makefile +++ /dev/null @@ -1,131 +0,0 @@ -MODULE = $(shell env GO111MODULE=on $(GO) list -m) -DATE ?= $(shell date +%FT%T%z) -VERSION ?= $(shell git describe --tags --always --dirty --match="v*" 2> /dev/null || \ - cat $(CURDIR)/.version 2> /dev/null || echo v0) -COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null) -BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) -PKGS = $(or $(PKG),$(shell env GO111MODULE=on $(GO) list ./...)) -TESTPKGS = $(shell env GO111MODULE=on $(GO) list -f \ - '{{ if or .TestGoFiles .XTestGoFiles }}{{ .ImportPath }}{{ end }}' \ - $(PKGS)) - -BIN := $(CURDIR)/.bin -BINARY = kubeip -TARGETOS := $(or $(TARGETOS), linux) -TARGETARCH := $(or $(TARGETARCH), amd64) - -GO = go -TIMEOUT = 15 -V = 0 -Q = $(if $(filter 1,$V),,@) -M = $(shell printf "\033[34;1m▶\033[0m") - -export CGO_ENABLED=0 -export GOPROXY=https://proxy.golang.org -export GOOS=$(TARGETOS) -export GOARCH=$(TARGETARCH) - -.PHONY: all -all: fmt lint binary - -.PHONY: binary -binary: $(BIN) ; $(info $(M) building $(GOOS)/$(GOARCH) binary...) @ ## Build program binary - $Q $(GO) build \ - -tags release \ - -ldflags '-X main.version=$(VERSION) -X main.buildDate=$(DATE)' \ - -o $(BIN)/$(basename $(MODULE)) main.go - -.PHONY: image -image: clean-image - @docker build -t "${BINARY}" -f Dockerfile . - -.PHONY: stop -stop: - @docker stop "${BINARY}" || true # Do not fail if container does not exist - -.PHONY: clean-image -clean-image: stop - @docker rmi "${BINARY}" || true # Do not fail if image does not exist - - -# Tools - -$(BIN): - @mkdir -p $@ -$(BIN)/%: | $(BIN) ; $(info $(M) building $(PACKAGE)...) - $Q tmp=$$(mktemp -d); \ - env GO111MODULE=off GOPATH=$$tmp GOBIN=$(BIN) $(GO) get $(PACKAGE) \ - || ret=$$?; \ - rm -rf $$tmp ; exit $$ret - -GOLINT = $(BIN)/golint -$(BIN)/golint: PACKAGE=golang.org/x/lint/golint - -GOCOV = $(BIN)/gocov -$(BIN)/gocov: PACKAGE=github.com/axw/gocov/... - -GOCOVXML = $(BIN)/gocov-xml -$(BIN)/gocov-xml: PACKAGE=github.com/AlekSi/gocov-xml - -GO2XUNIT = $(BIN)/go2xunit -$(BIN)/go2xunit: PACKAGE=github.com/tebeka/go2xunit - -# Tests - -TEST_TARGETS := test-default test-bench test-short test-verbose test-race -.PHONY: $(TEST_TARGETS) test-xml check test tests -test-bench: ARGS=-run=__absolutelynothing__ -bench=. ## Run benchmarks -test-short: ARGS=-short ## Run only short tests -test-verbose: ARGS=-v ## Run tests in verbose mode with coverage reporting -test-race: ARGS=-race ## Run tests with race detector -$(TEST_TARGETS): NAME=$(MAKECMDGOALS:test-%=%) -$(TEST_TARGETS): test -check test tests: fmt lint ; $(info $(M) running $(NAME:%=% )tests...) @ ## Run tests - $Q $(GO) test -timeout $(TIMEOUT)s $(ARGS) $(TESTPKGS) - -test-xml: fmt lint | $(GO2XUNIT) ; $(info $(M) running xUnit tests...) @ ## Run tests with xUnit output - $Q mkdir -p test - $Q 2>&1 $(GO) test -timeout $(TIMEOUT)s -v $(TESTPKGS) | tee test/tests.output - $(GO2XUNIT) -fail -input test/tests.output -output test/tests.xml - -COVERAGE_MODE = atomic -COVERAGE_PROFILE = $(COVERAGE_DIR)/profile.out -COVERAGE_XML = $(COVERAGE_DIR)/coverage.xml -COVERAGE_HTML = $(COVERAGE_DIR)/index.html -.PHONY: test-coverage test-coverage-tools -test-coverage-tools: | $(GOCOV) $(GOCOVXML) -test-coverage: COVERAGE_DIR := $(CURDIR)/test/coverage.$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") -test-coverage: fmt lint test-coverage-tools ; $(info $(M) running coverage tests...) @ ## Run coverage tests - $Q mkdir -p $(COVERAGE_DIR) - $Q $(GO) test \ - -coverpkg=$$($(GO) list -f '{{ join .Deps "\n" }}' $(TESTPKGS) | \ - grep '^$(MODULE)/' | \ - tr '\n' ',' | sed 's/,$$//') \ - -covermode=$(COVERAGE_MODE) \ - -coverprofile="$(COVERAGE_PROFILE)" $(TESTPKGS) - $Q $(GO) tool cover -html=$(COVERAGE_PROFILE) -o $(COVERAGE_HTML) - $Q $(GOCOV) convert $(COVERAGE_PROFILE) | $(GOCOVXML) > $(COVERAGE_XML) - -.PHONY: lint -lint: | $(GOLINT) ; $(info $(M) running golint...) @ ## Run golint - $Q $(GOLINT) -set_exit_status $(PKGS) - -.PHONY: fmt -fmt: ; $(info $(M) running gofmt...) @ ## Run gofmt on all source files - $Q $(GO) fmt $(PKGS) - -# Misc - -.PHONY: clean -clean: ; $(info $(M) cleaning...) @ ## Cleanup everything - @rm -rf $(BIN) - @rm -rf test/tests.* test/coverage.* - -.PHONY: help -help: - @grep -E '^[ a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ - awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' - -.PHONY: version -version: - @echo $(VERSION) diff --git a/README.md b/README.md index 3663fbf..498cd39 100644 --- a/README.md +++ b/README.md @@ -1,343 +1,275 @@ -![ci](https://github.com/doitintl/kubeip/workflows/build/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/doitintl/kubeip)](https://goreportcard.com/report/github.com/doitintl/kubeip) ![Docker Pulls](https://img.shields.io/docker/pulls/doitintl/kubeip) - -# What is kubeIP? - -Many applications need to be whitelisted by users based on a Source IP Address. As of today, Google Kubernetes Engine doesn't support assigning a static pool of IP addresses to the GKE cluster. Using kubeIP, this problem is solved by assigning GKE nodes external IP addresses from a predefined list. kubeIP monitors the Kubernetes API for new/removed nodes and applies the changes accordingly. - -# Deploy kubeIP (without building from source) - -If you just want to use kubeIP (instead of building it yourself from source), please follow the instructions in this section. You’ll need Kubernetes version 1.10 or newer. You'll also need the Google Cloud SDK. You can install the [Google Cloud SDK](https://cloud.google.com/sdk) (which also installs kubectl). - -To configure your Google Cloud SDK, set default project as: - -``` -gcloud config set project {your project_id} +![build](https://github.com/doitintl/kubeip/workflows/build/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/doitintl/kubeip)](https://goreportcard.com/report/github.com/doitintl/kubeip) ![Docker Pulls](https://img.shields.io/docker/pulls/doitintl/kubeip-agent) + +# KubeIP v2 + +Welcome to KubeIP v2, a complete overhaul of the popular [DoiT](https://www.doit.com/) +KubeIP [v1-main](https://github.com/doitintl/kubeip/tree/v1-main) open-source project, originally developed +by [Aviv Laufer](https://github.com/avivl). + +KubeIP v2 expands its support beyond Google Cloud (as in v1) to include AWS, and it's designed to be extendable to other cloud providers +that allow assigning static public IP to VMs. We've also transitioned from a Kubernetes controller to a standard DaemonSet, enhancing +reliability and ease of use. + +## What happens with KubeIP v1 + +KubeIP v1 is still available in the [v1-main](https://github.com/doitintl/kubeip/tree/v1-main) branch. No further development is planned. We +will fix critical bugs and security issues, but we will not add new features. + +## What KubeIP v2 does? + +Kubernetes' nodes don't necessarily need their own public IP addresses to communicate. However, there are certain situations where it's +beneficial for nodes in a node pool to have their own unique public IP addresses. + +For instance, in gaming applications, a console might need to establish a direct connection with a cloud virtual machine to reduce the +number of hops. + +Similarly, if you have multiple agents running on Kubernetes that need a direct server connection, and the server needs to whitelist all +agent IPs, having dedicated public IPs can be useful. These scenarios, among others, can be handled on a cloud-managed Kubernetes cluster +using Node Public IP. + +KubeIP is a utility that assigns a static public IP to each node it manages. The IP is allocated to the node's primary network interface, +chosen from a pool of reserved static IPs using platform-supported filtering and ordering. + +If there are no static public IPs left, KubeIP will hold on until one becomes available. When a node is removed, KubeIP releases the static +public IP back into the pool of reserved static IPs. + +## How to use KubeIP? + +Deploy KubeIP as a DaemonSet on your desired nodes using standard +Kubernetes [mechanism](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). Once deployed, KubeIP will assign a static +public IP +to each node it operates on. If no static public IP is available, KubeIP will wait until one becomes available. When a node is deleted, +KubeIP will release the static public IP. + +### Kubernetes Service Account + +KubeIP requires a Kubernetes service account with the following permissions: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeip-service-account + namespace: kube-system +--- + +piVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeip-cluster-role +rules: + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get" ] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeip-cluster-role-binding +subjects: + - kind: ServiceAccount + name: kubeip-service-account + namespace: kube-system +roleRef: + kind: ClusterRole + name: kubeip-cluster-role + apiGroup: rbac.authorization.k8s.io +``` + +### Kubernetes DaemonSet + +Deploy KubeIP as a DaemonSet on your desired nodes using standard Kubernetes selectors. Once deployed, KubeIP will assign a static public IP +to the node's primary network interface, selected from a list of reserved static IPs using platform-supported filtering. If no static public +IP is available, KubeIP will wait until one becomes available. When a node is deleted, KubeIP will release the static public IP. + +```yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubeip +spec: + selector: + matchLabels: + app: kubeip + template: + metadata: + labels: + app: kubeip + spec: + serviceAccountName: kubeip-service-account + terminationGracePeriodSeconds: 30 + priorityClassName: system-node-critical + nodeSelector: + kubeip.com/public: "true" + containers: + - name: kubeip + image: doitintl/kubeip-agent + resources: + requests: + cpu: 100m + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: FILTER + value: PUT_PLATFORM_SPECIFIC_FILTER_HERE + - name: LOG_LEVEL + value: debug + - name: LOG_JSON + value: "true" +``` + +### AWS + +Make sure that KubeIP DaemonSet is deployed on nodes that have a public IP (node running in public subnet) and uses a Kubernetes service +account [bound](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) to the IAM role with the following +permissions: + +```yaml +Version: '2012-10-17' +Statement: + - Effect: Allow + Action: + - ec2:AssociateAddress + - ec2:DisassociateAddress + - ec2:DescribeInstances + - ec2:DescribeAddresses + Resource: '*' +``` + +KubeIP supports filtering of reserved Elastic IPs using tags and Elastic IP properties. To use this feature, add the `filter` flag (or +set `FILTER` environment variable) to the KubeIP DaemonSet: + +```yaml +- name: FILTER + value: "Name=tag:env,Values=dev;Name=tag:app,Values=streamer" +``` + +KubeIP AWS filter supports the same filter syntax as the AWS `describe-addresses` command. For more information, +see [describe-addresses](https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-addresses.html#options). If you specify multiple +filters, they are joined with an `AND`, and the request returns only results that match all the specified filters. Multiple filters must be +separated by semicolons (`;`). + +### Google Cloud + +Ensure that the KubeIP DaemonSet is deployed on nodes with a public IP (nodes in a public subnet) and uses a Kubernetes service +account [bound](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) to an IAM role with the following permissions: + +```yaml +title: "KubeIP Role" +description: "KubeIP required permissions" +stage: "GA" +includedPermissions: + - compute.instances.addAccessConfig + - compute.instances.deleteAccessConfig + - compute.instances.get + - compute.addresses.get + - compute.addresses.list + - compute.addresses.use + - compute.zoneOperations.get + - compute.subnetworks.useExternalIp + - compute.projects.get ``` -Set the environment variables and make sure to configure before continuing: - -``` -export GCP_REGION= -export GCP_ZONE= -export GKE_CLUSTER_NAME= -export PROJECT_ID=$(gcloud config list --format 'value(core.project)') -export KUBEIP_NODEPOOL= -export KUBEIP_SELF_NODEPOOL= -``` +KubeIP Google Cloud filter supports the same filter syntax as the Google Cloud `gcloud compute addresses list` command. For more +information, see [gcloud topic filter](https://cloud.google.com/sdk/gcloud/reference/topic/filters). If you specify multiple filters, they +are joined with an `AND`, and the request returns only results that match all the specified filters. Multiple filters must be separated by +semicolons (`;`). -**Get credentials for yourself** +To use this feature, add the `filter` flag (or set `FILTER` environment variable) to the KubeIP DaemonSet: -If you haven't alrady, set up your GKE cluster credentials with -(replace `$GKE_CLUSTER_NAME` with your real GKE cluster name): - -``` -gcloud container clusters get-credentials $GKE_CLUSTER_NAME \ - --region $GCP_ZONE \ - --project $PROJECT_ID +```yaml +- name: FILTER + value: "labels.env=dev;labels.app=streamer" ``` -You will need admin access to the cluster to deploy to the `kube-system` namespace. -Either set the relevant IAM rolebinding for your user, or get RBAC permissions with: +## How to contribute to KubeIP? -``` -kubectl create clusterrolebinding cluster-admin-binding \ - --clusterrole cluster-admin --user `gcloud config list --format 'value(core.account)'` -``` +KubeIP is an open-source project, and we welcome your contributions! -**Creating an IAM Service Account** +## How to build KubeIP? -Create a Service Account with this command: +KubeIP is written in Go and can be built using standard Go tools. To build KubeIP, run the following command: -``` -gcloud iam service-accounts create kubeip-service-account --display-name "kubeIP" +```shell +make build ``` -Create and attach a custom kubeIP role to the service account by running the following commands: +## How to run KubeIP? -``` -gcloud iam roles create kubeip --project $PROJECT_ID --file roles.yaml +KubeIP is a standard command-line application. To explore the available options, run the following command: -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member=serviceAccount:kubeip-service-account@$PROJECT_ID.iam.gserviceaccount.com \ - --role=projects/$PROJECT_ID/roles/kubeip \ - --condition=None +```shell +kubeip-agent run --help ``` -**Getting credentials to the deployment** - -Note: If you use Workload Identity in your cluster, you do not need to upload a credential file. -You can just remove the GOOGLE_APPLICATION_CREDENTIALS environment variable from the manifest -and the Google Cloud SDK will pick up the credentials from the metadata server as per normal -operation. - -Generate the Key using the following command: -``` -gcloud iam service-accounts keys create key.json \ - --iam-account kubeip-service-account@$PROJECT_ID.iam.gserviceaccount.com -``` +```text +NAME: + kubeip-agent run - run agent -Create a Kubernetes secret object by running: -``` -kubectl create secret generic kubeip-key --from-file=key.json -n kube-system -``` +USAGE: + kubeip-agent run [command options] [arguments...] -**Create Static, Reserved IP Addresses:** +OPTIONS: + Configuration -Create as many static IP addresses for the number of nodes in your GKE cluster (this example creates 10 addresses) so you will have enough addresses when your cluster scales up (manually or automatically): + --filter value [ --filter value ] filter for the IP addresses [$FILTER] + --kubeconfig value path to Kubernetes configuration file (not needed if running in node) [$KUBECONFIG] + --node-name value Kubernetes node name (not needed if running in node) [$NODE_NAME] + --order-by value order by for the IP addresses [$ORDER_BY] + --project value name of the GCP project or the AWS account ID (not needed if running in node) [$PROJECT] + --region value name of the GCP region or the AWS region (not needed if running in node) [$REGION] + --release-on-exit release the static public IP address on exit (default: true) [$RELEASE_ON_EXIT] + --retry-attempts value number of attempts to assign the static public IP address (default: 10) [$RETRY_ATTEMPTS] + --retry-interval value when the agent fails to assign the static public IP address, it will retry after this interval (default: 5m0s) [$RETRY_INTERVAL] -``` -for i in {1..10}; do gcloud compute addresses create kubeip-ip$i --project=$PROJECT_ID --region=$GCP_REGION; done -``` + Development -Add labels to reserved IP addresses. A common practice is to assign a unique value per cluster (for example cluster name): + --develop-mode enable develop mode (default: false) [$DEV_MODE] -``` -for i in {1..10}; do gcloud beta compute addresses update kubeip-ip$i --update-labels kubeip=$GKE_CLUSTER_NAME --region $GCP_REGION; done -``` + Logging -``` -sed -i -e "s/reserved/$GKE_CLUSTER_NAME/g" -e "s/default-pool/$KUBEIP_NODEPOOL/g" deploy/kubeip-configmap.yaml + --json produce log in JSON format: Logstash and Splunk friendly (default: false) [$LOG_JSON] + --log-level value set log level (debug, info(*), warning, error, fatal, panic) (default: "info") [$LOG_LEVEL] ``` -Make sure the `deploy/kubeip-configmap.yaml` file contains the correct values: +## How to test KubeIP? - - The `KUBEIP_LABELVALUE` should be your GKE's cluster name - - The `KUBEIP_NODEPOOL` should match the name of your GKE node-pool on which kubeIP will operate - - The `KUBEIP_FORCEASSIGNMENT` - controls whether kubeIP should assign static IPs to existing nodes in the node-pool and defaults to true +To test KubeIP, create a pool of reserved static public IPs, ensuring that the pool has enough IPs to assign to all nodes that KubeIP will +operate on. Use labels to filter the pool of reserved static public IPs. -We recommend that KUBEIP_NODEPOOL should *NOT* be the same as KUBEIP_SELF_NODEPOOL +Next, create a Kubernetes cluster and deploy KubeIP as a DaemonSet on your desired nodes. Ensure that the nodes have a public IP (nodes in a +public subnet). Configure KubeIP to use the pool of reserved static public IPs, using filters and order by. +Finally, scale the number of nodes in the cluster and verify that KubeIP assigns a static public IP to each node. Scale down the number of +nodes in the cluster and verify that KubeIP releases the static public IP addresses. -If you would like to assign addresses to other node pools, then `KUBEIP_NODEPOOL` can be added to this nodepool `KUBEIP_ADDITIONALNODEPOOLS` as a comma separated list. -You should tag the addresses for this pool with the `KUBEIP_LABELKEY` value + `-node-pool` and assign the value of the node pool a name i.e., `kubeip-node-pool=my-node-pool` +#### AWS EKS Example -``` -sed -i -e "s/pool-kubip/$KUBEIP_SELF_NODEPOOL/g" deploy/kubeip-deployment.yaml -``` +The [examples/aws](examples/aws) folder contains a Terraform configuration that creates an EKS cluster and deploys KubeIP as a DaemonSet on +the cluster nodes in a public subnet. The Terraform configuration also creates a pool of reserved Elastic IPs and configures KubeIP to use +the pool of reserved static public IPs. -Deploy kubeIP by running: +To run the example, follow these steps: +```shell +cd examples/aws +terraform init +terraform apply ``` -kubectl apply -f deploy/. -``` - -Once you’ve assigned an IP address to a node kubeIP, a label will be created for that node `kubip_assigned` with the value of the IP address (`.` are replaced with `-`): - - `172.31.255.255 ==> 172-31-255-255` - -**Ordering IPs** - -KubeIP can order IPs based on the numeric value identified by `KUBEIP_ORDERBYLABELKEY`. -IPs are ordered in descending order if `KUBEIP_ORDERBYDESC` is set to true, ascending order otherwise. +#### Google Cloud GKE Example -Missing `KUBEIP_ORDERBYLABELKEY` or invalid values present on `KUBEIP_ORDERBYLABELKEY` will be assigned the lowest priority. +The [examples/gcp](examples/gcp) folder contains a Terraform configuration that creates a GKE cluster and deploys KubeIP as a DaemonSet on +the cluster nodes in a public subnet. The Terraform configuration also creates a pool of reserved static public IPs and configures KubeIP to +use the pool of reserved static public IPs. -When nodes are added, deleted or on tick, kubeIP will check whether the nodes have the most optimal IP assignment. What does this mean? +To run the example, follow these steps: -E.g. Let's assume Node1 has IP_A, Node2 has IP_B and IP_A > IP_B, when we scale the cluster down the cluster two things might happen -1. Node 1 is deleted which results in a sub-optimal IP assignment since Node2 has IP_B and IP_A > IP_B -2. Node 2 is deleted maintaining optimal order. - -In the first case Node 2 is re-assigned IP_A. - -To order the IPs reserved above in asc order use - -``` -for i in {1..10}; do gcloud beta compute addresses update kubeip-ip$i --update-labels priority=$i --region=$GCP_REGION; done -``` - -and set - -``` -KUBEIP_ORDERBYLABELKEY: "priority" -KUBEIP_ORDERBYDESC: "false" -``` - -**Copy Labels** - -KubeIP will also copy all labels from the IP being assigned over to the node if `KUBEIP_COPYLABELS` is set to true. - -This is typically helpful when we want to have node selection not based on IP but more semantic label keys and values. - -As an example let's label `kubeip-ip1` with `platform_whitelisted=true`, to do this we execute the following command - -``` -gcloud beta compute addresses update kubeip-ip1 --update-labels "platform_whitelisted=true" --region=$GCP_REGION; -``` - -Now, when a node is assigned the IP address of `kubeip-ip1` it will also be labelled with `platform_whitelisted=true` as well as the default `kubip_assigned`. - -An IP can have multiple labels, all will be copied over. - -**Clear Labels** - -When IPs get assigned or re-assigned to achieve optimal IP assignment we can configure the system to clear any previous labels. Set `KUBEIP_CLEARLABELS` flag to `true` if you want this behaviour. - -This feature is required when labels are not overlapping. E.g. let's assume we have the following tagged IPs; IP_A and IP_B, order by priority - -``` -IP_A test_a=value_a,test_b=value_b,priority=1 -IP_B test_c=value_c,priority=2 -``` -Let's assume that the assignment was as follows - -``` -IP_A => NodeA -IP_B => NodeB -``` - -At this point `NodeA` has labels `test_a=value_a,test_b=value_b` and `NodeB` has labels `test_c=value_c`. Note priority is not copied over. - -If `NodeA` is deleted a re-assignment needs to happen (due to the fact that IP_A > IP_B) and `NodeB` would have -- `test_a=value_a,test_b=value_b,test_c=value_c` if `KUBEIP_CLEARLABELS="false"` and -- `test_a=value_a,test_b=value_b` if `KUBEIP_CLEARLABELS="true"` - -Note that `test_c` is not an overlapping label and hence might cause problems if `KUBEIP_CLEARLABELS` is not set to `true`. - -**Dry Run Mode** - -Dry run mode allows debugging the operations performed by KubeIP without actually performing the operations. - -ONLY use this mode during development of new features on KubeIP. - - -# Deploy & Build From Source - -You need Kubernetes version 1.10 or newer. You also need Docker version and kubectl 1.10.x or newer installed on your machine, as well as the Google Cloud SDK. You can install the [Google Cloud SDK](https://cloud.google.com/sdk) (which also installs kubectl). - - -**Clone Git Repository** - -Make sure your $GOPATH is [configured](https://github.com/golang/go/wiki/SettingGOPATH). You'll need to clone this repository to your `$GOPATH/src` folder. - -``` -mkdir -p $GOPATH/src/doitintl/kubeip -git clone https://github.com/doitintl/kubeip.git $GOPATH/src/doitintl/kubeip -cd $GOPATH/src/doitintl/kubeip +```shell +cd examples/gcp +terraform init +terraform apply -var="project_id=" ``` - -**Set Environment Variables** - -Replace **us-central1** with the region where your GKE cluster resides and **kubeip-cluster** with your real GKE cluster name - -``` -export GCP_REGION=us-central1 -export GCP_ZONE=us-central1-b -export GKE_CLUSTER_NAME=kubeip-cluster -export PROJECT_ID=$(gcloud config list --format 'value(core.project)') -``` - -**Develop kubeIP Locally** - -Compile the kubeIP binary and run tests - -``` -make -``` - -**Build kubeIP's Container Image** - - -Compile the kubeIP binary and build the Docker image as following: - -``` -make image -``` - -Tag the image using: - -``` -docker tag kubeip gcr.io/$PROJECT_ID/kubeip -``` - -Finally, push the image to Google Container Registry with: - -``` -docker push gcr.io/$PROJECT_ID/kubeip -``` - -Alternatively, you can export `REGISTRY` to `gcr.io/$PROJECT_ID` and run the script `build-all-and-push.sh` which builds and publishes the docker image. - -**Create IAM Service Account and obtain the Key in JSON format** - -Create a Service Account with this command: - -``` -gcloud iam service-accounts create kubeip-service-account --display-name "kubeIP" -``` - -Create and attach the custom kubeIP role to the service account by running the following commands: - -``` -gcloud iam roles create kubeip --project $PROJECT_ID --file roles.yaml - -gcloud projects add-iam-policy-binding $PROJECT_ID --member serviceAccount:kubeip-service-account@$PROJECT_ID.iam.gserviceaccount.com --role projects/$PROJECT_ID/roles/kubeip -``` - -Generate the Key using the following command: - -``` -gcloud iam service-accounts keys create key.json \ - --iam-account kubeip-service-account@$PROJECT_ID.iam.gserviceaccount.com -``` - -**Create Kubernetes Secret** - -Get your GKE cluster credentaials with (replace *cluster_name* with your real GKE cluster name): - -``` -gcloud container clusters get-credentials $GKE_CLUSTER_NAME \ - --region $GCP_ZONE \ - --project $PROJECT_ID -``` - -Create a Kubernetes secret by running: - -``` -kubectl create secret generic kubeip-key --from-file=key.json -n kube-system -``` - -**We need to get RBAC permissions first with** -``` -kubectl create clusterrolebinding cluster-admin-binding \ - --clusterrole cluster-admin --user `gcloud config list --format 'value(core.account)'` -``` - -**Create static reserved IP addresses:** - -Create as many static IP addresses for the number of nodes in your GKE cluster (this example creates 10 addresses) so you will have enough addresses when your cluster scales up (automatically or manually): - -``` -for i in {1..10}; do gcloud compute addresses create kubeip-ip$i --project=$PROJECT_ID --region=$GCP_REGION; done -``` - -Add labels to reserved IP addresses. A common practice is to assign a unique value per cluster. You can use your cluster name for example: - -``` -for i in {1..10}; do gcloud beta compute addresses update kubeip-ip$i --update-labels kubeip=$GKE_CLUSTER_NAME --region $GCP_REGION; done -``` - -Adjust the deploy/kubeip-configmap.yaml with your GKE cluster name (replace the GKE-cluster-name with your real GKE cluster name): - -``` -sed -i -e "s/reserved/$GKE_CLUSTER_NAME/g" deploy/kubeip-configmap.yaml -``` - -Adjust the `deploy/kubeip-deployment.yaml` to reflect your real container image path: - - - Edit the `image` to match your container image path, i.e. `gcr.io/$PROJECT_ID/kubeip` - -By default, kubeIP will only manage the nodes in default-pool nodepool. If you'd like kubeIP to manage another node-pool, please update the `KUBEIP_NODEPOOL` setting in `deploy/kubeip-configmap.yaml` file before deploying. You can also update the `KUBEIP_LABELKEY` and `KUBEIP_LABELVALUE` to control which static external IP addresses the kubeIP will look for to assign to your nodes. - -The `KUBEIP_FORCEASSIGNMENT` (which defaults to true) will check on startup and every five minutes if there are nodes in the node-pool that are not assigned to a reserved address. If such nodes are found, then kubeIP will assign a reserved address (if one is available to them): - -Deploy kubeIP by running - -``` -kubectl apply -f deploy/. -``` - -References: - - - Event listening code was take from [kubewatch](https://github.com/bitnami-labs/kubewatch/) diff --git a/build-all-and-push.sh b/build-all-and-push.sh deleted file mode 100755 index aa37687..0000000 --- a/build-all-and-push.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -export DOCKER_BUILDKIT=1 -# BUILD -echo "Building" -make -# Make Image -echo "Creating Image" -make image -# Re-tagging docker file -echo "Tagging" -docker tag kubeip:latest ${REGISTRY}/kubeip:latest -# Pushing image -echo "Pushing Image" -gcloud docker -- push ${REGISTRY}kubeip:latest diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..ee94de0 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,328 @@ +package main + +import ( + "context" + "fmt" + "os" + "runtime" + "time" + + "github.com/doitintl/kubeip/internal/address" + "github.com/doitintl/kubeip/internal/config" + nd "github.com/doitintl/kubeip/internal/node" + "github.com/doitintl/kubeip/internal/types" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" +) + +type contextKey string + +const ( + developModeKey contextKey = "develop-mode" +) + +var ( + version = "dev" + buildDate string + gitCommit string + gitBranch string + errEmptyPath = errors.New("empty path") +) + +const ( + // DefaultRetryInterval is the default retry interval + defaultRetryInterval = time.Minute + defaultRetryAttempts = 60 +) + +func prepareLogger(level string, json bool) *logrus.Entry { + logger := logrus.New() + + // set debug log level + switch level { + case "debug", "DEBUG": + logger.SetLevel(logrus.DebugLevel) + case "info", "INFO": + logger.SetLevel(logrus.InfoLevel) + case "warning", "WARNING": + logger.SetLevel(logrus.WarnLevel) + case "error", "ERROR": + logger.SetLevel(logrus.ErrorLevel) + case "fatal", "FATAL": + logger.SetLevel(logrus.FatalLevel) + case "panic", "PANIC": + logger.SetLevel(logrus.PanicLevel) + default: + logger.SetLevel(logrus.WarnLevel) + } + + logger.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + }) + if json { + logger.SetFormatter(&logrus.JSONFormatter{}) + } + + // record the file name and line number of the log + logger.SetReportCaller(true) + + log := logger.WithFields(logrus.Fields{ + "version": version, + }) + + return log +} + +func assignAddress(c context.Context, log *logrus.Entry, assigner address.Assigner, node *types.Node, cfg *config.Config) error { + ctx, cancel := context.WithCancel(c) + defer cancel() + // retry counter + retryCounter := 0 + // ticker for retry interval + ticker := time.NewTicker(cfg.RetryInterval) + defer ticker.Stop() + + for { + err := assigner.Assign(ctx, node.Instance, node.Zone, cfg.Filter, cfg.OrderBy) + if err != nil { + log.WithError(err).Errorf("failed to assign static public IP address to node %s", node.Name) + if retryCounter < cfg.RetryAttempts { + retryCounter++ + log.Infof("retrying after %v", cfg.RetryInterval) + } else { + log.Infof("reached maximum number of retries (%d)", cfg.RetryAttempts) + return errors.Wrap(err, "reached maximum number of retries") + } + select { + case <-ticker.C: + continue + case <-ctx.Done(): + return errors.Wrap(err, "context is done") + } + } + break + } + return nil +} + +func run(c context.Context, log *logrus.Entry, cfg *config.Config) error { + ctx, cancel := context.WithCancel(c) + defer cancel() + // add debug mode to context + if cfg.DevelopMode { + ctx = context.WithValue(ctx, developModeKey, true) + } + log.WithField("develop-mode", cfg.DevelopMode).Infof("kubeip agent started") + + restconfig, err := retrieveKubeConfig(log, cfg) + if err != nil { + return errors.Wrap(err, "retrieving kube config") + } + + clientset, err := kubernetes.NewForConfig(restconfig) + if err != nil { + return errors.Wrap(err, "initializing kubernetes client") + } + + explorer := nd.NewExplorer(clientset) + n, err := explorer.GetNode(ctx, cfg.NodeName) + if err != nil { + return errors.Wrap(err, "getting node") + } + log.WithField("node", n).Debug("node discovery done") + + // assign static public IP address with retry (interval and attempts) + assigner, err := address.NewAssigner(ctx, log, n.Cloud, cfg) + if err != nil { + return errors.Wrap(err, "initializing assigner") + } + // assign static public IP address + errorCh := make(chan error) + go func() { + e := assignAddress(ctx, log, assigner, n, cfg) + if e != nil { + errorCh <- e + } + }() + + select { + case err = <-errorCh: + if err != nil { + return errors.Wrap(err, "assigning static public IP address") + } + case <-ctx.Done(): + log.Infof("kubeip agent stopped") + if cfg.ReleaseOnExit { + log.Infof("releasing static public IP address") + // use a different context for releasing the static public IP address since the main context is canceled + if err = assigner.Unassign(context.Background(), n.Instance, n.Zone); err != nil { + return errors.Wrap(err, "releasing static public IP address") + } + } + } + + return nil +} + +func runCmd(c *cli.Context) error { + // setup signal handler for graceful shutdown: SIGTERM, SIGINT + ctx := signals.SetupSignalHandler() + log := prepareLogger(c.String("log-level"), c.Bool("json")) + cfg := config.NewConfig(c) + + if err := run(ctx, log, cfg); err != nil { + log.Fatalf("eks-lens agent failed: %v", err) + } + + return nil +} + +func main() { + app := &cli.App{ + // use ";" instead of "," for slice flag separator + // AWS filter values can contain "," and shorthand filter format uses "," to separate Names and Values + SliceFlagSeparator: ";", + Commands: []*cli.Command{ + { + Name: "run", + Usage: "run agent", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "node-name", + Usage: "Kubernetes node name (not needed if running in node)", + EnvVars: []string{"NODE_NAME"}, + Category: "Configuration", + }, + &cli.StringFlag{ + Name: "project", + Usage: "name of the GCP project or the AWS account ID (not needed if running in node)", + EnvVars: []string{"PROJECT"}, + Category: "Configuration", + }, + &cli.StringFlag{ + Name: "region", + Usage: "name of the GCP region or the AWS region (not needed if running in node)", + EnvVars: []string{"REGION"}, + Category: "Configuration", + }, + &cli.PathFlag{ + Name: "kubeconfig", + Usage: "path to Kubernetes configuration file (not needed if running in node)", + EnvVars: []string{"KUBECONFIG"}, + Category: "Configuration", + }, + &cli.DurationFlag{ + Name: "retry-interval", + Usage: "when the agent fails to assign the static public IP address, it will retry after this interval", + Value: defaultRetryInterval, + EnvVars: []string{"RETRY_INTERVAL"}, + Category: "Configuration", + }, + &cli.StringSliceFlag{ + Name: "filter", + Usage: "filter for the IP addresses", + EnvVars: []string{"FILTER"}, + Category: "Configuration", + }, + &cli.StringFlag{ + Name: "order-by", + Usage: "order by for the IP addresses", + EnvVars: []string{"ORDER_BY"}, + Category: "Configuration", + }, + &cli.IntFlag{ + Name: "retry-attempts", + Usage: "number of attempts to assign the static public IP address", + Value: defaultRetryAttempts, + EnvVars: []string{"RETRY_ATTEMPTS"}, + Category: "Configuration", + }, + &cli.BoolFlag{ + Name: "release-on-exit", + Usage: "release the static public IP address on exit", + EnvVars: []string{"RELEASE_ON_EXIT"}, + Category: "Configuration", + Value: true, + }, + &cli.StringFlag{ + Name: "log-level", + Usage: "set log level (debug, info(*), warning, error, fatal, panic)", + Value: "info", + EnvVars: []string{"LOG_LEVEL"}, + Category: "Logging", + }, + &cli.BoolFlag{ + Name: "json", + Usage: "produce log in JSON format: Logstash and Splunk friendly", + EnvVars: []string{"LOG_JSON"}, + Category: "Logging", + }, + &cli.BoolFlag{ + Name: "develop-mode", + Usage: "enable develop mode", + EnvVars: []string{"DEV_MODE"}, + Category: "Development", + }, + }, + Action: runCmd, + }, + }, + Name: "kubeip-agent", + Usage: "replaces the node's public IP address with a static public IP address", + Version: version, + } + cli.VersionPrinter = func(c *cli.Context) { + fmt.Printf("kubeip-agent %s\n", version) + fmt.Printf(" Build date: %s\n", buildDate) + fmt.Printf(" Git commit: %s\n", gitCommit) + fmt.Printf(" Git branch: %s\n", gitBranch) + fmt.Printf(" Built with: %s\n", runtime.Version()) + } + + err := app.Run(os.Args) + if err != nil { + logrus.Fatal(err) + } +} + +func kubeConfigFromPath(kubepath string) (*rest.Config, error) { + if kubepath == "" { + return nil, errEmptyPath + } + + data, err := os.ReadFile(kubepath) + if err != nil { + return nil, errors.Wrapf(err, "reading kubeconfig at %s", kubepath) + } + + cfg, err := clientcmd.RESTConfigFromKubeConfig(data) + if err != nil { + return nil, errors.Wrapf(err, "building rest config from kubeconfig at %s", kubepath) + } + + return cfg, nil +} + +func retrieveKubeConfig(log logrus.FieldLogger, cfg *config.Config) (*rest.Config, error) { + kubeconfig, err := kubeConfigFromPath(cfg.KubeConfigPath) + if err != nil && !errors.Is(err, errEmptyPath) { + return nil, errors.Wrap(err, "retrieving kube config from path") + } + + if kubeconfig != nil { + log.Debug("using kube config from env variables") + return kubeconfig, nil + } + + inClusterConfig, err := rest.InClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "retrieving in node kube config") + } + log.Debug("using in node kube config") + return inClusterConfig, nil +} diff --git a/cmd/main_test.go b/cmd/main_test.go new file mode 100644 index 0000000..dda1580 --- /dev/null +++ b/cmd/main_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/doitintl/kubeip/internal/address" + "github.com/doitintl/kubeip/internal/config" + "github.com/doitintl/kubeip/internal/types" + mocks "github.com/doitintl/kubeip/mocks/address" + "github.com/pkg/errors" + tmock "github.com/stretchr/testify/mock" +) + +func Test_assignAddress(t *testing.T) { + type args struct { + c context.Context + assignerFn func(t *testing.T) address.Assigner + node *types.Node + cfg *config.Config + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "assign address successfully", + args: args{ + c: context.Background(), + assignerFn: func(t *testing.T) address.Assigner { + mock := mocks.NewAssigner(t) + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(nil) + return mock + }, + node: &types.Node{ + Name: "test-node", + Instance: "test-instance", + Region: "test-region", + Zone: "test-zone", + }, + cfg: &config.Config{ + Filter: []string{"test-filter"}, + OrderBy: "test-order-by", + RetryAttempts: 3, + RetryInterval: time.Millisecond, + }, + }, + }, + { + name: "assign address after a few retries", + args: args{ + c: context.Background(), + assignerFn: func(t *testing.T) address.Assigner { + mock := mocks.NewAssigner(t) + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("first error")).Once() + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("second error")).Once() + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(nil).Once() + return mock + }, + node: &types.Node{ + Name: "test-node", + Instance: "test-instance", + Region: "test-region", + Zone: "test-zone", + }, + cfg: &config.Config{ + Filter: []string{"test-filter"}, + OrderBy: "test-order-by", + RetryAttempts: 3, + RetryInterval: time.Millisecond, + }, + }, + }, + { + name: "error after a few retries and reached maximum number of retries", + args: args{ + c: context.Background(), + assignerFn: func(t *testing.T) address.Assigner { + mock := mocks.NewAssigner(t) + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("error")).Times(4) + return mock + }, + node: &types.Node{ + Name: "test-node", + Instance: "test-instance", + Region: "test-region", + Zone: "test-zone", + }, + cfg: &config.Config{ + Filter: []string{"test-filter"}, + OrderBy: "test-order-by", + RetryAttempts: 3, + RetryInterval: time.Millisecond, + }, + }, + wantErr: true, + }, + { + name: "error after a few retries and context is done", + args: args{ + c: func() context.Context { + ctx, _ := context.WithTimeout(context.Background(), 10*time.Millisecond) //nolint:govet + return ctx + }(), + assignerFn: func(t *testing.T) address.Assigner { + mock := mocks.NewAssigner(t) + mock.EXPECT().Assign(tmock.Anything, "test-instance", "test-zone", []string{"test-filter"}, "test-order-by").Return(errors.New("error")).Maybe() + return mock + }, + node: &types.Node{ + Name: "test-node", + Instance: "test-instance", + Region: "test-region", + Zone: "test-zone", + }, + cfg: &config.Config{ + Filter: []string{"test-filter"}, + OrderBy: "test-order-by", + RetryAttempts: 3, + RetryInterval: 15 * time.Millisecond, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + log := prepareLogger("debug", false) + assigner := tt.args.assignerFn(t) + if err := assignAddress(tt.args.c, log, assigner, tt.args.node, tt.args.cfg); (err != nil) != tt.wantErr { + t.Errorf("assignAddress() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/deploy/kubeip-configmap.yaml b/deploy/kubeip-configmap.yaml deleted file mode 100644 index fba262f..0000000 --- a/deploy/kubeip-configmap.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -data: - KUBEIP_LABELKEY: "kubeip" - KUBEIP_LABELVALUE: "reserved" - KUBEIP_NODEPOOL: "default-pool" - KUBEIP_FORCEASSIGNMENT: "true" - KUBEIP_ADDITIONALNODEPOOLS: "" - KUBEIP_TICKER: "5" - KUBEIP_ALLNODEPOOLS: "false" - KUBEIP_ORDERBYLABELKEY: "priority" - KUBEIP_ORDERBYDESC: "true" - KUBEIP_COPYLABELS: "false" - KUBEIP_CLEARLABELS: "false" - KUBEIP_DRYRUN: "false" -kind: ConfigMap -metadata: - labels: - app: kubeip - name: kubeip-config - namespace: kube-system diff --git a/deploy/kubeip-deployment.yaml b/deploy/kubeip-deployment.yaml deleted file mode 100644 index 8985c55..0000000 --- a/deploy/kubeip-deployment.yaml +++ /dev/null @@ -1,137 +0,0 @@ -# We need to get RBAC permissions first with -# kubectl create clusterrolebinding cluster-admin-binding \ -# --clusterrole cluster-admin --user `gcloud config list --format 'value(core.account)'` - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kubeip - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - app: kubeip - template: - metadata: - labels: - app: kubeip - spec: - priorityClassName: system-cluster-critical - nodeSelector: - cloud.google.com/gke-nodepool: pool-kubip - containers: - - name: "kubeip" - image: doitintl/kubeip:latest - imagePullPolicy: Always - volumeMounts: - - name: google-cloud-key - mountPath: /var/secrets/google - env: - - name: "KUBEIP_LABELKEY" - valueFrom: - configMapKeyRef: - key: "KUBEIP_LABELKEY" - name: "kubeip-config" - - name: "KUBEIP_LABELVALUE" - valueFrom: - configMapKeyRef: - key: "KUBEIP_LABELVALUE" - name: "kubeip-config" - - name: "KUBEIP_NODEPOOL" - valueFrom: - configMapKeyRef: - key: "KUBEIP_NODEPOOL" - name: "kubeip-config" - - name: "KUBEIP_FORCEASSIGNMENT" - valueFrom: - configMapKeyRef: - key: "KUBEIP_FORCEASSIGNMENT" - name: "kubeip-config" - - name: "KUBEIP_ORDERBYLABELKEY" - valueFrom: - configMapKeyRef: - key: "KUBEIP_ORDERBYLABELKEY" - name: "kubeip-config" - - name: "KUBEIP_ORDERBYDESC" - valueFrom: - configMapKeyRef: - key: "KUBEIP_ORDERBYDESC" - name: "kubeip-config" - - name: "KUBEIP_ADDITIONALNODEPOOLS" - valueFrom: - configMapKeyRef: - key: "KUBEIP_ADDITIONALNODEPOOLS" - name: "kubeip-config" - - name: "KUBEIP_COPYLABELS" - valueFrom: - configMapKeyRef: - key: "KUBEIP_COPYLABELS" - name: "kubeip-config" - - name: "KUBEIP_CLEARLABELS" - valueFrom: - configMapKeyRef: - key: "KUBEIP_CLEARLABELS" - name: "kubeip-config" - - name: "KUBEIP_DRYRUN" - valueFrom: - configMapKeyRef: - key: "KUBEIP_DRYRUN" - name: "kubeip-config" - - name: "KUBEIP_TICKER" - valueFrom: - configMapKeyRef: - key: "KUBEIP_TICKER" - name: "kubeip-config" - - name: "KUBEIP_ALLNODEPOOLS" - valueFrom: - configMapKeyRef: - key: "KUBEIP_ALLNODEPOOLS" - name: "kubeip-config" - - - - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /var/secrets/google/key.json - restartPolicy: Always - serviceAccountName: kubeip-sa - volumes: - - name: google-cloud-key - secret: - secretName: kubeip-key - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kubeip-sa - namespace: kube-system - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: kubeip-sa - namespace: kube-system -rules: -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get","list","watch","patch"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get","list","watch"] - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubeip-sa -subjects: - - kind: ServiceAccount - name: kubeip-sa - namespace: kube-system -roleRef: - kind: ClusterRole - name: kubeip-sa - apiGroup: rbac.authorization.k8s.io diff --git a/examples/aws/.terraform.lock.hcl b/examples/aws/.terraform.lock.hcl new file mode 100644 index 0000000..3e5955b --- /dev/null +++ b/examples/aws/.terraform.lock.hcl @@ -0,0 +1,105 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.21.0" + constraints = ">= 5.0.0" + hashes = [ + "h1:bCRZDV8QYPpl+zIU7karO77B4x2cvQ7UHKAGNvTFWQs=", + "zh:1ba1411e4f8c047950db94c236f146d4590790320c68320b4e56082d8746a507", + "zh:3185e4a34cfcad35dcf11439290a4bd0ad52d462eca2ab5d4940488a2db72833", + "zh:3c6b901f874b4d9a85301a653d0bd507b052992bd84fc81100f4e5f73b1adab7", + "zh:45d3fdbbc5804f295576b7155fdca527dedff17a014ed40c215af3bc60c329db", + "zh:47b64b453d2c373062e47a54f3df33335dc29bce6ddbbf2da9e7be768c560abe", + "zh:5cdf57ffd465288d9732d14ba13b377a8d389e0ba0ce3ac4773fd6fdfc09d6a1", + "zh:81ec4c662581a2446c78da7b27d7e0d5c2e4d50925294789ec13661817f4b5a4", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:ac248464fd4ce1f020c05f27e3182532a7d1af4b8185a4b4be8b906b30b0ca5a", + "zh:bbbedc6b6eaffcce0b31b397d607464f0c21c1b9406182163d504d3f392cc68d", + "zh:c2afc111f9503829ed055e2ae91d873670c57bd16acc1a3246ac3957f6998d4e", + "zh:cd3c8175b2152848113482da70e5b9c7cb4c951f2046fc0b832715300bd88b97", + "zh:cf89b0c09d426d489f9477209d4084e64ad1b598036284fa688b41de626b58e6", + "zh:d9d127637c3b9ff6e2d0a2c30f54bd48ab1de34f725a5df1a6a3d039b021e636", + "zh:dccca1090e4054d6558218406385fb0421ab4ac3b75e121641973be481a81f01", + ] +} + +provider "registry.terraform.io/hashicorp/cloudinit" { + version = "2.3.2" + constraints = ">= 2.0.0" + hashes = [ + "h1:ocyv0lvfyvzW4krenxV5CL4Jq5DiA3EUfoy8DR6zFMw=", + "zh:2487e498736ed90f53de8f66fe2b8c05665b9f8ff1506f751c5ee227c7f457d1", + "zh:3d8627d142942336cf65eea6eb6403692f47e9072ff3fa11c3f774a3b93130b3", + "zh:434b643054aeafb5df28d5529b72acc20c6f5ded24decad73b98657af2b53f4f", + "zh:436aa6c2b07d82aa6a9dd746a3e3a627f72787c27c80552ceda6dc52d01f4b6f", + "zh:458274c5aabe65ef4dbd61d43ce759287788e35a2da004e796373f88edcaa422", + "zh:54bc70fa6fb7da33292ae4d9ceef5398d637c7373e729ed4fce59bd7b8d67372", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:893ba267e18749c1a956b69be569f0d7bc043a49c3a0eb4d0d09a8e8b2ca3136", + "zh:95493b7517bce116f75cdd4c63b7c82a9d0d48ec2ef2f5eb836d262ef96d0aa7", + "zh:9ae21ab393be52e3e84e5cce0ef20e690d21f6c10ade7d9d9d22b39851bfeddc", + "zh:cc3b01ac2472e6d59358d54d5e4945032efbc8008739a6d4946ca1b621a16040", + "zh:f23bfe9758f06a1ec10ea3a81c9deedf3a7b42963568997d84a5153f35c5839a", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.23.0" + constraints = ">= 2.10.0" + hashes = [ + "h1:arTzD0XG/DswGCAx9JEttkSKe9RyyFW9W7UWcXF13dU=", + "zh:10488a12525ed674359585f83e3ee5e74818b5c98e033798351678b21b2f7d89", + "zh:1102ba5ca1a595f880e67102bbf999cc8b60203272a078a5b1e896d173f3f34b", + "zh:1347cf958ed3f3f80b3c7b3e23ddda3d6c6573a81847a8ee92b7df231c238bf6", + "zh:2cb18e9f5156bc1b1ee6bc580a709f7c2737d142722948f4a6c3c8efe757fa8d", + "zh:5506aa6f28dcca2a265ccf8e34478b5ec2cb43b867fe6d93b0158f01590fdadd", + "zh:6217a20686b631b1dcb448ee4bc795747ebc61b56fbe97a1ad51f375ebb0d996", + "zh:8accf916c00579c22806cb771e8909b349ffb7eb29d9c5468d0a3f3166c7a84a", + "zh:9379b0b54a0fa030b19c7b9356708ec8489e194c3b5e978df2d31368563308e5", + "zh:aa99c580890691036c2931841e88e7ee80d59ae52289c8c2c28ea0ac23e31520", + "zh:c57376d169875990ac68664d227fb69cd0037b92d0eba6921d757c3fd1879080", + "zh:e6068e3f94f6943b5586557b73f109debe19d1a75ca9273a681d22d1ce066579", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.9.1" + constraints = ">= 0.9.0" + hashes = [ + "h1:VxyoYYOCaJGDmLz4TruZQTSfQhvwEcMxvcKclWdnpbs=", + "zh:00a1476ecf18c735cc08e27bfa835c33f8ac8fa6fa746b01cd3bcbad8ca84f7f", + "zh:3007f8fc4a4f8614c43e8ef1d4b0c773a5de1dcac50e701d8abc9fdc8fcb6bf5", + "zh:5f79d0730fdec8cb148b277de3f00485eff3e9cf1ff47fb715b1c969e5bbd9d4", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8c8094689a2bed4bb597d24a418bbbf846e15507f08be447d0a5acea67c2265a", + "zh:a6d9206e95d5681229429b406bc7a9ba4b2d9b67470bda7df88fa161508ace57", + "zh:aa299ec058f23ebe68976c7581017de50da6204883950de228ed9246f309e7f1", + "zh:b129f00f45fba1991db0aa954a6ba48d90f64a738629119bfb8e9a844b66e80b", + "zh:ef6cecf5f50cda971c1b215847938ced4cb4a30a18095509c068643b14030b00", + "zh:f1f46a4f6c65886d2dd27b66d92632232adc64f92145bf8403fe64d5ffa5caea", + "zh:f79d6155cda7d559c60d74883a24879a01c4d5f6fd7e8d1e3250f3cd215fb904", + "zh:fd59fa73074805c3575f08cd627eef7acda14ab6dac2c135a66e7a38d262201c", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.4" + constraints = ">= 3.0.0" + hashes = [ + "h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=", + "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", + "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", + "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be", + "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5", + "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe", + "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e", + "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48", + "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8", + "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60", + "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e", + "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/examples/aws/eks.tf b/examples/aws/eks.tf new file mode 100644 index 0000000..a784f7d --- /dev/null +++ b/examples/aws/eks.tf @@ -0,0 +1,256 @@ +provider "aws" { + region = var.region +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + + name = var.vpc_name + cidr = var.vpc_cidr + azs = var.availability_zones + private_subnets = var.private_cidr_ranges + public_subnets = var.public_cidr_ranges + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + map_public_ip_on_launch = true + + tags = { + App = "kubeip" + Env = "demo" + } + public_subnet_tags = { + public = "true" + environment = "demo" + } + private_subnet_tags = { + public = "false" + environment = "demo" + } +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + + cluster_name = var.cluster_name + cluster_version = var.kubernetes_version + + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = concat(module.vpc.private_subnets, module.vpc.public_subnets) + + eks_managed_node_groups = { + eks_nodes_public = { + desired_size = 3 + max_size = 5 + min_size = 1 + + instance_types = ["t3a.small", "t3a.medium"] + capacity_type = "SPOT" + + labels = { + nodegroup = "public" + kubeip = "use" + } + + tags = { + Name = "public-node-group" + environment = "demo" + public = "true" + kubeip = "use" + } + + subnet_ids = module.vpc.public_subnets + } + + eks_nodes_private = { + desired_size = 1 + max_size = 5 + min_size = 1 + + instance_types = ["t3a.small", "t3a.medium"] + capacity_type = "SPOT" + + labels = { + nodegroup = "private" + kubeip = "ignore" + } + + tags = { + Name = "private-node-group" + environment = "demo" + } + + subnet_ids = module.vpc.private_subnets + } + } +} + +resource "aws_iam_policy" "kubeip-policy" { + name = "kubeip-policy" + description = "KubeIP required permissions" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:AssociateAddress", + "ec2:DisassociateAddress", + "ec2:DescribeInstances", + "ec2:DescribeAddresses" + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) +} + +module "kubeip_eks_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + role_name = "kubeip-eks-role" + + role_policy_arns = { + "kubeip-policy" = aws_iam_policy.kubeip-policy.arn + } + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:kubeip-service-account"] + } + } +} + +# 3 elastic IPs in the same region +resource "aws_eip" "kubeip" { + // default EIP limit is 5 (make sure to increase it if you need more) + count = 5 + + tags = { + Name = "kubeip-${count.index}" + environment = "demo" + kubeip = "reserved" + } +} + +data "aws_eks_cluster_auth" "kubeip_cluster_auth" { + name = module.eks.cluster_name +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.kubeip_cluster_auth.token +} + +resource "kubernetes_service_account" "kubeip_service_account" { + metadata { + name = "kubeip-service-account" + namespace = "kube-system" + annotations = { + "eks.amazonaws.com/role-arn" = module.kubeip_eks_role.iam_role_arn + } + } + depends_on = [module.eks] +} + +# Create cluster role with get node permission +resource "kubernetes_cluster_role" "kubeip_cluster_role" { + metadata { + name = "kubeip-cluster-role" + } + rule { + api_groups = ["*"] + resources = ["nodes"] + verbs = ["get"] + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, + module.eks + ] +} + +# Bind cluster role to kubeip service account +resource "kubernetes_cluster_role_binding" "kubeip_cluster_role_binding" { + metadata { + name = "kubeip-cluster-role-binding" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.kubeip_cluster_role.metadata[0].name + } + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.kubeip_service_account.metadata[0].name + namespace = kubernetes_service_account.kubeip_service_account.metadata[0].namespace + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, + kubernetes_cluster_role.kubeip_cluster_role + ] +} + + +# Deploy KubeIP DaemonSet +resource "kubernetes_daemonset" "kubeip_daemonset" { + metadata { + name = "kubeip-agent" + namespace = "kube-system" + labels = { + app = "kubeip" + } + } + spec { + selector { + match_labels = { + app = "kubeip" + } + } + template { + metadata { + labels = { + app = "kubeip" + } + } + spec { + service_account_name = "kubeip-service-account" + termination_grace_period_seconds = 30 + priority_class_name = "system-node-critical" + container { + name = "kubeip-agent" + image = "doitintl/kubeip-agent" + env { + name = "NODE_NAME" + value_from { + field_ref { + field_path = "spec.nodeName" + } + } + } + env { + name = "FILTER" + value = "Name=tag:kubeip,Values=reserved;Name=tag:environment,Values=demo" + } + env { + name = "LOG_LEVEL" + value = "debug" + } + resources { + requests = { + cpu = "100m" + } + } + } + node_selector = { + nodegroup = "public" + kubeip = "use" + } + } + } + } + depends_on = [kubernetes_service_account.kubeip_service_account] +} diff --git a/examples/aws/variables.tf b/examples/aws/variables.tf new file mode 100644 index 0000000..d8a590b --- /dev/null +++ b/examples/aws/variables.tf @@ -0,0 +1,39 @@ +variable "region" { + type = string + default = "us-west-2" +} + +variable "availability_zones" { + type = list(string) + default = ["us-west-2a", "us-west-2b", "us-west-2c"] +} + +variable "cluster_name" { + type = string + default = "kubeip-demo" +} + +variable "vpc_name" { + type = string + default = "kubeip-demo" +} + +variable "vpc_cidr" { + type = string + default = "10.0.0.0/16" +} + +variable "private_cidr_ranges" { + type = list(string) + default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] +} + +variable "public_cidr_ranges" { + type = list(string) + default = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] +} + +variable "kubernetes_version" { + type = string + default = "1.28" +} \ No newline at end of file diff --git a/examples/gcp/.terraform.lock.hcl b/examples/gcp/.terraform.lock.hcl new file mode 100644 index 0000000..271117d --- /dev/null +++ b/examples/gcp/.terraform.lock.hcl @@ -0,0 +1,60 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "4.84.0" + constraints = ">= 3.39.0, >= 3.53.0, < 5.0.0, < 6.0.0" + hashes = [ + "h1:fybaK74buTd4Ys2CUZm6jw7NXtSqtcLoW2jeNB4Ff2E=", + "zh:0b3e945fa76876c312bdddca7b18c93b734998febb616b2ebb84a0a299ae97c2", + "zh:1d47d00730fab764bddb6d548fed7e124739b0bcebb9f3b3c6aa247de55fb804", + "zh:29bff92b4375a35a7729248b3bc5db8991ca1b9ba640fc25b13700e12f99c195", + "zh:382353516e7d408a81f1a09a36f9015429be73ca3665367119aad88713209d9a", + "zh:78afa20e25a690d076eeaafd7879993ef9763a8a1b6762e2cbe42330464cc1fa", + "zh:8f6422e94de865669b33a2d9fb95a3e392e841988e890f7379a206e9d47e3415", + "zh:be5c7b52c893b971c860146aec643f7007f34430106f101eab686ed81eccbd26", + "zh:bfc37b641bf3378183eb3b8735554c3949a5cfaa8f76403d7eff38de1474b6d9", + "zh:c834f88dc8eb21af992871ed13a221015ae3b051aeca7386662071026f1546b4", + "zh:f3296c8c0d57dc28e23cf91717484264531655ac478d994584ebc73f70679471", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f8efe114ff4891776f48f7d2620b8d6963d3ddac6e42ce25bc761343da964c24", + ] +} + +provider "registry.terraform.io/hashicorp/google-beta" { + version = "5.1.0" + hashes = [ + "h1:d0u9a3m826V3QZ7XsycvlJGduERSi1eunLoHWZjRIb0=", + "zh:3369d685b81dfad4ee307c9729ea20f6593f1dee23bc785c167c6a0b6843f208", + "zh:40cef4bbbcefc4944843915a626ab39b734ba0e7dcbe57ea9faed1e935b73efb", + "zh:50dea4f57191c8a91dcbc9db3a09381899b43e06c3f2e6767792d8ce7711e8a1", + "zh:5406877b75fdf94daeed74a69e65f6e02c40880cf22f5d91c75ca69c3c7f435a", + "zh:7d9074317ca61384a86468d40e2f30f67eec5e44e87d2eac752cdaaed0a45e83", + "zh:9188b5492c70f3826f65134f7c74ce74a933ced6e28426e9d6a9358d8c33b13d", + "zh:b06dabf01ca9f9a0cf2c0613d00a212ae2b8c2b7d3e78057f52856e385483c87", + "zh:b7ac631dbd6efea37ca94bae7d0476a13243d884a8bd8eb2d39e3398c1e9b9ad", + "zh:c927efccfab1e3afb1fdc3ba141d0e04f67fffadb55346b2b4b272a1e358fe8a", + "zh:d418d657c7b95762b6d5caae993ccc18bf54c63dec08c5a03f0aeb53403440f4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f9e96ead5152fbb2df8571ee87810e7c3638df877ba5124e2b092faf4a3a641e", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.23.0" + hashes = [ + "h1:arTzD0XG/DswGCAx9JEttkSKe9RyyFW9W7UWcXF13dU=", + "zh:10488a12525ed674359585f83e3ee5e74818b5c98e033798351678b21b2f7d89", + "zh:1102ba5ca1a595f880e67102bbf999cc8b60203272a078a5b1e896d173f3f34b", + "zh:1347cf958ed3f3f80b3c7b3e23ddda3d6c6573a81847a8ee92b7df231c238bf6", + "zh:2cb18e9f5156bc1b1ee6bc580a709f7c2737d142722948f4a6c3c8efe757fa8d", + "zh:5506aa6f28dcca2a265ccf8e34478b5ec2cb43b867fe6d93b0158f01590fdadd", + "zh:6217a20686b631b1dcb448ee4bc795747ebc61b56fbe97a1ad51f375ebb0d996", + "zh:8accf916c00579c22806cb771e8909b349ffb7eb29d9c5468d0a3f3166c7a84a", + "zh:9379b0b54a0fa030b19c7b9356708ec8489e194c3b5e978df2d31368563308e5", + "zh:aa99c580890691036c2931841e88e7ee80d59ae52289c8c2c28ea0ac23e31520", + "zh:c57376d169875990ac68664d227fb69cd0037b92d0eba6921d757c3fd1879080", + "zh:e6068e3f94f6943b5586557b73f109debe19d1a75ca9273a681d22d1ce066579", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/examples/gcp/gke.tf b/examples/gcp/gke.tf new file mode 100644 index 0000000..cee99e4 --- /dev/null +++ b/examples/gcp/gke.tf @@ -0,0 +1,314 @@ +# Save state to local file +terraform { + backend "local" { + path = "terraform.tfstate" + } +} + +# Set the provider and credentials +provider "google" { + project = var.project_id + region = var.region +} + +# Create custom IAM Role +resource "google_project_iam_custom_role" "kubeip_role" { + role_id = "kubeip_role" + title = "KubeIP Role" + description = "KubeIP required permissions" + stage = "GA" + permissions = [ + "compute.instances.addAccessConfig", + "compute.instances.deleteAccessConfig", + "compute.instances.get", + "compute.addresses.get", + "compute.addresses.list", + "compute.addresses.use", + "compute.zoneOperations.get", + "compute.zoneOperations.list", + "compute.subnetworks.useExternalIp", + "compute.projects.get" + ] +} + +# Create custom IAM service account +resource "google_service_account" "kubeip_service_account" { + account_id = "kubeip-service-account" + display_name = "KubeIP Service Account" +} + +# Bind custom IAM Role to kubeip IAM service account +resource "google_project_iam_member" "kubeip_role_binding" { + role = google_project_iam_custom_role.kubeip_role.id + member = "serviceAccount:${google_service_account.kubeip_service_account.email}" + project = var.project_id +} + +# Bind workload identity to kubeip IAM service account +resource "google_service_account_iam_member" "kubeip_workload_identity_binding" { + service_account_id = google_service_account.kubeip_service_account.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project_id}.svc.id.goog[kube-system/kubeip-service-account]" +} + +# Create a VPC network +resource "google_compute_network" "vpc" { + name = var.vpc_name + auto_create_subnetworks = false +} + +# Create a public subnet +resource "google_compute_subnetwork" "kubeip_subnet" { + name = "kubeip-subnet" + network = google_compute_network.vpc.id + region = var.region + ip_cidr_range = var.subnet_range + private_ip_google_access = true + secondary_ip_range { + range_name = var.services_range_name + ip_cidr_range = var.services_range + } + secondary_ip_range { + range_name = var.pods_range_name + ip_cidr_range = var.pods_range + } +} + +# Create GKE cluster +resource "google_container_cluster" "kubeip_cluster" { + name = var.cluster_name + location = var.region + + initial_node_count = 1 + remove_default_node_pool = true + + network = google_compute_network.vpc.id + subnetwork = google_compute_subnetwork.kubeip_subnet.id + + ip_allocation_policy { + services_secondary_range_name = var.services_range_name + cluster_secondary_range_name = var.pods_range_name + } + + # Enable Workload Identity + workload_identity_config { + workload_pool = "${var.project_id}.svc.id.goog" + } +} + +# Create node pools +resource "google_container_node_pool" "public_node_pool" { + name = "public-node-pool" + location = google_container_cluster.kubeip_cluster.location + cluster = google_container_cluster.kubeip_cluster.name + initial_node_count = 1 + autoscaling { + min_node_count = 1 + max_node_count = 2 + location_policy = "ANY" + } + node_config { + machine_type = var.machine_type + spot = true + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + ] + metadata = { + disable-legacy-endpoints = "true" + } + workload_metadata_config { + mode = "GKE_METADATA" + } + labels = { + nodegroup = "public" + kubeip = "use" + } + resource_labels = { + environment = "demo" + kubeip = "use" + public = "true" + } + } +} + +resource "google_container_node_pool" "private_node_pool" { + name = "private-node-pool" + location = google_container_cluster.kubeip_cluster.location + cluster = google_container_cluster.kubeip_cluster.name + initial_node_count = 1 + autoscaling { + min_node_count = 1 + max_node_count = 2 + location_policy = "ANY" + } + node_config { + machine_type = var.machine_type + spot = true + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + ] + metadata = { + disable-legacy-endpoints = "true" + } + workload_metadata_config { + mode = "GKE_METADATA" + } + labels = { + nodegroup = "private" + kubeip = "ignore" + } + resource_labels = { + environment = "demo" + kubeip = "ignore" + public = "false" + } + } + network_config { + enable_private_nodes = true + } +} + +# Create static public IP addresses +resource "google_compute_address" "static_ip" { + provider = google-beta + project = var.project_id + count = 5 + name = "static-ip-${count.index}" + address_type = "EXTERNAL" + region = google_container_cluster.kubeip_cluster.location + labels = { + environment = "demo" + kubeip = "reserved" + } +} + +data "google_client_config" "provider" {} + +provider "kubernetes" { + host = "https://${google_container_cluster.kubeip_cluster.endpoint}" + token = data.google_client_config.provider.access_token + cluster_ca_certificate = base64decode( + google_container_cluster.kubeip_cluster.master_auth[0].cluster_ca_certificate, + ) +} + +# Create Kubernetes service account in kube-system namespace +resource "kubernetes_service_account" "kubeip_service_account" { + metadata { + name = "kubeip-service-account" + namespace = "kube-system" + annotations = { + "iam.gke.io/gcp-service-account" = google_service_account.kubeip_service_account.email + } + } + depends_on = [ + google_service_account.kubeip_service_account, + google_container_cluster.kubeip_cluster + ] +} + +# Create cluster role with get node permission +resource "kubernetes_cluster_role" "kubeip_cluster_role" { + metadata { + name = "kubeip-cluster-role" + } + rule { + api_groups = ["*"] + resources = ["nodes"] + verbs = ["get"] + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, + google_container_cluster.kubeip_cluster + ] +} + +# Bind cluster role to kubeip service account +resource "kubernetes_cluster_role_binding" "kubeip_cluster_role_binding" { + metadata { + name = "kubeip-cluster-role-binding" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.kubeip_cluster_role.metadata[0].name + } + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.kubeip_service_account.metadata[0].name + namespace = kubernetes_service_account.kubeip_service_account.metadata[0].namespace + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, + kubernetes_cluster_role.kubeip_cluster_role + ] +} + +# Deploy KubeIP DaemonSet +resource "kubernetes_daemonset" "kubeip_daemonset" { + metadata { + name = "kubeip-agent" + namespace = "kube-system" + labels = { + app = "kubeip" + } + } + spec { + selector { + match_labels = { + app = "kubeip" + } + } + template { + metadata { + labels = { + app = "kubeip" + } + } + spec { + service_account_name = "kubeip-service-account" + termination_grace_period_seconds = 30 + priority_class_name = "system-node-critical" + container { + name = "kubeip-agent" + image = "doitintl/kubeip-agent" + env { + name = "NODE_NAME" + value_from { + field_ref { + field_path = "spec.nodeName" + } + } + } + env { + name = "FILTER" + value = "labels.kubeip=reserved;labels.environment=demo" + } + env { + name = "LOG_LEVEL" + value = "debug" + } + evn { + name = "LOG_JSON" + value = "true" + } + resources { + requests = { + cpu = "100m" + } + } + } + node_selector = { + nodegroup = "public" + kubeip = "use" + } + } + } + } + depends_on = [ + kubernetes_service_account.kubeip_service_account, + google_container_cluster.kubeip_cluster + ] +} diff --git a/examples/gcp/variables.tf b/examples/gcp/variables.tf new file mode 100644 index 0000000..e03524b --- /dev/null +++ b/examples/gcp/variables.tf @@ -0,0 +1,48 @@ +variable "project_id" { + type = string +} + +variable "region" { + type = string + default = "us-central1" +} + +variable "cluster_name" { + type = string + default = "kubeip-demo" +} + +variable "vpc_name" { + type = string + default = "kubeip-demo" +} + +variable "subnet_range" { + type = string + default = "10.128.0.0/20" +} + +variable "pods_range" { + type = string + default = "10.128.64.0/18" +} + +variable "pods_range_name" { + type = string + default = "pods-range" +} + +variable "services_range_name" { + type = string + default = "services-range" +} + +variable "services_range" { + type = string + default = "10.128.32.0/20" +} + +variable "machine_type" { + type = string + default = "e2-medium" +} \ No newline at end of file diff --git a/go.mod b/go.mod index 4c2c3e8..16ad7d7 100644 --- a/go.mod +++ b/go.mod @@ -1,59 +1,85 @@ module github.com/doitintl/kubeip -go 1.20 +go 1.21 require ( cloud.google.com/go/compute/metadata v0.2.3 + github.com/aws/aws-sdk-go-v2 v1.21.1 + github.com/aws/aws-sdk-go-v2/config v1.18.44 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.124.0 github.com/pkg/errors v0.9.1 - github.com/sirupsen/logrus v1.8.1 - github.com/spf13/viper v1.0.2 - golang.org/x/net v0.7.0 - golang.org/x/oauth2 v0.4.0 - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac - google.golang.org/api v0.103.0 - k8s.io/api v0.22.2 - k8s.io/apimachinery v0.22.2 - k8s.io/client-go v0.22.2 + github.com/sirupsen/logrus v1.9.3 + github.com/stretchr/testify v1.8.3 + github.com/urfave/cli/v2 v2.25.7 + google.golang.org/api v0.143.0 + k8s.io/api v0.28.1 + k8s.io/apimachinery v0.28.1 + k8s.io/client-go v0.28.1 + sigs.k8s.io/controller-runtime v0.16.2 ) require ( - cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go/compute v1.23.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.42 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 // indirect + github.com/aws/smithy-go v1.15.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/go-logr/logr v0.4.0 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.9.0 // indirect - k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 264d6da..2547d5f 100644 --- a/go.sum +++ b/go.sum @@ -1,109 +1,85 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go-v2 v1.21.1 h1:wjHYshtPpYOZm+/mu3NhVgRRc0baM6LJZOmxPZ5Cwzs= +github.com/aws/aws-sdk-go-v2 v1.21.1/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= +github.com/aws/aws-sdk-go-v2/config v1.18.44 h1:U10NQ3OxiY0dGGozmVIENIDnCT0W432PWxk2VO8wGnY= +github.com/aws/aws-sdk-go-v2/config v1.18.44/go.mod h1:pHxnQBldd0heEdJmolLBk78D1Bf69YnKLY3LOpFImlU= +github.com/aws/aws-sdk-go-v2/credentials v1.13.42 h1:KMkjpZqcMOwtRHChVlHdNxTUUAC6NC/b58mRZDIdcRg= +github.com/aws/aws-sdk-go-v2/credentials v1.13.42/go.mod h1:7ltKclhvEB8305sBhrpls24HGxORl6qgnQqSJ314Uw8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12 h1:3j5lrl9kVQrJ1BU4O0z7MQ8sa+UXdiLuo4j0V+odNI8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12/go.mod h1:JbFpcHDBdsex1zpIKuVRorZSQiZEyc3MykNCcjgz174= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 h1:817VqVe6wvwE46xXy6YF5RywvjOX6U2zRQQ6IbQFK0s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42/go.mod h1:oDfgXoBBmj+kXnqxDDnIDnC56QBosglKp8ftRCTxR+0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 h1:7ZApaXzWbo8slc+W5TynuUlB4z66g44h7uqa3/d/BsY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36/go.mod h1:rwr4WnmFi3RJO0M4dxbJtgi9BPLMpVBMX1nUte5ha9U= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44 h1:quOJOqlbSfeJTboXLjYXM1M9T52LBXqLoTPlmsKLpBo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44/go.mod h1:LNy+P1+1LiRcCsVYr/4zG5n8zWFL0xsvZkOybjbftm8= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.124.0 h1:3VsdIKjFmyXFkKV21tgn49/dxSziWhjnx3YbqrDofXc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.124.0/go.mod h1:f2AJtWtbonV7cSBVdxfs6e68cponNukbBDvzc4WIASo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 h1:YXlm7LxwNlauqb2OrinWlcvtsflTzP8GaMvYfQBhoT4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36/go.mod h1:ou9ffqJ9hKOVZmjlC6kQ6oROAyG1M4yBKzR+9BKbDwk= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.1 h1:ZN3bxw9OYC5D6umLw6f57rNJfGfhg1DIAAcKpzyUTOE= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.1/go.mod h1:PieckvBoT5HtyB9AsJRrYZFY2Z+EyfVM/9zG6gbV8DQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2 h1:fSCCJuT5i6ht8TqGdZc5Q5K9pz/atrf7qH4iK5C9XzU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2/go.mod h1:5eNtr+vNc5vVd92q7SJ+U/HszsIdhZBEyi9dkMRKsp8= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 h1:ASNYk1ypWAxRhJjKS0jBnTUeDl7HROOpeSMu1xDA/I8= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.1/go.mod h1:2cnsAhVT3mqusovc2stUSUrSBGTcX9nh8Tu6xh//2eI= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -111,345 +87,172 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= -github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.0.2 h1:Ncr3ZIuJn322w2k1qmzXDnkLAdQMlJqBa9kfAH+irso= -github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.143.0 h1:o8cekTkqhywkbZT6p1UHJPZ9+9uuCAJs/KYomxZB8fA= +google.golang.org/api v0.143.0/go.mod h1:FoX9DO9hT7DLNn97OuoZAGSDuNAXdJRuGK98rSUgurk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -458,58 +261,41 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= -k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= -k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= -k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= -k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= +k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= +k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU= +sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/address/assigner.go b/internal/address/assigner.go new file mode 100644 index 0000000..88dd12c --- /dev/null +++ b/internal/address/assigner.go @@ -0,0 +1,30 @@ +package address + +import ( + "context" + "errors" + + "github.com/doitintl/kubeip/internal/config" + "github.com/doitintl/kubeip/internal/types" + "github.com/sirupsen/logrus" +) + +var ( + ErrUnknownCloudProvider = errors.New("unknown cloud provider") +) + +type Assigner interface { + Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error + Unassign(ctx context.Context, instanceID, zone string) error +} + +func NewAssigner(ctx context.Context, logger *logrus.Entry, provider types.CloudProvider, cfg *config.Config) (Assigner, error) { + if provider == types.CloudProviderAWS { + return NewAwsAssigner(ctx, logger, cfg.Region) + } else if provider == types.CloudProviderAzure { + return &azureAssigner{}, nil + } else if provider == types.CloudProviderGCP { + return NewGCPAssigner(ctx, logger, cfg.Project, cfg.Region) + } + return nil, ErrUnknownCloudProvider +} diff --git a/internal/address/aws.go b/internal/address/aws.go new file mode 100644 index 0000000..fa82012 --- /dev/null +++ b/internal/address/aws.go @@ -0,0 +1,304 @@ +package address + +import ( + "context" + "sort" + "strings" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/doitintl/kubeip/internal/cloud" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + shorthandFilterTokens = 2 +) + +type awsAssigner struct { + region string + logger *logrus.Entry + instanceGetter cloud.Ec2InstanceGetter + eipLister cloud.EipLister + eipAssigner cloud.EipAssigner +} + +func NewAwsAssigner(ctx context.Context, logger *logrus.Entry, region string) (Assigner, error) { + // initialize AWS client + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + if err != nil { + return nil, errors.Wrap(err, "failed to load AWS config") + } + + // create AWS client for EC2 service in the given region with default config and credentials + client := ec2.NewFromConfig(cfg) + + // initialize AWS instance getter + instanceGetter := cloud.NewEc2InstanceGetter(client) + + // initialize AWS elastic IP lister + eipLister := cloud.NewEipLister(client) + + // initialize AWS elastic IP assigner + eipAssigner := cloud.NewEipAssigner(client) + + return &awsAssigner{ + region: region, + logger: logger, + instanceGetter: instanceGetter, + eipLister: eipLister, + eipAssigner: eipAssigner, + }, nil +} + +// parseShorthandFilter parses shorthand filter string into filter name and values +// shorthand filter format: Name=string,Values=string,string ... +// https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-addresses.html#options +func parseShorthandFilter(filter string) (string, []string, error) { + // split filter by the first "," + exp := strings.SplitN(filter, ",", shorthandFilterTokens) + if len(exp) != shorthandFilterTokens { + return "", nil, errors.New("invalid filter format; supported format Name=string,Values=string,string,") + } + // get filter name + name := strings.Split(exp[0], "=") + if len(name) != 2 || name[0] != "Name" { + return "", nil, errors.New("invalid filter Name") + } + // get filter values + values := strings.Split(exp[1], "=") + if len(values) != 2 || values[0] != "Values" { + return "", nil, errors.New("invalid filter Values list") + } + listValues := strings.Split(values[1], ",") + return name[1], listValues, nil +} + +func sortAddressesByTag(addresses []types.Address, key string) { + sort.Slice(addresses, func(i, j int) bool { + if addresses[i].Tags == nil { + return false + } + if addresses[j].Tags == nil { + return true + } + for _, tag := range addresses[i].Tags { + if *tag.Key == key { + for _, tag2 := range addresses[j].Tags { + if *tag2.Key == key { + return *tag.Value < *tag2.Value + } + } + } + } + return false + }) +} + +// sortAddressesByField sorts addresses by the given field +// if sortBy is Tag:, sort addresses by tag value +func sortAddressesByField(addresses []types.Address, sortBy string) { + // if sortBy is Tag:, sort addresses by tag value + if strings.HasPrefix(sortBy, "Tag:") { + key := strings.TrimPrefix(sortBy, "Tag:") + sortAddressesByTag(addresses, key) + return // return if sortBy is Tag: + } + // sort addresses by orderBy field + switch sortBy { + case "AllocationId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].AllocationId < *addresses[j].AllocationId + }) + case "AssociationId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].AssociationId < *addresses[j].AssociationId + }) + case "Domain": + sort.Slice(addresses, func(i, j int) bool { + return addresses[i].Domain < addresses[j].Domain + }) + case "InstanceId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].InstanceId < *addresses[j].InstanceId + }) + case "NetworkInterfaceId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].NetworkInterfaceId < *addresses[j].NetworkInterfaceId + }) + case "NetworkInterfaceOwnerId": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].NetworkInterfaceOwnerId < *addresses[j].NetworkInterfaceOwnerId + }) + case "PrivateIpAddress": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].PrivateIpAddress < *addresses[j].PrivateIpAddress + }) + case "PublicIp": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].PublicIp < *addresses[j].PublicIp + }) + case "PublicIpv4Pool": + sort.Slice(addresses, func(i, j int) bool { + return *addresses[i].PublicIpv4Pool < *addresses[j].PublicIpv4Pool + }) + } +} + +func (a *awsAssigner) forceCheckAddressAssigned(ctx context.Context, allocationID string) (bool, error) { + // get elastic IP attached to the allocation ID + filters := make(map[string][]string) + filters["allocation-id"] = []string{allocationID} + addresses, err := a.eipLister.List(ctx, filters, true) + if err != nil { + return false, errors.Wrapf(err, "failed to list elastic IPs by allocation-id %s", allocationID) + } + if len(addresses) == 0 { + return false, nil + } + // check if the first address (and the only) is assigned + if addresses[0].AssociationId != nil { + return true, nil + } + return false, nil +} + +//nolint:funlen,gocyclo +func (a *awsAssigner) Assign(ctx context.Context, instanceID, _ string, filter []string, orderBy string) error { + // get elastic IP attached to the instance + filters := make(map[string][]string) + filters["instance-id"] = []string{instanceID} + addresses, err := a.eipLister.List(ctx, filters, true) + if err != nil { + return errors.Wrapf(err, "failed to list elastic IPs attached to instance %s", instanceID) + } + if len(addresses) > 0 { + a.logger.Infof("elastic IP %s is already attached to instance %s", *addresses[0].PublicIp, instanceID) + return nil + } + + // get available elastic IPs + filters = make(map[string][]string) + for _, f := range filter { + name, values, err2 := parseShorthandFilter(f) + if err2 != nil { + return errors.Wrapf(err2, "failed to parse filter %s", f) + } + filters[name] = values + } + addresses, err = a.eipLister.List(context.Background(), filters, false) + if err != nil { + return errors.Wrap(err, "failed to list available elastic IPs") + } + + // if no available elastic IPs, return error + if len(addresses) == 0 { + return errors.Errorf("no available elastic IPs") + } + + // log available addresses IPs + ips := make([]string, 0, len(addresses)) + for _, address := range addresses { + ips = append(ips, *address.PublicIp) + } + a.logger.WithField("addresses", ips).Debugf("found %d available addresses", len(addresses)) + + // get EC2 instance + instance, err := a.instanceGetter.Get(ctx, instanceID, a.region) + if err != nil { + return errors.Wrapf(err, "failed to get instance %s", instanceID) + } + // get network interface ID + if instance.NetworkInterfaces == nil || len(instance.NetworkInterfaces) == 0 { + return errors.Errorf("no network interfaces found for instance %s", instanceID) + } + // get primary network interface ID with public IP address (DeviceIndex == 0) + networkInterfaceID := "" + for _, ni := range instance.NetworkInterfaces { + if ni.Association != nil && ni.Association.PublicIp != nil && + ni.Attachment != nil && ni.Attachment.DeviceIndex != nil && *ni.Attachment.DeviceIndex == 0 { + networkInterfaceID = *ni.NetworkInterfaceId + break + } + } + if networkInterfaceID == "" { + return errors.Errorf("no network interfaces with public IP address found for instance %s", instanceID) + } + + // sort addresses by orderBy field + sortAddressesByField(addresses, orderBy) + + // try to assign all available addresses until one succeeds + // due to concurrency, it is possible that another kubeip instance will assign the same address + for i := range addresses { + // force check if address is already assigned (reduce the chance of assigning the same address by multiple kubeip instances) + var addressAssigned bool + addressAssigned, err = a.forceCheckAddressAssigned(ctx, *addresses[i].AllocationId) + if err != nil { + a.logger.WithError(err).Errorf("failed to check if address %s is assigned", *addresses[i].PublicIp) + a.logger.Debug("trying next address") + continue + } + if addressAssigned { + a.logger.WithField("address", addresses[i].PublicIp).Debug("address is already assigned") + a.logger.Debug("trying next address") + continue + } + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": *addresses[i].PublicIp, + "allocation_id": *addresses[i].AllocationId, + "networkInterfaceID": networkInterfaceID, + }).Debug("assigning elastic IP to the instance") + if err = a.eipAssigner.Assign(ctx, networkInterfaceID, *addresses[i].AllocationId); err != nil { + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": *addresses[i].PublicIp, + "allocation_id": *addresses[i].AllocationId, + "networkInterfaceID": networkInterfaceID, + }).Debug("failed to assign elastic IP to the instance") + a.logger.Debug("trying next address") + continue + } + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": *addresses[i].PublicIp, + "allocation_id": *addresses[i].AllocationId, + }).Info("elastic IP assigned to the instance") + break + } + if err != nil { + return errors.Wrap(err, "failed to assign elastic IP address") + } + return nil +} + +func (a *awsAssigner) Unassign(ctx context.Context, instanceID, _ string) error { + // get elastic IP attached to the instance + filters := make(map[string][]string) + filters["instance-id"] = []string{instanceID} + addresses, err := a.eipLister.List(ctx, filters, true) + if err != nil { + return errors.Wrapf(err, "failed to list elastic IPs attached to instance %s", instanceID) + } + if len(addresses) == 0 { + a.logger.Infof("no elastic IP attached to instance %s", instanceID) + return nil + } + + // unassign elastic IP from the instance + address := addresses[0] + if err = a.eipAssigner.Unassign(ctx, *address.AssociationId); err != nil { + return errors.Wrap(err, "failed to unassign elastic IP") + } + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": *address.PublicIp, + "allocation_id": *address.AllocationId, + "associationId": *address.AssociationId, + }).Info("elastic IP unassigned from the instance") + + return nil +} diff --git a/internal/address/aws_test.go b/internal/address/aws_test.go new file mode 100644 index 0000000..9aab050 --- /dev/null +++ b/internal/address/aws_test.go @@ -0,0 +1,590 @@ +package address + +import ( + "context" + "reflect" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/doitintl/kubeip/internal/cloud" + mocks "github.com/doitintl/kubeip/mocks/cloud" + "github.com/sirupsen/logrus" +) + +func Test_sortAddressesByTag(t *testing.T) { + type args struct { + addresses []types.Address + key string + } + tests := []struct { + name string + args args + want []types.Address + }{ + { + name: "Test case 1: Sort addresses by tag value", + args: args{ + addresses: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + }, + key: "Name", + }, + want: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + }, + }, + { + name: "Test case 2: Addresses with no tags", + args: args{ + addresses: []types.Address{ + {}, + {}, + }, + key: "Name", + }, + want: []types.Address{ + {}, + {}, + }, + }, + { + name: "Test case 3: Key not found in tags", + args: args{ + addresses: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + }, + key: "NonExistentKey", + }, + want: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + }, + }, + { + name: "Test case 4: One address with tags, one without", + args: args{ + addresses: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + {}, + }, + key: "Name", + }, + want: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + {}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sortAddressesByTag(tt.args.addresses, tt.args.key) + if !reflect.DeepEqual(tt.args.addresses, tt.want) { + t.Errorf("sortAddressesByTag() = %v, want %v", tt.args.addresses, tt.want) + } + }) + } +} + +func Test_sortAddressesByField(t *testing.T) { + type args struct { + addresses []types.Address + sortBy string + } + tests := []struct { + name string + args args + want []types.Address + }{ + { + name: "Test case 1: Sort addresses by tag value", + args: args{ + addresses: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + }, + sortBy: "Tag:Name", + }, + want: []types.Address{ + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("A"), + }, + }, + }, + { + Tags: []types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("B"), + }, + }, + }, + }, + }, + { + name: "Test case 2: Sort addresses by AllocationId", + args: args{ + addresses: []types.Address{ + { + AllocationId: aws.String("b"), + }, + { + AllocationId: aws.String("a"), + }, + }, + sortBy: "AllocationId", + }, + want: []types.Address{ + { + AllocationId: aws.String("a"), + }, + { + AllocationId: aws.String("b"), + }, + }, + }, + { + name: "Test case 3: Sort addresses by PublicIp", + args: args{ + addresses: []types.Address{ + { + PublicIp: aws.String("192.168.1.2"), + }, + { + PublicIp: aws.String("192.168.1.1"), + }, + }, + sortBy: "PublicIp", + }, + want: []types.Address{ + { + PublicIp: aws.String("192.168.1.1"), + }, + { + PublicIp: aws.String("192.168.1.2"), + }, + }, + }, + { + name: "Test case 4: Sort addresses by InstanceId", + args: args{ + addresses: []types.Address{ + { + InstanceId: aws.String("i-0abcd1234efgh5678"), + }, + { + InstanceId: aws.String("i-0abcd1234efgh5679"), + }, + }, + sortBy: "InstanceId", + }, + want: []types.Address{ + { + InstanceId: aws.String("i-0abcd1234efgh5678"), + }, + { + InstanceId: aws.String("i-0abcd1234efgh5679"), + }, + }, + }, + { + name: "Test case 5: Sort addresses by Domain", + args: args{ + addresses: []types.Address{ + { + Domain: types.DomainTypeVpc, + }, + { + Domain: types.DomainTypeStandard, + }, + }, + sortBy: "Domain", + }, + want: []types.Address{ + { + Domain: types.DomainTypeStandard, + }, + { + Domain: types.DomainTypeVpc, + }, + }, + }, + { + name: "Test case 6: Sort addresses by NetworkInterfaceId", + args: args{ + addresses: []types.Address{ + { + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5679"), + }, + { + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5678"), + }, + }, + sortBy: "NetworkInterfaceId", + }, + want: []types.Address{ + { + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5678"), + }, + { + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5679"), + }, + }, + }, + { + name: "Test case 7: Sort addresses by NetworkInterfaceOwnerId", + args: args{ + addresses: []types.Address{ + { + NetworkInterfaceOwnerId: aws.String("123456789013"), + }, + { + NetworkInterfaceOwnerId: aws.String("123456789012"), + }, + }, + sortBy: "NetworkInterfaceOwnerId", + }, + want: []types.Address{ + { + NetworkInterfaceOwnerId: aws.String("123456789012"), + }, + { + NetworkInterfaceOwnerId: aws.String("123456789013"), + }, + }, + }, + { + name: "Test case 8: Sort addresses by AssociationId", + args: args{ + addresses: []types.Address{ + { + AssociationId: aws.String("b"), + }, + { + AssociationId: aws.String("a"), + }, + }, + sortBy: "AssociationId", + }, + want: []types.Address{ + { + AssociationId: aws.String("a"), + }, + { + AssociationId: aws.String("b"), + }, + }, + }, + { + name: "Test case 9: Sort addresses by PrivateIpAddress", + args: args{ + addresses: []types.Address{ + { + PrivateIpAddress: aws.String("10.10.0.3"), + }, + { + PrivateIpAddress: aws.String("10.10.0.1"), + }, + }, + sortBy: "PrivateIpAddress", + }, + want: []types.Address{ + { + PrivateIpAddress: aws.String("10.10.0.1"), + }, + { + PrivateIpAddress: aws.String("10.10.0.3"), + }, + }, + }, + { + name: "Test case 10: Sort addresses by PublicIpv4Pool", + args: args{ + addresses: []types.Address{ + { + PublicIpv4Pool: aws.String("amazon"), + }, + { + PublicIpv4Pool: aws.String("aws"), + }, + }, + sortBy: "PublicIpv4Pool", + }, + want: []types.Address{ + { + PublicIpv4Pool: aws.String("amazon"), + }, + { + PublicIpv4Pool: aws.String("aws"), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sortAddressesByField(tt.args.addresses, tt.args.sortBy) + if !reflect.DeepEqual(tt.args.addresses, tt.want) { + t.Errorf("sortAddressesByField() = %v, want %v", tt.args.addresses, tt.want) + } + }) + } +} + +func Test_awsAssigner_Assign(t *testing.T) { + type args struct { + ctx context.Context + instanceID string + filter []string + orderBy string + } + type fields struct { + region string + logger *logrus.Entry + instanceGetterFn func(t *testing.T, args *args) cloud.Ec2InstanceGetter + eipListerFn func(t *testing.T, args *args) cloud.EipLister + eipAssignerFn func(t *testing.T, args *args) cloud.EipAssigner + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "assign EIP to instance", + fields: fields{ + region: "us-east-1", + logger: logrus.NewEntry(logrus.New()), + instanceGetterFn: func(t *testing.T, args *args) cloud.Ec2InstanceGetter { + mock := mocks.NewEc2InstanceGetter(t) + mock.EXPECT().Get(args.ctx, args.instanceID, "us-east-1").Return(&types.Instance{ + InstanceId: aws.String(args.instanceID), + NetworkInterfaces: []types.InstanceNetworkInterface{ + { + Association: &types.InstanceNetworkInterfaceAssociation{ + PublicIp: aws.String("135.64.10.1"), + }, + Attachment: &types.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int32(0), + }, + NetworkInterfaceId: aws.String("eni-0abcd1234efgh5678"), + }, + }, + }, nil) + return mock + }, + eipListerFn: func(t *testing.T, args *args) cloud.EipLister { + mock := mocks.NewEipLister(t) + mock.EXPECT().List(args.ctx, map[string][]string{ + "instance-id": {args.instanceID}, + }, true).Return([]types.Address{}, nil).Once() + mock.EXPECT().List(args.ctx, map[string][]string{ + "tag:env": {"test"}, + "tag:kubeip": {"reserved"}, + }, false).Return([]types.Address{ + { + AllocationId: aws.String("eipalloc-0abcd1234efgh5678"), + PublicIp: aws.String("100.0.0.1"), + Tags: []types.Tag{ + { + Key: aws.String("env"), + Value: aws.String("test"), + }, + { + Key: aws.String("kubeip"), + Value: aws.String("reserved"), + }, + }, + }, + { + AllocationId: aws.String("eipalloc-0abcd1234efgh5679"), + PublicIp: aws.String("100.0.0.2"), + Tags: []types.Tag{ + { + Key: aws.String("env"), + Value: aws.String("test"), + }, + { + Key: aws.String("kubeip"), + Value: aws.String("reserved"), + }, + }, + }, + }, nil).Once() + mock.EXPECT().List(args.ctx, map[string][]string{ + "allocation-id": {"eipalloc-0abcd1234efgh5678"}, + }, true).Return([]types.Address{ + { + AllocationId: aws.String("eipalloc-0abcd1234efgh5678"), + PublicIp: aws.String("100.0.0.1"), + Tags: []types.Tag{ + { + Key: aws.String("env"), + Value: aws.String("test"), + }, + { + Key: aws.String("kubeip"), + Value: aws.String("reserved"), + }, + }, + }, + }, nil).Once() + return mock + }, + eipAssignerFn: func(t *testing.T, args *args) cloud.EipAssigner { + mock := mocks.NewEipAssigner(t) + mock.EXPECT().Assign(args.ctx, "eni-0abcd1234efgh5678", "eipalloc-0abcd1234efgh5678").Return(nil) + return mock + }, + }, + args: args{ + ctx: context.Background(), + instanceID: "i-0abcd1234efgh5678", + filter: []string{ + "Name=tag:env,Values=test", + "Name=tag:kubeip,Values=reserved", + }, + orderBy: "PublicIp", + }, + }, + { + name: "instance already has EIP assigned", + fields: fields{ + region: "us-east-1", + logger: logrus.NewEntry(logrus.New()), + instanceGetterFn: func(t *testing.T, args *args) cloud.Ec2InstanceGetter { + return nil + }, + eipAssignerFn: func(t *testing.T, args *args) cloud.EipAssigner { + return nil + }, + eipListerFn: func(t *testing.T, args *args) cloud.EipLister { + mock := mocks.NewEipLister(t) + mock.EXPECT().List(args.ctx, map[string][]string{ + "instance-id": {args.instanceID}, + }, true).Return([]types.Address{ + { + AllocationId: aws.String("eipalloc-0abcd1234efgh5678"), + PublicIp: aws.String("100.0.0.1"), + Tags: []types.Tag{ + { + Key: aws.String("env"), + Value: aws.String("test"), + }, + }, + }, + }, nil) + return mock + }, + }, + args: args{ + ctx: context.Background(), + instanceID: "i-0abcd1234efgh5678", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &awsAssigner{ + region: tt.fields.region, + logger: tt.fields.logger, + instanceGetter: tt.fields.instanceGetterFn(t, &tt.args), + eipLister: tt.fields.eipListerFn(t, &tt.args), + eipAssigner: tt.fields.eipAssignerFn(t, &tt.args), + } + if err := a.Assign(tt.args.ctx, tt.args.instanceID, "", tt.args.filter, tt.args.orderBy); (err != nil) != tt.wantErr { + t.Errorf("Assign() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/internal/address/azure.go b/internal/address/azure.go new file mode 100644 index 0000000..79d870f --- /dev/null +++ b/internal/address/azure.go @@ -0,0 +1,14 @@ +package address + +import "context" + +type azureAssigner struct { +} + +func (a *azureAssigner) Assign(_ context.Context, _, _ string, _ []string, _ string) error { + return nil +} + +func (a *azureAssigner) Unassign(_ context.Context, _, _ string) error { + return nil +} diff --git a/internal/address/gcp.go b/internal/address/gcp.go new file mode 100644 index 0000000..b8443a8 --- /dev/null +++ b/internal/address/gcp.go @@ -0,0 +1,363 @@ +package address + +import ( + "context" + "fmt" + "strings" + "time" + + "cloud.google.com/go/compute/metadata" + "github.com/doitintl/kubeip/internal/cloud" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "google.golang.org/api/compute/v1" +) + +const ( + operationDone = "DONE" // operation status DONE + inUseStatus = "IN_USE" + reservedStatus = "RESERVED" // static IP addresses that are reserved but not currently in use + defaultTimeout = 10 * time.Minute + defaultNetworkInterface = "nic0" + defaultNetworkName = "External NAT" + accessConfigType = "ONE_TO_ONE_NAT" + accessConfigKind = "compute#accessConfig" +) + +type gcpAssigner struct { + lister cloud.Lister + waiter cloud.ZoneWaiter + addressManager cloud.AddressManager + instanceGetter cloud.InstanceGetter + project string + region string + logger *logrus.Entry +} + +type operationError struct { + name string + err *compute.OperationError +} + +func newOperationError(name string, err *compute.OperationError) *operationError { + return &operationError{name: name, err: err} +} + +func isOperationError(err error) bool { + _, ok := err.(*operationError) //nolint:errorlint + return ok +} + +func joinErrorMessages(operationError *compute.OperationError) string { + if operationError == nil || len(operationError.Errors) == 0 { + return "" + } + messages := make([]string, 0, len(operationError.Errors)) + for _, errorItem := range operationError.Errors { + messages = append(messages, errorItem.Message) + } + return strings.Join(messages, "; ") +} + +func (e *operationError) Error() string { + if e.err == nil { + return "" + } + return fmt.Sprintf("operation %s failed with error %v", e.name, joinErrorMessages(e.err)) +} + +func NewGCPAssigner(ctx context.Context, logger *logrus.Entry, project, region string) (Assigner, error) { + // initialize Google Cloud client + client, err := compute.NewService(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to initialize Google Cloud client") + } + + // get project ID from metadata server + if project == "" { + project, err = metadata.ProjectID() + if err != nil { + return nil, errors.Wrap(err, "failed to get project ID from metadata server") + } + } + + // get region from metadata server + if region == "" { + region, err = metadata.InstanceAttributeValue("cluster-location") + if err != nil { + return nil, errors.Wrap(err, "failed to get region from metadata server") + } + // if cluster-location is zone, extract region from zone + if len(region) > 3 && region[len(region)-3] == '-' { + region = region[:len(region)-3] + } + } + + return &gcpAssigner{ + lister: cloud.NewLister(client), + waiter: cloud.NewZoneWaiter(client), + addressManager: cloud.NewAddressManager(client), + instanceGetter: cloud.NewInstanceGetter(client), + project: project, + region: region, + logger: logger, + }, nil +} + +func (a *gcpAssigner) waitForOperation(c context.Context, op *compute.Operation, zone string, timeout time.Duration) error { + if op == nil { + a.logger.Warn("operation is nil") + return nil + } + // Create a context that will be cancelled with timeout + ctx, cancel := context.WithTimeout(c, timeout) + defer cancel() + + var err error + name := op.Name + for op.Status != operationDone { + // Pass the cancellable context to the Wait method + op, err = a.waiter.Wait(a.project, zone, name).Context(ctx).Do() + if err != nil { + // If the context was cancelled, return a timeout error + if errors.Is(err, context.Canceled) { + return errors.New("operation timed out") + } + return errors.Wrapf(err, "failed to get operation %s", name) + } + // If the operation has an error, return it + if op != nil && op.Error != nil { + return newOperationError(op.Name, op.Error) + } + } + return nil +} + +func (a *gcpAssigner) deleteInstanceAddress(ctx context.Context, instance *compute.Instance, zone string) error { + // Check if the instance has at least one network interface + if len(instance.NetworkInterfaces) == 0 { + a.logger.WithField("instance", instance.Name).Info("instance has no network interfaces") + return nil + } + // get instance network interface + networkInterface := instance.NetworkInterfaces[0] + // get instance network interface access config + if len(networkInterface.AccessConfigs) == 0 { + a.logger.WithField("instance", instance.Name).Info("instance network interface has no access configs") + return nil + } + accessConfig := networkInterface.AccessConfigs[0] + // get instance network interface access config name + accessConfigName := accessConfig.Name + // delete instance network interface access config + a.logger.WithFields(logrus.Fields{ + "instance": instance.Name, + "address": accessConfig.NatIP, + }).Infof("deleting public IP address from instance") + op, err := a.addressManager.DeleteAccessConfig(a.project, zone, instance.Name, accessConfigName, networkInterface.Name) + if err != nil { + return errors.Wrapf(err, "failed to delete access config %s from instance %s", accessConfigName, instance.Name) + } + // wait for operation to complete + if err = a.waitForOperation(ctx, op, zone, defaultTimeout); err != nil { + // return error if operation failed + if isOperationError(err) { + return err + } + // log error and continue (ignore non-operation errors) + a.logger.WithError(err).Errorf("failed waiting for operation %s", op.Name) + } + return nil +} + +func (a *gcpAssigner) addInstanceAddress(ctx context.Context, instance *compute.Instance, zone string, address *compute.Address) error { + // empty address means ephemeral public IP address + natIP := "" + name := defaultNetworkName + kind := "ephemeral" + if address != nil { + natIP = address.Address + name = address.Name + kind = "static" + } + // add instance network interface access config + a.logger.WithFields(logrus.Fields{ + "instance": instance.Name, + "address-name": name, + "address-ip": natIP, + "kind": kind, + }).Info("adding public IP address to instance") + op, err := a.addressManager.AddAccessConfig(a.project, zone, instance.Name, defaultNetworkInterface, &compute.AccessConfig{ + Name: name, + Type: accessConfigType, + Kind: accessConfigKind, + NatIP: natIP, + }) + if err != nil { + return errors.Wrapf(err, "failed to add access config %s to instance %s", name, instance.Name) + } + // wait for operation to complete + if err = a.waitForOperation(ctx, op, zone, defaultTimeout); err != nil { + // return error if operation failed + if isOperationError(err) { + return err + } + // log error and continue (ignore non-operation errors) + a.logger.WithError(err).Errorf("failed waiting for operation %s", op.Name) + } + return nil +} + +func (a *gcpAssigner) forceCheckAddressAssigned(region, addressName string) (bool, error) { + address, err := a.addressManager.GetAddress(a.project, region, addressName) + if err != nil { + return false, errors.Wrapf(err, "failed to get address %s", addressName) + } + return address.Status == inUseStatus, nil +} + +//nolint:gocyclo +func (a *gcpAssigner) Assign(ctx context.Context, instanceID, zone string, filter []string, orderBy string) error { + // check if instance already has a public static IP address assigned + instance, err := a.instanceGetter.Get(a.project, zone, instanceID) + if err != nil { + return errors.Wrapf(err, "failed to get instance %s", instanceID) + } + assigned, err := a.listAddresses(nil, "", inUseStatus) + if err != nil { + return errors.Wrap(err, "failed to list assigned addresses") + } + if len(assigned) > 0 { + for _, address := range assigned { + for _, user := range address.Users { + if user == instance.SelfLink { + a.logger.WithFields(logrus.Fields{ + "instance": instanceID, + "address": address.Address, + }).Infof("instance already has a static public IP address assigned") + return nil + } + } + } + } + + // get available reserved public IP addresses + addresses, err := a.listAddresses(filter, orderBy, reservedStatus) + if err != nil { + return errors.Wrap(err, "failed to list available addresses") + } + if len(addresses) == 0 { + return errors.Errorf("no available addresses") + } + // log available addresses IPs + ips := make([]string, 0, len(addresses)) + for _, address := range addresses { + ips = append(ips, address.Address) + } + a.logger.WithField("addresses", ips).Debugf("found %d available addresses", len(addresses)) + + // delete current ephemeral public IP address + if err = a.deleteInstanceAddress(ctx, instance, zone); err != nil { + return errors.Wrap(err, "failed to delete current public IP address") + } + + // try to assign all available addresses until one succeeds + // due to concurrency, it is possible that another kubeip instance will assign the same address + for _, address := range addresses { + // force check if address is already assigned (reduce the chance of assigning the same address by multiple kubeip instances) + var addressAssigned bool + addressAssigned, err = a.forceCheckAddressAssigned(a.region, address.Name) + if err != nil { + a.logger.WithError(err).Errorf("failed to check if address %s is assigned", address.Address) + a.logger.Debug("trying next address") + continue + } + if addressAssigned { + a.logger.WithField("address", address.Address).Debug("address is already assigned") + a.logger.Debug("trying next address") + continue + } + // assign address to the instance and try the next address if it fails + if err = a.addInstanceAddress(ctx, instance, zone, address); err != nil { + a.logger.WithError(err).Errorf("failed to assign static public IP address %s", address.Address) + a.logger.Debug("trying next address") + continue + } + // break the loop after successfully assigning an address + break + } + if err != nil { + return errors.Wrap(err, "failed to assign static public IP address") + } + return nil +} + +func (a *gcpAssigner) listAddresses(filter []string, orderBy, status string) ([]*compute.Address, error) { + call := a.lister.List(a.project, a.region) + // Initialize filters with known filters + filters := []string{ + fmt.Sprintf("(status=%s)", status), + "(addressType=EXTERNAL)", + } + + // filter addresses by provided filter: labels.key=value + for _, f := range filter { + filters = append(filters, fmt.Sprintf("(%s)", f)) + } + // set the filter + call = call.Filter(strings.Join(filters, " ")) + // sort addresses by + if orderBy != "" { + call = call.OrderBy(orderBy) + } + // get all addresses + var addresses []*compute.Address + for { + list, err := call.Do() + if err != nil { + return nil, errors.Wrap(err, "failed to list available addresses") + } + addresses = append(addresses, list.Items...) + if list.NextPageToken == "" { + return addresses, nil + } + call = call.PageToken(list.NextPageToken) + } +} + +func (a *gcpAssigner) Unassign(ctx context.Context, instanceID, zone string) error { + // get the instance details + instance, err := a.instanceGetter.Get(a.project, zone, instanceID) + if err != nil { + return errors.Wrapf(err, "failed to get instance %s", instanceID) + } + // list all assigned addresses + assigned, err := a.listAddresses(nil, "", inUseStatus) + if err != nil { + return errors.Wrap(err, "failed to list assigned addresses") + } + // if there are assigned addresses, check if they are assigned to the instance + if len(assigned) > 0 { + for _, address := range assigned { + for _, user := range address.Users { + if user == instance.SelfLink { + // release/remove current static public IP address + if err = a.deleteInstanceAddress(ctx, instance, zone); err != nil { + return errors.Wrap(err, "failed to delete current public IP address") + } + // assign ephemeral public IP address to the instance (pass nil address) + for { + // retry until ephemeral public IP address is assigned + // sometime this operation fails and needs to be retried + if err = a.addInstanceAddress(ctx, instance, zone, nil); err != nil { + a.logger.WithError(err).Error("failed to assign ephemeral public IP address, retrying") + continue + } + return nil + } + } + } + } + } + return nil +} diff --git a/internal/address/gcp_test.go b/internal/address/gcp_test.go new file mode 100644 index 0000000..761eda8 --- /dev/null +++ b/internal/address/gcp_test.go @@ -0,0 +1,423 @@ +package address + +import ( + "context" + "reflect" + "testing" + "time" + + "github.com/doitintl/kubeip/internal/cloud" + mocks "github.com/doitintl/kubeip/mocks/cloud" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + tmock "github.com/stretchr/testify/mock" + "google.golang.org/api/compute/v1" +) + +func Test_gcpAssigner_listAddresses(t *testing.T) { + type fields struct { + listerFn func(t *testing.T) cloud.Lister + project string + region string + } + type args struct { + filter []string + orderBy string + status string + } + tests := []struct { + name string + fields fields + args args + want []*compute.Address + wantErr bool + }{ + { + name: "list addresses successfully", + fields: fields{ + project: "test-project", + region: "test-region", + listerFn: func(t *testing.T) cloud.Lister { + mock := mocks.NewLister(t) + mockCall := mocks.NewListCall(t) + mock.EXPECT().List("test-project", "test-region").Return(mockCall) + mockCall.EXPECT().Filter("(status=RESERVED) (addressType=EXTERNAL) (test-filter-1) (test-filter-2)").Return(mockCall) + mockCall.EXPECT().OrderBy("test-order-by").Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-1", Status: "RESERVED", Address: "10.10.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-2", Status: "RESERVED", Address: "10.10.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, nil) + return mock + }, + }, + args: args{ + filter: []string{"test-filter-1", "test-filter-2"}, + orderBy: "test-order-by", + status: "RESERVED", + }, + want: []*compute.Address{ + {Name: "test-address-1", Status: "RESERVED", Address: "10.10.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-2", Status: "RESERVED", Address: "10.10.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, + { + name: "list addresses with multiple pages successfully", + fields: fields{ + project: "test-project", + region: "test-region", + listerFn: func(t *testing.T) cloud.Lister { + mock := mocks.NewLister(t) + mockCall := mocks.NewListCall(t) + mock.EXPECT().List("test-project", "test-region").Return(mockCall) + mockCall.EXPECT().Filter("(status=RESERVED) (addressType=EXTERNAL) (test-filter-1) (test-filter-2)").Return(mockCall) + mockCall.EXPECT().OrderBy("test-order-by").Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-1", Status: "RESERVED", Address: "10.10.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-2", Status: "RESERVED", Address: "10.10.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + NextPageToken: "test-next-page-token", + }, nil).Once() + mockCall.EXPECT().PageToken("test-next-page-token").Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-3", Status: "RESERVED", Address: "10.10.0.3", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-4", Status: "RESERVED", Address: "10.10.0.4", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, nil).Once() + return mock + }, + }, + args: args{ + filter: []string{"test-filter-1", "test-filter-2"}, + orderBy: "test-order-by", + status: "RESERVED", + }, + want: []*compute.Address{ + {Name: "test-address-1", Status: "RESERVED", Address: "10.10.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-2", Status: "RESERVED", Address: "10.10.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-3", Status: "RESERVED", Address: "10.10.0.3", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-4", Status: "RESERVED", Address: "10.10.0.4", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, + } + for _, tt := range tests { + logger := logrus.NewEntry(logrus.New()) + t.Run(tt.name, func(t *testing.T) { + a := &gcpAssigner{ + lister: tt.fields.listerFn(t), + project: tt.fields.project, + region: tt.fields.region, + logger: logger, + } + got, err := a.listAddresses(tt.args.filter, tt.args.orderBy, tt.args.status) + if (err != nil) != tt.wantErr { + t.Errorf("listAddresses() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("listAddresses() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_gcpAssigner_waitForOperation(t *testing.T) { + type fields struct { + waiterFn func(t *testing.T) cloud.ZoneWaiter + project string + } + type args struct { + op *compute.Operation + zone string + timeout time.Duration + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "wait for operation successfully", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.Operation{Status: "DONE"}, nil) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond, + }, + }, + { + name: "wait for operation with a few retries successfully", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.Operation{Status: "RUNNING"}, nil).Times(2) + mockCall.EXPECT().Do().Return(&compute.Operation{Status: "DONE"}, nil) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond * 2, + }, + }, + { + name: "wait for operation with timeout", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(nil, context.Canceled) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond, + }, + wantErr: true, + }, + { + name: "wait for operation with error", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(nil, errors.New("test-error")) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond, + }, + wantErr: true, + }, + { + name: "wait for operation with error in operation", + fields: fields{ + project: "test-project", + waiterFn: func(t *testing.T) cloud.ZoneWaiter { + mock := mocks.NewZoneWaiter(t) + mockCall := mocks.NewWaitCall(t) + mock.EXPECT().Wait("test-project", "test-zone", "test-operation").Return(mockCall) + mockCall.EXPECT().Context(tmock.Anything).Return(mockCall) + mockCall.EXPECT().Do().Return(&compute.Operation{Status: "DONE", Error: &compute.OperationError{Errors: []*compute.OperationErrorErrors{{Code: "123", Message: "test-error"}}}}, nil) + return mock + }, + }, + args: args{ + op: &compute.Operation{Name: "test-operation", Status: "RUNNING"}, + zone: "test-zone", + timeout: time.Millisecond, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := logrus.NewEntry(logrus.New()) + waiter := tt.fields.waiterFn(t) + a := &gcpAssigner{ + waiter: waiter, + project: tt.fields.project, + logger: logger, + } + if err := a.waitForOperation(context.TODO(), tt.args.op, tt.args.zone, tt.args.timeout); (err != nil) != tt.wantErr { + t.Errorf("waitForOperation() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_gcpAssigner_deleteInstanceAddress(t *testing.T) { + type args struct { + ctx context.Context + instance *compute.Instance + zone string + } + type fields struct { + addressManagerFn func(t *testing.T, args *args) cloud.AddressManager + project string + region string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "delete instance address successfully", + fields: fields{ + project: "test-project", + region: "test-region", + addressManagerFn: func(t *testing.T, args *args) cloud.AddressManager { + mock := mocks.NewAddressManager(t) + networkInterfaceName := args.instance.NetworkInterfaces[0].Name + accessConfigName := args.instance.NetworkInterfaces[0].AccessConfigs[0].Name + mock.EXPECT().DeleteAccessConfig("test-project", "", args.instance.Name, accessConfigName, networkInterfaceName).Return(&compute.Operation{Name: "test-operation", Status: "DONE"}, nil) + return mock + }, + }, + args: args{ + ctx: context.TODO(), + instance: &compute.Instance{ + Name: "test-instance", + Zone: "test-zone", + NetworkInterfaces: []*compute.NetworkInterface{ + { + Name: "test-network-interface", + AccessConfigs: []*compute.AccessConfig{ + {Name: "test-access-config", NatIP: "100.0.0.1"}, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := logrus.NewEntry(logrus.New()) + a := &gcpAssigner{ + addressManager: tt.fields.addressManagerFn(t, &tt.args), + project: tt.fields.project, + region: tt.fields.region, + logger: logger, + } + if err := a.deleteInstanceAddress(tt.args.ctx, tt.args.instance, tt.args.zone); (err != nil) != tt.wantErr { + t.Errorf("deleteInstanceAddress() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_gcpAssigner_Assign(t *testing.T) { + type fields struct { + listerFn func(t *testing.T) cloud.Lister + addressManagerFn func(t *testing.T) cloud.AddressManager + instanceGetterFn func(t *testing.T) cloud.InstanceGetter + project string + region string + } + type args struct { + ctx context.Context + instanceID string + zone string + filter []string + orderBy string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "assign static IP address successfully", + fields: fields{ + project: "test-project", + region: "test-region", + listerFn: func(t *testing.T) cloud.Lister { + mock := mocks.NewLister(t) + mockCall := mocks.NewListCall(t) + mock.EXPECT().List("test-project", "test-region").Return(mockCall) + mockCall.EXPECT().Filter("(status=IN_USE) (addressType=EXTERNAL)").Return(mockCall).Once() + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-1", Status: inUseStatus, Address: "100.0.0.1", NetworkTier: "PREMIUM", AddressType: "EXTERNAL", Users: []string{"self-link-test-instance-1"}}, + {Name: "test-address-2", Status: inUseStatus, Address: "100.0.0.2", NetworkTier: "PREMIUM", AddressType: "EXTERNAL", Users: []string{"self-link-test-instance-2"}}, + }, + }, nil).Once() + mockCall.EXPECT().Filter("(status=RESERVED) (addressType=EXTERNAL) (test-filter-1) (test-filter-2)").Return(mockCall).Once() + mockCall.EXPECT().OrderBy("test-order-by").Return(mockCall).Once() + mockCall.EXPECT().Do().Return(&compute.AddressList{ + Items: []*compute.Address{ + {Name: "test-address-3", Status: reservedStatus, Address: "100.0.0.3", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + {Name: "test-address-4", Status: reservedStatus, Address: "100.0.0.4", NetworkTier: "PREMIUM", AddressType: "EXTERNAL"}, + }, + }, nil).Once() + return mock + }, + instanceGetterFn: func(t *testing.T) cloud.InstanceGetter { + mock := mocks.NewInstanceGetter(t) + mock.EXPECT().Get("test-project", "test-zone", "test-instance-0").Return(&compute.Instance{ + Name: "test-instance-0", + Zone: "test-zone", + NetworkInterfaces: []*compute.NetworkInterface{ + { + Name: "test-network-interface", + AccessConfigs: []*compute.AccessConfig{ + {Name: "test-access-config", NatIP: "200.0.0.1", Type: accessConfigType, Kind: accessConfigKind}, + }, + }, + }, + }, nil) + return mock + }, + addressManagerFn: func(t *testing.T) cloud.AddressManager { + mock := mocks.NewAddressManager(t) + mock.EXPECT().DeleteAccessConfig("test-project", "test-zone", "test-instance-0", "test-access-config", "test-network-interface").Return(&compute.Operation{Name: "test-operation", Status: "DONE"}, nil) + mock.EXPECT().AddAccessConfig("test-project", "test-zone", "test-instance-0", defaultNetworkInterface, &compute.AccessConfig{ + Name: "test-address-3", + Type: accessConfigType, + Kind: accessConfigKind, + NatIP: "100.0.0.3", + }).Return(&compute.Operation{Name: "test-operation", Status: "DONE"}, nil) + mock.EXPECT().GetAddress("test-project", "test-region", "test-address-3").Return(&compute.Address{Name: "test-address-3", Status: reservedStatus}, nil) + return mock + }, + }, + args: args{ + ctx: context.TODO(), + instanceID: "test-instance-0", + zone: "test-zone", + filter: []string{"test-filter-1", "test-filter-2"}, + orderBy: "test-order-by", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := logrus.NewEntry(logrus.New()) + a := &gcpAssigner{ + lister: tt.fields.listerFn(t), + addressManager: tt.fields.addressManagerFn(t), + instanceGetter: tt.fields.instanceGetterFn(t), + project: tt.fields.project, + region: tt.fields.region, + logger: logger, + } + if err := a.Assign(tt.args.ctx, tt.args.instanceID, tt.args.zone, tt.args.filter, tt.args.orderBy); (err != nil) != tt.wantErr { + t.Errorf("Assign() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/internal/cloud/aws_address.go b/internal/cloud/aws_address.go new file mode 100644 index 0000000..de0b2f2 --- /dev/null +++ b/internal/cloud/aws_address.go @@ -0,0 +1,52 @@ +package cloud + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/pkg/errors" +) + +type EipAssigner interface { + Assign(ctx context.Context, networkInterfaceID, allocationID string) error + Unassign(ctx context.Context, associationID string) error +} + +type eipAssigner struct { + client *ec2.Client +} + +func NewEipAssigner(client *ec2.Client) EipAssigner { + return &eipAssigner{client: client} +} + +func (a *eipAssigner) Assign(ctx context.Context, networkInterfaceID, allocationID string) error { + // associate elastic IP with the instance + input := &ec2.AssociateAddressInput{ + AllocationId: &allocationID, + NetworkInterfaceId: &networkInterfaceID, + AllowReassociation: aws.Bool(false), // do not allow reassociation of the elastic IP + } + + _, err := a.client.AssociateAddress(ctx, input) + if err != nil { + return errors.Wrap(err, "failed to associate elastic IP with the instance") + } + + return nil +} + +func (a *eipAssigner) Unassign(ctx context.Context, associationID string) error { + // disassociate elastic IP from the instance + input := &ec2.DisassociateAddressInput{ + AssociationId: &associationID, + } + + _, err := a.client.DisassociateAddress(ctx, input) + if err != nil { + return errors.Wrap(err, "failed to disassociate elastic IP from the instance") + } + + return nil +} diff --git a/internal/cloud/aws_instance.go b/internal/cloud/aws_instance.go new file mode 100644 index 0000000..4f90cb2 --- /dev/null +++ b/internal/cloud/aws_instance.go @@ -0,0 +1,40 @@ +package cloud + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" +) + +type Ec2InstanceGetter interface { + Get(ctx context.Context, instanceID, region string) (*types.Instance, error) +} + +type ec2InstanceGetter struct { + client *ec2.Client +} + +func NewEc2InstanceGetter(client *ec2.Client) Ec2InstanceGetter { + return &ec2InstanceGetter{client: client} +} + +func (g *ec2InstanceGetter) Get(ctx context.Context, instanceID, _ string) (*types.Instance, error) { + input := &ec2.DescribeInstancesInput{ + InstanceIds: []string{ + instanceID, + }, + } + + resp, err := g.client.DescribeInstances(ctx, input) + if err != nil { + return nil, errors.Wrap(err, "failed to describe instances, %v") + } + + if len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 { + return nil, errors.Wrap(err, "no instances found for the given id") + } + + return &resp.Reservations[0].Instances[0], nil +} diff --git a/internal/cloud/aws_lister.go b/internal/cloud/aws_lister.go new file mode 100644 index 0000000..64247b6 --- /dev/null +++ b/internal/cloud/aws_lister.go @@ -0,0 +1,53 @@ +package cloud + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/pkg/errors" +) + +type EipLister interface { + List(ctx context.Context, filter map[string][]string, inUse bool) ([]types.Address, error) +} + +type eipLister struct { + client *ec2.Client +} + +func NewEipLister(client *ec2.Client) EipLister { + return &eipLister{client: client} +} + +func (l *eipLister) List(ctx context.Context, filter map[string][]string, inUse bool) ([]types.Address, error) { + // create filter for DescribeAddressesInput + filters := make([]types.Filter, 0, len(filter)+1) + for k, v := range filter { + key := k + filters = append(filters, types.Filter{ + Name: &key, + Values: v, + }) + } + + // list all elastic IPs in the region matching the filter + input := &ec2.DescribeAddressesInput{ + Filters: filters, + } + list, err := l.client.DescribeAddresses(ctx, input) + if err != nil { + return nil, errors.Wrap(err, "failed to list elastic IPs") + } + + filtered := make([]types.Address, 0, len(list.Addresses)) + // API does not support filtering by association ID equal to nil + // filter addresses based on whether they are in use or not + for _, address := range list.Addresses { + if (inUse && address.AssociationId != nil) || (!inUse && address.AssociationId == nil) { + filtered = append(filtered, address) + } + } + + return filtered, nil +} diff --git a/internal/cloud/gcp_address.go b/internal/cloud/gcp_address.go new file mode 100644 index 0000000..90532cc --- /dev/null +++ b/internal/cloud/gcp_address.go @@ -0,0 +1,31 @@ +package cloud + +import ( + "google.golang.org/api/compute/v1" +) + +type AddressManager interface { + AddAccessConfig(project string, zone string, instance string, networkInterface string, accessconfig *compute.AccessConfig) (*compute.Operation, error) + DeleteAccessConfig(project string, zone string, instance string, accessConfig string, networkInterface string) (*compute.Operation, error) + GetAddress(project, region, name string) (*compute.Address, error) +} + +type addressManager struct { + client *compute.Service +} + +func NewAddressManager(client *compute.Service) AddressManager { + return &addressManager{client: client} +} + +func (m *addressManager) AddAccessConfig(project, zone, instance, networkInterface string, accessconfig *compute.AccessConfig) (*compute.Operation, error) { + return m.client.Instances.AddAccessConfig(project, zone, instance, networkInterface, accessconfig).Do() //nolint:wrapcheck +} + +func (m *addressManager) DeleteAccessConfig(project, zone, instance, accessConfig, networkInterface string) (*compute.Operation, error) { + return m.client.Instances.DeleteAccessConfig(project, zone, instance, accessConfig, networkInterface).Do() //nolint:wrapcheck +} + +func (m *addressManager) GetAddress(project, region, name string) (*compute.Address, error) { + return m.client.Addresses.Get(project, region, name).Do() //nolint:wrapcheck +} diff --git a/internal/cloud/gcp_instance.go b/internal/cloud/gcp_instance.go new file mode 100644 index 0000000..10e7d6b --- /dev/null +++ b/internal/cloud/gcp_instance.go @@ -0,0 +1,19 @@ +package cloud + +import "google.golang.org/api/compute/v1" + +type InstanceGetter interface { + Get(projectID, zone, instance string) (*compute.Instance, error) +} + +type instanceGetter struct { + client *compute.Service +} + +func NewInstanceGetter(client *compute.Service) InstanceGetter { + return &instanceGetter{client: client} +} + +func (g *instanceGetter) Get(projectID, zone, instance string) (*compute.Instance, error) { + return g.client.Instances.Get(projectID, zone, instance).Do() //nolint:wrapcheck +} diff --git a/internal/cloud/gcp_lister.go b/internal/cloud/gcp_lister.go new file mode 100644 index 0000000..6adc75a --- /dev/null +++ b/internal/cloud/gcp_lister.go @@ -0,0 +1,46 @@ +package cloud + +import "google.golang.org/api/compute/v1" + +type ListCall interface { + Filter(filter string) ListCall + OrderBy(orderBy string) ListCall + PageToken(pageToken string) ListCall + Do() (*compute.AddressList, error) +} + +type Lister interface { + List(projectID, region string) ListCall +} + +func NewLister(client *compute.Service) Lister { + return &gcpLister{client: client} +} + +type gcpLister struct { + client *compute.Service +} + +type gcpListCall struct { + call *compute.AddressesListCall +} + +func (l *gcpLister) List(projectID, region string) ListCall { + return &gcpListCall{l.client.Addresses.List(projectID, region)} +} + +func (c *gcpListCall) Filter(filter string) ListCall { + return &gcpListCall{c.call.Filter(filter)} +} + +func (c *gcpListCall) OrderBy(orderBy string) ListCall { + return &gcpListCall{c.call.OrderBy(orderBy)} +} + +func (c *gcpListCall) PageToken(pageToken string) ListCall { + return &gcpListCall{c.call.PageToken(pageToken)} +} + +func (c *gcpListCall) Do() (*compute.AddressList, error) { + return c.call.Do() //nolint:wrapcheck +} diff --git a/internal/cloud/gcp_waiter.go b/internal/cloud/gcp_waiter.go new file mode 100644 index 0000000..0875eee --- /dev/null +++ b/internal/cloud/gcp_waiter.go @@ -0,0 +1,40 @@ +package cloud + +import ( + "context" + + "google.golang.org/api/compute/v1" +) + +type WaitCall interface { + Context(ctx context.Context) WaitCall + Do() (*compute.Operation, error) +} + +type ZoneWaiter interface { + Wait(projectID, region, operationName string) WaitCall +} + +type zoneWaiter struct { + client *compute.Service +} + +type zoneWaitCall struct { + call *compute.ZoneOperationsWaitCall +} + +func NewZoneWaiter(client *compute.Service) ZoneWaiter { + return &zoneWaiter{client: client} +} + +func (w *zoneWaiter) Wait(projectID, region, operationName string) WaitCall { + return &zoneWaitCall{w.client.ZoneOperations.Wait(projectID, region, operationName)} +} + +func (c *zoneWaitCall) Context(ctx context.Context) WaitCall { + return &zoneWaitCall{c.call.Context(ctx)} +} + +func (c *zoneWaitCall) Do() (*compute.Operation, error) { + return c.call.Do() //nolint:wrapcheck +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..c50f41a --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,45 @@ +package config + +import ( + "time" + + "github.com/urfave/cli/v2" +) + +type Config struct { + // KubeConfigPath is the path to the kubeconfig file + KubeConfigPath string `json:"kubeconfig"` + // NodeName is the name of the Kubernetes node + NodeName string `json:"node-name"` + // Project is the name of the GCP project or the AWS account ID + Project string `json:"project"` + // Region is the name of the GCP region or the AWS region + Region string `json:"region"` + // DevelopMode mode + DevelopMode bool `json:"develop-mode"` + // Filter is the filter for the IP addresses + Filter []string `json:"filter"` + // OrderBy is the order by for the IP addresses + OrderBy string `json:"order-by"` + // Retry interval + RetryInterval time.Duration `json:"retry-interval"` + // Retry attempts + RetryAttempts int `json:"retry-attempts"` + // ReleaseOnExit releases the IP address on exit + ReleaseOnExit bool `json:"release-on-exit"` +} + +func NewConfig(c *cli.Context) *Config { + var cfg Config + cfg.KubeConfigPath = c.String("kubeconfig") + cfg.NodeName = c.String("node-name") + cfg.DevelopMode = c.Bool("develop-mode") + cfg.RetryInterval = c.Duration("retry-interval") + cfg.RetryAttempts = c.Int("retry-attempts") + cfg.Filter = c.StringSlice("filter") + cfg.OrderBy = c.String("order-by") + cfg.Project = c.String("project") + cfg.Region = c.String("region") + cfg.ReleaseOnExit = c.Bool("release-on-exit") + return &cfg +} diff --git a/internal/node/explorer.go b/internal/node/explorer.go new file mode 100644 index 0000000..8b0f0aa --- /dev/null +++ b/internal/node/explorer.go @@ -0,0 +1,171 @@ +package node + +import ( + "context" + "net" + "os" + "strings" + + "github.com/doitintl/kubeip/internal/types" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + minProviderIDTokens = 2 + podInfoDir = "/etc/podinfo/" + awsPoolLabel = "eks.amazonaws.com/nodegroup" + azurePoolLabel = "node.kubernetes.io/instancegroup" + gcpPoolLabel = "cloud.google.com/gke-nodepool" + regionLabel = "topology.kubernetes.io/region" + zoneLabel = "topology.kubernetes.io/zone" +) + +type Explorer interface { + GetNode(ctx context.Context, nodeName string) (*types.Node, error) +} + +type explorer struct { + client kubernetes.Interface +} + +func getNodeName(file string) (string, error) { + // get node name from file + nodeName, err := os.ReadFile(file) + if err != nil { + return "", errors.Wrapf(err, "failed to read %s", file) + } + return string(nodeName), nil +} + +func NewExplorer(client kubernetes.Interface) Explorer { + return &explorer{ + client: client, + } +} + +func getCloudProvider(providerID string) (types.CloudProvider, error) { + if strings.HasPrefix(providerID, "aws://") { + return types.CloudProviderAWS, nil + } + if strings.HasPrefix(providerID, "azure://") { + return types.CloudProviderAzure, nil + } + if strings.HasPrefix(providerID, "gce://") { + return types.CloudProviderGCP, nil + } + return "", errors.Errorf("unsupported cloud provider: %s", providerID) +} + +func getInstance(providerID string) (string, error) { + s := strings.Split(providerID, "/") + if len(s) < minProviderIDTokens { + return "", errors.Errorf("failed to get instance ID") + } + return s[len(s)-1], nil +} + +func getNodePool(providerID types.CloudProvider, labels map[string]string) (string, error) { + var ok bool + var pool string + if providerID == types.CloudProviderAWS { + pool, ok = labels[awsPoolLabel] + } else if providerID == types.CloudProviderAzure { + pool, ok = labels[azurePoolLabel] + } else if providerID == types.CloudProviderGCP { + pool, ok = labels[gcpPoolLabel] + } else { + return "", errors.Errorf("unsupported cloud provider: %s", providerID) + } + if !ok { + return "", errors.Errorf("failed to get node pool") + } + return pool, nil +} + +func getAddresses(addresses []v1.NodeAddress) ([]net.IP, []net.IP, error) { + var externalIPs []net.IP + var internalIPs []net.IP + for _, address := range addresses { + if address.Type != v1.NodeExternalIP && address.Type != v1.NodeInternalIP { + continue + } + ip := net.ParseIP(address.Address) + if ip == nil { + return nil, nil, errors.Errorf("failed to parse IP address: %s", address.Address) + } + if address.Type == v1.NodeExternalIP { + externalIPs = append(externalIPs, ip) + } else if address.Type == v1.NodeInternalIP { + internalIPs = append(internalIPs, ip) + } + } + return externalIPs, internalIPs, nil +} + +// GetNode returns the node object +func (d *explorer) GetNode(ctx context.Context, nodeName string) (*types.Node, error) { + // get node name from downward API if nodeName is empty + if nodeName == "" { + var err error + nodeName, err = getNodeName(podInfoDir + "nodeName") + if err != nil { + return nil, errors.Wrap(err, "failed to get node name from downward API") + } + } + + // get node object from API server + n, err := d.client.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "failed to get kubernetes node") + } + + // get cloud provider from node spec + cloudProvider, err := getCloudProvider(n.Spec.ProviderID) + if err != nil { + return nil, errors.Wrap(err, "failed to get cloud provider") + } + + // get instance ID from provider ID + instance, err := getInstance(n.Spec.ProviderID) + if err != nil { + return nil, errors.Wrap(err, "failed to get instance ID") + } + + // get node region from node labels + region, ok := n.Labels[regionLabel] + if !ok { + return nil, errors.Errorf("failed to get node region") + } + + // get node zone from node labels + zone, ok := n.Labels[zoneLabel] + if !ok { + return nil, errors.Errorf("failed to get node zone") + } + + // get node pool from node labels + pool, err := getNodePool(cloudProvider, n.Labels) + if err != nil { + return nil, errors.Wrap(err, "failed to get node pool") + } + + // get node addresses + externalIPs, internalIPs, err := getAddresses(n.Status.Addresses) + if err != nil { + return nil, errors.Wrap(err, "failed to get node addresses") + } + + return &types.Node{ + Name: nodeName, + Instance: instance, + Cloud: cloudProvider, + Region: region, + Zone: zone, + Pool: pool, + ExternalIPs: externalIPs, + InternalIPs: internalIPs, + }, nil +} diff --git a/internal/node/explorer_test.go b/internal/node/explorer_test.go new file mode 100644 index 0000000..89a1a9b --- /dev/null +++ b/internal/node/explorer_test.go @@ -0,0 +1,470 @@ +package node + +import ( + "context" + "net" + "os" + "reflect" + "testing" + + "github.com/doitintl/kubeip/internal/types" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) + +func Test_getNodeName(t *testing.T) { + tests := []struct { + name string + nodeName string + tearUp func(name string) (*os.File, error) + tearDown func(file string) error + want string + wantErr bool + }{ + { + name: "get node name from .run/podinfo/nodeName file", + nodeName: "test-node", + tearUp: func(name string) (*os.File, error) { + // Setup: create a temporary file and write some data to it + tmpfile, err := os.CreateTemp("", "nodeName") + if err != nil { + return nil, err + } + if _, err = tmpfile.Write([]byte(name)); err != nil { + return nil, err + } + if err = tmpfile.Close(); err != nil { + return nil, err + } + return tmpfile, nil + }, + tearDown: func(file string) error { + return os.Remove(file) + }, + want: "test-node", + }, + { + name: "no such file or directory", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileName := "" + if tt.tearUp != nil { + f, err := tt.tearUp(tt.nodeName) + if err != nil { + t.Fatal(err) + } + fileName = f.Name() + defer func() { + if tt.tearDown != nil { + err = tt.tearDown(fileName) + if err != nil { + t.Fatal(err) + } + } + }() + } + got, err := getNodeName(fileName) + if (err != nil) != tt.wantErr { + t.Errorf("getNodeName() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getNodeName() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getCloudProvider(t *testing.T) { + type args struct { + providerID string + } + tests := []struct { + name string + args args + want types.CloudProvider + wantErr bool + }{ + { + name: "aws", + args: args{ + providerID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + want: types.CloudProviderAWS, + }, + { + name: "azure", + args: args{ + providerID: "azure:///subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/aks-agentpool-12345678-vmss_0", + }, + want: types.CloudProviderAzure, + }, + { + name: "gcp", + args: args{ + providerID: "gce:///projects/123456789012/zones/us-west1-b/instances/gke-cluster-1-default-pool-12345678-0v0v", + }, + want: types.CloudProviderGCP, + }, + { + name: "unsupported", + args: args{ + providerID: "unsupported", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getCloudProvider(tt.args.providerID) + if (err != nil) != tt.wantErr { + t.Errorf("getCloudProvider() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getCloudProvider() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getNodePool(t *testing.T) { + type args struct { + providerID types.CloudProvider + labels map[string]string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "aws", + args: args{ + providerID: types.CloudProviderAWS, + labels: map[string]string{ + "eks.amazonaws.com/nodegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + want: "test-node-pool", + }, + { + name: "azure", + args: args{ + providerID: types.CloudProviderAzure, + labels: map[string]string{ + "node.kubernetes.io/instancegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + want: "test-node-pool", + }, + { + name: "gcp", + args: args{ + providerID: types.CloudProviderGCP, + labels: map[string]string{ + "cloud.google.com/gke-nodepool": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + want: "test-node-pool", + }, + { + name: "unsupported", + args: args{ + providerID: "unsupported", + labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + wantErr: true, + }, + { + name: "no node pool", + args: args{ + providerID: types.CloudProviderAWS, + labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getNodePool(tt.args.providerID, tt.args.labels) + if (err != nil) != tt.wantErr { + t.Errorf("getNodePool() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getNodePool() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getAddresses(t *testing.T) { + type args struct { + addresses []v1.NodeAddress + } + tests := []struct { + name string + args args + want []net.IP + want1 []net.IP + wantErr bool + }{ + { + name: "external and internal IPs", + args: args{ + addresses: []v1.NodeAddress{ + { + Type: v1.NodeExternalIP, + Address: "132.64.12.125", + }, + { + Type: v1.NodeInternalIP, + Address: "10.10.0.1", + }, + }, + }, + want: []net.IP{ + net.ParseIP("132.64.12.125"), + }, + want1: []net.IP{ + net.ParseIP("10.10.0.1"), + }, + }, + { + name: "no external IPs", + args: args{ + addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, + }, + }, + want1: []net.IP{net.ParseIP("10.0.0.1")}, + }, + { + name: "no internal IPs", + args: args{ + addresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "132.10.10.1"}, + }, + }, + want: []net.IP{net.ParseIP("132.10.10.1")}, + }, + { + name: "invalid IP address", + args: args{ + addresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "invalid"}, + }, + }, + wantErr: true, + }, + { + name: "skip unsupported IP type", + args: args{ + addresses: []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: "test-node"}, + {Type: v1.NodeInternalDNS, Address: "test-node-internal"}, + {Type: v1.NodeExternalIP, Address: "132.10.10.1"}, + }, + }, + want: []net.IP{net.ParseIP("132.10.10.1")}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := getAddresses(tt.args.addresses) + if (err != nil) != tt.wantErr { + t.Errorf("getAddresses() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getAddresses() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("getAddresses() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_explorer_GetNode(t *testing.T) { + type fields struct { + client kubernetes.Interface + } + type args struct { + nodeName string + } + tests := []struct { + name string + fields fields + args args + want *types.Node + wantErr bool + }{ + { + name: "get node", + fields: fields{ + client: fake.NewSimpleClientset(&v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "eks.amazonaws.com/nodegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + "topology.kubernetes.io/region": "us-west-2", + "topology.kubernetes.io/zone": "us-west-2b", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "132.10.10.1"}, + {Type: v1.NodeInternalIP, Address: "10.10.0.1"}, + }, + }, + }), + }, + args: args{ + nodeName: "test-node", + }, + want: &types.Node{ + Name: "test-node", + Instance: "i-06d71a5ffc05cc325", + Cloud: types.CloudProviderAWS, + Pool: "test-node-pool", + Region: "us-west-2", + Zone: "us-west-2b", + ExternalIPs: []net.IP{ + net.ParseIP("132.10.10.1"), + }, + InternalIPs: []net.IP{ + net.ParseIP("10.10.0.1"), + }, + }, + }, + { + name: "failed to get region", + fields: fields{ + client: fake.NewSimpleClientset(&v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "eks.amazonaws.com/nodegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + }), + }, + args: args{ + nodeName: "test-node", + }, + wantErr: true, + }, + { + name: "failed to get zone", + fields: fields{ + client: fake.NewSimpleClientset(&v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "eks.amazonaws.com/nodegroup": "test-node-pool", + "beta.kubernetes.io/os": "linux", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + }), + }, + args: args{ + nodeName: "test-node", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &explorer{ + client: tt.fields.client, + } + got, err := d.GetNode(context.Background(), tt.args.nodeName) + if (err != nil) != tt.wantErr { + t.Errorf("GetNode() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetNode() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getInstance(t *testing.T) { + type args struct { + providerID string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "aws", + args: args{ + providerID: "aws:///us-west-2b/i-06d71a5ffc05cc325", + }, + want: "i-06d71a5ffc05cc325", + }, + { + name: "azure", + args: args{ + providerID: "azure:///subscriptions/12345678-1234-1234-1234-123456789012/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/aks-agentpool-12345678-vmss_0", + }, + want: "aks-agentpool-12345678-vmss_0", + }, + { + name: "gcp", + args: args{ + providerID: "gce:///projects/123456789012/zones/us-west1-b/instances/gke-cluster-1-default-pool-12345678-0v0v", + }, + want: "gke-cluster-1-default-pool-12345678-0v0v", + }, + { + name: "unsupported", + args: args{ + providerID: "unsupported", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getInstance(tt.args.providerID) + if (err != nil) != tt.wantErr { + t.Errorf("getInstance() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getInstance() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/types/node.go b/internal/types/node.go new file mode 100644 index 0000000..286cf1f --- /dev/null +++ b/internal/types/node.go @@ -0,0 +1,30 @@ +package types + +import ( + "fmt" + "net" +) + +type CloudProvider string + +const ( + CloudProviderGCP CloudProvider = "gcp" + CloudProviderAWS CloudProvider = "aws" + CloudProviderAzure CloudProvider = "azure" +) + +type Node struct { + Name string + Instance string + Cloud CloudProvider + Pool string + Region string + Zone string + ExternalIPs []net.IP + InternalIPs []net.IP +} + +// Stringer interface: all fields with name and value +func (n *Node) String() string { + return fmt.Sprintf("%+v", *n) +} diff --git a/main.go b/main.go deleted file mode 100644 index 5dbb7c1..0000000 --- a/main.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package main - -import ( - c "github.com/doitintl/kubeip/pkg/client" - cfg "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/kipcompute" - "github.com/sirupsen/logrus" -) - -var config *cfg.Config -var version string -var buildDate string - -func main() { - config, _ = cfg.NewConfig() - logrus.Info("kubeIP version: ", version) - logrus.Info(config) - cluster, err := kipcompute.ClusterName() - if err != nil { - logrus.Fatal(err) - panic(err) - } - projectID, err := kipcompute.ProjectName() - if err != nil { - logrus.Fatal(err) - panic(err) - } - logrus.Info(config.AdditionalNodePools) - logrus.WithFields(logrus.Fields{ - "Cluster name": cluster, - "Project name": projectID, - "Version": version, - "Build Date": buildDate, - }).Info("kubeIP is starting") - err = c.Run(config) - if err != nil { - logrus.Fatal(err) - panic(err) - } -} diff --git a/makefile b/makefile new file mode 100644 index 0000000..14add89 --- /dev/null +++ b/makefile @@ -0,0 +1,85 @@ +# Go parameters +GOCMD=go +GOBUILD=$(GOCMD) build +GORUN=$(GOCMD) run +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get +GOTOOL=$(GOCMD) tool +GOLINT=golangci-lint +GOMOCK=mockery +LINT_CONFIG = $(CURDIR)/.golangci.yaml + +BIN=$(CURDIR)/.bin +BINARY_NAME=kubeip-agent +TARGETOS := $(or $(TARGETOS), linux) +TARGETARCH := $(or $(TARGETARCH), amd64) + +DATE ?= $(shell date +%FT%T%z) + +# get version from environment variable if set or use git describe (match SemVer) +VERSION := $(if $(VERSION),$(VERSION),$(shell git describe --tags --always --dirty --match="[0-9]*.[0-9]*.[0-9]*" 2> /dev/null || \ + cat $(CURDIR)/.version 2> /dev/null || echo v0)) + +# get commit from environment variable if set or use git commit +COMMIT := $(if $(COMMIT),$(COMMIT),$(shell git rev-parse --short HEAD 2>/dev/null)) +# get branch from environment variable if set or use git branch +BRANCH := $(if $(BRANCH),$(BRANCH),$(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)) + +Q = $(if $(filter 1,$V),,@) +M = $(shell printf "\033[34;1m▶\033[0m") + +export CGO_ENABLED=0 +export GOOS=$(TARGETOS) +export GOARCH=$(TARGETARCH) + +# main task +all: lint test build ; $(info $(M) build, test and deploy ...) @ ## release cycle + +# Tools +setup-lint: + $(GOCMD) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2 +setup-mockery: + $(GOCMD) install github.com/vektra/mockery/v2@v2.35.2 + +# Tasks + +build: ; $(info $(M) building $(GOOS)/$(GOARCH) binary...) @ ## build with local Go SDK + $(GOBUILD) -v \ + -tags release \ + -ldflags '-s -w -X main.version=$(VERSION) -X main.buildDate=$(DATE) -X main.gitCommit=$(COMMIT) -X main.gitBranch=$(BRANCH)' \ + -o $(BIN)/$(BINARY_NAME) ./cmd/. + +lint: setup-lint; $(info $(M) running golangci-lint ...) @ ## run golangci-lint linters + # updating path since golangci-lint is looking for go binary and this may lead to + # conflict when multiple go versions are installed + $Q $(GOLINT) run -v -c $(LINT_CONFIG) --out-format checkstyle ./... > golangci-lint.out + $Q cat golangci-lint.out + +mock: setup-mockery ; $(info $(M) running mockery ...) @ ## run mockery to generate mocks + $Q $(GOMOCK) --dir internal --all --keeptree --with-expecter + +test: ; $(info $(M) running test ...) @ ## run tests with coverage + $Q $(GOCMD) fmt ./... + $Q $(GOTEST) -v -cover ./... -coverprofile=coverage.out + $Q $(GOTOOL) cover -func=coverage.out + +test-json: ; $(info $(M) running test output JSON ...) @ ## run tests with JSON report and coverage + $Q $(GOTEST) -v -cover ./... -coverprofile=coverage.out -json > test-report.out + +precommit: lint test ; $(info $(M) test and lint ...) @ ## release cycle: test > lint + +testview: ; $(info $(M) generating coverage report ...) @ ## generate HTML coverage report + $(GOTOOL) cover -html=coverage.out + +clean: ; $(info $(M) cleaning...) @ ## cleanup everything + $Q $(GOCLEAN) + @rm -rf $(BIN) + @rm -rf test/tests.* test/coverage.* + +run: ; $(info $(M) running ...) @ ## run locally + $Q $(GORUN) -v cmd/main.go + +help: ## display help + @grep -E '^[ a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' diff --git a/mocks/address/Assigner.go b/mocks/address/Assigner.go new file mode 100644 index 0000000..15140f7 --- /dev/null +++ b/mocks/address/Assigner.go @@ -0,0 +1,126 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// Assigner is an autogenerated mock type for the Assigner type +type Assigner struct { + mock.Mock +} + +type Assigner_Expecter struct { + mock *mock.Mock +} + +func (_m *Assigner) EXPECT() *Assigner_Expecter { + return &Assigner_Expecter{mock: &_m.Mock} +} + +// Assign provides a mock function with given fields: ctx, instanceID, zone, filter, orderBy +func (_m *Assigner) Assign(ctx context.Context, instanceID string, zone string, filter []string, orderBy string) error { + ret := _m.Called(ctx, instanceID, zone, filter, orderBy) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, []string, string) error); ok { + r0 = rf(ctx, instanceID, zone, filter, orderBy) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Assigner_Assign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Assign' +type Assigner_Assign_Call struct { + *mock.Call +} + +// Assign is a helper method to define mock.On call +// - ctx context.Context +// - instanceID string +// - zone string +// - filter []string +// - orderBy string +func (_e *Assigner_Expecter) Assign(ctx interface{}, instanceID interface{}, zone interface{}, filter interface{}, orderBy interface{}) *Assigner_Assign_Call { + return &Assigner_Assign_Call{Call: _e.mock.On("Assign", ctx, instanceID, zone, filter, orderBy)} +} + +func (_c *Assigner_Assign_Call) Run(run func(ctx context.Context, instanceID string, zone string, filter []string, orderBy string)) *Assigner_Assign_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].([]string), args[4].(string)) + }) + return _c +} + +func (_c *Assigner_Assign_Call) Return(_a0 error) *Assigner_Assign_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Assigner_Assign_Call) RunAndReturn(run func(context.Context, string, string, []string, string) error) *Assigner_Assign_Call { + _c.Call.Return(run) + return _c +} + +// Unassign provides a mock function with given fields: ctx, instanceID, zone +func (_m *Assigner) Unassign(ctx context.Context, instanceID string, zone string) error { + ret := _m.Called(ctx, instanceID, zone) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, instanceID, zone) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Assigner_Unassign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Unassign' +type Assigner_Unassign_Call struct { + *mock.Call +} + +// Unassign is a helper method to define mock.On call +// - ctx context.Context +// - instanceID string +// - zone string +func (_e *Assigner_Expecter) Unassign(ctx interface{}, instanceID interface{}, zone interface{}) *Assigner_Unassign_Call { + return &Assigner_Unassign_Call{Call: _e.mock.On("Unassign", ctx, instanceID, zone)} +} + +func (_c *Assigner_Unassign_Call) Run(run func(ctx context.Context, instanceID string, zone string)) *Assigner_Unassign_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *Assigner_Unassign_Call) Return(_a0 error) *Assigner_Unassign_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Assigner_Unassign_Call) RunAndReturn(run func(context.Context, string, string) error) *Assigner_Unassign_Call { + _c.Call.Return(run) + return _c +} + +// NewAssigner creates a new instance of Assigner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAssigner(t interface { + mock.TestingT + Cleanup(func()) +}) *Assigner { + mock := &Assigner{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/AddressManager.go b/mocks/cloud/AddressManager.go new file mode 100644 index 0000000..617d6d7 --- /dev/null +++ b/mocks/cloud/AddressManager.go @@ -0,0 +1,207 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + compute "google.golang.org/api/compute/v1" +) + +// AddressManager is an autogenerated mock type for the AddressManager type +type AddressManager struct { + mock.Mock +} + +type AddressManager_Expecter struct { + mock *mock.Mock +} + +func (_m *AddressManager) EXPECT() *AddressManager_Expecter { + return &AddressManager_Expecter{mock: &_m.Mock} +} + +// AddAccessConfig provides a mock function with given fields: project, zone, instance, networkInterface, accessconfig +func (_m *AddressManager) AddAccessConfig(project string, zone string, instance string, networkInterface string, accessconfig *compute.AccessConfig) (*compute.Operation, error) { + ret := _m.Called(project, zone, instance, networkInterface, accessconfig) + + var r0 *compute.Operation + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string, string, *compute.AccessConfig) (*compute.Operation, error)); ok { + return rf(project, zone, instance, networkInterface, accessconfig) + } + if rf, ok := ret.Get(0).(func(string, string, string, string, *compute.AccessConfig) *compute.Operation); ok { + r0 = rf(project, zone, instance, networkInterface, accessconfig) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Operation) + } + } + + if rf, ok := ret.Get(1).(func(string, string, string, string, *compute.AccessConfig) error); ok { + r1 = rf(project, zone, instance, networkInterface, accessconfig) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AddressManager_AddAccessConfig_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddAccessConfig' +type AddressManager_AddAccessConfig_Call struct { + *mock.Call +} + +// AddAccessConfig is a helper method to define mock.On call +// - project string +// - zone string +// - instance string +// - networkInterface string +// - accessconfig *compute.AccessConfig +func (_e *AddressManager_Expecter) AddAccessConfig(project interface{}, zone interface{}, instance interface{}, networkInterface interface{}, accessconfig interface{}) *AddressManager_AddAccessConfig_Call { + return &AddressManager_AddAccessConfig_Call{Call: _e.mock.On("AddAccessConfig", project, zone, instance, networkInterface, accessconfig)} +} + +func (_c *AddressManager_AddAccessConfig_Call) Run(run func(project string, zone string, instance string, networkInterface string, accessconfig *compute.AccessConfig)) *AddressManager_AddAccessConfig_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string), args[3].(string), args[4].(*compute.AccessConfig)) + }) + return _c +} + +func (_c *AddressManager_AddAccessConfig_Call) Return(_a0 *compute.Operation, _a1 error) *AddressManager_AddAccessConfig_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AddressManager_AddAccessConfig_Call) RunAndReturn(run func(string, string, string, string, *compute.AccessConfig) (*compute.Operation, error)) *AddressManager_AddAccessConfig_Call { + _c.Call.Return(run) + return _c +} + +// DeleteAccessConfig provides a mock function with given fields: project, zone, instance, accessConfig, networkInterface +func (_m *AddressManager) DeleteAccessConfig(project string, zone string, instance string, accessConfig string, networkInterface string) (*compute.Operation, error) { + ret := _m.Called(project, zone, instance, accessConfig, networkInterface) + + var r0 *compute.Operation + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string, string, string) (*compute.Operation, error)); ok { + return rf(project, zone, instance, accessConfig, networkInterface) + } + if rf, ok := ret.Get(0).(func(string, string, string, string, string) *compute.Operation); ok { + r0 = rf(project, zone, instance, accessConfig, networkInterface) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Operation) + } + } + + if rf, ok := ret.Get(1).(func(string, string, string, string, string) error); ok { + r1 = rf(project, zone, instance, accessConfig, networkInterface) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AddressManager_DeleteAccessConfig_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteAccessConfig' +type AddressManager_DeleteAccessConfig_Call struct { + *mock.Call +} + +// DeleteAccessConfig is a helper method to define mock.On call +// - project string +// - zone string +// - instance string +// - accessConfig string +// - networkInterface string +func (_e *AddressManager_Expecter) DeleteAccessConfig(project interface{}, zone interface{}, instance interface{}, accessConfig interface{}, networkInterface interface{}) *AddressManager_DeleteAccessConfig_Call { + return &AddressManager_DeleteAccessConfig_Call{Call: _e.mock.On("DeleteAccessConfig", project, zone, instance, accessConfig, networkInterface)} +} + +func (_c *AddressManager_DeleteAccessConfig_Call) Run(run func(project string, zone string, instance string, accessConfig string, networkInterface string)) *AddressManager_DeleteAccessConfig_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string), args[3].(string), args[4].(string)) + }) + return _c +} + +func (_c *AddressManager_DeleteAccessConfig_Call) Return(_a0 *compute.Operation, _a1 error) *AddressManager_DeleteAccessConfig_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AddressManager_DeleteAccessConfig_Call) RunAndReturn(run func(string, string, string, string, string) (*compute.Operation, error)) *AddressManager_DeleteAccessConfig_Call { + _c.Call.Return(run) + return _c +} + +// GetAddress provides a mock function with given fields: project, region, name +func (_m *AddressManager) GetAddress(project string, region string, name string) (*compute.Address, error) { + ret := _m.Called(project, region, name) + + var r0 *compute.Address + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string) (*compute.Address, error)); ok { + return rf(project, region, name) + } + if rf, ok := ret.Get(0).(func(string, string, string) *compute.Address); ok { + r0 = rf(project, region, name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Address) + } + } + + if rf, ok := ret.Get(1).(func(string, string, string) error); ok { + r1 = rf(project, region, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// AddressManager_GetAddress_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetAddress' +type AddressManager_GetAddress_Call struct { + *mock.Call +} + +// GetAddress is a helper method to define mock.On call +// - project string +// - region string +// - name string +func (_e *AddressManager_Expecter) GetAddress(project interface{}, region interface{}, name interface{}) *AddressManager_GetAddress_Call { + return &AddressManager_GetAddress_Call{Call: _e.mock.On("GetAddress", project, region, name)} +} + +func (_c *AddressManager_GetAddress_Call) Run(run func(project string, region string, name string)) *AddressManager_GetAddress_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *AddressManager_GetAddress_Call) Return(_a0 *compute.Address, _a1 error) *AddressManager_GetAddress_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *AddressManager_GetAddress_Call) RunAndReturn(run func(string, string, string) (*compute.Address, error)) *AddressManager_GetAddress_Call { + _c.Call.Return(run) + return _c +} + +// NewAddressManager creates a new instance of AddressManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAddressManager(t interface { + mock.TestingT + Cleanup(func()) +}) *AddressManager { + mock := &AddressManager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/Ec2InstanceGetter.go b/mocks/cloud/Ec2InstanceGetter.go new file mode 100644 index 0000000..f94bd73 --- /dev/null +++ b/mocks/cloud/Ec2InstanceGetter.go @@ -0,0 +1,93 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + context "context" + + types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + mock "github.com/stretchr/testify/mock" +) + +// Ec2InstanceGetter is an autogenerated mock type for the Ec2InstanceGetter type +type Ec2InstanceGetter struct { + mock.Mock +} + +type Ec2InstanceGetter_Expecter struct { + mock *mock.Mock +} + +func (_m *Ec2InstanceGetter) EXPECT() *Ec2InstanceGetter_Expecter { + return &Ec2InstanceGetter_Expecter{mock: &_m.Mock} +} + +// Get provides a mock function with given fields: ctx, instanceID, region +func (_m *Ec2InstanceGetter) Get(ctx context.Context, instanceID string, region string) (*types.Instance, error) { + ret := _m.Called(ctx, instanceID, region) + + var r0 *types.Instance + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (*types.Instance, error)); ok { + return rf(ctx, instanceID, region) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) *types.Instance); ok { + r0 = rf(ctx, instanceID, region) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Instance) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, instanceID, region) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Ec2InstanceGetter_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type Ec2InstanceGetter_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - ctx context.Context +// - instanceID string +// - region string +func (_e *Ec2InstanceGetter_Expecter) Get(ctx interface{}, instanceID interface{}, region interface{}) *Ec2InstanceGetter_Get_Call { + return &Ec2InstanceGetter_Get_Call{Call: _e.mock.On("Get", ctx, instanceID, region)} +} + +func (_c *Ec2InstanceGetter_Get_Call) Run(run func(ctx context.Context, instanceID string, region string)) *Ec2InstanceGetter_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *Ec2InstanceGetter_Get_Call) Return(_a0 *types.Instance, _a1 error) *Ec2InstanceGetter_Get_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Ec2InstanceGetter_Get_Call) RunAndReturn(run func(context.Context, string, string) (*types.Instance, error)) *Ec2InstanceGetter_Get_Call { + _c.Call.Return(run) + return _c +} + +// NewEc2InstanceGetter creates a new instance of Ec2InstanceGetter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEc2InstanceGetter(t interface { + mock.TestingT + Cleanup(func()) +}) *Ec2InstanceGetter { + mock := &Ec2InstanceGetter{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/EipAssigner.go b/mocks/cloud/EipAssigner.go new file mode 100644 index 0000000..18cda75 --- /dev/null +++ b/mocks/cloud/EipAssigner.go @@ -0,0 +1,123 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// EipAssigner is an autogenerated mock type for the EipAssigner type +type EipAssigner struct { + mock.Mock +} + +type EipAssigner_Expecter struct { + mock *mock.Mock +} + +func (_m *EipAssigner) EXPECT() *EipAssigner_Expecter { + return &EipAssigner_Expecter{mock: &_m.Mock} +} + +// Assign provides a mock function with given fields: ctx, networkInterfaceID, allocationID +func (_m *EipAssigner) Assign(ctx context.Context, networkInterfaceID string, allocationID string) error { + ret := _m.Called(ctx, networkInterfaceID, allocationID) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, networkInterfaceID, allocationID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// EipAssigner_Assign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Assign' +type EipAssigner_Assign_Call struct { + *mock.Call +} + +// Assign is a helper method to define mock.On call +// - ctx context.Context +// - networkInterfaceID string +// - allocationID string +func (_e *EipAssigner_Expecter) Assign(ctx interface{}, networkInterfaceID interface{}, allocationID interface{}) *EipAssigner_Assign_Call { + return &EipAssigner_Assign_Call{Call: _e.mock.On("Assign", ctx, networkInterfaceID, allocationID)} +} + +func (_c *EipAssigner_Assign_Call) Run(run func(ctx context.Context, networkInterfaceID string, allocationID string)) *EipAssigner_Assign_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *EipAssigner_Assign_Call) Return(_a0 error) *EipAssigner_Assign_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *EipAssigner_Assign_Call) RunAndReturn(run func(context.Context, string, string) error) *EipAssigner_Assign_Call { + _c.Call.Return(run) + return _c +} + +// Unassign provides a mock function with given fields: ctx, associationID +func (_m *EipAssigner) Unassign(ctx context.Context, associationID string) error { + ret := _m.Called(ctx, associationID) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { + r0 = rf(ctx, associationID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// EipAssigner_Unassign_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Unassign' +type EipAssigner_Unassign_Call struct { + *mock.Call +} + +// Unassign is a helper method to define mock.On call +// - ctx context.Context +// - associationID string +func (_e *EipAssigner_Expecter) Unassign(ctx interface{}, associationID interface{}) *EipAssigner_Unassign_Call { + return &EipAssigner_Unassign_Call{Call: _e.mock.On("Unassign", ctx, associationID)} +} + +func (_c *EipAssigner_Unassign_Call) Run(run func(ctx context.Context, associationID string)) *EipAssigner_Unassign_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *EipAssigner_Unassign_Call) Return(_a0 error) *EipAssigner_Unassign_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *EipAssigner_Unassign_Call) RunAndReturn(run func(context.Context, string) error) *EipAssigner_Unassign_Call { + _c.Call.Return(run) + return _c +} + +// NewEipAssigner creates a new instance of EipAssigner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEipAssigner(t interface { + mock.TestingT + Cleanup(func()) +}) *EipAssigner { + mock := &EipAssigner{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/EipLister.go b/mocks/cloud/EipLister.go new file mode 100644 index 0000000..1cd651f --- /dev/null +++ b/mocks/cloud/EipLister.go @@ -0,0 +1,93 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + context "context" + + types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + mock "github.com/stretchr/testify/mock" +) + +// EipLister is an autogenerated mock type for the EipLister type +type EipLister struct { + mock.Mock +} + +type EipLister_Expecter struct { + mock *mock.Mock +} + +func (_m *EipLister) EXPECT() *EipLister_Expecter { + return &EipLister_Expecter{mock: &_m.Mock} +} + +// List provides a mock function with given fields: ctx, filter, inUse +func (_m *EipLister) List(ctx context.Context, filter map[string][]string, inUse bool) ([]types.Address, error) { + ret := _m.Called(ctx, filter, inUse) + + var r0 []types.Address + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, map[string][]string, bool) ([]types.Address, error)); ok { + return rf(ctx, filter, inUse) + } + if rf, ok := ret.Get(0).(func(context.Context, map[string][]string, bool) []types.Address); ok { + r0 = rf(ctx, filter, inUse) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]types.Address) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, map[string][]string, bool) error); ok { + r1 = rf(ctx, filter, inUse) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// EipLister_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' +type EipLister_List_Call struct { + *mock.Call +} + +// List is a helper method to define mock.On call +// - ctx context.Context +// - filter map[string][]string +// - inUse bool +func (_e *EipLister_Expecter) List(ctx interface{}, filter interface{}, inUse interface{}) *EipLister_List_Call { + return &EipLister_List_Call{Call: _e.mock.On("List", ctx, filter, inUse)} +} + +func (_c *EipLister_List_Call) Run(run func(ctx context.Context, filter map[string][]string, inUse bool)) *EipLister_List_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(map[string][]string), args[2].(bool)) + }) + return _c +} + +func (_c *EipLister_List_Call) Return(_a0 []types.Address, _a1 error) *EipLister_List_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *EipLister_List_Call) RunAndReturn(run func(context.Context, map[string][]string, bool) ([]types.Address, error)) *EipLister_List_Call { + _c.Call.Return(run) + return _c +} + +// NewEipLister creates a new instance of EipLister. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEipLister(t interface { + mock.TestingT + Cleanup(func()) +}) *EipLister { + mock := &EipLister{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/InstanceGetter.go b/mocks/cloud/InstanceGetter.go new file mode 100644 index 0000000..2a72783 --- /dev/null +++ b/mocks/cloud/InstanceGetter.go @@ -0,0 +1,91 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + compute "google.golang.org/api/compute/v1" +) + +// InstanceGetter is an autogenerated mock type for the InstanceGetter type +type InstanceGetter struct { + mock.Mock +} + +type InstanceGetter_Expecter struct { + mock *mock.Mock +} + +func (_m *InstanceGetter) EXPECT() *InstanceGetter_Expecter { + return &InstanceGetter_Expecter{mock: &_m.Mock} +} + +// Get provides a mock function with given fields: projectID, zone, instance +func (_m *InstanceGetter) Get(projectID string, zone string, instance string) (*compute.Instance, error) { + ret := _m.Called(projectID, zone, instance) + + var r0 *compute.Instance + var r1 error + if rf, ok := ret.Get(0).(func(string, string, string) (*compute.Instance, error)); ok { + return rf(projectID, zone, instance) + } + if rf, ok := ret.Get(0).(func(string, string, string) *compute.Instance); ok { + r0 = rf(projectID, zone, instance) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Instance) + } + } + + if rf, ok := ret.Get(1).(func(string, string, string) error); ok { + r1 = rf(projectID, zone, instance) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// InstanceGetter_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type InstanceGetter_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +// - projectID string +// - zone string +// - instance string +func (_e *InstanceGetter_Expecter) Get(projectID interface{}, zone interface{}, instance interface{}) *InstanceGetter_Get_Call { + return &InstanceGetter_Get_Call{Call: _e.mock.On("Get", projectID, zone, instance)} +} + +func (_c *InstanceGetter_Get_Call) Run(run func(projectID string, zone string, instance string)) *InstanceGetter_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *InstanceGetter_Get_Call) Return(_a0 *compute.Instance, _a1 error) *InstanceGetter_Get_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *InstanceGetter_Get_Call) RunAndReturn(run func(string, string, string) (*compute.Instance, error)) *InstanceGetter_Get_Call { + _c.Call.Return(run) + return _c +} + +// NewInstanceGetter creates a new instance of InstanceGetter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewInstanceGetter(t interface { + mock.TestingT + Cleanup(func()) +}) *InstanceGetter { + mock := &InstanceGetter{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/ListCall.go b/mocks/cloud/ListCall.go new file mode 100644 index 0000000..a3d483c --- /dev/null +++ b/mocks/cloud/ListCall.go @@ -0,0 +1,222 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + cloud "github.com/doitintl/kubeip/internal/cloud" + compute "google.golang.org/api/compute/v1" + + mock "github.com/stretchr/testify/mock" +) + +// ListCall is an autogenerated mock type for the ListCall type +type ListCall struct { + mock.Mock +} + +type ListCall_Expecter struct { + mock *mock.Mock +} + +func (_m *ListCall) EXPECT() *ListCall_Expecter { + return &ListCall_Expecter{mock: &_m.Mock} +} + +// Do provides a mock function with given fields: +func (_m *ListCall) Do() (*compute.AddressList, error) { + ret := _m.Called() + + var r0 *compute.AddressList + var r1 error + if rf, ok := ret.Get(0).(func() (*compute.AddressList, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *compute.AddressList); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.AddressList) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListCall_Do_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Do' +type ListCall_Do_Call struct { + *mock.Call +} + +// Do is a helper method to define mock.On call +func (_e *ListCall_Expecter) Do() *ListCall_Do_Call { + return &ListCall_Do_Call{Call: _e.mock.On("Do")} +} + +func (_c *ListCall_Do_Call) Run(run func()) *ListCall_Do_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ListCall_Do_Call) Return(_a0 *compute.AddressList, _a1 error) *ListCall_Do_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *ListCall_Do_Call) RunAndReturn(run func() (*compute.AddressList, error)) *ListCall_Do_Call { + _c.Call.Return(run) + return _c +} + +// Filter provides a mock function with given fields: filter +func (_m *ListCall) Filter(filter string) cloud.ListCall { + ret := _m.Called(filter) + + var r0 cloud.ListCall + if rf, ok := ret.Get(0).(func(string) cloud.ListCall); ok { + r0 = rf(filter) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.ListCall) + } + } + + return r0 +} + +// ListCall_Filter_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Filter' +type ListCall_Filter_Call struct { + *mock.Call +} + +// Filter is a helper method to define mock.On call +// - filter string +func (_e *ListCall_Expecter) Filter(filter interface{}) *ListCall_Filter_Call { + return &ListCall_Filter_Call{Call: _e.mock.On("Filter", filter)} +} + +func (_c *ListCall_Filter_Call) Run(run func(filter string)) *ListCall_Filter_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *ListCall_Filter_Call) Return(_a0 cloud.ListCall) *ListCall_Filter_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ListCall_Filter_Call) RunAndReturn(run func(string) cloud.ListCall) *ListCall_Filter_Call { + _c.Call.Return(run) + return _c +} + +// OrderBy provides a mock function with given fields: orderBy +func (_m *ListCall) OrderBy(orderBy string) cloud.ListCall { + ret := _m.Called(orderBy) + + var r0 cloud.ListCall + if rf, ok := ret.Get(0).(func(string) cloud.ListCall); ok { + r0 = rf(orderBy) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.ListCall) + } + } + + return r0 +} + +// ListCall_OrderBy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'OrderBy' +type ListCall_OrderBy_Call struct { + *mock.Call +} + +// OrderBy is a helper method to define mock.On call +// - orderBy string +func (_e *ListCall_Expecter) OrderBy(orderBy interface{}) *ListCall_OrderBy_Call { + return &ListCall_OrderBy_Call{Call: _e.mock.On("OrderBy", orderBy)} +} + +func (_c *ListCall_OrderBy_Call) Run(run func(orderBy string)) *ListCall_OrderBy_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *ListCall_OrderBy_Call) Return(_a0 cloud.ListCall) *ListCall_OrderBy_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ListCall_OrderBy_Call) RunAndReturn(run func(string) cloud.ListCall) *ListCall_OrderBy_Call { + _c.Call.Return(run) + return _c +} + +// PageToken provides a mock function with given fields: pageToken +func (_m *ListCall) PageToken(pageToken string) cloud.ListCall { + ret := _m.Called(pageToken) + + var r0 cloud.ListCall + if rf, ok := ret.Get(0).(func(string) cloud.ListCall); ok { + r0 = rf(pageToken) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.ListCall) + } + } + + return r0 +} + +// ListCall_PageToken_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PageToken' +type ListCall_PageToken_Call struct { + *mock.Call +} + +// PageToken is a helper method to define mock.On call +// - pageToken string +func (_e *ListCall_Expecter) PageToken(pageToken interface{}) *ListCall_PageToken_Call { + return &ListCall_PageToken_Call{Call: _e.mock.On("PageToken", pageToken)} +} + +func (_c *ListCall_PageToken_Call) Run(run func(pageToken string)) *ListCall_PageToken_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *ListCall_PageToken_Call) Return(_a0 cloud.ListCall) *ListCall_PageToken_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ListCall_PageToken_Call) RunAndReturn(run func(string) cloud.ListCall) *ListCall_PageToken_Call { + _c.Call.Return(run) + return _c +} + +// NewListCall creates a new instance of ListCall. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewListCall(t interface { + mock.TestingT + Cleanup(func()) +}) *ListCall { + mock := &ListCall{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/Lister.go b/mocks/cloud/Lister.go new file mode 100644 index 0000000..e2f5f14 --- /dev/null +++ b/mocks/cloud/Lister.go @@ -0,0 +1,80 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + cloud "github.com/doitintl/kubeip/internal/cloud" + mock "github.com/stretchr/testify/mock" +) + +// Lister is an autogenerated mock type for the Lister type +type Lister struct { + mock.Mock +} + +type Lister_Expecter struct { + mock *mock.Mock +} + +func (_m *Lister) EXPECT() *Lister_Expecter { + return &Lister_Expecter{mock: &_m.Mock} +} + +// List provides a mock function with given fields: projectID, region +func (_m *Lister) List(projectID string, region string) cloud.ListCall { + ret := _m.Called(projectID, region) + + var r0 cloud.ListCall + if rf, ok := ret.Get(0).(func(string, string) cloud.ListCall); ok { + r0 = rf(projectID, region) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.ListCall) + } + } + + return r0 +} + +// Lister_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' +type Lister_List_Call struct { + *mock.Call +} + +// List is a helper method to define mock.On call +// - projectID string +// - region string +func (_e *Lister_Expecter) List(projectID interface{}, region interface{}) *Lister_List_Call { + return &Lister_List_Call{Call: _e.mock.On("List", projectID, region)} +} + +func (_c *Lister_List_Call) Run(run func(projectID string, region string)) *Lister_List_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string)) + }) + return _c +} + +func (_c *Lister_List_Call) Return(_a0 cloud.ListCall) *Lister_List_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Lister_List_Call) RunAndReturn(run func(string, string) cloud.ListCall) *Lister_List_Call { + _c.Call.Return(run) + return _c +} + +// NewLister creates a new instance of Lister. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewLister(t interface { + mock.TestingT + Cleanup(func()) +}) *Lister { + mock := &Lister{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/WaitCall.go b/mocks/cloud/WaitCall.go new file mode 100644 index 0000000..8516a60 --- /dev/null +++ b/mocks/cloud/WaitCall.go @@ -0,0 +1,136 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + cloud "github.com/doitintl/kubeip/internal/cloud" + compute "google.golang.org/api/compute/v1" + + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// WaitCall is an autogenerated mock type for the WaitCall type +type WaitCall struct { + mock.Mock +} + +type WaitCall_Expecter struct { + mock *mock.Mock +} + +func (_m *WaitCall) EXPECT() *WaitCall_Expecter { + return &WaitCall_Expecter{mock: &_m.Mock} +} + +// Context provides a mock function with given fields: ctx +func (_m *WaitCall) Context(ctx context.Context) cloud.WaitCall { + ret := _m.Called(ctx) + + var r0 cloud.WaitCall + if rf, ok := ret.Get(0).(func(context.Context) cloud.WaitCall); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.WaitCall) + } + } + + return r0 +} + +// WaitCall_Context_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Context' +type WaitCall_Context_Call struct { + *mock.Call +} + +// Context is a helper method to define mock.On call +// - ctx context.Context +func (_e *WaitCall_Expecter) Context(ctx interface{}) *WaitCall_Context_Call { + return &WaitCall_Context_Call{Call: _e.mock.On("Context", ctx)} +} + +func (_c *WaitCall_Context_Call) Run(run func(ctx context.Context)) *WaitCall_Context_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *WaitCall_Context_Call) Return(_a0 cloud.WaitCall) *WaitCall_Context_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *WaitCall_Context_Call) RunAndReturn(run func(context.Context) cloud.WaitCall) *WaitCall_Context_Call { + _c.Call.Return(run) + return _c +} + +// Do provides a mock function with given fields: +func (_m *WaitCall) Do() (*compute.Operation, error) { + ret := _m.Called() + + var r0 *compute.Operation + var r1 error + if rf, ok := ret.Get(0).(func() (*compute.Operation, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *compute.Operation); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*compute.Operation) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// WaitCall_Do_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Do' +type WaitCall_Do_Call struct { + *mock.Call +} + +// Do is a helper method to define mock.On call +func (_e *WaitCall_Expecter) Do() *WaitCall_Do_Call { + return &WaitCall_Do_Call{Call: _e.mock.On("Do")} +} + +func (_c *WaitCall_Do_Call) Run(run func()) *WaitCall_Do_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *WaitCall_Do_Call) Return(_a0 *compute.Operation, _a1 error) *WaitCall_Do_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *WaitCall_Do_Call) RunAndReturn(run func() (*compute.Operation, error)) *WaitCall_Do_Call { + _c.Call.Return(run) + return _c +} + +// NewWaitCall creates a new instance of WaitCall. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewWaitCall(t interface { + mock.TestingT + Cleanup(func()) +}) *WaitCall { + mock := &WaitCall{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/cloud/ZoneWaiter.go b/mocks/cloud/ZoneWaiter.go new file mode 100644 index 0000000..7e7612d --- /dev/null +++ b/mocks/cloud/ZoneWaiter.go @@ -0,0 +1,81 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + cloud "github.com/doitintl/kubeip/internal/cloud" + mock "github.com/stretchr/testify/mock" +) + +// ZoneWaiter is an autogenerated mock type for the ZoneWaiter type +type ZoneWaiter struct { + mock.Mock +} + +type ZoneWaiter_Expecter struct { + mock *mock.Mock +} + +func (_m *ZoneWaiter) EXPECT() *ZoneWaiter_Expecter { + return &ZoneWaiter_Expecter{mock: &_m.Mock} +} + +// Wait provides a mock function with given fields: projectID, region, operationName +func (_m *ZoneWaiter) Wait(projectID string, region string, operationName string) cloud.WaitCall { + ret := _m.Called(projectID, region, operationName) + + var r0 cloud.WaitCall + if rf, ok := ret.Get(0).(func(string, string, string) cloud.WaitCall); ok { + r0 = rf(projectID, region, operationName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(cloud.WaitCall) + } + } + + return r0 +} + +// ZoneWaiter_Wait_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Wait' +type ZoneWaiter_Wait_Call struct { + *mock.Call +} + +// Wait is a helper method to define mock.On call +// - projectID string +// - region string +// - operationName string +func (_e *ZoneWaiter_Expecter) Wait(projectID interface{}, region interface{}, operationName interface{}) *ZoneWaiter_Wait_Call { + return &ZoneWaiter_Wait_Call{Call: _e.mock.On("Wait", projectID, region, operationName)} +} + +func (_c *ZoneWaiter_Wait_Call) Run(run func(projectID string, region string, operationName string)) *ZoneWaiter_Wait_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *ZoneWaiter_Wait_Call) Return(_a0 cloud.WaitCall) *ZoneWaiter_Wait_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ZoneWaiter_Wait_Call) RunAndReturn(run func(string, string, string) cloud.WaitCall) *ZoneWaiter_Wait_Call { + _c.Call.Return(run) + return _c +} + +// NewZoneWaiter creates a new instance of ZoneWaiter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewZoneWaiter(t interface { + mock.TestingT + Cleanup(func()) +}) *ZoneWaiter { + mock := &ZoneWaiter{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/node/Explorer.go b/mocks/node/Explorer.go new file mode 100644 index 0000000..85c961f --- /dev/null +++ b/mocks/node/Explorer.go @@ -0,0 +1,93 @@ +// Code generated by mockery v2.30.16. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + types "github.com/doitintl/kubeip/internal/types" +) + +// Explorer is an autogenerated mock type for the Explorer type +type Explorer struct { + mock.Mock +} + +type Explorer_Expecter struct { + mock *mock.Mock +} + +func (_m *Explorer) EXPECT() *Explorer_Expecter { + return &Explorer_Expecter{mock: &_m.Mock} +} + +// GetNode provides a mock function with given fields: ctx, nodeName +func (_m *Explorer) GetNode(ctx context.Context, nodeName string) (*types.Node, error) { + ret := _m.Called(ctx, nodeName) + + var r0 *types.Node + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*types.Node, error)); ok { + return rf(ctx, nodeName) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *types.Node); ok { + r0 = rf(ctx, nodeName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Node) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, nodeName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Explorer_GetNode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNode' +type Explorer_GetNode_Call struct { + *mock.Call +} + +// GetNode is a helper method to define mock.On call +// - ctx context.Context +// - nodeName string +func (_e *Explorer_Expecter) GetNode(ctx interface{}, nodeName interface{}) *Explorer_GetNode_Call { + return &Explorer_GetNode_Call{Call: _e.mock.On("GetNode", ctx, nodeName)} +} + +func (_c *Explorer_GetNode_Call) Run(run func(ctx context.Context, nodeName string)) *Explorer_GetNode_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *Explorer_GetNode_Call) Return(_a0 *types.Node, _a1 error) *Explorer_GetNode_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Explorer_GetNode_Call) RunAndReturn(run func(context.Context, string) (*types.Node, error)) *Explorer_GetNode_Call { + _c.Call.Return(run) + return _c +} + +// NewExplorer creates a new instance of Explorer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewExplorer(t interface { + mock.TestingT + Cleanup(func()) +}) *Explorer { + mock := &Explorer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/client/run.go b/pkg/client/run.go deleted file mode 100644 index ee670cc..0000000 --- a/pkg/client/run.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package client - -import ( - cfg "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/controller" -) - -// Run start kubeip controller -func Run(config *cfg.Config) error { - return controller.Start(config) -} diff --git a/pkg/config/config.go b/pkg/config/config.go deleted file mode 100644 index 7ed8000..0000000 --- a/pkg/config/config.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package config - -import ( - "strings" - "time" - - "github.com/spf13/viper" -) - -// Config kubeip configuration -type Config struct { - LabelKey string - LabelValue string - NodePool string - ForceAssignment bool - AdditionalNodePools []string - Ticker time.Duration - AllNodePools bool - OrderByLabelKey string - OrderByDesc bool - CopyLabels bool - ClearLabels bool - DryRun bool -} - -func setConfigDefaults() { - viper.SetDefault("LabelKey", "kubeip") - viper.SetDefault("LabelValue", "reserved") - viper.SetDefault("NodePool", "default-pool") - viper.SetDefault("ForceAssignment", true) - viper.SetDefault("ForceAssignment", true) - viper.SetDefault("AdditionalNodePools", "") - viper.SetDefault("Ticker", 5) - viper.SetDefault("AllNodePools", false) - viper.SetDefault("OrderByLabelKey", "priority") - viper.SetDefault("OrderByDesc", true) - viper.SetDefault("CopyLabels", true) - viper.SetDefault("ClearLabels", true) - viper.SetDefault("DryRun", false) -} - -// NewConfig initialize kubeip configuration -func NewConfig() (*Config, error) { - var AdditionalNodePools []string - viper.SetEnvPrefix("kubeip") - viper.AutomaticEnv() - setConfigDefaults() - AdditionalNodePoolsStr := viper.GetString("additionalnodepools") - if len(AdditionalNodePoolsStr) > 0 { - AdditionalNodePools = strings.Split(AdditionalNodePoolsStr, ",") - } - - c := Config{ - LabelKey: viper.GetString("labelkey"), - LabelValue: viper.GetString("labelvalue"), - NodePool: viper.GetString("nodepool"), - ForceAssignment: viper.GetBool("forceassignment"), - AdditionalNodePools: AdditionalNodePools, - Ticker: viper.GetDuration("ticker"), - AllNodePools: viper.GetBool("allnodepools"), - OrderByLabelKey: viper.GetString("orderbylabelkey"), - OrderByDesc: viper.GetBool("orderbydesc"), - CopyLabels: viper.GetBool("copylabels"), - ClearLabels: viper.GetBool("clearlabels"), - DryRun: viper.GetBool("dryrun"), - } - return &c, nil -} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go deleted file mode 100644 index 2da3de7..0000000 --- a/pkg/controller/controller.go +++ /dev/null @@ -1,502 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package controller - -import ( - "fmt" - "os" - "os/signal" - "strings" - "syscall" - "time" - - "github.com/pkg/errors" - "golang.org/x/time/rate" - - cfg "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/kipcompute" - "github.com/doitintl/kubeip/pkg/types" - "github.com/doitintl/kubeip/pkg/utils" - "github.com/sirupsen/logrus" - "golang.org/x/net/context" - api_v1 "k8s.io/api/core/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" -) - -// AddressInstanceTuple object -type AddressInstanceTuple struct { - address string - instance types.Instance -} - -// Controller object -type Controller struct { - logger *logrus.Entry - clientset kubernetes.Interface - queue workqueue.RateLimitingInterface - informer cache.SharedIndexInformer - instance chan<- types.Instance - projectID string - clusterName string - config *cfg.Config - ticker *time.Ticker - processing bool -} - -// Event indicate the informerEvent -type Event struct { - key string - eventType string - resourceType string -} - -var serverStartTime time.Time - -const maxRetries = 5 - -const prefix = "kube-system/kube-proxy-" - -// Start kubeip controller -func Start(config *cfg.Config) error { - var kubeClient kubernetes.Interface - _, err := rest.InClusterConfig() - if err != nil { - return errors.Wrap(err, "Can not get kubernetes config") - } - kubeClient, err = utils.GetClient() - if err != nil { - return errors.Wrap(err, "Can not get kubernetes API") - } - informer := cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options meta_v1.ListOptions) (runtime.Object, error) { - return kubeClient.CoreV1().Pods(meta_v1.NamespaceAll).List(context.Background(), options) - }, - WatchFunc: func(options meta_v1.ListOptions) (watch.Interface, error) { - return kubeClient.CoreV1().Pods(meta_v1.NamespaceAll).Watch(context.Background(), options) - }, - }, - &api_v1.Pod{}, - 0, // Skip resync - cache.Indexers{}, - ) - - c := newResourceController(kubeClient, informer, "node") - c.projectID, err = kipcompute.ProjectName() - if err != nil { - return errors.Wrap(err, "Can not get project name") - } - c.clusterName, err = kipcompute.ClusterName() - if err != nil { - return errors.Wrap(err, "Can not get cluster name") - } - c.config = config - c.ticker = time.NewTicker(c.config.Ticker * time.Minute) - stopCh := make(chan struct{}) - defer close(stopCh) - // TODO Set size - instance := make(chan types.Instance, 100) - c.instance = instance - go c.Run(stopCh) - go c.forceAssignment() - kipcompute.Kubeip(instance, c.config) - sigterm := make(chan os.Signal, 1) - signal.Notify(sigterm, syscall.SIGTERM) - signal.Notify(sigterm, syscall.SIGINT) - <-sigterm - - return nil -} - -func newResourceController(client kubernetes.Interface, informer cache.SharedIndexInformer, resourceType string) *Controller { - queue := workqueue.NewRateLimitingQueue(workqueue.NewMaxOfRateLimiter( - workqueue.NewItemExponentialFailureRateLimiter(1*time.Second, 100*time.Second), - // 10 qps, 100 bucket size. This is only for retry speed, and it's only the overall factor (not per item) - &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, - )) - var newEvent Event - var err error - informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - newEvent.key, err = cache.MetaNamespaceKeyFunc(obj) - newEvent.eventType = "create" - newEvent.resourceType = resourceType - if err == nil { - queue.Add(newEvent) - } - }, - DeleteFunc: func(obj interface{}) { - newEvent.key, err = cache.MetaNamespaceKeyFunc(obj) - newEvent.eventType = "delete" - newEvent.resourceType = resourceType - if err == nil { - queue.Add(newEvent) - } - }, - }) - - return &Controller{ - logger: logrus.WithField("pkg", "kubeip-"+resourceType), - clientset: client, - informer: informer, - queue: queue, - processing: false, - } -} - -// Run starts the kubeip controller -func (c *Controller) Run(stopCh <-chan struct{}) { - defer utilruntime.HandleCrash() - defer c.queue.ShutDown() - - c.logger.Info("Starting kubeip controller") - serverStartTime = time.Now().Local() - - go c.informer.Run(stopCh) - - if !cache.WaitForCacheSync(stopCh, c.HasSynced) { - utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) - return - } - - c.logger.Info("kubeip controller synced and ready") - - wait.Until(c.runWorker, time.Second, stopCh) -} - -// HasSynced is required for the cache.Controller interface. -func (c *Controller) HasSynced() bool { - return c.informer.HasSynced() -} - -// LastSyncResourceVersion is required for the cache.Controller interface. -func (c *Controller) LastSyncResourceVersion() string { - return c.informer.LastSyncResourceVersion() -} - -func (c *Controller) runWorker() { - for c.processNextItem() { - // continue looping - } -} - -func (c *Controller) processNextItem() bool { - newEvent, quit := c.queue.Get() - - if quit { - return false - } - defer c.queue.Done(newEvent) - err := c.processItem(newEvent.(Event)) - if err == nil { - // No error, reset the ratelimit counters - c.queue.Forget(newEvent) - } else if c.queue.NumRequeues(newEvent) < maxRetries { - c.logger.Errorf("Error processing %s (will retry): %v", newEvent.(Event).key, err) - c.queue.AddRateLimited(newEvent) - } else { - // err != nil and too many retries - c.logger.Errorf("Error processing %s (giving up): %v", newEvent.(Event).key, err) - c.queue.Forget(newEvent) - utilruntime.HandleError(err) - } - - return true -} - -func (c *Controller) isNodePoolMonitored(pool string) bool { - if c.config.AllNodePools { - return true - } - if strings.EqualFold(pool, c.config.NodePool) { - return true - } - for _, ns := range c.config.AdditionalNodePools { - if strings.EqualFold(pool, ns) { - return true - } - } - return false -} -func (c *Controller) processItem(newEvent Event) error { - obj, _, err := c.informer.GetIndexer().GetByKey(newEvent.key) - if err != nil { - return fmt.Errorf("error fetching object with key %s from store: %v", newEvent.key, err) - } - // get object's metadata - objectMeta := utils.GetObjectMetaData(obj) - - // process events based on its type - switch newEvent.eventType { - case "delete": - if strings.HasPrefix(newEvent.key, prefix) { - node := newEvent.key[len(prefix):] - logrus.WithFields(logrus.Fields{"pkg": "kubeip-" + newEvent.resourceType, "function": "processItem"}).Infof("Processing removal to %v: %s ", newEvent.resourceType, node) - // A node has been deleted... we need to check whether the assignment is still optimal - c.forceAssignmentOnce(true) - return nil - } - case "create": - // compare CreationTimestamp and serverStartTime and alert only on latest events - // Could be Replaced by using Delta or DeltaFIFO - if objectMeta.CreationTimestamp.Sub(serverStartTime).Seconds() > 0 { - if strings.HasPrefix(newEvent.key, prefix) { - kubeClient, err := utils.GetClient() - if err != nil { - return errors.Wrap(err, "Can not get kubernetes API") - } - node := newEvent.key[len(prefix):] - var options meta_v1.GetOptions - options.Kind = "Node" - options.APIVersion = "1" - nodeMeta, err := kubeClient.CoreV1().Nodes().Get(context.Background(), node, options) - if err != nil { - return errors.Wrap(err, "Can not get node") - } - - labels := nodeMeta.Labels - var pool string - var ok bool - if pool, ok = labels["cloud.google.com/gke-nodepool"]; ok { - logrus.Infof("Node pool found %s", pool) - if !c.isNodePoolMonitored(pool) { - return nil - } - } else { - return errors.New("Did not find node pool. ") - } - var inst types.Instance - if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { - logrus.Infof("Zone pool found %s", nodeZone) - inst.Zone = nodeZone - } else { - return errors.New("did not find zone") - } - logrus.WithFields(logrus.Fields{"pkg": "kubeip-" + newEvent.resourceType, "function": "processItem"}).Infof("Processing add to %v: %s ", newEvent.resourceType, node) - inst.Name = node - inst.ProjectID = c.projectID - inst.Pool = pool - c.instance <- inst - logrus.WithFields(logrus.Fields{"pkg": "kubeip-" + newEvent.resourceType, "function": "processItem"}).Infof("Processing node %s of cluster %s in zone %s", node, c.clusterName, inst.Zone) - return nil - } - } - - } - return nil -} - -func isNodeReady(node api_v1.Node) bool { - for _, condition := range node.Status.Conditions { - if condition.Type == api_v1.NodeReady { - // If the node is unknown we assume that it is ready, we do not want to do IP changes so rapidly. - return condition.Status == api_v1.ConditionTrue || condition.Status == api_v1.ConditionUnknown - } - } - return false -} - -func (c *Controller) processAllNodes(shouldCheckOptimalIPAssignment bool) error { - kubeClient, err := utils.GetClient() - if err != nil { - return errors.Wrap(err, "Can not get kubernetes API") - } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Info("Collecting Node List...") - nodelist, _ := kubeClient.CoreV1().Nodes().List(context.Background(), meta_v1.ListOptions{}) - var pool string - var ok bool - var nodesOfInterest []types.Instance - - for _, node := range nodelist.Items { - labels := node.GetLabels() - if pool, ok = labels["cloud.google.com/gke-nodepool"]; ok { - if !c.isNodePoolMonitored(pool) { - continue - } - } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Info("Did not found node pool") - continue - } - var inst types.Instance - if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { - inst.Zone = nodeZone - } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Info("Did not find zone") - continue - } - - inst.ProjectID = c.projectID - inst.Name = node.GetName() - inst.Pool = pool - - // If node is not ready we will basically remove the node IP just in case - if !isNodeReady(node) { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Node %s in zone %s is not ready, removing IP so we can reuse it. ", inst.Name, inst.Zone) - // Delete the IP we will re-assign this - err = kipcompute.DeleteIP(c.projectID, inst.Zone, inst.Name, c.config) - if err != nil { - return errors.Wrap(err, "Can not delete IP") - } - continue - } - - nodesOfInterest = append(nodesOfInterest, inst) - } - - // Should we check that the IPs assigned to the current nodes are in fact the best possible IPs to assign? - if shouldCheckOptimalIPAssignment { - // Determining the required IP per region - regionsCount := make(map[string][]types.Instance) - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Collected %d Nodes of interest...calculating number of IPs required", len(nodesOfInterest)) - for _, inst := range nodesOfInterest { - zone := inst.Zone - region := zone[:len(zone)-2] - regionsCount[region] = append(regionsCount[region], inst) - } - - // Determining the most optimal nodes per region. - for region, instances := range regionsCount { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Collected %d Nodes of interest...processing %d nodes instances within region %s", len(nodesOfInterest), len(instances), region) - - addresses, err := kipcompute.GetAllAddresses(c.projectID, region, false, c.config) - if err != nil { - return errors.Wrap(err, "Can not get all addresses") - } - - var topMostAddresses []string - for _, address := range addresses.Items[:utils.Min(len(instances), len(addresses.Items))] { - topMostAddresses = append(topMostAddresses, address.Address) - } - - // Retrieve all addresses in the region. - var usedAddresses []AddressInstanceTuple - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Retrieving addresses used in project %s in region %s", c.projectID, region) - for _, instance := range instances { - address, err := kipcompute.GetAddressUsedByInstance(c.projectID, instance.Name, instance.Zone, c.config) - if err != nil { - return errors.Wrap(err, "Can not get address used by instance") - } - usedAddresses = append(usedAddresses, AddressInstanceTuple{ - address, - instance, - }) - } - - // Perform subtraction - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Project %s in region %s should use the following IPs %s... Checking that the instances follow these assignments", c.projectID, region, topMostAddresses) - var toRemove []AddressInstanceTuple - for _, usedAddress := range usedAddresses { - if usedAddress.address != "0.0.0.0" && !utils.Contains(topMostAddresses, usedAddress.address) { - toRemove = append(toRemove, usedAddress) - } - } - - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Found %d Addresses to remove project %s in region %s. Addresses %s", len(toRemove), c.projectID, region, toRemove) - if len(toRemove) > 0 { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Found %d ips %s in region %s which are not part of the top most addresses %s", len(toRemove), toRemove, region, topMostAddresses) - for _, remove := range toRemove { - - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Instance %s in project %s in region %s uses suboptimal IP %s... Removing so we reassign", remove.instance.Name, c.projectID, region, toRemove) - // Delete the IP we will re-assign this - err = kipcompute.DeleteIP(c.projectID, remove.instance.Zone, remove.instance.Name, c.config) - if err != nil { - return errors.Wrap(err, "Can not delete IP") - } - } - } - } - } - - for _, inst := range nodesOfInterest { - if !kipcompute.IsInstanceUsesReservedIP(c.projectID, inst.Name, inst.Zone, c.config) { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "processAllNodes"}).Infof("Found unassigned node %s in pool %s", inst.Name, inst.Pool) - c.instance <- inst - } - } - return nil -} - -func (c *Controller) forceAssignmentOnce(shouldCheckOptimalIPAssignment bool) { - if !c.processing { - c.processing = true - if c.config.ForceAssignment { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "forceAssignmentOnce"}).Info("Starting forceAssignmentOnce") - c.processAllNodes(shouldCheckOptimalIPAssignment) - } - c.assignMissingTags() - c.processing = false - } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "forceAssignmentOnce"}).Info("Skipping forceAssignmentOnce ... already in progress") - } -} - -func (c *Controller) forceAssignment() { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "forceAssignment"}).Info("Processing initial force assignment check") - c.forceAssignmentOnce(true) - for range c.ticker.C { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "forceAssignment"}).Info("Tick received for force assignment check") - c.forceAssignmentOnce(false) - } -} - -func (c *Controller) assignMissingTags() error { - nodePools := append(c.config.AdditionalNodePools, c.config.NodePool) - - kubeClient, err := utils.GetClient() - if err != nil { - return errors.Wrap(err, "Can not get kubernetes API") - } - - for _, pool := range nodePools { - label := fmt.Sprintf("!kubip_assigned,cloud.google.com/gke-nodepool=%s", pool) - nodelist, err := kubeClient.CoreV1().Nodes().List(context.Background(), meta_v1.ListOptions{ - LabelSelector: label, - }) - if err != nil { - logrus.Error(err) - continue - - } - for _, node := range nodelist.Items { - labels := node.GetLabels() - if nodeZone, ok := labels["failure-domain.beta.kubernetes.io/zone"]; ok { - if err != nil { - logrus.WithError(err).Error("Could not get node zone") - continue - } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "assignMissingTags"}).Infof("Found node without tag %s", node.GetName()) - kipcompute.AddTagIfMissing(c.projectID, node.GetName(), nodeZone, c.config) - - } else { - continue - } - } - } - return nil -} diff --git a/pkg/kipcompute/compute.go b/pkg/kipcompute/compute.go deleted file mode 100644 index 5e75aa6..0000000 --- a/pkg/kipcompute/compute.go +++ /dev/null @@ -1,434 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package kipcompute - -import ( - "fmt" - "math" - "sort" - "strconv" - "strings" - "time" - - "cloud.google.com/go/compute/metadata" - cfg "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/types" - "github.com/doitintl/kubeip/pkg/utils" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/net/context" - "golang.org/x/oauth2/google" - "google.golang.org/api/compute/v0.beta" - "google.golang.org/api/container/v1" -) - -// ClusterName get GKE cluster name from metadata -func ClusterName() (string, error) { - return metadata.InstanceAttributeValue("cluster-name") -} - -// ProjectName get GCP project name from metadata -func ProjectName() (string, error) { - return metadata.ProjectID() -} - -func getPriorityOrder(address *compute.Address, config *cfg.Config) int { - var defaultValue int - if config.OrderByDesc { - defaultValue = math.MinInt - } else { - defaultValue = math.MaxInt - } - - strVal, ok := address.Labels[config.OrderByLabelKey] - if ok { - intVal, err := strconv.Atoi(strVal) - if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "getPriorityOrder"}).Errorf("Address %s has errors. Failed to convert order by label value %s with value %s to integer", address.Name, config.OrderByLabelKey, strVal, err) - return defaultValue - } - return intVal - - } - - return defaultValue -} - -// GetAllAddresses retrieves all addresses matching the query. -func GetAllAddresses(projectID string, region string, filterJustReserved bool, config *cfg.Config) (*compute.AddressList, error) { - return getAllAddresses(projectID, region, config.NodePool, filterJustReserved, config) -} - -func getAllAddresses(projectID string, region string, pool string, filterJustReserved bool, config *cfg.Config) (*compute.AddressList, error) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Error(err) - return nil, err - } - computeService, err := compute.New(hc) - if err != nil { - logrus.Error(err) - return nil, err - } - var filter string - if config.AllNodePools || strings.EqualFold(pool, config.NodePool) { - filter = "(labels." + config.LabelKey + "=" + config.LabelValue + ")" + " AND (-labels." + config.LabelKey + "-node-pool:*)" - } else { - filter = "(labels." + config.LabelKey + "=" + config.LabelValue + ")" + " AND " + "(labels." + config.LabelKey + "-node-pool=" + pool + ")" - } - - var computedFilter string - if filterJustReserved { - computedFilter = "(status=RESERVED) AND " + filter - } else { - computedFilter = filter - } - - var addresses *compute.AddressList - addresses, err = computeService.Addresses.List(projectID, region).Filter(computedFilter).Do() - - if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "getAllAddresses"}).Errorf("Failed to list IP addresses: %q", err) - return nil, err - } - - // Right now the SDK does not support filter and order together, so we do it programmatically. - sort.SliceStable(addresses.Items, func(i, j int) bool { - address1 := addresses.Items[i] - address2 := addresses.Items[j] - val1 := getPriorityOrder(address1, config) - val2 := getPriorityOrder(address2, config) - if config.OrderByDesc { - return val1 > val2 - } - return val1 < val2 - }) - - return addresses, nil -} - -func findFreeAddress(projectID string, region string, pool string, config *cfg.Config) (types.IPAddress, error) { - addresses, err := getAllAddresses(projectID, region, pool, true, config) - if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "findFreeAddress"}).Errorf("Failed to list IP addresses in region %s: %q", region, err) - return types.IPAddress{IP: "", Labels: map[string]string{}}, err - } - - if len(addresses.Items) != 0 { - address := addresses.Items[0] - return types.IPAddress{IP: address.Address, Labels: address.Labels}, nil - } - return types.IPAddress{IP: "", Labels: map[string]string{}}, errors.New("no free address found") - -} - -// DeleteIP delete current IP on GKE node -func DeleteIP(projectID string, zone string, instance string, config *cfg.Config) error { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - } - - computeService, err := compute.New(hc) - if err != nil { - logrus.Infof(err.Error()) - return err - } - inst, err := computeService.Instances.Get(projectID, zone, instance).Do() - if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "DeleteIP"}).Errorf("Instance not found %s zone %s: %q", instance, zone, err) - return err - } - if len(inst.NetworkInterfaces) > 0 && len(inst.NetworkInterfaces[0].AccessConfigs) > 0 { - accessConfigName := inst.NetworkInterfaces[0].AccessConfigs[0].Name - if config.DryRun { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "DeleteIP"}).Infof("Deleted Access Config for %s zone %s new ip %s", instance, zone, accessConfigName) - } else { - op, err := computeService.Instances.DeleteAccessConfig(projectID, zone, instance, accessConfigName, "nic0").Do() - if err != nil { - logrus.Errorf("DeleteAccessConfig %q", err) - return err - } - err = waitForCompilation(projectID, zone, op) - if err != nil { - return err - } - } - - } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "DeleteIP"}).Infof("Deleted IP for %s zone %s", instance, zone) - // Delete an prior tags. - utils.TagNode(instance, types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, config) - return nil -} - -func addIP(projectID string, zone string, instance string, pool string, addr types.IPAddress, config *cfg.Config) error { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - } - - computeService, err := compute.New(hc) - if err != nil { - logrus.Infof(err.Error()) - return err - } - - if config.DryRun { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "addIP"}).Infof("Added Access Config for %s zone %s new ip %s", instance, zone, addr.IP) - } else { - accessConfig := &compute.AccessConfig{ - Name: "External NAT", - Type: "ONE_TO_ONE_NAT", - NatIP: addr.IP, - Kind: "compute#accessConfig", - } - op, err := computeService.Instances.AddAccessConfig(projectID, zone, instance, "nic0", accessConfig).Do() - if err != nil { - logrus.Errorf("AddAccessConfig %q", err) - return err - } - err = waitForCompilation(projectID, zone, op) - if err != nil { - return err - } - } - - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "addIP"}).Infof("Added IP for %s zone %s new ip %s", instance, zone, addr.IP) - return nil -} - -func replaceIP(projectID string, zone string, instance string, pool string, config *cfg.Config) error { - // wait for node to be ready with timeout of 5 minutes: this should be enough for the node to be ready - err := utils.WaitForNodeReady(instance, 5*time.Minute) - if err != nil { - return errors.Wrap(err, "error waiting for node to be ready") - } - - region := zone[:len(zone)-2] - addr, err := findFreeAddress(projectID, region, pool, config) - // Check if we found address. - if err != nil { - logrus.WithError(err).Error("can not find free address") - return err - } - - err = DeleteIP(projectID, zone, instance, config) - if err != nil { - logrus.WithError(err).Error("can not delete ip") - return err - } - - err = addIP(projectID, zone, instance, pool, addr, config) - if err != nil { - logrus.WithError(err).Error("can not add ip") - return err - } - - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "replaceIP"}).Infof("Replaced IP for %s zone %s new ip %s", instance, zone, addr.IP) - oldNode, err := utils.GetNodeByIP(addr.IP) - if err != nil { - utils.TagNode(oldNode, types.IPAddress{IP: "0.0.0.0", Labels: map[string]string{}}, config) - } - utils.TagNode(instance, addr, config) - return nil - -} - -func waitForCompilation(projectID string, zone string, operation *compute.Operation) (err error) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - return err - } - computeService, err := compute.New(hc) - if err != nil { - logrus.Fatalf("Could not get create compute service: %v", err) - return err - } - for { - op, err := computeService.ZoneOperations.Get(projectID, zone, operation.Name).Do() - if err != nil { - logrus.Errorf("ZoneOperations.Get %q %s", err, operation.Name) - return err - } - if strings.ToLower(op.Status) != "done" { - time.Sleep(2 * time.Second) - } else { - return nil - } - } -} - -// GetAddressUsedByInstance returns the IP used by this instance or the broadcast address otherwise. -func GetAddressUsedByInstance(projectID string, instance string, zone string, config *cfg.Config) (string, error) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - return "", err - } - - computeService, err := compute.New(hc) - if err != nil { - logrus.Fatalf("Could not get create compute service: %v", err) - return "", err - } - - region := zone[:len(zone)-2] - filter := "(labels." + config.LabelKey + "=" + config.LabelValue + ")" - addresses, err := computeService.Addresses.List(projectID, region).Filter(filter).Do() - if err != nil { - logrus.Fatalf("Could not list addresses for instance %s: %v", instance, err) - return "", err - } - - for _, addr := range addresses.Items { - if len(addr.Users) > 0 && strings.Contains(addr.Users[0], instance) { - return addr.Address, nil - } - } - - return "0.0.0.0", nil -} - -// IsInstanceUsesReservedIP test if GKE node is using reserved IP -func IsInstanceUsesReservedIP(projectID string, instance string, zone string, config *cfg.Config) bool { - ctx := context.Background() - hc, err := google.DefaultClient(ctx, container.CloudPlatformScope) - if err != nil { - logrus.Error(err) - return false - } - computeService, err := compute.New(hc) - if err != nil { - logrus.Error(err) - return false - } - region := zone[:len(zone)-2] - filter := "(labels." + config.LabelKey + "=" + config.LabelValue + ")" - addresses, err := computeService.Addresses.List(projectID, region).Filter("(status=IN_USE) AND " + filter).Do() - if err != nil { - logrus.Error(err) - return false - } - - for _, addr := range addresses.Items { - if strings.Contains(addr.Users[0], instance) { - return true - } - } - return false -} - -// Kubeip replace GKE node IP -func Kubeip(instance <-chan types.Instance, config *cfg.Config) { - for { - inst := <-instance - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "Kubeip"}).Infof("Working on %s in zone %s", inst.Name, inst.Zone) - _ = replaceIP(inst.ProjectID, inst.Zone, inst.Name, inst.Pool, config) - } -} - -func getAddressDetails(ip string, region string, projectID string, config *cfg.Config) (types.IPAddress, error) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Error(err) - return types.IPAddress{IP: "", Labels: map[string]string{}}, err - } - computeService, err := compute.New(hc) - if err != nil { - logrus.Error(err) - return types.IPAddress{IP: "", Labels: map[string]string{}}, err - } - filter := "address=" + "\"" + ip + "\"" - - addresses, err := computeService.Addresses.List(projectID, region).Filter(filter).Do() - if err != nil { - logrus.Error(err) - return types.IPAddress{IP: "", Labels: map[string]string{}}, err - } - - if len(addresses.Items) != 1 { - address := addresses.Items[0] - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "getAddressDetails"}).Infof("Node ip is reserved %s %s", ip, fmt.Sprint(address.Labels)) - return types.IPAddress{IP: address.Address, Labels: address.Labels}, nil - } - - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "getAddressDetails"}).Errorf("More than one address found %s", ip) - return types.IPAddress{IP: "", Labels: map[string]string{}}, fmt.Errorf("more than one address found for ip %s", ip) -} - -func isAddressReserved(ip string, region string, projectID string, config *cfg.Config) bool { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Error(err) - return false - } - computeService, err := compute.New(hc) - if err != nil { - logrus.Error(err) - return false - } - filter := "address=" + "\"" + ip + "\"" - addresses, err := computeService.Addresses.List(projectID, region).Filter(filter).Do() - if err != nil { - logrus.Error(err) - return false - } - - if len(addresses.Items) != 0 { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "isAddressReserved"}).Infof("Node ip is reserved %s", ip) - return true - } - return false -} - -// AddTagIfMissing add GKE node tag if missing -func AddTagIfMissing(projectID string, instance string, zone string, config *cfg.Config) { - hc, err := google.DefaultClient(context.Background(), container.CloudPlatformScope) - if err != nil { - logrus.Fatalf("Could not get authenticated client: %v", err) - return - } - computeService, err := compute.New(hc) - if err != nil { - return - } - inst, err := computeService.Instances.Get(projectID, zone, instance).Do() - if err != nil { - return - } - var ip string - for _, config := range inst.NetworkInterfaces[0].AccessConfigs { - if config.NatIP != "" { - ip = config.NatIP - } - } - if isAddressReserved(ip, zone[:len(zone)-2], projectID, config) { - addressDetails, err := getAddressDetails(ip, zone, projectID, config) - if err != nil { - return - } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "AddTagIfMissing"}).Infof("Tagging %s", instance) - utils.TagNode(instance, addressDetails, config) - } - -} diff --git a/pkg/types/types.go b/pkg/types/types.go deleted file mode 100644 index 54068de..0000000 --- a/pkg/types/types.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package types - -// Instance GKE Instance VM -type Instance struct { - ProjectID string - Name string - Zone string - Pool string -} - -// IPAddress GKE IP -type IPAddress struct { - IP string - Name string - Labels map[string]string -} diff --git a/pkg/utils/k8sutil.go b/pkg/utils/k8sutil.go deleted file mode 100644 index 2ebe29d..0000000 --- a/pkg/utils/k8sutil.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright © 2021 DoiT International -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package utils - -import ( - "bytes" - "fmt" - "strings" - "time" - - cfg "github.com/doitintl/kubeip/pkg/config" - "github.com/doitintl/kubeip/pkg/types" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/net/context" - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - apiv1 "k8s.io/api/core/v1" - extv1beta1 "k8s.io/api/extensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - typesv1 "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -// Min helper method to determine the minimum between two numbers -func Min(a, b int) int { - if a < b { - return a - } - return b -} - -// Contains helper method to determine if a string is contained in an array -func Contains(s []string, e string) bool { - for _, a := range s { - if strings.EqualFold(a, e) { - return true - } - } - return false -} - -// GetClient returns a k8s clientset to the request from inside of cluster -func GetClient() (kubernetes.Interface, error) { - config, err := rest.InClusterConfig() - if err != nil { - return nil, errors.Wrap(err, "Can not get kubernetes config") - } - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, errors.Wrap(err, "Can not create kubernetes clientset") - } - - return clientset, nil -} - -// GetObjectMetaData returns metadata of a given k8s object -func GetObjectMetaData(obj interface{}) metav1.ObjectMeta { - - var objectMeta metav1.ObjectMeta - - switch object := obj.(type) { - case *appsv1.Deployment: - objectMeta = object.ObjectMeta - case *apiv1.ReplicationController: - objectMeta = object.ObjectMeta - case *appsv1.ReplicaSet: - objectMeta = object.ObjectMeta - case *appsv1.DaemonSet: - objectMeta = object.ObjectMeta - case *apiv1.Service: - objectMeta = object.ObjectMeta - case *apiv1.Pod: - objectMeta = object.ObjectMeta - case *batchv1.Job: - objectMeta = object.ObjectMeta - case *apiv1.PersistentVolume: - objectMeta = object.ObjectMeta - case *apiv1.Namespace: - objectMeta = object.ObjectMeta - case *apiv1.Secret: - objectMeta = object.ObjectMeta - case *extv1beta1.Ingress: - objectMeta = object.ObjectMeta - } - return objectMeta -} - -func clearLabels(m map[string]string, config *cfg.Config) string { - stringBuffer := new(bytes.Buffer) - for key := range m { - if !strings.EqualFold(key, config.OrderByLabelKey) && - !strings.EqualFold(key, config.LabelKey) && - !strings.Contains(key, "kubip_assigned") && - !strings.Contains(key, "kubernetes") && - !strings.Contains(key, "google") && - !strings.Contains(key, "gke") { - fmt.Fprintf(stringBuffer, " ,\"%s\":null", key) - } - } - return stringBuffer.String() -} - -func createLabelKeyValuePairs(m map[string]string, config *cfg.Config) string { - stringBuffer := new(bytes.Buffer) - for key, value := range m { - if !strings.EqualFold(key, config.OrderByLabelKey) && - !strings.EqualFold(key, config.LabelKey) && - !strings.Contains(key, "kubip_assigned") && - !strings.Contains(key, "kubernetes") && - !strings.Contains(key, "google") && - !strings.Contains(key, "gke") { - fmt.Fprintf(stringBuffer, " ,\"%s\":\"%s\"", key, value) - } - } - return stringBuffer.String() -} - -// TagNode tag GKE node with "kubip_assigned" label (with typo) and also copy the labels present on the address if the copyLabels flag is set to true -func TagNode(node string, ip types.IPAddress, config *cfg.Config) error { - kubeClient, err := GetClient() - if err != nil { - return errors.Wrap(err, "Can not get kubernetes API") - } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s", node, ip.IP) - dashIP := strings.Replace(ip.IP, ".", "-", 4) - var labelString string - - if config.CopyLabels { - var labelsToClear string - if config.ClearLabels { - result, err := kubeClient.CoreV1().Nodes().Get(context.Background(), node, metav1.GetOptions{}) - if err != nil { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).WithError(err).Warnf("Can not get node %s", node) - } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Clear label tag for node %s with ip %s and clear tags %s", node, ip.IP, result.Labels) - createLabelKeyValuePairs(result.Labels, config) - labelsToClear = clearLabels(result.Labels, config) - } - } else { - labelsToClear = "" - } - - labelString = "{" + "\"" + "kubip_assigned" + "\":\"" + dashIP + "\"" + labelsToClear + createLabelKeyValuePairs(ip.Labels, config) + "}" - } else { - labelString = "{" + "\"" + "kubip_assigned" + "\":\"" + dashIP + "\"" + "}" - } - patch := fmt.Sprintf(`{"metadata":{"labels":%v}}`, labelString) - - if config.DryRun { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) - } else { - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "tagNode"}).Infof("Tagging node %s as %s with tags %s ", node, ip.IP, labelString) - _, err = kubeClient.CoreV1().Nodes().Patch(context.Background(), node, typesv1.MergePatchType, []byte(patch), metav1.PatchOptions{}) - if err != nil { - return errors.Wrap(err, "Can not patch node") - } - } - return nil -} - -// GetNodeByIP get GKE node by IP -func GetNodeByIP(ip string) (string, error) { - kubeClient, err := GetClient() - if err != nil { - return "", errors.Wrap(err, "Can not get kubernetes API") - } - dashIP := strings.Replace(ip, ".", "-", 4) - label := fmt.Sprintf("kubip_assigned=%v", dashIP) - l, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ - LabelSelector: label, - }) - if err != nil { - logrus.Error(err) - return "", err - } - if len(l.Items) == 0 { - return "", errors.New("did not found matching node with IP") - } - return l.Items[0].GetName(), nil - -} - -func isNodeReady(conditions []apiv1.NodeCondition) bool { - for _, condition := range conditions { - if condition.Type == apiv1.NodeReady { - return condition.Status == apiv1.ConditionTrue - } - } - return false -} - -// WaitForNodeReady wait for node to be ready -func WaitForNodeReady(node string, timeout time.Duration) error { - kubeClient, err := GetClient() - if err != nil { - return errors.Wrap(err, "Can not get kubernetes API") - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - for { - select { - case <-ctx.Done(): - return errors.New("timeout waiting for node to be ready") - default: - var options metav1.GetOptions - options.Kind = "Node" - options.APIVersion = "1" - n, err := kubeClient.CoreV1().Nodes().Get(context.Background(), node, options) - if err != nil { - return errors.Wrap(err, "can not get node") - } - if isNodeReady(n.Status.Conditions) { - return nil - } - logrus.WithFields(logrus.Fields{"pkg": "kubeip", "function": "waitForNodeReady"}).Infof("waiting for node %s to be ready", node) - time.Sleep(5 * time.Second) - } - } -} diff --git a/roles.yaml b/roles.yaml deleted file mode 100644 index c2e031c..0000000 --- a/roles.yaml +++ /dev/null @@ -1,16 +0,0 @@ -title: "kubeip" -description: "required permissions to run KubeIP" -stage: "GA" -includedPermissions: -- compute.addresses.list -- compute.instances.addAccessConfig -- compute.instances.deleteAccessConfig -- compute.instances.get -- compute.instances.list -- compute.projects.get -- container.clusters.get -- container.clusters.list -- resourcemanager.projects.get -- compute.networks.useExternalIp -- compute.subnetworks.useExternalIp -- compute.addresses.use diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..3df65b0 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,21 @@ +sonar.projectKey=doitintl_kubeip +sonar.organization=doitintl +sonar.host.url=https://sonarcloud.io +# This is the name and version displayed in the SonarCloud UI. +sonar.projectName=kubeip +#sonar.projectVersion=1.0 +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +# This is where the source code is located. +sonar.sources=. +sonar.exclusions=**/*_test.go,mocks/**/*.go,**/mock_*.go +# This is where the tests are located. +sonar.tests=. +sonar.test.inclusions=**/*_test.go +# Test coverage report +sonar.go.coverage.reportPaths=coverage.out +# Test report json file +sonar.go.tests.reportPaths=test-report.out +# golangci-lint report json file +sonar.go.golangci-lint.reportPaths=golangci-lint.out +# Encoding of the source code. Default is default system encoding +sonar.sourceEncoding=UTF-8 \ No newline at end of file diff --git a/test.sh b/test.sh deleted file mode 100644 index 26a7641..0000000 --- a/test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -xe -if [ -z "$REGION" ]; then - echo REGION not defined! - exit -fi -if [ -z "$CLUSTER" ]; then - echo CLUSTER not defined! - exit -fi - -NODES=`gcloud container node-pools describe default-pool --cluster $CLUSTER|grep initialNodeCount|awk '{print $2}'` -NEW_NODES=$(($NODES + 1)) -gcloud compute addresses create kubeip-test-1 --region $REGION -gcloud beta compute addresses update kubeip-test-1 --update-labels kubeip=reserved --region us-central1 -IP1=`gcloud compute addresses describe kubeip-test-1 --region $REGION|grep address:|awk '{print $2}'` -gcloud compute addresses create kubeip-test-2 --region $REGION -gcloud beta compute addresses update kubeip-test-2 --update-labels kubeip=reserved --region us-central1 -IP2=`gcloud compute addresses describe kubeip-test-2 --region $REGION|grep address:|awk '{print $2}'` -gcloud beta container clusters resize $CLUSTER --node-pool default-pool --size $NEW_NODES --quiet - -STATUS1=`gcloud compute addresses describe kubeip-test-1 --region $REGION|grep status|awk '{print $2}'` -STATUS2=`gcloud compute addresses describe kubeip-test-2 --region $REGION|grep status|awk '{print $2}'` -echo 'expecting one IP IN_USE and one RESERVED' -echo 'Results:' -echo $STATUS1 '--' $STATUS2 - -gcloud beta container clusters resize $CLUSTER --node-pool default-pool --size $NODES --quiet -gcloud compute addresses delete kubeip-test-1 --region $REGION --quiet -gcloud compute addresses delete kubeip-test-2 --region $REGION --quiet