From df796bf07bd1c48ef42e54bb306e78b6eedc81c7 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 26 Mar 2024 14:31:44 +0100 Subject: [PATCH 01/93] init tests --- .gitignore | 5 + test/README.md | 27 + test/src/Makefile | 20 + .../fixtures/fixtures.eu-central-1.eks.tfvars | 5 + test/src/go.mod | 99 ++ test/src/go.sum | 1149 +++++++++++++++++ test/src/modules_eks_cluster_test.go | 36 + 7 files changed, 1341 insertions(+) create mode 100644 .gitignore create mode 100644 test/README.md create mode 100644 test/src/Makefile create mode 100644 test/src/fixtures/fixtures.eu-central-1.eks.tfvars create mode 100644 test/src/go.mod create mode 100644 test/src/go.sum create mode 100644 test/src/modules_eks_cluster_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..50dd07a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.vscode +.idea +terraform.tfstate +.terraform +.terraform.lock.hcl \ No newline at end of file diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..64b01d7f --- /dev/null +++ b/test/README.md @@ -0,0 +1,27 @@ +## Requirements + +```bash +brew update +brew install opentofu +```[README.md](..%2FREADME.md) + +Make sure you have awscli installed and configured +Make sure you have an AWS profile setup and a region: +```bash +# install aws cli +asdf global awscli "yourversion" + +# sso login +aws sso login --profile SystemAdministrator-*** + +export AWS_DEFAULT_PROFILE=SystemAdministrator-**** +export AWS_REGION=eu-central-1 +``` + +test with +```bash +go test -v -timeout 120m +``` + +# TODO: https://github.com/gruntwork-io/cloud-nuke every weekend +# => we should have a dedicated tenant for CI \ No newline at end of file diff --git a/test/src/Makefile b/test/src/Makefile new file mode 100644 index 00000000..f83775e1 --- /dev/null +++ b/test/src/Makefile @@ -0,0 +1,20 @@ +#!make +.DEFAULT_GOAL= help + +.PHONY : init +init: ## Initialize tests + @exit 0 + +.PHONY : test +test: init ## Run tests + go mod download + go test -v -timeout 120m + +.PHONY : clean +clean: ## Clean up files + rm -rf ../../*/*/*.tfstate* + +# see https://suva.sh/posts/well-documented-makefiles/ +.PHONY : help +help: ## Show this help prompt. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/test/src/fixtures/fixtures.eu-central-1.eks.tfvars b/test/src/fixtures/fixtures.eu-central-1.eks.tfvars new file mode 100644 index 00000000..51969296 --- /dev/null +++ b/test/src/fixtures/fixtures.eu-central-1.eks.tfvars @@ -0,0 +1,5 @@ +region = "eu-central-1" +name = "cluster-name" + +cluster_service_ipv4_cidr = "10.190.0.0/16" +cluster_node_ipv4_cidr = "10.192.0.0/16" \ No newline at end of file diff --git a/test/src/go.mod b/test/src/go.mod new file mode 100644 index 00000000..230c9097 --- /dev/null +++ b/test/src/go.mod @@ -0,0 +1,99 @@ +module github.com/camunda/camunda-tf-eks-module + +go 1.22.0 + +require ( + github.com/aws/aws-sdk-go v1.51.7 + github.com/gruntwork-io/terratest v0.46.13 + github.com/stretchr/testify v1.9.0 + k8s.io/api v0.29.3 + k8s.io/apimachinery v0.29.3 + k8s.io/client-go v0.29.3 + sigs.k8s.io/aws-iam-authenticator v0.6.19 +) + +require ( + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute v1.20.1 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v0.13.0 // indirect + cloud.google.com/go/storage v1.29.0 // indirect + github.com/agext/levenshtein v1.2.3 // indirect + github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/s2a-go v0.1.4 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-getter v1.7.1 // indirect + github.com/hashicorp/go-multierror v1.1.0 // indirect + github.com/hashicorp/go-safetemp v1.0.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/hcl/v2 v2.9.1 // indirect + github.com/hashicorp/terraform-json v0.13.0 // indirect + github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.15.11 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/tmccombs/hcl2json v0.3.3 // indirect + github.com/ulikunitz/xz v0.5.10 // indirect + github.com/zclconf/go-cty v1.9.1 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.17.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/api v0.126.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/grpc v1.56.3 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/test/src/go.sum b/test/src/go.sum new file mode 100644 index 00000000..381ae3cf --- /dev/null +++ b/test/src/go.sum @@ -0,0 +1,1149 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= +github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= +github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.51.7 h1:RRjxHhx9RCjw5AhgpmmShq3F4JDlleSkyhYMQ2xUAe8= +github.com/aws/aws-sdk-go v1.51.7/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/gruntwork-io/terratest v0.46.13 h1:FDaEoZ7DtkomV8pcwLdBV/VsytdjnPRqJkIriYEYwjs= +github.com/gruntwork-io/terratest v0.46.13/go.mod h1:8sxu3Qup8TxtbzOHzq0MUrQffJj/G61/OwlsReaCwpo= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY= +github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= +github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= +github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl/v2 v2.9.1 h1:eOy4gREY0/ZQHNItlfuEZqtcQbXIxzojlP301hDpnac= +github.com/hashicorp/hcl/v2 v2.9.1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= +github.com/hashicorp/terraform-json v0.13.0 h1:Li9L+lKD1FO5RVFRM1mMMIBDoUHslOniyEi5CM+FWGY= +github.com/hashicorp/terraform-json v0.13.0/go.mod h1:y5OdLBCT+rxbwnpxZs9kGL7R9ExU76+cpdY8zHwoazk= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= +github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ= +github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= +github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= +github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty v1.8.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty v1.9.1 h1:viqrgQwFl5UpSxc046qblj78wZXVDFnSOufaOTER+cc= +github.com/zclconf/go-cty v1.9.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= +k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= +k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= +k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= +k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/aws-iam-authenticator v0.6.19 h1:c35TtOSHAP2B5qUJr1yS6IQSTuL24ymdE/snnysHH+A= +sigs.k8s.io/aws-iam-authenticator v0.6.19/go.mod h1:oAKrGtldY2rr0nyqm1449raoy77O5Cgc5vOWEkba+Qo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go new file mode 100644 index 00000000..9addbdf7 --- /dev/null +++ b/test/src/modules_eks_cluster_test.go @@ -0,0 +1,36 @@ +package test + +import ( + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/runtime" + "testing" +) + +// Test the Terraform module in modules/eks-cluster using Terratest. +func TestModulesEKSCluster(t *testing.T) { + + /* randId := strings.ToLower(random.UniqueId()) + attributes := []string{randId}*/ + + terraformOptions := &terraform.Options{ + // The path to where our Terraform code is located + TerraformDir: "../../modules/eks-cluster", + Upgrade: false, + // Variables to pass to our Terraform code using -var-file options + VarFiles: []string{"../../test/src/fixtures/fixtures.eu-central-1.eks.tfvars"}, + Vars: map[string]interface{}{}, + } + + // At the end of the test, run `terraform destroy` to clean up any resources that were created + defer terraform.Destroy(t, terraformOptions) + + // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(t, terraformOptions) + }) + + // This will run `terraform init` and `terraform apply` and fail the test if there are any errors + errTfApply, _ := terraform.InitAndApplyE(t, terraformOptions) + require.Nil(t, errTfApply) +} From d98bc49321a8b0fbde398f14978dc028d3668f5a Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:55:59 +0100 Subject: [PATCH 02/93] continue tests --- test/README.md | 11 +++++++++- test/src/modules_eks_cluster_test.go | 31 ++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/test/README.md b/test/README.md index 64b01d7f..e2a7d4bc 100644 --- a/test/README.md +++ b/test/README.md @@ -23,5 +23,14 @@ test with go test -v -timeout 120m ``` +### Troubleshooting + +```bash +# make sure you don't have test clusters running since a while + +eksctl get clusters +``` + # TODO: https://github.com/gruntwork-io/cloud-nuke every weekend -# => we should have a dedicated tenant for CI \ No newline at end of file +# => we should have a dedicated tenant for CI +# => sometimes, EKS deletion fails with error: DeleteCluster ResourceInUseException: Cluster has nodegroups attached terraform \ No newline at end of file diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index 9addbdf7..d0b3002b 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -1,17 +1,21 @@ package test import ( + "fmt" + "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/runtime" + "strings" "testing" ) // Test the Terraform module in modules/eks-cluster using Terratest. func TestModulesEKSCluster(t *testing.T) { - /* randId := strings.ToLower(random.UniqueId()) - attributes := []string{randId}*/ + randId := strings.ToLower(random.UniqueId()) + clusterName := fmt.Sprintf("cluster-%s", randId) terraformOptions := &terraform.Options{ // The path to where our Terraform code is located @@ -19,7 +23,9 @@ func TestModulesEKSCluster(t *testing.T) { Upgrade: false, // Variables to pass to our Terraform code using -var-file options VarFiles: []string{"../../test/src/fixtures/fixtures.eu-central-1.eks.tfvars"}, - Vars: map[string]interface{}{}, + Vars: map[string]interface{}{ + "name": clusterName, + }, } // At the end of the test, run `terraform destroy` to clean up any resources that were created @@ -31,6 +37,23 @@ func TestModulesEKSCluster(t *testing.T) { }) // This will run `terraform init` and `terraform apply` and fail the test if there are any errors - errTfApply, _ := terraform.InitAndApplyE(t, terraformOptions) + _, errTfApply := terraform.InitAndApplyE(t, terraformOptions) require.Nil(t, errTfApply) + + // Do some basic not empty tests on outputs + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_endpoint")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_security_group_id")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_security_group_arn")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_primary_security_group_id")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_iam_role_name")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_iam_role_arn")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "ebs_cs_arn")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "external_dns_arn")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "vpc_id")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_vpc_cidr_blocks")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "public_vpc_cidr_blocks")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_subnet_ids")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "default_security_group_id")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "vpc_main_route_table_id")) + assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_route_table_ids")) } From af7cbb8db61aafa91843091f1a6a2ad2eca2fa26 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 27 Mar 2024 15:49:50 +0100 Subject: [PATCH 03/93] update tests --- .gitignore | 1 + modules/eks-cluster/variables.tf | 2 +- test/README.md | 4 +- .../fixtures/fixtures.eu-central-1.eks.tfvars | 3 - test/src/modules_eks_cluster_test.go | 220 +++++++++++++++++- 5 files changed, 218 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 50dd07a4..fc7727ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .vscode .idea terraform.tfstate +terraform.tfstate.backup .terraform .terraform.lock.hcl \ No newline at end of file diff --git a/modules/eks-cluster/variables.tf b/modules/eks-cluster/variables.tf index d875e982..4b209c79 100644 --- a/modules/eks-cluster/variables.tf +++ b/modules/eks-cluster/variables.tf @@ -40,7 +40,7 @@ variable "np_instance_types" { variable "np_disk_size" { type = number - description = "Disk size of the nodes on the default node poool" + description = "Disk size of the nodes on the default node pool" default = 20 } diff --git a/test/README.md b/test/README.md index e2a7d4bc..df504073 100644 --- a/test/README.md +++ b/test/README.md @@ -9,7 +9,7 @@ Make sure you have awscli installed and configured Make sure you have an AWS profile setup and a region: ```bash # install aws cli -asdf global awscli "yourversion" +brew install awscli # sso login aws sso login --profile SystemAdministrator-*** @@ -20,7 +20,7 @@ export AWS_REGION=eu-central-1 test with ```bash -go test -v -timeout 120m +make test ``` ### Troubleshooting diff --git a/test/src/fixtures/fixtures.eu-central-1.eks.tfvars b/test/src/fixtures/fixtures.eu-central-1.eks.tfvars index 51969296..368e61f3 100644 --- a/test/src/fixtures/fixtures.eu-central-1.eks.tfvars +++ b/test/src/fixtures/fixtures.eu-central-1.eks.tfvars @@ -1,5 +1,2 @@ -region = "eu-central-1" -name = "cluster-name" - cluster_service_ipv4_cidr = "10.190.0.0/16" cluster_node_ipv4_cidr = "10.192.0.0/16" \ No newline at end of file diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index d0b3002b..4572677d 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -1,21 +1,80 @@ package test +// adapted from https://github.com/cloudposse/terraform-aws-eks-cluster/blob/main/test/src/examples_complete_test.go + import ( + "encoding/base64" "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/kms" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/aws-iam-authenticator/pkg/token" "strings" + "sync/atomic" "testing" + "time" ) -// Test the Terraform module in modules/eks-cluster using Terratest. +func newClientSet(cluster *eks.Cluster) (*kubernetes.Clientset, error) { + gen, err := token.NewGenerator(true, false) + if err != nil { + return nil, err + } + opts := &token.GetTokenOptions{ + ClusterID: aws.StringValue(cluster.Name), + } + tok, err := gen.GetWithOptions(opts) + if err != nil { + return nil, err + } + ca, err := base64.StdEncoding.DecodeString(aws.StringValue(cluster.CertificateAuthority.Data)) + if err != nil { + return nil, err + } + clientSet, err := kubernetes.NewForConfig( + &rest.Config{ + Host: aws.StringValue(cluster.Endpoint), + BearerToken: tok.Token, + TLSClientConfig: rest.TLSClientConfig{ + CAData: ca, + }, + }, + ) + if err != nil { + return nil, err + } + return clientSet, nil +} + +// Function to check a specific VPC attribute +func checkVpcAttribute(ec2Svc *ec2.EC2, vpcID, attributeName string) (*ec2.DescribeVpcAttributeOutput, error) { + input := &ec2.DescribeVpcAttributeInput{ + VpcId: aws.String(vpcID), + Attribute: aws.String(attributeName), + } + + return ec2Svc.DescribeVpcAttribute(input) +} + +// Test the Terraform module in modules/eks-cluster. func TestModulesEKSCluster(t *testing.T) { randId := strings.ToLower(random.UniqueId()) clusterName := fmt.Sprintf("cluster-%s", randId) + region := "eu-central-1" terraformOptions := &terraform.Options{ // The path to where our Terraform code is located @@ -24,7 +83,8 @@ func TestModulesEKSCluster(t *testing.T) { // Variables to pass to our Terraform code using -var-file options VarFiles: []string{"../../test/src/fixtures/fixtures.eu-central-1.eks.tfvars"}, Vars: map[string]interface{}{ - "name": clusterName, + "name": clusterName, + "region": region, }, } @@ -37,23 +97,171 @@ func TestModulesEKSCluster(t *testing.T) { }) // This will run `terraform init` and `terraform apply` and fail the test if there are any errors - _, errTfApply := terraform.InitAndApplyE(t, terraformOptions) - require.Nil(t, errTfApply) + // then it will re-run apply to make sure that out tf is idempotent + _, errTfApply := terraform.InitAndApplyAndIdempotentE(t, terraformOptions) + assert.NoError(t, errTfApply) // Do some basic not empty tests on outputs assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_endpoint")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_security_group_id")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_security_group_arn")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_primary_security_group_id")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_iam_role_name")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_iam_role_arn")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "ebs_cs_arn")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "external_dns_arn")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "vpc_id")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_vpc_cidr_blocks")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "public_vpc_cidr_blocks")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_subnet_ids")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "default_security_group_id")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "vpc_main_route_table_id")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_route_table_ids")) + + assert.Equal(t, fmt.Sprintf("%s-eks-iam-role", clusterName), terraform.Output(t, terraformOptions, "cluster_iam_role_name")) + + // this is a split(6)[0..2] of the base cluster_node_ipv4_cidr = "10.192.0.0/16" + expectedPrivateVpcCidrBlocks := []string([]string{"10.192.0.0/19", "10.192.32.0/19", "10.192.64.0/19"}) + assert.Equal(t, terraform.Output(t, terraformOptions, "private_vpc_cidr_blocks"), expectedPrivateVpcCidrBlocks) + + // this is a split(6)[3..5] of the base cluster_node_ipv4_cidr = "10.192.0.0/16" + expectedPublicVpcCidrBlocks := []string([]string{"10.192.96.0/19", "10.192.128.0/19", "10.192.160.0/19"}) + assert.Equal(t, terraform.Output(t, terraformOptions, "public_vpc_cidr_blocks"), expectedPublicVpcCidrBlocks) + + // Wait for the worker nodes to join the cluster + // https://github.com/kubernetes/client-go + // https://www.rushtehrani.com/post/using-kubernetes-api + // https://rancher.com/using-kubernetes-api-go-kubecon-2017-session-recap + // https://gianarb.it/blog/kubernetes-shared-informer + // https://stackoverflow.com/questions/60547409/unable-to-obtain-kubeconfig-of-an-aws-eks-cluster-in-go-code/60573982#60573982 + fmt.Println("Waiting for worker nodes to join the EKS cluster") + + sess := session.Must(session.NewSession(&aws.Config{ + Region: aws.String(region), + })) + + // list your services here + eksSvc := eks.New(sess) + iamSvc := iam.New(sess) + ec2Svc := ec2.New(sess) + kmsSvc := kms.New(sess) + + inputEKS := &eks.DescribeClusterInput{ + Name: aws.String(clusterName), + } + + result, err := eksSvc.DescribeCluster(inputEKS) + assert.NoError(t, err) + + clientSet, err := newClientSet(result.Cluster) + assert.NoError(t, err) + + factory := informers.NewSharedInformerFactory(clientSet, 0) + informer := factory.Core().V1().Nodes().Informer() + stopChannel := make(chan struct{}) + var countOfWorkerNodes uint64 = 0 + + _, errEventHandler := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + node := obj.(*corev1.Node) + fmt.Printf("Worker Node %s has joined the EKS cluster at %s\n", node.Name, node.CreationTimestamp) + atomic.AddUint64(&countOfWorkerNodes, 1) + if countOfWorkerNodes > 1 { + close(stopChannel) + } + }, + }) + require.NoError(t, errEventHandler) + + go informer.Run(stopChannel) + + select { + case <-stopChannel: + msg := "All worker nodes have joined the EKS cluster" + fmt.Println(msg) + case <-time.After(5 * time.Minute): + msg := "Not all worker nodes have joined the EKS cluster" + fmt.Println(msg) + assert.Fail(t, msg) + } + + // Verify list of addons installed on the EKS + expectedEKSAddons := []string{"coredns", "kube-proxy", "vpc-cni", "aws-ebs-csi-driver"} + inputDescribeAddons := &eks.ListAddonsInput{} + outputEKSAddons, errEKSAddons := eksSvc.ListAddons(inputDescribeAddons) + require.NoError(t, errEKSAddons) + + // perform the diff + presenceAddonsMap := make(map[string]bool) + for _, addon := range outputEKSAddons.Addons { + presenceAddonsMap[*addon] = true + } + for _, addonName := range expectedEKSAddons { + assert.Truef(t, presenceAddonsMap[addonName], "Addon %s not installed on the EKS cluster", addonName) + } + + // Verifies EKS roles + roleNames := []string{ + fmt.Sprintf("%s-cert-manager-role", clusterName), + fmt.Sprintf("%s-external-dns-role", clusterName), + fmt.Sprintf("%s-ebs-cs-role", clusterName), + fmt.Sprintf("%s-eks-iam-role", clusterName), + } + + for _, roleName := range roleNames { + input := &iam.GetRoleInput{ + RoleName: aws.String(roleName), + } + + _, err := iamSvc.GetRole(input) + assert.NoErrorf(t, err, "Failed to get IAM EKS role %s", roleName) + } + + // verifies the VPC + + vpcName := fmt.Sprintf("%s-vpc", clusterName) + + inputVPC := &ec2.DescribeVpcsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("tag:Name"), + Values: []*string{aws.String(vpcName)}, + }, + }, + } + + outputVPC, errVPC := ec2Svc.DescribeVpcs(inputVPC) + require.NoError(t, errVPC) + + assert.Equal(t, len(outputVPC.Vpcs), 1) + + vpc := outputVPC.Vpcs[0] + vpcID := *vpc.VpcId + assert.NotEmpty(t, vpcID) + + // WIP + /* val, err := checkVpcAttribute(ec2Svc, vpcID, "mapPublicIpOnLaunch") + assert.NoError(t, err) + assert.Equal(t, true, *val) + checkVpcAttribute(ec2Svc, vpcID, "enableDnsSupport") + checkVpcAttribute(ec2Svc, vpcID, "enableDnsHostnames") + */ + + keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) + inputKMS := &kms.ListKeysInput{} + outputKMSList, errKMSList := kmsSvc.ListKeys(inputKMS) + assert.NoError(t, errKMSList) + + // Check if the key corresponding to the description exists + keyFound := false + for _, key := range outputKMSList.Keys { + keyDetails, err := kmsSvc.DescribeKey(&kms.DescribeKeyInput{ + KeyId: key.KeyId, + }) + require.NoErrorf(t, err, "Failed to describe key %s", *key.KeyId) + + keyFound = *keyDetails.KeyMetadata.Description == keyDescription + if keyFound { + break + } + } + assert.Truef(t, keyFound, "Failed to find key %s", keyDescription) } From 6c1ce08316133374ff3ad597128be0f277f6f983 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:47:57 +0100 Subject: [PATCH 04/93] update to aws sdk, refactoring tests --- test/README.md | 3 + test/src/Makefile | 2 +- test/src/go.mod | 21 ++- test/src/go.sum | 44 +++++- test/src/modules_eks_cluster_test.go | 205 +++++++++++++-------------- test/src/utils/helper.go | 11 ++ test/src/utils/utils.go | 65 +++++++++ 7 files changed, 244 insertions(+), 107 deletions(-) create mode 100644 test/src/utils/helper.go create mode 100644 test/src/utils/utils.go diff --git a/test/README.md b/test/README.md index df504073..21887c75 100644 --- a/test/README.md +++ b/test/README.md @@ -30,6 +30,9 @@ make test eksctl get clusters ``` +# TODO: implement db pod +# todo: tests weekly +# see https://github.com/camunda/c8-multi-region/blob/main/.github/workflows/nightly_aws_region_cleanup.yml # TODO: https://github.com/gruntwork-io/cloud-nuke every weekend # => we should have a dedicated tenant for CI diff --git a/test/src/Makefile b/test/src/Makefile index f83775e1..4d944493 100644 --- a/test/src/Makefile +++ b/test/src/Makefile @@ -6,7 +6,7 @@ init: ## Initialize tests @exit 0 .PHONY : test -test: init ## Run tests +test: init clean ## Run tests go mod download go test -v -timeout 120m diff --git a/test/src/go.mod b/test/src/go.mod index 230c9097..779a50f6 100644 --- a/test/src/go.mod +++ b/test/src/go.mod @@ -3,9 +3,16 @@ module github.com/camunda/camunda-tf-eks-module go 1.22.0 require ( - github.com/aws/aws-sdk-go v1.51.7 + github.com/aws/aws-sdk-go v1.44.332 + github.com/aws/aws-sdk-go-v2 v1.26.0 + github.com/aws/aws-sdk-go-v2/config v1.27.9 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.154.0 + github.com/aws/aws-sdk-go-v2/service/eks v1.41.2 + github.com/aws/aws-sdk-go-v2/service/iam v1.31.3 + github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 github.com/gruntwork-io/terratest v0.46.13 github.com/stretchr/testify v1.9.0 + go.uber.org/zap v1.27.0 k8s.io/api v0.29.3 k8s.io/apimachinery v0.29.3 k8s.io/client-go v0.29.3 @@ -20,6 +27,17 @@ require ( cloud.google.com/go/storage v1.29.0 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect + github.com/aws/smithy-go v1.20.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -72,6 +90,7 @@ require ( github.com/ulikunitz/xz v0.5.10 // indirect github.com/zclconf/go-cty v1.9.1 // indirect go.opencensus.io v0.24.0 // indirect + go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect diff --git a/test/src/go.sum b/test/src/go.sum index 381ae3cf..f7ec33ac 100644 --- a/test/src/go.sum +++ b/test/src/go.sum @@ -202,8 +202,42 @@ github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/ github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.51.7 h1:RRjxHhx9RCjw5AhgpmmShq3F4JDlleSkyhYMQ2xUAe8= -github.com/aws/aws-sdk-go v1.51.7/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.44.332 h1:Ze+98F41+LxoJUdsisAFThV+0yYYLYw17/Vt0++nFYM= +github.com/aws/aws-sdk-go v1.44.332/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA= +github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= +github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg= +github.com/aws/aws-sdk-go-v2/config v1.27.9/go.mod h1:dK1FQfpwpql83kbD873E9vz4FyAxuJtR22wzoXn3qq0= +github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMNQVNtNdUqf6cItao= +github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.154.0 h1:+OJ9EhHaqjtA4YTTbxxLxMffrWuGWh0qMaBmGJTLSSg= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.154.0/go.mod h1:TeZ9dVQzGaLG+SBIgdLIDbJ6WmfFvksLeG3EHGnNfZM= +github.com/aws/aws-sdk-go-v2/service/eks v1.41.2 h1:0X5g5H8YyW9QVtlp6j+ZGHl/h0ZS58jiLRXabyiB5uw= +github.com/aws/aws-sdk-go-v2/service/eks v1.41.2/go.mod h1:T2MBMUUCoSEvHuKPplubyQJbWNghbHhx3ToJpLoipDs= +github.com/aws/aws-sdk-go-v2/service/iam v1.31.3 h1:cJn9Snros9WmDA7/qCCN7jSkowcu1CqnwhFpv4ipHEE= +github.com/aws/aws-sdk-go-v2/service/iam v1.31.3/go.mod h1:+nAQlxsBxPFf6GrL93lvCuv5PxSTX3GO0RYrURyzl/Q= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU= +github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 h1:yS0JkEdV6h9JOo8sy2JSpjX+i7vsKifU8SIeHrqiDhU= +github.com/aws/aws-sdk-go-v2/service/kms v1.30.0/go.mod h1:+I8VUUSVD4p5ISQtzpgSva4I8cJ4SQ4b1dcBcof7O+g= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0= +github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= +github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -567,6 +601,12 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index 4572677d..2b75a66f 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -1,80 +1,67 @@ package test -// adapted from https://github.com/cloudposse/terraform-aws-eks-cluster/blob/main/test/src/examples_complete_test.go - import ( - "encoding/base64" + "context" "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/eks" - "github.com/aws/aws-sdk-go/service/iam" - "github.com/aws/aws-sdk-go/service/kms" - "github.com/gruntwork-io/terratest/modules/random" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/eks" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/camunda/camunda-tf-eks-module/utils" "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "sigs.k8s.io/aws-iam-authenticator/pkg/token" - "strings" "sync/atomic" "testing" "time" ) -func newClientSet(cluster *eks.Cluster) (*kubernetes.Clientset, error) { - gen, err := token.NewGenerator(true, false) - if err != nil { - return nil, err - } - opts := &token.GetTokenOptions{ - ClusterID: aws.StringValue(cluster.Name), - } - tok, err := gen.GetWithOptions(opts) - if err != nil { - return nil, err - } - ca, err := base64.StdEncoding.DecodeString(aws.StringValue(cluster.CertificateAuthority.Data)) - if err != nil { - return nil, err - } - clientSet, err := kubernetes.NewForConfig( - &rest.Config{ - Host: aws.StringValue(cluster.Endpoint), - BearerToken: tok.Token, - TLSClientConfig: rest.TLSClientConfig{ - CAData: ca, - }, - }, - ) - if err != nil { - return nil, err - } - return clientSet, nil -} +func TestTearsUp(t *testing.T) { + // log + logger := zaptest.NewLogger(t) + sugar := logger.Sugar() -// Function to check a specific VPC attribute -func checkVpcAttribute(ec2Svc *ec2.EC2, vpcID, attributeName string) (*ec2.DescribeVpcAttributeOutput, error) { - input := &ec2.DescribeVpcAttributeInput{ - VpcId: aws.String(vpcID), - Attribute: aws.String(attributeName), - } + //randId := strings.ToLower(random.UniqueId()) + randId := "leo" + clusterName := fmt.Sprintf("cluster-%s", randId) + region := "eu-central-1" + sugar.Infow("Creating EKS cluster...") + + terraformOptions := TearsUpEKS(t, sugar, clusterName, region, "") + + // test suite + testEksCluster(t, sugar, terraformOptions) - return ec2Svc.DescribeVpcAttribute(input) + // At the end of the test, run `terraform destroy` to clean up any resources that were created + /* defer terraform.Destroy(t, terraformOptions) + + // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(t, terraformOptions) + })*/ + + TearsDown(t, sugar) } -// Test the Terraform module in modules/eks-cluster. -func TestModulesEKSCluster(t *testing.T) { +// TearsUpEKS spawns a new EKS Cluster with a random name from a fixture file +func TearsUpEKS(t *testing.T, sugar *zap.SugaredLogger, clusterName, region, kubernetesVersion string) *terraform.Options { + varsConfig := map[string]interface{}{ + "name": clusterName, + "region": region, + } - randId := strings.ToLower(random.UniqueId()) - clusterName := fmt.Sprintf("cluster-%s", randId) - region := "eu-central-1" + if kubernetesVersion != "" { + varsConfig["kubernetes_version"] = kubernetesVersion + } + + sugar.Infow("TF vars", varsConfig) terraformOptions := &terraform.Options{ // The path to where our Terraform code is located @@ -82,24 +69,23 @@ func TestModulesEKSCluster(t *testing.T) { Upgrade: false, // Variables to pass to our Terraform code using -var-file options VarFiles: []string{"../../test/src/fixtures/fixtures.eu-central-1.eks.tfvars"}, - Vars: map[string]interface{}{ - "name": clusterName, - "region": region, - }, + Vars: varsConfig, } - // At the end of the test, run `terraform destroy` to clean up any resources that were created - defer terraform.Destroy(t, terraformOptions) - - // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created - defer runtime.HandleCrash(func(i interface{}) { - terraform.Destroy(t, terraformOptions) - }) - // This will run `terraform init` and `terraform apply` and fail the test if there are any errors // then it will re-run apply to make sure that out tf is idempotent - _, errTfApply := terraform.InitAndApplyAndIdempotentE(t, terraformOptions) - assert.NoError(t, errTfApply) + terraform.InitAndApplyAndIdempotent(t, terraformOptions) + return terraformOptions +} + +func TearsDown(t *testing.T, sugar *zap.SugaredLogger) { + sugar.Infow("Tests completed") +} + +// Test the Terraform module in modules/eks-cluster. +func testEksCluster(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *terraform.Options) { + clusterName := terraformOptions.Vars["name"].(string) + sugar.Infow("Testing status of the EKS cluster", clusterName) // Do some basic not empty tests on outputs assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_endpoint")) @@ -119,12 +105,12 @@ func TestModulesEKSCluster(t *testing.T) { assert.Equal(t, fmt.Sprintf("%s-eks-iam-role", clusterName), terraform.Output(t, terraformOptions, "cluster_iam_role_name")) // this is a split(6)[0..2] of the base cluster_node_ipv4_cidr = "10.192.0.0/16" - expectedPrivateVpcCidrBlocks := []string([]string{"10.192.0.0/19", "10.192.32.0/19", "10.192.64.0/19"}) - assert.Equal(t, terraform.Output(t, terraformOptions, "private_vpc_cidr_blocks"), expectedPrivateVpcCidrBlocks) + expectedPrivateVpcCidrBlocks := "[10.192.0.0/19 10.192.32.0/19 10.192.64.0/19]" + assert.Equal(t, expectedPrivateVpcCidrBlocks, terraform.Output(t, terraformOptions, "private_vpc_cidr_blocks")) // this is a split(6)[3..5] of the base cluster_node_ipv4_cidr = "10.192.0.0/16" - expectedPublicVpcCidrBlocks := []string([]string{"10.192.96.0/19", "10.192.128.0/19", "10.192.160.0/19"}) - assert.Equal(t, terraform.Output(t, terraformOptions, "public_vpc_cidr_blocks"), expectedPublicVpcCidrBlocks) + expectedPublicVpcCidrBlocks := "[10.192.96.0/19 10.192.128.0/19 10.192.160.0/19]" + assert.Equal(t, expectedPublicVpcCidrBlocks, terraform.Output(t, terraformOptions, "public_vpc_cidr_blocks")) // Wait for the worker nodes to join the cluster // https://github.com/kubernetes/client-go @@ -134,24 +120,25 @@ func TestModulesEKSCluster(t *testing.T) { // https://stackoverflow.com/questions/60547409/unable-to-obtain-kubeconfig-of-an-aws-eks-cluster-in-go-code/60573982#60573982 fmt.Println("Waiting for worker nodes to join the EKS cluster") - sess := session.Must(session.NewSession(&aws.Config{ - Region: aws.String(region), - })) + sess, err := utils.GetAwsClient() + require.NoErrorf(t, err, "Failed to get aws client") // list your services here - eksSvc := eks.New(sess) - iamSvc := iam.New(sess) - ec2Svc := ec2.New(sess) - kmsSvc := kms.New(sess) + eksSvc := eks.NewFromConfig(sess) + iamSvc := iam.NewFromConfig(sess) + ec2Svc := ec2.NewFromConfig(sess) + kmsSvc := kms.NewFromConfig(sess) + + var expectedNodesCount uint64 = 3 inputEKS := &eks.DescribeClusterInput{ Name: aws.String(clusterName), } - result, err := eksSvc.DescribeCluster(inputEKS) + result, err := eksSvc.DescribeCluster(context.TODO(), inputEKS) assert.NoError(t, err) - clientSet, err := newClientSet(result.Cluster) + clientSet, err := utils.NewClientSet(result.Cluster) assert.NoError(t, err) factory := informers.NewSharedInformerFactory(clientSet, 0) @@ -164,14 +151,19 @@ func TestModulesEKSCluster(t *testing.T) { node := obj.(*corev1.Node) fmt.Printf("Worker Node %s has joined the EKS cluster at %s\n", node.Name, node.CreationTimestamp) atomic.AddUint64(&countOfWorkerNodes, 1) - if countOfWorkerNodes > 1 { - close(stopChannel) + if countOfWorkerNodes >= expectedNodesCount { + stopChannel <- struct{}{} // send close signal } }, }) require.NoError(t, errEventHandler) go informer.Run(stopChannel) + go func() { + // wait to receive a signal to close the channel + <-stopChannel + close(stopChannel) + }() select { case <-stopChannel: @@ -185,14 +177,16 @@ func TestModulesEKSCluster(t *testing.T) { // Verify list of addons installed on the EKS expectedEKSAddons := []string{"coredns", "kube-proxy", "vpc-cni", "aws-ebs-csi-driver"} - inputDescribeAddons := &eks.ListAddonsInput{} - outputEKSAddons, errEKSAddons := eksSvc.ListAddons(inputDescribeAddons) + inputDescribeAddons := &eks.ListAddonsInput{ + ClusterName: aws.String(clusterName), + } + outputEKSAddons, errEKSAddons := eksSvc.ListAddons(context.TODO(), inputDescribeAddons) require.NoError(t, errEKSAddons) // perform the diff presenceAddonsMap := make(map[string]bool) for _, addon := range outputEKSAddons.Addons { - presenceAddonsMap[*addon] = true + presenceAddonsMap[addon] = true } for _, addonName := range expectedEKSAddons { assert.Truef(t, presenceAddonsMap[addonName], "Addon %s not installed on the EKS cluster", addonName) @@ -211,7 +205,7 @@ func TestModulesEKSCluster(t *testing.T) { RoleName: aws.String(roleName), } - _, err := iamSvc.GetRole(input) + _, err := iamSvc.GetRole(context.TODO(), input) assert.NoErrorf(t, err, "Failed to get IAM EKS role %s", roleName) } @@ -220,15 +214,15 @@ func TestModulesEKSCluster(t *testing.T) { vpcName := fmt.Sprintf("%s-vpc", clusterName) inputVPC := &ec2.DescribeVpcsInput{ - Filters: []*ec2.Filter{ + Filters: []types.Filter{ { Name: aws.String("tag:Name"), - Values: []*string{aws.String(vpcName)}, + Values: []string{vpcName}, }, }, } - outputVPC, errVPC := ec2Svc.DescribeVpcs(inputVPC) + outputVPC, errVPC := ec2Svc.DescribeVpcs(context.TODO(), inputVPC) require.NoError(t, errVPC) assert.Equal(t, len(outputVPC.Vpcs), 1) @@ -237,26 +231,27 @@ func TestModulesEKSCluster(t *testing.T) { vpcID := *vpc.VpcId assert.NotEmpty(t, vpcID) - // WIP - /* val, err := checkVpcAttribute(ec2Svc, vpcID, "mapPublicIpOnLaunch") - assert.NoError(t, err) - assert.Equal(t, true, *val) - checkVpcAttribute(ec2Svc, vpcID, "enableDnsSupport") - checkVpcAttribute(ec2Svc, vpcID, "enableDnsHostnames") - */ + // todo: implement + /* valAttr, errAttr := utils.CheckVpcAttribute(ec2Svc, vpcID, types.VpcAttributeNameEnableDnsHostnames) + assert.NoError(t, errAttr) + assert.Equal(t, true, *valAttr) + valAttr, errAttr = utils.CheckVpcAttribute(ec2Svc, vpcID, types.VpcAttributeNameEnableDnsSupport) + assert.NoError(t, errAttr) + assert.Equal(t, true, *valAttr)*/ + // key keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) inputKMS := &kms.ListKeysInput{} - outputKMSList, errKMSList := kmsSvc.ListKeys(inputKMS) + outputKMSList, errKMSList := kmsSvc.ListKeys(context.TODO(), inputKMS) assert.NoError(t, errKMSList) // Check if the key corresponding to the description exists keyFound := false for _, key := range outputKMSList.Keys { - keyDetails, err := kmsSvc.DescribeKey(&kms.DescribeKeyInput{ + keyDetails, errKey := kmsSvc.DescribeKey(context.TODO(), &kms.DescribeKeyInput{ KeyId: key.KeyId, }) - require.NoErrorf(t, err, "Failed to describe key %s", *key.KeyId) + require.NoErrorf(t, errKey, "Failed to describe key %s", *key.KeyId) keyFound = *keyDetails.KeyMetadata.Description == keyDescription if keyFound { @@ -265,3 +260,7 @@ func TestModulesEKSCluster(t *testing.T) { } assert.Truef(t, keyFound, "Failed to find key %s", keyDescription) } + +// todo: test upgrade path +// todo: test auroradb integration +// todo: test inputs diff --git a/test/src/utils/helper.go b/test/src/utils/helper.go new file mode 100644 index 00000000..8216c370 --- /dev/null +++ b/test/src/utils/helper.go @@ -0,0 +1,11 @@ +package utils + +import "os" + +func GetEnv(key, fallback string) string { + value, exists := os.LookupEnv(key) + if !exists { + value = fallback + } + return value +} diff --git a/test/src/utils/utils.go b/test/src/utils/utils.go new file mode 100644 index 00000000..08ecc505 --- /dev/null +++ b/test/src/utils/utils.go @@ -0,0 +1,65 @@ +package utils + +import ( + "context" + "encoding/base64" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + types2 "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/eks/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/aws-iam-authenticator/pkg/token" +) + +func NewClientSet(cluster *types.Cluster) (*kubernetes.Clientset, error) { + gen, err := token.NewGenerator(true, false) + if err != nil { + return nil, err + } + opts := &token.GetTokenOptions{ + ClusterID: *cluster.Name, + } + tok, err := gen.GetWithOptions(opts) + if err != nil { + return nil, err + } + ca, err := base64.StdEncoding.DecodeString(*cluster.CertificateAuthority.Data) + if err != nil { + return nil, err + } + clientSet, err := kubernetes.NewForConfig( + &rest.Config{ + Host: *cluster.Endpoint, + BearerToken: tok.Token, + TLSClientConfig: rest.TLSClientConfig{ + CAData: ca, + }, + }, + ) + if err != nil { + return nil, err + } + return clientSet, nil +} + +func GetAwsClient() (aws.Config, error) { + awsProfile := GetEnv("AWS_PROFILE", GetEnv("AWS_DEFAULT_PROFILE", "infex")) + region := GetEnv("AWS_REGION", "eu-central-1") + + return config.LoadDefaultConfig(context.TODO(), + config.WithRegion(region), + config.WithSharedConfigProfile(awsProfile), + ) +} + +// CheckVpcAttribute Function to check a specific VPC attribute +func CheckVpcAttribute(ec2Svc *ec2.Client, vpcID string, attributeName types2.VpcAttributeName) (*ec2.DescribeVpcAttributeOutput, error) { + input := &ec2.DescribeVpcAttributeInput{ + VpcId: &vpcID, + Attribute: attributeName, + } + + return ec2Svc.DescribeVpcAttribute(context.TODO(), input) +} From 34cf10ec21700eef011719c1888fe5faea3091e1 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:58:18 +0100 Subject: [PATCH 05/93] extract cluster --- test/src/modules_eks_cluster_test.go | 42 ++--------------------- test/src/utils/utils.go | 51 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 40 deletions(-) diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index 2b75a66f..f6d2d366 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -15,10 +15,6 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/tools/cache" - "sync/atomic" "testing" "time" ) @@ -138,42 +134,8 @@ func testEksCluster(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *te result, err := eksSvc.DescribeCluster(context.TODO(), inputEKS) assert.NoError(t, err) - clientSet, err := utils.NewClientSet(result.Cluster) - assert.NoError(t, err) - - factory := informers.NewSharedInformerFactory(clientSet, 0) - informer := factory.Core().V1().Nodes().Informer() - stopChannel := make(chan struct{}) - var countOfWorkerNodes uint64 = 0 - - _, errEventHandler := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - node := obj.(*corev1.Node) - fmt.Printf("Worker Node %s has joined the EKS cluster at %s\n", node.Name, node.CreationTimestamp) - atomic.AddUint64(&countOfWorkerNodes, 1) - if countOfWorkerNodes >= expectedNodesCount { - stopChannel <- struct{}{} // send close signal - } - }, - }) - require.NoError(t, errEventHandler) - - go informer.Run(stopChannel) - go func() { - // wait to receive a signal to close the channel - <-stopChannel - close(stopChannel) - }() - - select { - case <-stopChannel: - msg := "All worker nodes have joined the EKS cluster" - fmt.Println(msg) - case <-time.After(5 * time.Minute): - msg := "Not all worker nodes have joined the EKS cluster" - fmt.Println(msg) - assert.Fail(t, msg) - } + errClusterReady := utils.WaitUntilClusterIsReady(result.Cluster, 5*time.Minute, expectedNodesCount) + require.NoError(t, errClusterReady) // Verify list of addons installed on the EKS expectedEKSAddons := []string{"coredns", "kube-proxy", "vpc-cni", "aws-ebs-csi-driver"} diff --git a/test/src/utils/utils.go b/test/src/utils/utils.go index 08ecc505..8ef13d96 100644 --- a/test/src/utils/utils.go +++ b/test/src/utils/utils.go @@ -3,14 +3,21 @@ package utils import ( "context" "encoding/base64" + "errors" + "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" types2 "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/eks/types" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" "sigs.k8s.io/aws-iam-authenticator/pkg/token" + "sync/atomic" + "time" ) func NewClientSet(cluster *types.Cluster) (*kubernetes.Clientset, error) { @@ -63,3 +70,47 @@ func CheckVpcAttribute(ec2Svc *ec2.Client, vpcID string, attributeName types2.Vp return ec2Svc.DescribeVpcAttribute(context.TODO(), input) } + +func WaitUntilClusterIsReady(cluster *types.Cluster, timeout time.Duration, expectedNodesCount uint64) error { + clientSet, err := NewClientSet(cluster) + if err != nil { + return err + } + + factory := informers.NewSharedInformerFactory(clientSet, 0) + informer := factory.Core().V1().Nodes().Informer() + stopChannel := make(chan struct{}) + var countOfWorkerNodes uint64 = 0 + + _, errEventHandler := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + node := obj.(*corev1.Node) + fmt.Printf("Worker Node %s has joined the EKS cluster at %s\n", node.Name, node.CreationTimestamp) + atomic.AddUint64(&countOfWorkerNodes, 1) + if countOfWorkerNodes >= expectedNodesCount { + stopChannel <- struct{}{} // send close signal + } + }, + }) + if errEventHandler != nil { + return errEventHandler + } + + go informer.Run(stopChannel) + go func() { + // wait to receive a signal to close the channel + <-stopChannel + close(stopChannel) + }() + + select { + case <-stopChannel: + msg := "All worker nodes have joined the Kube cluster" + fmt.Println(msg) + case <-time.After(timeout): + msg := "Not all worker nodes have joined the Kube cluster" + fmt.Println(msg) + return errors.New(msg) + } + return nil +} From 9c809d8b83aaaf9ef9efcc3f047b59071071b440 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 29 Mar 2024 11:38:50 +0100 Subject: [PATCH 06/93] wip tests --- .gitignore | 2 +- .pre-commit-config.yaml | 10 ++ modules/eks-cluster/README.md | 2 +- test/README.md | 16 ++- .../fixtures/fixtures.default.aurora.tfvars | 38 +++++++ ...eks.tfvars => fixtures.default.eks.tfvars} | 2 +- test/src/modules_eks_cluster_test.go | 98 ++++++++++++------- test/src/utils/utils.go | 21 ++-- 8 files changed, 134 insertions(+), 55 deletions(-) create mode 100644 test/src/fixtures/fixtures.default.aurora.tfvars rename test/src/fixtures/{fixtures.eu-central-1.eks.tfvars => fixtures.default.eks.tfvars} (50%) diff --git a/.gitignore b/.gitignore index fc7727ea..9f316be3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ terraform.tfstate terraform.tfstate.backup .terraform -.terraform.lock.hcl \ No newline at end of file +.terraform.lock.hcl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8fb7415c..d09dcc8a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,12 +2,14 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-added-large-files + - repo: https://github.com/antonbabenko/pre-commit-terraform rev: v1.88.3 hooks: @@ -21,4 +23,12 @@ repos: - --hook-config=--create-file-if-not-exist=true - --hook-config=--add-to-existing-file=true - --args=--config=.lint/terraform_docs/.terraform-docs.yml + +- repo: https://github.com/dnephin/pre-commit-golang + rev: v0.5.1 + hooks: + - id: go-fmt + - id: validate-toml + - id: no-go-testing + - id: go-mod-tidy ... diff --git a/modules/eks-cluster/README.md b/modules/eks-cluster/README.md index 1f44d346..af6990f5 100644 --- a/modules/eks-cluster/README.md +++ b/modules/eks-cluster/README.md @@ -60,7 +60,7 @@ module "eks_cluster" { | [np\_ami\_type](#input\_np\_ami\_type) | Amazon Machine Image | `string` | `"AL2_x86_64"` | no | | [np\_capacity\_type](#input\_np\_capacity\_type) | Allows setting the capacity type to ON\_DEMAND or SPOT to determine stable nodes | `string` | `"ON_DEMAND"` | no | | [np\_desired\_node\_count](#input\_np\_desired\_node\_count) | Actual number of nodes for the default node pool. Min-Max will be used for autoscaling | `number` | `4` | no | -| [np\_disk\_size](#input\_np\_disk\_size) | Disk size of the nodes on the default node poool | `number` | `20` | no | +| [np\_disk\_size](#input\_np\_disk\_size) | Disk size of the nodes on the default node pool | `number` | `20` | no | | [np\_instance\_types](#input\_np\_instance\_types) | Allow passing a list of instance types for the auto scaler to select from when scaling the default node pool | `list(string)` | 
[
  "m6i.xlarge"
]
| no | | [np\_max\_node\_count](#input\_np\_max\_node\_count) | Maximum number of nodes for the default node pool | `number` | `10` | no | | [np\_min\_node\_count](#input\_np\_min\_node\_count) | Minimum number of nodes for the default node pool | `number` | `1` | no | diff --git a/test/README.md b/test/README.md index 21887c75..6c746560 100644 --- a/test/README.md +++ b/test/README.md @@ -18,9 +18,17 @@ export AWS_DEFAULT_PROFILE=SystemAdministrator-**** export AWS_REGION=eu-central-1 ``` -test with +If you want to have a non-random cluster uid: +``` +export TEST_CLUSTER_ID="myTest" +``` + +test with ```bash make test + +# or just test one case +go test -v -timeout 120m -run TestDefaultEKS ``` ### Troubleshooting @@ -28,12 +36,12 @@ make test ```bash # make sure you don't have test clusters running since a while -eksctl get clusters +eksctl get clusters ``` # TODO: implement db pod -# todo: tests weekly +# todo: tests weekly # see https://github.com/camunda/c8-multi-region/blob/main/.github/workflows/nightly_aws_region_cleanup.yml # TODO: https://github.com/gruntwork-io/cloud-nuke every weekend # => we should have a dedicated tenant for CI -# => sometimes, EKS deletion fails with error: DeleteCluster ResourceInUseException: Cluster has nodegroups attached terraform \ No newline at end of file +# => sometimes, EKS deletion fails with error: DeleteCluster ResourceInUseException: Cluster has nodegroups attached terraform diff --git a/test/src/fixtures/fixtures.default.aurora.tfvars b/test/src/fixtures/fixtures.default.aurora.tfvars new file mode 100644 index 00000000..4cda94a6 --- /dev/null +++ b/test/src/fixtures/fixtures.default.aurora.tfvars @@ -0,0 +1,38 @@ + + + + +variable "username" { + type = string + description = "The username for the postgres admin user. Important: secret value!" + sensitive = true +} + +variable "password" { + type = string + description = "The password for the postgres admin user. Important: secret value!" + sensitive = true +} + +variable "subnet_ids" { + type = list(string) + description = "The subnet IDs to create the cluster in. For easier usage we are passing through the subnet IDs from the AWS EKS Cluster module." +} + +# pass through from root +variable "vpc_id" { + description = "The VPC ID to create the cluster in. For easier usage we are passing through the VPC ID from the AWS EKS Cluster module." +} + +# Allows defining whether IAM auth should be activated for IRSA usage +variable "iam_auth_enabled" { + default = false + type = bool + description = "Determines whether IAM auth should be activated for IRSA usage" +} + +variable "default_database_name" { + type = string + default = "camunda" + description = "The name for the automatically created database on cluster creation." +} diff --git a/test/src/fixtures/fixtures.eu-central-1.eks.tfvars b/test/src/fixtures/fixtures.default.eks.tfvars similarity index 50% rename from test/src/fixtures/fixtures.eu-central-1.eks.tfvars rename to test/src/fixtures/fixtures.default.eks.tfvars index 368e61f3..c8258284 100644 --- a/test/src/fixtures/fixtures.eu-central-1.eks.tfvars +++ b/test/src/fixtures/fixtures.default.eks.tfvars @@ -1,2 +1,2 @@ cluster_service_ipv4_cidr = "10.190.0.0/16" -cluster_node_ipv4_cidr = "10.192.0.0/16" \ No newline at end of file +cluster_node_ipv4_cidr = "10.192.0.0/16" diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index f6d2d366..9ccfa1b3 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -10,30 +10,33 @@ import ( "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/camunda/camunda-tf-eks-module/utils" + "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" + "strings" "testing" "time" ) -func TestTearsUp(t *testing.T) { +// TestDefaultEKS spawns an EKS cluster with the default parameters and checks the parameters +func TestDefaultEKS(t *testing.T) { // log logger := zaptest.NewLogger(t) sugar := logger.Sugar() - //randId := strings.ToLower(random.UniqueId()) - randId := "leo" - clusterName := fmt.Sprintf("cluster-%s", randId) + clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) + clusterName := fmt.Sprintf("cluster-%s", clusterSuffix) region := "eu-central-1" sugar.Infow("Creating EKS cluster...") + expectedCapacity := 4 - terraformOptions := TearsUpEKS(t, sugar, clusterName, region, "") + terraformOptions := SpawnEKS(t, sugar, expectedCapacity, clusterName, region, "") // test suite - testEksCluster(t, sugar, terraformOptions) + baseChecksEKS(t, sugar, terraformOptions, uint64(expectedCapacity)) // At the end of the test, run `terraform destroy` to clean up any resources that were created /* defer terraform.Destroy(t, terraformOptions) @@ -46,11 +49,53 @@ func TestTearsUp(t *testing.T) { TearsDown(t, sugar) } -// TearsUpEKS spawns a new EKS Cluster with a random name from a fixture file -func TearsUpEKS(t *testing.T, sugar *zap.SugaredLogger, clusterName, region, kubernetesVersion string) *terraform.Options { +// TestCustomEKSAndRDS spawns a custom EKS cluster with custom parameters, and spawns a +// pg client pod that will test connection to AuroraDB +func TestCustomEKSAndRDS(t *testing.T) { + +} + +// TestUpgradeEKS starts from a version of EKS, deploy a simple chart, upgrade the cluster +// and check that everything is working as expected +func TestUpgradeEKS(t *testing.T) { + // log + logger := zaptest.NewLogger(t) + sugar := logger.Sugar() + + clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) + clusterName := fmt.Sprintf("cluster-%s", clusterSuffix) + region := "eu-central-1" + sugar.Infow("Creating EKS cluster...") + expectedCapacity := 4 + + terraformOptions := SpawnEKS(t, sugar, expectedCapacity, clusterName, region, "1.27") + + // test suite + baseChecksEKS(t, sugar, terraformOptions, 3) + + // upgrade the cluster + terraformOptions = SpawnEKS(t, sugar, expectedCapacity, clusterName, region, "1.28") + + // check everything works as expected + baseChecksEKS(t, sugar, terraformOptions, 3) + + // At the end of the test, run `terraform destroy` to clean up any resources that were created + /* defer terraform.Destroy(t, terraformOptions) + + // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(t, terraformOptions) + })*/ + + TearsDown(t, sugar) +} + +// SpawnEKS spawns a new EKS Cluster with a random name from a fixture file +func SpawnEKS(t *testing.T, sugar *zap.SugaredLogger, desiredNodeCount int, clusterName, region, kubernetesVersion string) *terraform.Options { varsConfig := map[string]interface{}{ - "name": clusterName, - "region": region, + "name": clusterName, + "region": region, + "np_desired_node_count": desiredNodeCount, } if kubernetesVersion != "" { @@ -64,7 +109,7 @@ func TearsUpEKS(t *testing.T, sugar *zap.SugaredLogger, clusterName, region, kub TerraformDir: "../../modules/eks-cluster", Upgrade: false, // Variables to pass to our Terraform code using -var-file options - VarFiles: []string{"../../test/src/fixtures/fixtures.eu-central-1.eks.tfvars"}, + VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, Vars: varsConfig, } @@ -75,11 +120,11 @@ func TearsUpEKS(t *testing.T, sugar *zap.SugaredLogger, clusterName, region, kub } func TearsDown(t *testing.T, sugar *zap.SugaredLogger) { - sugar.Infow("Tests completed") + sugar.Infow("All tests completed") } -// Test the Terraform module in modules/eks-cluster. -func testEksCluster(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *terraform.Options) { +// baseChecksEKS checks the defaults of an EKS cluster +func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *terraform.Options, expectedNodesCount uint64) { clusterName := terraformOptions.Vars["name"].(string) sugar.Infow("Testing status of the EKS cluster", clusterName) @@ -98,6 +143,7 @@ func testEksCluster(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *te assert.NotEmpty(t, terraform.Output(t, terraformOptions, "vpc_main_route_table_id")) assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_route_table_ids")) + // test IAM roles assert.Equal(t, fmt.Sprintf("%s-eks-iam-role", clusterName), terraform.Output(t, terraformOptions, "cluster_iam_role_name")) // this is a split(6)[0..2] of the base cluster_node_ipv4_cidr = "10.192.0.0/16" @@ -108,14 +154,6 @@ func testEksCluster(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *te expectedPublicVpcCidrBlocks := "[10.192.96.0/19 10.192.128.0/19 10.192.160.0/19]" assert.Equal(t, expectedPublicVpcCidrBlocks, terraform.Output(t, terraformOptions, "public_vpc_cidr_blocks")) - // Wait for the worker nodes to join the cluster - // https://github.com/kubernetes/client-go - // https://www.rushtehrani.com/post/using-kubernetes-api - // https://rancher.com/using-kubernetes-api-go-kubecon-2017-session-recap - // https://gianarb.it/blog/kubernetes-shared-informer - // https://stackoverflow.com/questions/60547409/unable-to-obtain-kubeconfig-of-an-aws-eks-cluster-in-go-code/60573982#60573982 - fmt.Println("Waiting for worker nodes to join the EKS cluster") - sess, err := utils.GetAwsClient() require.NoErrorf(t, err, "Failed to get aws client") @@ -125,8 +163,6 @@ func testEksCluster(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *te ec2Svc := ec2.NewFromConfig(sess) kmsSvc := kms.NewFromConfig(sess) - var expectedNodesCount uint64 = 3 - inputEKS := &eks.DescribeClusterInput{ Name: aws.String(clusterName), } @@ -134,6 +170,8 @@ func testEksCluster(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *te result, err := eksSvc.DescribeCluster(context.TODO(), inputEKS) assert.NoError(t, err) + // Wait for the worker nodes to join the cluster + sugar.Infow("Waiting for worker nodes to join the EKS cluster") errClusterReady := utils.WaitUntilClusterIsReady(result.Cluster, 5*time.Minute, expectedNodesCount) require.NoError(t, errClusterReady) @@ -189,18 +227,6 @@ func testEksCluster(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *te assert.Equal(t, len(outputVPC.Vpcs), 1) - vpc := outputVPC.Vpcs[0] - vpcID := *vpc.VpcId - assert.NotEmpty(t, vpcID) - - // todo: implement - /* valAttr, errAttr := utils.CheckVpcAttribute(ec2Svc, vpcID, types.VpcAttributeNameEnableDnsHostnames) - assert.NoError(t, errAttr) - assert.Equal(t, true, *valAttr) - valAttr, errAttr = utils.CheckVpcAttribute(ec2Svc, vpcID, types.VpcAttributeNameEnableDnsSupport) - assert.NoError(t, errAttr) - assert.Equal(t, true, *valAttr)*/ - // key keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) inputKMS := &kms.ListKeysInput{} diff --git a/test/src/utils/utils.go b/test/src/utils/utils.go index 8ef13d96..d8d242bf 100644 --- a/test/src/utils/utils.go +++ b/test/src/utils/utils.go @@ -7,8 +7,6 @@ import ( "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ec2" - types2 "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/eks/types" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" @@ -20,6 +18,7 @@ import ( "time" ) +// NewClientSet generate a kubernetes.Clientset from an aws Cluster func NewClientSet(cluster *types.Cluster) (*kubernetes.Clientset, error) { gen, err := token.NewGenerator(true, false) if err != nil { @@ -51,6 +50,7 @@ func NewClientSet(cluster *types.Cluster) (*kubernetes.Clientset, error) { return clientSet, nil } +// GetAwsClient returns an aws.Config client from the env variables `AWS_PROFILE` and `AWS_REGION` func GetAwsClient() (aws.Config, error) { awsProfile := GetEnv("AWS_PROFILE", GetEnv("AWS_DEFAULT_PROFILE", "infex")) region := GetEnv("AWS_REGION", "eu-central-1") @@ -61,17 +61,14 @@ func GetAwsClient() (aws.Config, error) { ) } -// CheckVpcAttribute Function to check a specific VPC attribute -func CheckVpcAttribute(ec2Svc *ec2.Client, vpcID string, attributeName types2.VpcAttributeName) (*ec2.DescribeVpcAttributeOutput, error) { - input := &ec2.DescribeVpcAttributeInput{ - VpcId: &vpcID, - Attribute: attributeName, - } - - return ec2Svc.DescribeVpcAttribute(context.TODO(), input) -} - +// WaitUntilClusterIsReady waits until the kube cluster is read or returns an error func WaitUntilClusterIsReady(cluster *types.Cluster, timeout time.Duration, expectedNodesCount uint64) error { + // https://github.com/kubernetes/client-go + // https://www.rushtehrani.com/post/using-kubernetes-api + // https://rancher.com/using-kubernetes-api-go-kubecon-2017-session-recap + // https://gianarb.it/blog/kubernetes-shared-informer + // https://stackoverflow.com/questions/60547409/unable-to-obtain-kubeconfig-of-an-aws-eks-cluster-in-go-code/60573982#60573982 + clientSet, err := NewClientSet(cluster) if err != nil { return err From c2fdef11dd8ffd7cf65a0b1263e4ade56612b0b2 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:01:26 +0100 Subject: [PATCH 07/93] add k8s cluster config todo --- test/README.md | 5 + test/src/fixtures/postgres-client.yml | 46 +++++++ test/src/modules_eks_cluster_test.go | 184 +++++++++++++++++++++----- 3 files changed, 205 insertions(+), 30 deletions(-) create mode 100644 test/src/fixtures/postgres-client.yml diff --git a/test/README.md b/test/README.md index 6c746560..d17863f4 100644 --- a/test/README.md +++ b/test/README.md @@ -23,6 +23,11 @@ If you want to have a non-random cluster uid: export TEST_CLUSTER_ID="myTest" ``` +If you don't want to delete the ressources at the end: +``` +CLEAN_CLUSTER_AT_THE_END=false +``` + test with ```bash make test diff --git a/test/src/fixtures/postgres-client.yml b/test/src/fixtures/postgres-client.yml new file mode 100644 index 00000000..b1bda672 --- /dev/null +++ b/test/src/fixtures/postgres-client.yml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-client + labels: + app: postgres-client +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-client + template: + metadata: + labels: + app: postgres-client + spec: + containers: + - name: postgres-client + image: ubuntu:latest + command: ["sh", "-c", "apt-get update && apt-get install -y python3 python3-pip && python3 -m pip install awscli && AWS_PG_PASSWORD=$(aws rds generate-db-auth-token --hostname $AURORA_ENDPOINT --port $AURORA_PORT --region $AWS_REGION --username $AURORA_USERNAME) && psql -h $AURORA_ENDPOINT -p $AURORA_PORT \"dbname=$AURORA_DB_NAME user=$AURORA_USERNAME password=$AWS_PG_PASSWORD\" -c 'SELECT version();' && sleep 3600"] + env: + - name: AURORA_ENDPOINT + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_endpoint + - name: AURORA_USERNAME + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_username + - name: AURORA_PORT + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_port + - name: AWS_REGION + valueFrom: + configMapKeyRef: + name: aurora-config + key: aws_region + - name: AURORA_DB_NAME + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_db_name diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index 9ccfa1b3..de22ea3d 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -10,12 +10,17 @@ import ( "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/camunda/camunda-tf-eks-module/utils" + "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zaptest" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/runtime" "strings" "testing" "time" @@ -33,26 +38,111 @@ func TestDefaultEKS(t *testing.T) { sugar.Infow("Creating EKS cluster...") expectedCapacity := 4 - terraformOptions := SpawnEKS(t, sugar, expectedCapacity, clusterName, region, "") + varsConfig := map[string]interface{}{ + "name": clusterName, + "region": region, + "np_desired_node_count": expectedCapacity, + } + + terraformOptions := SpawnEKS(t, sugar, varsConfig) // test suite baseChecksEKS(t, sugar, terraformOptions, uint64(expectedCapacity)) - // At the end of the test, run `terraform destroy` to clean up any resources that were created - /* defer terraform.Destroy(t, terraformOptions) - - // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created - defer runtime.HandleCrash(func(i interface{}) { - terraform.Destroy(t, terraformOptions) - })*/ - TearsDown(t, sugar) } // TestCustomEKSAndRDS spawns a custom EKS cluster with custom parameters, and spawns a // pg client pod that will test connection to AuroraDB func TestCustomEKSAndRDS(t *testing.T) { + // log + logger := zaptest.NewLogger(t) + sugar := logger.Sugar() + + clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) + clusterName := fmt.Sprintf("cluster-rds-%s", clusterSuffix) + region := "eu-central-1" + sugar.Infow("Creating EKS cluster...") + expectedCapacity := 3 + + varsConfigEKS := map[string]interface{}{ + "name": clusterName, + "region": region, + "np_desired_node_count": expectedCapacity, + } + + terraformOptions := SpawnEKS(t, sugar, varsConfigEKS) + + // Wait for the worker nodes to join the cluster + sess, err := utils.GetAwsClient() + require.NoErrorf(t, err, "Failed to get aws client") + + // list your services here + eksSvc := eks.NewFromConfig(sess) + + inputEKS := &eks.DescribeClusterInput{ + Name: aws.String(clusterName), + } + + result, err := eksSvc.DescribeCluster(context.TODO(), inputEKS) + assert.NoError(t, err) + + sugar.Infow("Waiting for worker nodes to join the EKS cluster") + errClusterReady := utils.WaitUntilClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) + require.NoError(t, errClusterReady) + + // Spawn RDS within the EKS VPC/subnet + publicBlocks := strings.Fields(strings.Trim(terraform.Output(t, terraformOptions, "public_vpc_cidr_blocks"), "[]")) + privateBlocks := strings.Fields(strings.Trim(terraform.Output(t, terraformOptions, "private_vpc_cidr_blocks"), "[]")) + + auroraUsername := "myuser" + auroraPassword := "mypassword123secure" + varsConfigAurora := map[string]interface{}{ + "username": auroraUsername, + "password": auroraPassword, + "cluster_name": fmt.Sprintf("postgres-%s", clusterSuffix), + "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, + "vpc_id": result.Cluster.ResourcesVpcConfig.VpcId, + "cidr_blocks": append(publicBlocks, privateBlocks...), + "iam_auth_enabled": true, + } + + terraformOptionsRDS := SpawnAurora(t, sugar, varsConfigAurora) + auroraEndpoint := terraform.Output(t, terraformOptionsRDS, "aurora_endpoint") + assert.NotEmpty(t, auroraEndpoint) + + // Launch a pod on the cluster and test the RDS + kubeClient, err := utils.NewClientSet(result.Cluster) + require.NoError(t, err) + + // create the configmap + namespace := "postgres-client" + k8s.CreateNamespace(t, kubeClient, namespace) + + // todo: +https: //github.com/camunda/c8-multi-region/blob/main/test/internal/helpers/aws/helpers.go#L242 + + configMapPostgres := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aurora-config", + Namespace: namespace, + }, + Data: map[string]string{ + "aurora_endpoint": auroraEndpoint, + "aurora_username": auroraUsername, + "aurora_port": "5432", + "aws_region": region, + "aurora_db_name": auroraEndpoint, + }, + } + _, err = kubeClient.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMapPostgres, metav1.CreateOptions{}) + if err != nil { + if !errors.IsAlreadyExists(err) { + require.NoError(t, err) + } + } + TearsDown(t, sugar) } // TestUpgradeEKS starts from a version of EKS, deploy a simple chart, upgrade the cluster @@ -63,56 +153,90 @@ func TestUpgradeEKS(t *testing.T) { sugar := logger.Sugar() clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) - clusterName := fmt.Sprintf("cluster-%s", clusterSuffix) + clusterName := fmt.Sprintf("cluster-upgrade-%s", clusterSuffix) region := "eu-central-1" sugar.Infow("Creating EKS cluster...") expectedCapacity := 4 - terraformOptions := SpawnEKS(t, sugar, expectedCapacity, clusterName, region, "1.27") + varsConfig := map[string]interface{}{ + "name": clusterName, + "region": region, + "np_desired_node_count": expectedCapacity, + "kubernetes_version": "1.27", + } + + terraformOptions := SpawnEKS(t, sugar, varsConfig) // test suite baseChecksEKS(t, sugar, terraformOptions, 3) // upgrade the cluster - terraformOptions = SpawnEKS(t, sugar, expectedCapacity, clusterName, region, "1.28") + varsConfig["kubernetes_version"] = "1.28" + + terraformOptions = SpawnEKS(t, sugar, varsConfig) // check everything works as expected baseChecksEKS(t, sugar, terraformOptions, 3) - // At the end of the test, run `terraform destroy` to clean up any resources that were created - /* defer terraform.Destroy(t, terraformOptions) - - // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created - defer runtime.HandleCrash(func(i interface{}) { - terraform.Destroy(t, terraformOptions) - })*/ - TearsDown(t, sugar) } -// SpawnEKS spawns a new EKS Cluster with a random name from a fixture file -func SpawnEKS(t *testing.T, sugar *zap.SugaredLogger, desiredNodeCount int, clusterName, region, kubernetesVersion string) *terraform.Options { - varsConfig := map[string]interface{}{ - "name": clusterName, - "region": region, - "np_desired_node_count": desiredNodeCount, +// SpawnEKS spawns a new EKS Cluster from a default fixture file +func SpawnEKS(t *testing.T, sugar *zap.SugaredLogger, varsConfig map[string]interface{}) *terraform.Options { + sugar.Infow("TF vars", varsConfig) + + terraformOptions := &terraform.Options{ + // The path to where our Terraform code is located + TerraformDir: "../../modules/eks-cluster", + Upgrade: false, + // Variables to pass to our Terraform code using -var-file options + VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, + Vars: varsConfig, } - if kubernetesVersion != "" { - varsConfig["kubernetes_version"] = kubernetesVersion + cleanClusterAtTheEnd := utils.GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") + + if cleanClusterAtTheEnd == "true" { + // At the end of the test, run `terraform destroy` to clean up any resources that were created + defer terraform.Destroy(t, terraformOptions) + + // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(t, terraformOptions) + }) } + // This will run `terraform init` and `terraform apply` and fail the test if there are any errors + // then it will re-run apply to make sure that out tf is idempotent + terraform.InitAndApplyAndIdempotent(t, terraformOptions) + return terraformOptions +} + +// SpawnAurora spawns a new Aurora RDS from a default fixture file +func SpawnAurora(t *testing.T, sugar *zap.SugaredLogger, varsConfig map[string]interface{}) *terraform.Options { sugar.Infow("TF vars", varsConfig) terraformOptions := &terraform.Options{ // The path to where our Terraform code is located - TerraformDir: "../../modules/eks-cluster", + TerraformDir: "../../modules/aurora", Upgrade: false, // Variables to pass to our Terraform code using -var-file options - VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, + VarFiles: []string{"../../test/src/fixtures/fixtures.default.aurora.tfvars"}, Vars: varsConfig, } + cleanClusterAtTheEnd := utils.GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") + + if cleanClusterAtTheEnd == "true" { + // At the end of the test, run `terraform destroy` to clean up any resources that were created + defer terraform.Destroy(t, terraformOptions) + + // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(t, terraformOptions) + }) + } + // This will run `terraform init` and `terraform apply` and fail the test if there are any errors // then it will re-run apply to make sure that out tf is idempotent terraform.InitAndApplyAndIdempotent(t, terraformOptions) From d807dce79016c96bf16f660815de3d01b0990fc4 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 29 Mar 2024 18:23:05 +0100 Subject: [PATCH 08/93] implement rds test --- .../fixtures/fixtures.default.aurora.tfvars | 38 -------------- test/src/fixtures/postgres-client.yml | 29 +++++++---- test/src/go.mod | 14 +++++- test/src/go.sum | 36 +++++++++++++ test/src/modules_eks_cluster_test.go | 34 +++++++++++-- test/src/utils/kube.go | 50 +++++++++++++++++++ test/src/utils/utils.go | 12 ++++- 7 files changed, 157 insertions(+), 56 deletions(-) create mode 100644 test/src/utils/kube.go diff --git a/test/src/fixtures/fixtures.default.aurora.tfvars b/test/src/fixtures/fixtures.default.aurora.tfvars index 4cda94a6..e69de29b 100644 --- a/test/src/fixtures/fixtures.default.aurora.tfvars +++ b/test/src/fixtures/fixtures.default.aurora.tfvars @@ -1,38 +0,0 @@ - - - - -variable "username" { - type = string - description = "The username for the postgres admin user. Important: secret value!" - sensitive = true -} - -variable "password" { - type = string - description = "The password for the postgres admin user. Important: secret value!" - sensitive = true -} - -variable "subnet_ids" { - type = list(string) - description = "The subnet IDs to create the cluster in. For easier usage we are passing through the subnet IDs from the AWS EKS Cluster module." -} - -# pass through from root -variable "vpc_id" { - description = "The VPC ID to create the cluster in. For easier usage we are passing through the VPC ID from the AWS EKS Cluster module." -} - -# Allows defining whether IAM auth should be activated for IRSA usage -variable "iam_auth_enabled" { - default = false - type = bool - description = "Determines whether IAM auth should be activated for IRSA usage" -} - -variable "default_database_name" { - type = string - default = "camunda" - description = "The name for the automatically created database on cluster creation." -} diff --git a/test/src/fixtures/postgres-client.yml b/test/src/fixtures/postgres-client.yml index b1bda672..14ad6eea 100644 --- a/test/src/fixtures/postgres-client.yml +++ b/test/src/fixtures/postgres-client.yml @@ -1,23 +1,30 @@ -apiVersion: apps/v1 -kind: Deployment +apiVersion: batch/v1 +kind: Job metadata: name: postgres-client labels: app: postgres-client spec: - replicas: 1 - selector: - matchLabels: - app: postgres-client template: - metadata: - labels: - app: postgres-client spec: + restartPolicy: Never containers: - name: postgres-client - image: ubuntu:latest - command: ["sh", "-c", "apt-get update && apt-get install -y python3 python3-pip && python3 -m pip install awscli && AWS_PG_PASSWORD=$(aws rds generate-db-auth-token --hostname $AURORA_ENDPOINT --port $AURORA_PORT --region $AWS_REGION --username $AURORA_USERNAME) && psql -h $AURORA_ENDPOINT -p $AURORA_PORT \"dbname=$AURORA_DB_NAME user=$AURORA_USERNAME password=$AWS_PG_PASSWORD\" -c 'SELECT version();' && sleep 3600"] + image: ubuntu:latest # TODO: create the irsa user on the db (see kc) + command: + - sh + - "-c" + - | + /bin/bash <<'EOF' + + apt-get update && + apt-get install -y python3 python3-pip build-essential postgresql-client && + python3 -m pip install awscli && + aws rds generate-db-auth-token --hostname $AURORA_ENDPOINT --port $AURORA_PORT --region $AWS_REGION --username $AURORA_USERNAME + AWS_PG_PASSWORD=$(aws rds generate-db-auth-token --hostname $AURORA_ENDPOINT --port $AURORA_PORT --region $AWS_REGION --username $AURORA_USERNAME) && + psql -h $AURORA_ENDPOINT -p $AURORA_PORT "dbname=$AURORA_DB_NAME user=$AURORA_USERNAME password=$AWS_PG_PASSWORD" -c 'SELECT version();' + + EOF env: - name: AURORA_ENDPOINT valueFrom: diff --git a/test/src/go.mod b/test/src/go.mod index 779a50f6..fd0f38bd 100644 --- a/test/src/go.mod +++ b/test/src/go.mod @@ -3,7 +3,6 @@ module github.com/camunda/camunda-tf-eks-module go 1.22.0 require ( - github.com/aws/aws-sdk-go v1.44.332 github.com/aws/aws-sdk-go-v2 v1.26.0 github.com/aws/aws-sdk-go-v2/config v1.27.9 github.com/aws/aws-sdk-go-v2/service/ec2 v1.154.0 @@ -27,6 +26,7 @@ require ( cloud.google.com/go/storage v1.29.0 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/aws/aws-sdk-go v1.44.332 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect @@ -40,13 +40,17 @@ require ( github.com/aws/smithy-go v1.20.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect + github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -58,6 +62,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.11.0 // indirect + github.com/gruntwork-io/go-commons v0.8.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-getter v1.7.1 // indirect @@ -66,6 +71,7 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl/v2 v2.9.1 // indirect github.com/hashicorp/terraform-json v0.13.0 // indirect + github.com/imdario/mergo v0.3.11 // indirect github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -77,17 +83,23 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pquerna/otp v1.2.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/tmccombs/hcl2json v0.3.3 // indirect github.com/ulikunitz/xz v0.5.10 // indirect + github.com/urfave/cli v1.22.2 // indirect github.com/zclconf/go-cty v1.9.1 // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.10.0 // indirect diff --git a/test/src/go.sum b/test/src/go.sum index f7ec33ac..738d03de 100644 --- a/test/src/go.sum +++ b/test/src/go.sum @@ -201,6 +201,8 @@ github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:o github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.332 h1:Ze+98F41+LxoJUdsisAFThV+0yYYLYw17/Vt0++nFYM= github.com/aws/aws-sdk-go v1.44.332/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= @@ -244,6 +246,9 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -264,6 +269,9 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -281,7 +289,11 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 h1:skJKxRtNmevLqnayafdLe2AsenqRupVmzZSqrvb5caU= +github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -301,6 +313,8 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= @@ -417,7 +431,10 @@ github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMd github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/gruntwork-io/go-commons v0.8.0 h1:k/yypwrPqSeYHevLlEDmvmgQzcyTwrlZGRaxEM6G0ro= +github.com/gruntwork-io/go-commons v0.8.0/go.mod h1:gtp0yTtIBExIZp7vyIV9I0XQkVwiQZze678hvDXof78= github.com/gruntwork-io/terratest v0.46.13 h1:FDaEoZ7DtkomV8pcwLdBV/VsytdjnPRqJkIriYEYwjs= github.com/gruntwork-io/terratest v0.46.13/go.mod h1:8sxu3Qup8TxtbzOHzq0MUrQffJj/G61/OwlsReaCwpo= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -441,6 +458,8 @@ github.com/hashicorp/terraform-json v0.13.0 h1:Li9L+lKD1FO5RVFRM1mMMIBDoUHslOniy github.com/hashicorp/terraform-json v0.13.0/go.mod h1:y5OdLBCT+rxbwnpxZs9kGL7R9ExU76+cpdY8zHwoazk= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -480,8 +499,12 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -496,6 +519,8 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -507,6 +532,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= @@ -516,6 +543,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok= +github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -546,8 +575,12 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -576,6 +609,8 @@ github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uL github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= @@ -755,6 +790,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index de22ea3d..6b2d2c63 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" + "os/exec" "strings" "testing" "time" @@ -97,12 +98,13 @@ func TestCustomEKSAndRDS(t *testing.T) { auroraUsername := "myuser" auroraPassword := "mypassword123secure" + varsConfigAurora := map[string]interface{}{ "username": auroraUsername, "password": auroraPassword, "cluster_name": fmt.Sprintf("postgres-%s", clusterSuffix), "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, - "vpc_id": result.Cluster.ResourcesVpcConfig.VpcId, + "vpc_id": *result.Cluster.ResourcesVpcConfig.VpcId, "cidr_blocks": append(publicBlocks, privateBlocks...), "iam_auth_enabled": true, } @@ -115,12 +117,24 @@ func TestCustomEKSAndRDS(t *testing.T) { kubeClient, err := utils.NewClientSet(result.Cluster) require.NoError(t, err) + // create kubeconfig + cmd := exec.Command("aws", "eks", "--region", region, "update-kubeconfig", "--name", clusterName, "--profile", utils.GetAwsProfile(), "--kubeconfig", "kubeconfig") + _, errCmdKubeProfile := cmd.Output() + require.NoError(t, errCmdKubeProfile) + // create the configmap namespace := "postgres-client" - k8s.CreateNamespace(t, kubeClient, namespace) + pgKubeCtlOptions := k8s.NewKubectlOptions("", "kubeconfig", namespace) + _, errFindNamespace := k8s.GetNamespaceE(t, pgKubeCtlOptions, namespace) + if errFindNamespace != nil { + if errors.IsNotFound(err) { + k8s.CreateNamespace(t, pgKubeCtlOptions, namespace) + } else { + require.NoError(t, errFindNamespace) + } + } - // todo: -https: //github.com/camunda/c8-multi-region/blob/main/test/internal/helpers/aws/helpers.go#L242 + // todo: https: //github.com/camunda/c8-multi-region/blob/main/test/internal/helpers/aws/helpers.go#L242 configMapPostgres := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -141,6 +155,18 @@ https: //github.com/camunda/c8-multi-region/blob/main/test/internal/helpers/aws/ require.NoError(t, err) } } + // cleanup existing jobs + jobListOptions := metav1.ListOptions{LabelSelector: "app=postgres-client"} + existingJobs := k8s.ListJobs(t, pgKubeCtlOptions, jobListOptions) + for _, job := range existingJobs { + err := kubeClient.BatchV1().Jobs(namespace).Delete(context.TODO(), job.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + } + + k8s.KubectlApply(t, pgKubeCtlOptions, "../../test/src/fixtures/postgres-client.yml") + + errJob := utils.WaitForJobCompletion(kubeClient, namespace, "postgres-client", 5*time.Minute, jobListOptions) + require.NoError(t, errJob) TearsDown(t, sugar) } diff --git a/test/src/utils/kube.go b/test/src/utils/kube.go new file mode 100644 index 00000000..ce406896 --- /dev/null +++ b/test/src/utils/kube.go @@ -0,0 +1,50 @@ +package utils + +import ( + "context" + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "time" + + batchv1 "k8s.io/api/batch/v1" +) + +func WaitForJobCompletion(clientset *kubernetes.Clientset, namespace, jobName string, timeout time.Duration, listOptions metav1.ListOptions) error { + // Create a context + ctx := context.Background() + + // Watch for job events + watch, err := clientset.BatchV1().Jobs(namespace).Watch(ctx, listOptions) + if err != nil { + return fmt.Errorf("Error creating watcher: %v", err) + } + defer watch.Stop() + + // Channel to receive events + eventChan := watch.ResultChan() + + // Loop to wait for job completion + for { + select { + case event := <-eventChan: + job, ok := event.Object.(*batchv1.Job) + if !ok { + return fmt.Errorf("Unexpected event received: %v", event) + } + if job.Name == jobName { + // Check if the job has completed + if job.Status.CompletionTime != nil { + // The job has completed, check if it succeeded + if job.Status.Succeeded == 1 { + return nil // Job completed successfully + } else { + return fmt.Errorf("Job completed with errors") + } + } + } + case <-time.After(timeout): + return fmt.Errorf("Timeout: job did not complete after %v minutes", timeout) + } + } +} diff --git a/test/src/utils/utils.go b/test/src/utils/utils.go index d8d242bf..53681478 100644 --- a/test/src/utils/utils.go +++ b/test/src/utils/utils.go @@ -50,10 +50,18 @@ func NewClientSet(cluster *types.Cluster) (*kubernetes.Clientset, error) { return clientSet, nil } +func GetAwsProfile() string { + return GetEnv("AWS_PROFILE", GetEnv("AWS_DEFAULT_PROFILE", "infex")) +} + +func GetAwsRegion() string { + return GetEnv("AWS_REGION", "eu-central-1") +} + // GetAwsClient returns an aws.Config client from the env variables `AWS_PROFILE` and `AWS_REGION` func GetAwsClient() (aws.Config, error) { - awsProfile := GetEnv("AWS_PROFILE", GetEnv("AWS_DEFAULT_PROFILE", "infex")) - region := GetEnv("AWS_REGION", "eu-central-1") + awsProfile := GetAwsProfile() + region := GetAwsRegion() return config.LoadDefaultConfig(context.TODO(), config.WithRegion(region), From 791280d90852312c104b1d97397330c0289c554f Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:18:55 +0200 Subject: [PATCH 09/93] wip tests eks rds --- .github/workflows/nighly_aws_cleanup.yml | 80 ++++++++++++ test/src/fixtures/fixtures.default.eks.tfvars | 1 + test/src/fixtures/postgres-client.bak.yml | 74 ++++++++++++ test/src/fixtures/postgres-client.yml | 33 +++-- .../fixtures/scripts/create_aurora_pg_db.sh | 10 ++ test/src/go.mod | 13 +- test/src/go.sum | 26 ++-- test/src/modules_eks_cluster_test.go | 114 +++++++++++++++--- 8 files changed, 307 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/nighly_aws_cleanup.yml create mode 100644 test/src/fixtures/postgres-client.bak.yml create mode 100644 test/src/fixtures/scripts/create_aurora_pg_db.sh diff --git a/.github/workflows/nighly_aws_cleanup.yml b/.github/workflows/nighly_aws_cleanup.yml new file mode 100644 index 00000000..16ee454a --- /dev/null +++ b/.github/workflows/nighly_aws_cleanup.yml @@ -0,0 +1,80 @@ +--- +name: Nightly AWS Region Cleanup + +on: + schedule: + - cron: '0 5 * * 1-5' + workflow_dispatch: + +env: + AWS_PROFILE: infex + +jobs: + aws-nightly-cleanup: + runs-on: ubuntu-latest + # Other dependencies from hosted runner + # AWS CLI + # indirectly node used by actions + + steps: + - uses: actions/checkout@v4 + + - name: Import Secrets + id: secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_ADDR }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + exportEnv: false + secrets: | + secret/data/products/infrastructure-experience/ci/common AWS_ACCESS_KEY; + secret/data/products/infrastructure-experience/ci/common AWS_SECRET_KEY; + + # Official action does not support profiles + - name: Add profile credentials to ~/.aws/credentials + run: | + aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set region eu-west-2 --profile ${{ env.AWS_PROFILE }} + + - name: Install Cloud Nuke + run: | + wget https://github.com/gruntwork-io/cloud-nuke/releases/download/v0.33.0/cloud-nuke_linux_amd64 + chmod +x cloud-nuke_linux_amd64 + + # This is likely to fail, therefore we ignore the error + # We're ignoring ec2_dhcp_option as they couldn't be deleted + # cloudtrail is managed by IT and can't be deleted either + - name: Run Cloud Nuke + timeout-minutes: 45 + env: + DISABLE_TELEMETRY: "true" + run: | + ./cloud-nuke_linux_amd64 aws \ + --region eu-west-2 \ + --region eu-west-3 \ + --force \ + --newer-than 4h \ + --exclude-resource-type ec2_dhcp_option \ + --exclude-resource-type cloudtrail || true + + # Following will delete global resources and things that cloud-nuke does not support + - name: Delete additional AWS resources + timeout-minutes: 15 + run: .github/workflows/scripts/aws_cleanup.sh + + # The second run should remove the remaining resources (VPCs) and fail if there's anything left + - name: Run Cloud Nuke + timeout-minutes: 45 + env: + DISABLE_TELEMETRY: "true" + run: | + ./cloud-nuke_linux_amd64 aws \ + --region eu-west-2 \ + --region eu-west-3 \ + --force \ + --newer-than 4h \ + --exclude-resource-type ec2_dhcp_option \ + --exclude-resource-type cloudtrail diff --git a/test/src/fixtures/fixtures.default.eks.tfvars b/test/src/fixtures/fixtures.default.eks.tfvars index c8258284..61614898 100644 --- a/test/src/fixtures/fixtures.default.eks.tfvars +++ b/test/src/fixtures/fixtures.default.eks.tfvars @@ -1,2 +1,3 @@ cluster_service_ipv4_cidr = "10.190.0.0/16" cluster_node_ipv4_cidr = "10.192.0.0/16" +np_instance_types = ["t2.medium"] diff --git a/test/src/fixtures/postgres-client.bak.yml b/test/src/fixtures/postgres-client.bak.yml new file mode 100644 index 00000000..24f37b6c --- /dev/null +++ b/test/src/fixtures/postgres-client.bak.yml @@ -0,0 +1,74 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: postgres-client + labels: + app: postgres-client +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: postgres-client + image: ubuntu:latest + command: + - sh + - "-c" + - | + /bin/bash <<'EOF' + set -o pipefail && \ + apt-get update && \ + apt-get install -y python3 python3-pip build-essential postgresql-client && \ + echo "Creating IRSA db user" && \ + mkdir -p /tmp/scripts && cp /scripts/create_aurora_pg_db.sh /tmp/scripts/create_aurora_pg_db.sh && chmod +x /tmp/scripts/create_aurora_pg_db.sh && /tmp/scripts/create_aurora_pg_db.sh && \ + echo "Testing connection using IRSA" && \ + python3 -m pip install awscli && \ + AWS_PG_PASSWORD=$(aws rds generate-db-auth-token --hostname $AURORA_ENDPOINT --port $AURORA_PORT --region $AWS_REGION --username $AURORA_USERNAME_IRSA) && \ + psql -h $AURORA_ENDPOINT -p $AURORA_PORT "dbname=$AURORA_DB_NAME user=$AURORA_USERNAME_IRSA password=$AWS_PG_PASSWORD" -c 'SELECT version();' + + EOF + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + env: + - name: AURORA_ENDPOINT + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_endpoint + - name: AURORA_USERNAME + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_username + - name: AURORA_USERNAME_IRSA + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_username_irsa + - name: AURORA_PASSWORD + valueFrom: + secretKeyRef: + name: aurora-secret + key: aurora_password + - name: AURORA_PORT + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_port + - name: AWS_REGION + valueFrom: + configMapKeyRef: + name: aurora-config + key: aws_region + - name: AURORA_DB_NAME + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_db_name + volumes: + - name: scripts + configMap: + name: postgres-scripts diff --git a/test/src/fixtures/postgres-client.yml b/test/src/fixtures/postgres-client.yml index 14ad6eea..95edbb70 100644 --- a/test/src/fixtures/postgres-client.yml +++ b/test/src/fixtures/postgres-client.yml @@ -5,26 +5,29 @@ metadata: labels: app: postgres-client spec: + backoffLimit: 0 template: spec: restartPolicy: Never containers: - name: postgres-client - image: ubuntu:latest # TODO: create the irsa user on the db (see kc) + image: ubuntu:latest command: - sh - "-c" - | /bin/bash <<'EOF' - - apt-get update && - apt-get install -y python3 python3-pip build-essential postgresql-client && - python3 -m pip install awscli && - aws rds generate-db-auth-token --hostname $AURORA_ENDPOINT --port $AURORA_PORT --region $AWS_REGION --username $AURORA_USERNAME - AWS_PG_PASSWORD=$(aws rds generate-db-auth-token --hostname $AURORA_ENDPOINT --port $AURORA_PORT --region $AWS_REGION --username $AURORA_USERNAME) && - psql -h $AURORA_ENDPOINT -p $AURORA_PORT "dbname=$AURORA_DB_NAME user=$AURORA_USERNAME password=$AWS_PG_PASSWORD" -c 'SELECT version();' + set -o pipefail && \ + apt-get update && \ + apt-get install -y python3 python3-pip build-essential postgresql-client && \ + echo "Creating IRSA db user" && \ + mkdir -p /tmp/scripts && cp /scripts/create_aurora_pg_db.sh /tmp/scripts/create_aurora_pg_db.sh && chmod +x /tmp/scripts/create_aurora_pg_db.sh && /tmp/scripts/create_aurora_pg_db.sh EOF + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true env: - name: AURORA_ENDPOINT valueFrom: @@ -36,6 +39,16 @@ spec: configMapKeyRef: name: aurora-config key: aurora_username + - name: AURORA_USERNAME_IRSA + valueFrom: + configMapKeyRef: + name: aurora-config + key: aurora_username_irsa + - name: AURORA_PASSWORD + valueFrom: + secretKeyRef: + name: aurora-secret + key: aurora_password - name: AURORA_PORT valueFrom: configMapKeyRef: @@ -51,3 +64,7 @@ spec: configMapKeyRef: name: aurora-config key: aurora_db_name + volumes: + - name: scripts + configMap: + name: postgres-scripts diff --git a/test/src/fixtures/scripts/create_aurora_pg_db.sh b/test/src/fixtures/scripts/create_aurora_pg_db.sh new file mode 100644 index 00000000..beb67b16 --- /dev/null +++ b/test/src/fixtures/scripts/create_aurora_pg_db.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# see https://github.com/camunda/infra-core/tree/opensearch-cluster/camunda-opensearch#user-setup +psql -h $AURORA_ENDPOINT -p $AURORA_PORT "dbname=$AURORA_DB_NAME user=$AURORA_USERNAME password=$AURORA_PASSWORD" \ + -c "CREATE USER \"${AURORA_USERNAME_IRSA}\" WITH LOGIN;" \ + -c "GRANT rds_iam TO \"${AURORA_USERNAME_IRSA}\";" \ + -c "GRANT rds_superuser TO \"${AURORA_USERNAME_IRSA}\";" \ + -c "GRANT ALL PRIVILEGES ON DATABASE \"${AURORA_DB_NAME}\" TO \"${AURORA_USERNAME_IRSA}\";" \ + -c "SELECT aurora_version();" \ + -c "SELECT version();" -c "\du" diff --git a/test/src/go.mod b/test/src/go.mod index fd0f38bd..41ef451c 100644 --- a/test/src/go.mod +++ b/test/src/go.mod @@ -3,12 +3,13 @@ module github.com/camunda/camunda-tf-eks-module go 1.22.0 require ( - github.com/aws/aws-sdk-go-v2 v1.26.0 + github.com/aws/aws-sdk-go-v2 v1.26.1 github.com/aws/aws-sdk-go-v2/config v1.27.9 github.com/aws/aws-sdk-go-v2/service/ec2 v1.154.0 github.com/aws/aws-sdk-go-v2/service/eks v1.41.2 github.com/aws/aws-sdk-go-v2/service/iam v1.31.3 github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 + github.com/aws/aws-sdk-go-v2/service/rds v1.76.1 github.com/gruntwork-io/terratest v0.46.13 github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.27.0 @@ -29,15 +30,15 @@ require ( github.com/aws/aws-sdk-go v1.44.332 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect - github.com/aws/smithy-go v1.20.1 // indirect + github.com/aws/smithy-go v1.20.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect diff --git a/test/src/go.sum b/test/src/go.sum index 738d03de..0a115fbb 100644 --- a/test/src/go.sum +++ b/test/src/go.sum @@ -206,18 +206,18 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.332 h1:Ze+98F41+LxoJUdsisAFThV+0yYYLYw17/Vt0++nFYM= github.com/aws/aws-sdk-go v1.44.332/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA= -github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= +github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= +github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/config v1.27.9 h1:gRx/NwpNEFSk+yQlgmk1bmxxvQ5TyJ76CWXs9XScTqg= github.com/aws/aws-sdk-go-v2/config v1.27.9/go.mod h1:dK1FQfpwpql83kbD873E9vz4FyAxuJtR22wzoXn3qq0= github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMNQVNtNdUqf6cItao= github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/service/ec2 v1.154.0 h1:+OJ9EhHaqjtA4YTTbxxLxMffrWuGWh0qMaBmGJTLSSg= @@ -226,20 +226,22 @@ github.com/aws/aws-sdk-go-v2/service/eks v1.41.2 h1:0X5g5H8YyW9QVtlp6j+ZGHl/h0ZS github.com/aws/aws-sdk-go-v2/service/eks v1.41.2/go.mod h1:T2MBMUUCoSEvHuKPplubyQJbWNghbHhx3ToJpLoipDs= github.com/aws/aws-sdk-go-v2/service/iam v1.31.3 h1:cJn9Snros9WmDA7/qCCN7jSkowcu1CqnwhFpv4ipHEE= github.com/aws/aws-sdk-go-v2/service/iam v1.31.3/go.mod h1:+nAQlxsBxPFf6GrL93lvCuv5PxSTX3GO0RYrURyzl/Q= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 h1:yS0JkEdV6h9JOo8sy2JSpjX+i7vsKifU8SIeHrqiDhU= github.com/aws/aws-sdk-go-v2/service/kms v1.30.0/go.mod h1:+I8VUUSVD4p5ISQtzpgSva4I8cJ4SQ4b1dcBcof7O+g= +github.com/aws/aws-sdk-go-v2/service/rds v1.76.1 h1:6NeDO4UyHun2N5Lmkv2yW1sNblRLRjq3j6Azv7kUyuM= +github.com/aws/aws-sdk-go-v2/service/rds v1.76.1/go.mod h1:Rw15qGaGWu3jO0dOz7JyvdOEjgae//YrJxVWLYGynvg= github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8= github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3/go.mod h1:b+qdhjnxj8GSR6t5YfphOffeoQSQ1KmpoVVuBn+PWxs= github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg= github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0= -github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= -github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index 6b2d2c63..9a645eae 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/camunda/camunda-tf-eks-module/utils" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/random" @@ -21,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" + "os" "os/exec" "strings" "testing" @@ -80,6 +82,7 @@ func TestCustomEKSAndRDS(t *testing.T) { // list your services here eksSvc := eks.NewFromConfig(sess) + rdsSvc := rds.NewFromConfig(sess) inputEKS := &eks.DescribeClusterInput{ Name: aws.String(clusterName), @@ -98,15 +101,17 @@ func TestCustomEKSAndRDS(t *testing.T) { auroraUsername := "myuser" auroraPassword := "mypassword123secure" + auroraDatabase := "camunda" varsConfigAurora := map[string]interface{}{ - "username": auroraUsername, - "password": auroraPassword, - "cluster_name": fmt.Sprintf("postgres-%s", clusterSuffix), - "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, - "vpc_id": *result.Cluster.ResourcesVpcConfig.VpcId, - "cidr_blocks": append(publicBlocks, privateBlocks...), - "iam_auth_enabled": true, + "username": auroraUsername, + "password": auroraPassword, + "default_database_name": auroraDatabase, + "cluster_name": fmt.Sprintf("postgres-%s", clusterSuffix), + "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, + "vpc_id": *result.Cluster.ResourcesVpcConfig.VpcId, + "cidr_blocks": append(publicBlocks, privateBlocks...), + "iam_auth_enabled": true, } terraformOptionsRDS := SpawnAurora(t, sugar, varsConfigAurora) @@ -127,7 +132,7 @@ func TestCustomEKSAndRDS(t *testing.T) { pgKubeCtlOptions := k8s.NewKubectlOptions("", "kubeconfig", namespace) _, errFindNamespace := k8s.GetNamespaceE(t, pgKubeCtlOptions, namespace) if errFindNamespace != nil { - if errors.IsNotFound(err) { + if errors.IsNotFound(errFindNamespace) { k8s.CreateNamespace(t, pgKubeCtlOptions, namespace) } else { require.NoError(t, errFindNamespace) @@ -142,19 +147,61 @@ func TestCustomEKSAndRDS(t *testing.T) { Namespace: namespace, }, Data: map[string]string{ - "aurora_endpoint": auroraEndpoint, - "aurora_username": auroraUsername, - "aurora_port": "5432", - "aws_region": region, - "aurora_db_name": auroraEndpoint, + "aurora_endpoint": auroraEndpoint, + "aurora_username": auroraUsername, + "aurora_username_irsa": fmt.Sprintf("%s-irsa", auroraUsername), + "aurora_port": "5432", + "aws_region": region, + "aurora_db_name": auroraDatabase, }, } + + err = kubeClient.CoreV1().ConfigMaps(namespace).Delete(context.Background(), configMapPostgres.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + require.NoError(t, err) + } _, err = kubeClient.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMapPostgres, metav1.CreateOptions{}) - if err != nil { - if !errors.IsAlreadyExists(err) { - require.NoError(t, err) - } + k8s.WaitUntilConfigMapAvailable(t, pgKubeCtlOptions, configMapPostgres.Name, 6, 10*time.Second) + + // create the secret + secretPostgres := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aurora-secret", + Namespace: namespace, + }, + StringData: map[string]string{ + "aurora_password": auroraPassword, + }, + } + err = kubeClient.CoreV1().Secrets(namespace).Delete(context.Background(), configMapPostgres.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + require.NoError(t, err) + } + _, err = kubeClient.CoreV1().Secrets(namespace).Create(context.Background(), secretPostgres, metav1.CreateOptions{}) + k8s.WaitUntilSecretAvailable(t, pgKubeCtlOptions, secretPostgres.Name, 6, 10*time.Second) + + // add the scripts + scriptPath := "../../test/src/fixtures/scripts/create_aurora_pg_db.sh" + scriptContent, err := os.ReadFile(scriptPath) + require.NoError(t, err) + + configMapScript := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-scripts", + Namespace: namespace, + }, + Data: map[string]string{ + "create_aurora_pg_db.sh": string(scriptContent), + }, + } + + err = kubeClient.CoreV1().ConfigMaps(namespace).Delete(context.Background(), configMapScript.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + require.NoError(t, err) } + _, err = kubeClient.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMapScript, metav1.CreateOptions{}) + k8s.WaitUntilConfigMapAvailable(t, pgKubeCtlOptions, configMapScript.Name, 6, 10*time.Second) + // cleanup existing jobs jobListOptions := metav1.ListOptions{LabelSelector: "app=postgres-client"} existingJobs := k8s.ListJobs(t, pgKubeCtlOptions, jobListOptions) @@ -168,7 +215,38 @@ func TestCustomEKSAndRDS(t *testing.T) { errJob := utils.WaitForJobCompletion(kubeClient, namespace, "postgres-client", 5*time.Minute, jobListOptions) require.NoError(t, errJob) - TearsDown(t, sugar) + // TODO: apply https://kubedemy.io/aws-eks-part-13-setup-iam-roles-for-service-accounts-irsa to setup iam + + // RDS test that cluster parameters are applied as expected + describeDBInput := &rds.DescribeDBInstancesInput{ + DBInstanceIdentifier: varsConfigAurora["cluster_name"].(*string), + } + + describeDBOutput, err := rdsSvc.DescribeDBInstances(context.TODO(), describeDBInput) + require.NoError(t, err) + + // todo : finish the tests + + assert.Equal(t, varsConfigAurora["iam_auth_enabled"].(bool), describeDBOutput.DBInstances[0].IAMDatabaseAuthenticationEnabled) + // EKS test that cluster parameters are applied as expected + + // count nb of nodes + nodes, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + require.NoError(t, err) + assert.Equal(t, expectedCapacity, len(nodes.Items)) + + // verifies for each node, the flavor and the region + expectedInstanceType := "t2.medium" + for _, node := range nodes.Items { + regionNode, _ := node.Labels["failure-domain.beta.kubernetes.io/region"] + instanceType, _ := node.Annotations["node.kubernetes.io/instance-type"] + for _, addr := range node.Status.Addresses { + if addr.Type == "InternalIP" { + assert.Equal(t, region, regionNode) + assert.Equal(t, expectedInstanceType, instanceType) + } + } + } } // TestUpgradeEKS starts from a version of EKS, deploy a simple chart, upgrade the cluster From 7eae05705bf810fd95e7244386bdd753f6f6b287 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:21:24 +0200 Subject: [PATCH 10/93] fix string --- test/src/modules_eks_cluster_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index 9a645eae..d759aa3d 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -219,7 +219,7 @@ func TestCustomEKSAndRDS(t *testing.T) { // RDS test that cluster parameters are applied as expected describeDBInput := &rds.DescribeDBInstancesInput{ - DBInstanceIdentifier: varsConfigAurora["cluster_name"].(*string), + DBInstanceIdentifier: aws.String(varsConfigAurora["cluster_name"].(string)), } describeDBOutput, err := rdsSvc.DescribeDBInstances(context.TODO(), describeDBInput) From 43c60a0436bc4b7effb878996f27c5b5a4f55b96 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:07:41 +0200 Subject: [PATCH 11/93] completed aurora db test --- test/src/modules_eks_cluster_test.go | 93 ++++++++++++-------- test/src/utils/aws.go | 26 ++++++ test/src/utils/kube.go | 102 ++++++++++++++++++++++ test/src/utils/utils.go | 121 --------------------------- 4 files changed, 184 insertions(+), 158 deletions(-) create mode 100644 test/src/utils/aws.go delete mode 100644 test/src/utils/utils.go diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index d759aa3d..21fa3504 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -23,7 +23,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" "os" - "os/exec" "strings" "testing" "time" @@ -88,11 +87,11 @@ func TestCustomEKSAndRDS(t *testing.T) { Name: aws.String(clusterName), } - result, err := eksSvc.DescribeCluster(context.TODO(), inputEKS) + result, err := eksSvc.DescribeCluster(context.Background(), inputEKS) assert.NoError(t, err) sugar.Infow("Waiting for worker nodes to join the EKS cluster") - errClusterReady := utils.WaitUntilClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) + errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) require.NoError(t, errClusterReady) // Spawn RDS within the EKS VPC/subnet @@ -118,16 +117,13 @@ func TestCustomEKSAndRDS(t *testing.T) { auroraEndpoint := terraform.Output(t, terraformOptionsRDS, "aurora_endpoint") assert.NotEmpty(t, auroraEndpoint) - // Launch a pod on the cluster and test the RDS - kubeClient, err := utils.NewClientSet(result.Cluster) + // Test of the RDS connection is performed by launching a pod on the cluster and test the pg connection + kubeClient, err := utils.NewKubeClientSet(result.Cluster) require.NoError(t, err) - // create kubeconfig - cmd := exec.Command("aws", "eks", "--region", region, "update-kubeconfig", "--name", clusterName, "--profile", utils.GetAwsProfile(), "--kubeconfig", "kubeconfig") - _, errCmdKubeProfile := cmd.Output() - require.NoError(t, errCmdKubeProfile) + utils.GenerateKubeConfigFromAWS(t, region, clusterName, utils.GetAwsProfile(), "kubeconfig") - // create the configmap + // ensure postgres-client namespace exists namespace := "postgres-client" pgKubeCtlOptions := k8s.NewKubectlOptions("", "kubeconfig", namespace) _, errFindNamespace := k8s.GetNamespaceE(t, pgKubeCtlOptions, namespace) @@ -139,8 +135,7 @@ func TestCustomEKSAndRDS(t *testing.T) { } } - // todo: https: //github.com/camunda/c8-multi-region/blob/main/test/internal/helpers/aws/helpers.go#L242 - + // deploy the postgres-client ConfigMap configMapPostgres := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "aurora-config", @@ -163,7 +158,7 @@ func TestCustomEKSAndRDS(t *testing.T) { _, err = kubeClient.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMapPostgres, metav1.CreateOptions{}) k8s.WaitUntilConfigMapAvailable(t, pgKubeCtlOptions, configMapPostgres.Name, 6, 10*time.Second) - // create the secret + // create the secret for aurora pg password secretPostgres := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "aurora-secret", @@ -180,7 +175,7 @@ func TestCustomEKSAndRDS(t *testing.T) { _, err = kubeClient.CoreV1().Secrets(namespace).Create(context.Background(), secretPostgres, metav1.CreateOptions{}) k8s.WaitUntilSecretAvailable(t, pgKubeCtlOptions, secretPostgres.Name, 6, 10*time.Second) - // add the scripts + // add the scripts as a ConfigMap scriptPath := "../../test/src/fixtures/scripts/create_aurora_pg_db.sh" scriptContent, err := os.ReadFile(scriptPath) require.NoError(t, err) @@ -206,29 +201,55 @@ func TestCustomEKSAndRDS(t *testing.T) { jobListOptions := metav1.ListOptions{LabelSelector: "app=postgres-client"} existingJobs := k8s.ListJobs(t, pgKubeCtlOptions, jobListOptions) for _, job := range existingJobs { - err := kubeClient.BatchV1().Jobs(namespace).Delete(context.TODO(), job.Name, metav1.DeleteOptions{}) + err := kubeClient.BatchV1().Jobs(namespace).Delete(context.Background(), job.Name, metav1.DeleteOptions{}) assert.NoError(t, err) } + // deploy the postgres-client Job to test the connection k8s.KubectlApply(t, pgKubeCtlOptions, "../../test/src/fixtures/postgres-client.yml") - errJob := utils.WaitForJobCompletion(kubeClient, namespace, "postgres-client", 5*time.Minute, jobListOptions) require.NoError(t, errJob) - // TODO: apply https://kubedemy.io/aws-eks-part-13-setup-iam-roles-for-service-accounts-irsa to setup iam + // TODO: test IRSA apply https://kubedemy.io/aws-eks-part-13-setup-iam-roles-for-service-accounts-irsa to setup iam - // RDS test that cluster parameters are applied as expected - describeDBInput := &rds.DescribeDBInstancesInput{ - DBInstanceIdentifier: aws.String(varsConfigAurora["cluster_name"].(string)), + // Retrieve RDS information + describeDBClusterInput := &rds.DescribeDBClustersInput{ + DBClusterIdentifier: aws.String(varsConfigAurora["cluster_name"].(string)), } + describeDBClusterOutput, err := rdsSvc.DescribeDBClusters(context.Background(), describeDBClusterInput) + require.NoError(t, err) - describeDBOutput, err := rdsSvc.DescribeDBInstances(context.TODO(), describeDBInput) + expectedRDSAZ := []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"} + assert.Equal(t, varsConfigAurora["iam_auth_enabled"].(bool), *describeDBClusterOutput.DBClusters[0].IAMDatabaseAuthenticationEnabled) + assert.Equal(t, varsConfigAurora["username"].(string), *describeDBClusterOutput.DBClusters[0].MasterUsername) + assert.Equal(t, auroraDatabase, *describeDBClusterOutput.DBClusters[0].DatabaseName) + assert.Equal(t, int32(5432), *describeDBClusterOutput.DBClusters[0].Port) + assert.Equal(t, "15.4", *describeDBClusterOutput.DBClusters[0].EngineVersion) + assert.ElementsMatch(t, expectedRDSAZ, describeDBClusterOutput.DBClusters[0].AvailabilityZones) + assert.Equal(t, varsConfigAurora["cluster_name"].(string), *describeDBClusterOutput.DBClusters[0].DBClusterIdentifier) + + // Some of the tests are performed on the first instance of the cluster + describeDBInstanceInput := &rds.DescribeDBInstancesInput{ + DBInstanceIdentifier: describeDBClusterOutput.DBClusters[0].DBClusterMembers[0].DBInstanceIdentifier, + } + describeDBInstanceOutput, err := rdsSvc.DescribeDBInstances(context.Background(), describeDBInstanceInput) require.NoError(t, err) - // todo : finish the tests + assert.Equal(t, "db.t3.medium", *describeDBInstanceOutput.DBInstances[0].DBInstanceClass) + assert.Equal(t, true, *describeDBInstanceOutput.DBInstances[0].AutoMinorVersionUpgrade) + assert.Equal(t, "aurora-postgresql", *describeDBInstanceOutput.DBInstances[0].Engine) + assert.Equal(t, "rds-ca-2019", *describeDBInstanceOutput.DBInstances[0].CertificateDetails.CAIdentifier) + assert.Equal(t, varsConfigAurora["vpc_id"].(string), *describeDBInstanceOutput.DBInstances[0].DBSubnetGroup.VpcId) + assert.Contains(t, *describeDBInstanceOutput.DBInstances[0].AvailabilityZone, region) + + // construct the subnet ids + actualSubnetIds := make([]string, len(describeDBInstanceOutput.DBInstances[0].DBSubnetGroup.Subnets)) + for id, subnet := range describeDBInstanceOutput.DBInstances[0].DBSubnetGroup.Subnets { + actualSubnetIds[id] = *subnet.SubnetIdentifier + } + assert.ElementsMatch(t, varsConfigAurora["subnet_ids"].([]string), actualSubnetIds) - assert.Equal(t, varsConfigAurora["iam_auth_enabled"].(bool), describeDBOutput.DBInstances[0].IAMDatabaseAuthenticationEnabled) - // EKS test that cluster parameters are applied as expected + // EKS test that custom cluster parameters are applied as expected // count nb of nodes nodes, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) @@ -239,7 +260,7 @@ func TestCustomEKSAndRDS(t *testing.T) { expectedInstanceType := "t2.medium" for _, node := range nodes.Items { regionNode, _ := node.Labels["failure-domain.beta.kubernetes.io/region"] - instanceType, _ := node.Annotations["node.kubernetes.io/instance-type"] + instanceType, _ := node.Labels["node.kubernetes.io/instance-type"] for _, addr := range node.Status.Addresses { if addr.Type == "InternalIP" { assert.Equal(t, region, regionNode) @@ -287,7 +308,7 @@ func TestUpgradeEKS(t *testing.T) { // SpawnEKS spawns a new EKS Cluster from a default fixture file func SpawnEKS(t *testing.T, sugar *zap.SugaredLogger, varsConfig map[string]interface{}) *terraform.Options { - sugar.Infow("TF vars", varsConfig) + sugar.Infow("TF vars", "vars", varsConfig) terraformOptions := &terraform.Options{ // The path to where our Terraform code is located @@ -318,7 +339,7 @@ func SpawnEKS(t *testing.T, sugar *zap.SugaredLogger, varsConfig map[string]inte // SpawnAurora spawns a new Aurora RDS from a default fixture file func SpawnAurora(t *testing.T, sugar *zap.SugaredLogger, varsConfig map[string]interface{}) *terraform.Options { - sugar.Infow("TF vars", varsConfig) + sugar.Infow("TF vars", "vars", varsConfig) terraformOptions := &terraform.Options{ // The path to where our Terraform code is located @@ -354,7 +375,7 @@ func TearsDown(t *testing.T, sugar *zap.SugaredLogger) { // baseChecksEKS checks the defaults of an EKS cluster func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *terraform.Options, expectedNodesCount uint64) { clusterName := terraformOptions.Vars["name"].(string) - sugar.Infow("Testing status of the EKS cluster", clusterName) + sugar.Infow("Testing status of the EKS cluster", "clusterName", clusterName) // Do some basic not empty tests on outputs assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_endpoint")) @@ -395,12 +416,12 @@ func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *ter Name: aws.String(clusterName), } - result, err := eksSvc.DescribeCluster(context.TODO(), inputEKS) + result, err := eksSvc.DescribeCluster(context.Background(), inputEKS) assert.NoError(t, err) // Wait for the worker nodes to join the cluster sugar.Infow("Waiting for worker nodes to join the EKS cluster") - errClusterReady := utils.WaitUntilClusterIsReady(result.Cluster, 5*time.Minute, expectedNodesCount) + errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, expectedNodesCount) require.NoError(t, errClusterReady) // Verify list of addons installed on the EKS @@ -408,7 +429,7 @@ func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *ter inputDescribeAddons := &eks.ListAddonsInput{ ClusterName: aws.String(clusterName), } - outputEKSAddons, errEKSAddons := eksSvc.ListAddons(context.TODO(), inputDescribeAddons) + outputEKSAddons, errEKSAddons := eksSvc.ListAddons(context.Background(), inputDescribeAddons) require.NoError(t, errEKSAddons) // perform the diff @@ -433,7 +454,7 @@ func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *ter RoleName: aws.String(roleName), } - _, err := iamSvc.GetRole(context.TODO(), input) + _, err := iamSvc.GetRole(context.Background(), input) assert.NoErrorf(t, err, "Failed to get IAM EKS role %s", roleName) } @@ -450,7 +471,7 @@ func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *ter }, } - outputVPC, errVPC := ec2Svc.DescribeVpcs(context.TODO(), inputVPC) + outputVPC, errVPC := ec2Svc.DescribeVpcs(context.Background(), inputVPC) require.NoError(t, errVPC) assert.Equal(t, len(outputVPC.Vpcs), 1) @@ -458,13 +479,13 @@ func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *ter // key keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) inputKMS := &kms.ListKeysInput{} - outputKMSList, errKMSList := kmsSvc.ListKeys(context.TODO(), inputKMS) + outputKMSList, errKMSList := kmsSvc.ListKeys(context.Background(), inputKMS) assert.NoError(t, errKMSList) // Check if the key corresponding to the description exists keyFound := false for _, key := range outputKMSList.Keys { - keyDetails, errKey := kmsSvc.DescribeKey(context.TODO(), &kms.DescribeKeyInput{ + keyDetails, errKey := kmsSvc.DescribeKey(context.Background(), &kms.DescribeKeyInput{ KeyId: key.KeyId, }) require.NoErrorf(t, errKey, "Failed to describe key %s", *key.KeyId) @@ -478,5 +499,3 @@ func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *ter } // todo: test upgrade path -// todo: test auroradb integration -// todo: test inputs diff --git a/test/src/utils/aws.go b/test/src/utils/aws.go new file mode 100644 index 00000000..f29c6741 --- /dev/null +++ b/test/src/utils/aws.go @@ -0,0 +1,26 @@ +package utils + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" +) + +func GetAwsProfile() string { + return GetEnv("AWS_PROFILE", GetEnv("AWS_DEFAULT_PROFILE", "infex")) +} + +func GetAwsRegion() string { + return GetEnv("AWS_REGION", "eu-central-1") +} + +// GetAwsClient returns an aws.Config client from the env variables `AWS_PROFILE` and `AWS_REGION` +func GetAwsClient() (aws.Config, error) { + awsProfile := GetAwsProfile() + region := GetAwsRegion() + + return config.LoadDefaultConfig(context.TODO(), + config.WithRegion(region), + config.WithSharedConfigProfile(awsProfile), + ) +} diff --git a/test/src/utils/kube.go b/test/src/utils/kube.go index ce406896..edf0ef6a 100644 --- a/test/src/utils/kube.go +++ b/test/src/utils/kube.go @@ -2,9 +2,21 @@ package utils import ( "context" + "encoding/base64" + "errors" "fmt" + "github.com/aws/aws-sdk-go-v2/service/eks/types" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "os/exec" + "sigs.k8s.io/aws-iam-authenticator/pkg/token" + "sync/atomic" + "testing" "time" batchv1 "k8s.io/api/batch/v1" @@ -35,6 +47,7 @@ func WaitForJobCompletion(clientset *kubernetes.Clientset, namespace, jobName st if job.Name == jobName { // Check if the job has completed if job.Status.CompletionTime != nil { + // The job has completed, check if it succeeded // The job has completed, check if it succeeded if job.Status.Succeeded == 1 { return nil // Job completed successfully @@ -48,3 +61,92 @@ func WaitForJobCompletion(clientset *kubernetes.Clientset, namespace, jobName st } } } + +func GenerateKubeConfigFromAWS(t *testing.T, region, clusterName, awsProfile, configOutputPath string) { + cmd := exec.Command("aws", "eks", "--region", region, "update-kubeconfig", "--name", clusterName, "--profile", awsProfile, "--kubeconfig", configOutputPath) + _, errCmdKubeProfile := cmd.Output() + require.NoError(t, errCmdKubeProfile) +} + +// WaitUntilKubeClusterIsReady waits until the kube cluster is read or returns an error +func WaitUntilKubeClusterIsReady(cluster *types.Cluster, timeout time.Duration, expectedNodesCount uint64) error { + // https://github.com/kubernetes/client-go + // https://www.rushtehrani.com/post/using-kubernetes-api + // https://rancher.com/using-kubernetes-api-go-kubecon-2017-session-recap + // https://gianarb.it/blog/kubernetes-shared-informer + // https://stackoverflow.com/questions/60547409/unable-to-obtain-kubeconfig-of-an-aws-eks-cluster-in-go-code/60573982#60573982 + + clientSet, err := NewKubeClientSet(cluster) + if err != nil { + return err + } + + factory := informers.NewSharedInformerFactory(clientSet, 0) + informer := factory.Core().V1().Nodes().Informer() + stopChannel := make(chan struct{}) + var countOfWorkerNodes uint64 = 0 + + _, errEventHandler := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + node := obj.(*corev1.Node) + fmt.Printf("Worker Node %s has joined the EKS cluster at %s\n", node.Name, node.CreationTimestamp) + atomic.AddUint64(&countOfWorkerNodes, 1) + if countOfWorkerNodes >= expectedNodesCount { + stopChannel <- struct{}{} // send close signal + } + }, + }) + if errEventHandler != nil { + return errEventHandler + } + + go informer.Run(stopChannel) + go func() { + // wait to receive a signal to close the channel + <-stopChannel + close(stopChannel) + }() + + select { + case <-stopChannel: + msg := "All worker nodes have joined the Kube cluster" + fmt.Println(msg) + case <-time.After(timeout): + msg := "Not all worker nodes have joined the Kube cluster" + fmt.Println(msg) + return errors.New(msg) + } + return nil +} + +// NewKubeClientSet generate a kubernetes.Clientset from an EKS Cluster +func NewKubeClientSet(cluster *types.Cluster) (*kubernetes.Clientset, error) { + gen, err := token.NewGenerator(true, false) + if err != nil { + return nil, err + } + opts := &token.GetTokenOptions{ + ClusterID: *cluster.Name, + } + tok, err := gen.GetWithOptions(opts) + if err != nil { + return nil, err + } + ca, err := base64.StdEncoding.DecodeString(*cluster.CertificateAuthority.Data) + if err != nil { + return nil, err + } + clientSet, err := kubernetes.NewForConfig( + &rest.Config{ + Host: *cluster.Endpoint, + BearerToken: tok.Token, + TLSClientConfig: rest.TLSClientConfig{ + CAData: ca, + }, + }, + ) + if err != nil { + return nil, err + } + return clientSet, nil +} diff --git a/test/src/utils/utils.go b/test/src/utils/utils.go deleted file mode 100644 index 53681478..00000000 --- a/test/src/utils/utils.go +++ /dev/null @@ -1,121 +0,0 @@ -package utils - -import ( - "context" - "encoding/base64" - "errors" - "fmt" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/eks/types" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - "sigs.k8s.io/aws-iam-authenticator/pkg/token" - "sync/atomic" - "time" -) - -// NewClientSet generate a kubernetes.Clientset from an aws Cluster -func NewClientSet(cluster *types.Cluster) (*kubernetes.Clientset, error) { - gen, err := token.NewGenerator(true, false) - if err != nil { - return nil, err - } - opts := &token.GetTokenOptions{ - ClusterID: *cluster.Name, - } - tok, err := gen.GetWithOptions(opts) - if err != nil { - return nil, err - } - ca, err := base64.StdEncoding.DecodeString(*cluster.CertificateAuthority.Data) - if err != nil { - return nil, err - } - clientSet, err := kubernetes.NewForConfig( - &rest.Config{ - Host: *cluster.Endpoint, - BearerToken: tok.Token, - TLSClientConfig: rest.TLSClientConfig{ - CAData: ca, - }, - }, - ) - if err != nil { - return nil, err - } - return clientSet, nil -} - -func GetAwsProfile() string { - return GetEnv("AWS_PROFILE", GetEnv("AWS_DEFAULT_PROFILE", "infex")) -} - -func GetAwsRegion() string { - return GetEnv("AWS_REGION", "eu-central-1") -} - -// GetAwsClient returns an aws.Config client from the env variables `AWS_PROFILE` and `AWS_REGION` -func GetAwsClient() (aws.Config, error) { - awsProfile := GetAwsProfile() - region := GetAwsRegion() - - return config.LoadDefaultConfig(context.TODO(), - config.WithRegion(region), - config.WithSharedConfigProfile(awsProfile), - ) -} - -// WaitUntilClusterIsReady waits until the kube cluster is read or returns an error -func WaitUntilClusterIsReady(cluster *types.Cluster, timeout time.Duration, expectedNodesCount uint64) error { - // https://github.com/kubernetes/client-go - // https://www.rushtehrani.com/post/using-kubernetes-api - // https://rancher.com/using-kubernetes-api-go-kubecon-2017-session-recap - // https://gianarb.it/blog/kubernetes-shared-informer - // https://stackoverflow.com/questions/60547409/unable-to-obtain-kubeconfig-of-an-aws-eks-cluster-in-go-code/60573982#60573982 - - clientSet, err := NewClientSet(cluster) - if err != nil { - return err - } - - factory := informers.NewSharedInformerFactory(clientSet, 0) - informer := factory.Core().V1().Nodes().Informer() - stopChannel := make(chan struct{}) - var countOfWorkerNodes uint64 = 0 - - _, errEventHandler := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - node := obj.(*corev1.Node) - fmt.Printf("Worker Node %s has joined the EKS cluster at %s\n", node.Name, node.CreationTimestamp) - atomic.AddUint64(&countOfWorkerNodes, 1) - if countOfWorkerNodes >= expectedNodesCount { - stopChannel <- struct{}{} // send close signal - } - }, - }) - if errEventHandler != nil { - return errEventHandler - } - - go informer.Run(stopChannel) - go func() { - // wait to receive a signal to close the channel - <-stopChannel - close(stopChannel) - }() - - select { - case <-stopChannel: - msg := "All worker nodes have joined the Kube cluster" - fmt.Println(msg) - case <-time.After(timeout): - msg := "Not all worker nodes have joined the Kube cluster" - fmt.Println(msg) - return errors.New(msg) - } - return nil -} From a59b2069e827ebf6c78e3dbec8e1b6ed6717c584 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:26:11 +0200 Subject: [PATCH 12/93] wip upgrade tests --- test/README.md | 11 ++- test/src/fixtures/postgres-client.bak.yml | 2 + test/src/fixtures/whoami-deployment.yml | 40 ++++++++ test/src/modules_eks_cluster_test.go | 113 ++++++++++++++++++---- test/src/utils/kube.go | 16 ++- 5 files changed, 156 insertions(+), 26 deletions(-) create mode 100644 test/src/fixtures/whoami-deployment.yml diff --git a/test/README.md b/test/README.md index d17863f4..4fbab2fd 100644 --- a/test/README.md +++ b/test/README.md @@ -43,10 +43,15 @@ go test -v -timeout 120m -run TestDefaultEKS eksctl get clusters ``` -# TODO: implement db pod -# todo: tests weekly + + +You can change the default deployment region: +```bash +export TESTS_CLUSTER_REGION "eu-west-1" +``` +# TODO : tests weekly # see https://github.com/camunda/c8-multi-region/blob/main/.github/workflows/nightly_aws_region_cleanup.yml -# TODO: https://github.com/gruntwork-io/cloud-nuke every weekend +# TODO : https://github.com/gruntwork-io/cloud-nuke every weekend # => we should have a dedicated tenant for CI # => sometimes, EKS deletion fails with error: DeleteCluster ResourceInUseException: Cluster has nodegroups attached terraform diff --git a/test/src/fixtures/postgres-client.bak.yml b/test/src/fixtures/postgres-client.bak.yml index 24f37b6c..6cb35533 100644 --- a/test/src/fixtures/postgres-client.bak.yml +++ b/test/src/fixtures/postgres-client.bak.yml @@ -1,3 +1,5 @@ +# this manifest contains a version with the IRSA connection check, it is currently listed as a TODO +# it may be implemented or dropped depending if it's relevant or not to test IRSA connection for the db apiVersion: batch/v1 kind: Job metadata: diff --git a/test/src/fixtures/whoami-deployment.yml b/test/src/fixtures/whoami-deployment.yml new file mode 100644 index 00000000..4ff4d643 --- /dev/null +++ b/test/src/fixtures/whoami-deployment.yml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: whoami + namespace: example +spec: + replicas: 1 + selector: + matchLabels: + app: whoami + template: + metadata: + labels: + app: whoami + spec: + containers: + - name: whoami + image: containous/whoami + ports: + - containerPort: 80 + readinessProbe: + httpGet: + path: /health + port: 80 + initialDelaySeconds: 10 + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: whoami-service + namespace: example +spec: + selector: + app: whoami + ports: + - protocol: TCP + port: 80 + targetPort: 80 +--- diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index 21fa3504..babee9ef 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/camunda/camunda-tf-eks-module/utils" + http_helper "github.com/gruntwork-io/terratest/modules/http-helper" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" @@ -36,7 +37,7 @@ func TestDefaultEKS(t *testing.T) { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) clusterName := fmt.Sprintf("cluster-%s", clusterSuffix) - region := "eu-central-1" + region := utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") sugar.Infow("Creating EKS cluster...") expectedCapacity := 4 @@ -63,7 +64,7 @@ func TestCustomEKSAndRDS(t *testing.T) { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) clusterName := fmt.Sprintf("cluster-rds-%s", clusterSuffix) - region := "eu-central-1" + region := utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") sugar.Infow("Creating EKS cluster...") expectedCapacity := 3 @@ -121,19 +122,12 @@ func TestCustomEKSAndRDS(t *testing.T) { kubeClient, err := utils.NewKubeClientSet(result.Cluster) require.NoError(t, err) - utils.GenerateKubeConfigFromAWS(t, region, clusterName, utils.GetAwsProfile(), "kubeconfig") + kubeConfigPath := "kubeconfig-eks-rds" + utils.GenerateKubeConfigFromAWS(t, region, clusterName, utils.GetAwsProfile(), kubeConfigPath) - // ensure postgres-client namespace exists namespace := "postgres-client" - pgKubeCtlOptions := k8s.NewKubectlOptions("", "kubeconfig", namespace) - _, errFindNamespace := k8s.GetNamespaceE(t, pgKubeCtlOptions, namespace) - if errFindNamespace != nil { - if errors.IsNotFound(errFindNamespace) { - k8s.CreateNamespace(t, pgKubeCtlOptions, namespace) - } else { - require.NoError(t, errFindNamespace) - } - } + pgKubeCtlOptions := k8s.NewKubectlOptions("", kubeConfigPath, namespace) + utils.CreateIfNotExistsNamespace(t, pgKubeCtlOptions, namespace) // deploy the postgres-client ConfigMap configMapPostgres := &corev1.ConfigMap{ @@ -219,7 +213,7 @@ func TestCustomEKSAndRDS(t *testing.T) { describeDBClusterOutput, err := rdsSvc.DescribeDBClusters(context.Background(), describeDBClusterInput) require.NoError(t, err) - expectedRDSAZ := []string{"eu-central-1a", "eu-central-1b", "eu-central-1c"} + expectedRDSAZ := []string{fmt.Sprintf("%sa", region), fmt.Sprintf("%sb", region), fmt.Sprintf("%sc", region)} assert.Equal(t, varsConfigAurora["iam_auth_enabled"].(bool), *describeDBClusterOutput.DBClusters[0].IAMDatabaseAuthenticationEnabled) assert.Equal(t, varsConfigAurora["username"].(string), *describeDBClusterOutput.DBClusters[0].MasterUsername) assert.Equal(t, auroraDatabase, *describeDBClusterOutput.DBClusters[0].DatabaseName) @@ -279,9 +273,9 @@ func TestUpgradeEKS(t *testing.T) { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) clusterName := fmt.Sprintf("cluster-upgrade-%s", clusterSuffix) - region := "eu-central-1" + region := utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") sugar.Infow("Creating EKS cluster...") - expectedCapacity := 4 + expectedCapacity := 3 varsConfig := map[string]interface{}{ "name": clusterName, @@ -290,18 +284,95 @@ func TestUpgradeEKS(t *testing.T) { "kubernetes_version": "1.27", } - terraformOptions := SpawnEKS(t, sugar, varsConfig) + SpawnEKS(t, sugar, varsConfig) - // test suite - baseChecksEKS(t, sugar, terraformOptions, 3) + // Wait for the worker nodes to join the cluster + sess, err := utils.GetAwsClient() + require.NoErrorf(t, err, "Failed to get aws client") + + // list your services here + eksSvc := eks.NewFromConfig(sess) + + inputEKS := &eks.DescribeClusterInput{ + Name: aws.String(clusterName), + } + + result, err := eksSvc.DescribeCluster(context.Background(), inputEKS) + assert.NoError(t, err) + + sugar.Infow("Waiting for worker nodes to join the EKS cluster") + errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) + require.NoError(t, errClusterReady) + + kubeConfigPath := "kubeconfig-upgrade-eks" + utils.GenerateKubeConfigFromAWS(t, region, clusterName, utils.GetAwsProfile(), kubeConfigPath) + + // test suite: deploy a pod and check it is healthy + namespace := "example" + kubeCtlOptions := k8s.NewKubectlOptions("", kubeConfigPath, namespace) + utils.CreateIfNotExistsNamespace(t, kubeCtlOptions, namespace) + + // deploy the postgres-client Job to test the connection + k8s.KubectlApply(t, kubeCtlOptions, "../../test/src/fixtures/whoami-deployment.yml") + + k8s.WaitUntilServiceAvailable(t, kubeCtlOptions, "whoami-service", 10, 1*time.Second) + + // Now we verify that the service will successfully boot and start serving requests + localPort1 := 8081 + + service := k8s.GetService(t, kubeCtlOptions, "whoami-service") + portForwardProc1 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort1, 80) + defer portForwardProc1.Close() + + // wait for the port forward to be ready + time.Sleep(5 * time.Second) + localURL := fmt.Sprintf("http://127.0.0.1:%d", localPort1) + + http_helper.HttpGetWithRetryWithCustomValidation( + t, + localURL, + nil, + 30, + 10*time.Second, + func(statusCode int, body string) bool { + return statusCode == 200 + }, + ) // upgrade the cluster varsConfig["kubernetes_version"] = "1.28" + SpawnEKS(t, sugar, varsConfig) - terraformOptions = SpawnEKS(t, sugar, varsConfig) + sugar.Infow("Waiting for worker nodes to join the EKS cluster after the upgrade") + errClusterReady = utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) + require.NoError(t, errClusterReady) // check everything works as expected - baseChecksEKS(t, sugar, terraformOptions, 3) + k8s.WaitUntilServiceAvailable(t, kubeCtlOptions, "whoami-service", 10, 1*time.Second) + + // Now we verify that the service will successfully boot and start serving requests + localPort2 := 8082 + + service = k8s.GetService(t, kubeCtlOptions, "whoami-service") + portForwardProc2 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort2, 80) + defer portForwardProc2.Close() + + // wait for the port forward to be ready + time.Sleep(5 * time.Second) + localURL = fmt.Sprintf("http://127.0.0.1:%d", localPort2) + + http_helper.HttpGetWithRetryWithCustomValidation( + t, + localURL, + nil, + 30, + 10*time.Second, + func(statusCode int, body string) bool { + return statusCode == 200 + }, + ) + + // TODO: assert version of the cluster after the upgrade TearsDown(t, sugar) } diff --git a/test/src/utils/kube.go b/test/src/utils/kube.go index edf0ef6a..bc97c78a 100644 --- a/test/src/utils/kube.go +++ b/test/src/utils/kube.go @@ -3,11 +3,12 @@ package utils import ( "context" "encoding/base64" - "errors" "fmt" "github.com/aws/aws-sdk-go-v2/service/eks/types" + "github.com/gruntwork-io/terratest/modules/k8s" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -62,6 +63,17 @@ func WaitForJobCompletion(clientset *kubernetes.Clientset, namespace, jobName st } } +func CreateIfNotExistsNamespace(t *testing.T, kubeCtlOptions *k8s.KubectlOptions, namespace string) { + _, errFindNamespace := k8s.GetNamespaceE(t, kubeCtlOptions, namespace) + if errFindNamespace != nil { + if errors.IsNotFound(errFindNamespace) { + k8s.CreateNamespace(t, kubeCtlOptions, namespace) + } else { + require.NoError(t, errFindNamespace) + } + } +} + func GenerateKubeConfigFromAWS(t *testing.T, region, clusterName, awsProfile, configOutputPath string) { cmd := exec.Command("aws", "eks", "--region", region, "update-kubeconfig", "--name", clusterName, "--profile", awsProfile, "--kubeconfig", configOutputPath) _, errCmdKubeProfile := cmd.Output() @@ -114,7 +126,7 @@ func WaitUntilKubeClusterIsReady(cluster *types.Cluster, timeout time.Duration, case <-time.After(timeout): msg := "Not all worker nodes have joined the Kube cluster" fmt.Println(msg) - return errors.New(msg) + return errors.NewResourceExpired(msg) } return nil } From d79104138ae15efb1ea95673ae313ce00f311add Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:40:25 +0200 Subject: [PATCH 13/93] WIP tests --- test/src/modules_eks_cluster_test.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go index babee9ef..51a18991 100644 --- a/test/src/modules_eks_cluster_test.go +++ b/test/src/modules_eks_cluster_test.go @@ -304,6 +304,8 @@ func TestUpgradeEKS(t *testing.T) { errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) require.NoError(t, errClusterReady) + assert.Equal(t, varsConfig["kubernetes_version"], *result.Cluster.Version) + kubeConfigPath := "kubeconfig-upgrade-eks" utils.GenerateKubeConfigFromAWS(t, region, clusterName, utils.GetAwsProfile(), kubeConfigPath) @@ -318,19 +320,19 @@ func TestUpgradeEKS(t *testing.T) { k8s.WaitUntilServiceAvailable(t, kubeCtlOptions, "whoami-service", 10, 1*time.Second) // Now we verify that the service will successfully boot and start serving requests - localPort1 := 8081 + localPort1 := 8883 service := k8s.GetService(t, kubeCtlOptions, "whoami-service") portForwardProc1 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort1, 80) defer portForwardProc1.Close() + portForwardProc1.ForwardPort(t) // wait for the port forward to be ready time.Sleep(5 * time.Second) - localURL := fmt.Sprintf("http://127.0.0.1:%d", localPort1) http_helper.HttpGetWithRetryWithCustomValidation( t, - localURL, + fmt.Sprintf("http://%s", portForwardProc1.Endpoint()), nil, 30, 10*time.Second, @@ -347,23 +349,28 @@ func TestUpgradeEKS(t *testing.T) { errClusterReady = utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) require.NoError(t, errClusterReady) + // check version of the upgraded cluster + result, err = eksSvc.DescribeCluster(context.Background(), inputEKS) + assert.NoError(t, err) + assert.Equal(t, varsConfig["kubernetes_version"], *result.Cluster.Version) + // check everything works as expected k8s.WaitUntilServiceAvailable(t, kubeCtlOptions, "whoami-service", 10, 1*time.Second) // Now we verify that the service will successfully boot and start serving requests - localPort2 := 8082 + localPort2 := 8887 service = k8s.GetService(t, kubeCtlOptions, "whoami-service") portForwardProc2 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort2, 80) defer portForwardProc2.Close() + portForwardProc2.ForwardPort(t) // wait for the port forward to be ready time.Sleep(5 * time.Second) - localURL = fmt.Sprintf("http://127.0.0.1:%d", localPort2) http_helper.HttpGetWithRetryWithCustomValidation( t, - localURL, + fmt.Sprintf("http://%s", portForwardProc2.Endpoint()), nil, 30, 10*time.Second, @@ -372,8 +379,6 @@ func TestUpgradeEKS(t *testing.T) { }, ) - // TODO: assert version of the cluster after the upgrade - TearsDown(t, sugar) } @@ -568,5 +573,3 @@ func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *ter } assert.Truef(t, keyFound, "Failed to find key %s", keyDescription) } - -// todo: test upgrade path From 025599854996af033ab9ba6de71e7025eaa2b678 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:12:43 +0200 Subject: [PATCH 14/93] refacto tests in TestSuites to be reviewed --- .github/workflows/nighly_aws_cleanup.yml | 80 ---- test/src/.tool-versions | 1 + test/src/custom_eks_rds_test.go | 283 +++++++++++ test/src/default_eks_test.go | 210 +++++++++ test/src/modules_eks_cluster_test.go | 575 ----------------------- test/src/upgrade_eks_test.go | 183 ++++++++ test/src/utils/aws.go | 77 +++ test/src/utils/helper.go | 4 +- 8 files changed, 757 insertions(+), 656 deletions(-) delete mode 100644 .github/workflows/nighly_aws_cleanup.yml create mode 100644 test/src/.tool-versions create mode 100644 test/src/custom_eks_rds_test.go create mode 100644 test/src/default_eks_test.go delete mode 100644 test/src/modules_eks_cluster_test.go create mode 100644 test/src/upgrade_eks_test.go diff --git a/.github/workflows/nighly_aws_cleanup.yml b/.github/workflows/nighly_aws_cleanup.yml deleted file mode 100644 index 16ee454a..00000000 --- a/.github/workflows/nighly_aws_cleanup.yml +++ /dev/null @@ -1,80 +0,0 @@ ---- -name: Nightly AWS Region Cleanup - -on: - schedule: - - cron: '0 5 * * 1-5' - workflow_dispatch: - -env: - AWS_PROFILE: infex - -jobs: - aws-nightly-cleanup: - runs-on: ubuntu-latest - # Other dependencies from hosted runner - # AWS CLI - # indirectly node used by actions - - steps: - - uses: actions/checkout@v4 - - - name: Import Secrets - id: secrets - uses: hashicorp/vault-action@v3 - with: - url: ${{ secrets.VAULT_ADDR }} - method: approle - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - exportEnv: false - secrets: | - secret/data/products/infrastructure-experience/ci/common AWS_ACCESS_KEY; - secret/data/products/infrastructure-experience/ci/common AWS_SECRET_KEY; - - # Official action does not support profiles - - name: Add profile credentials to ~/.aws/credentials - run: | - aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} - aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} - aws configure set region eu-west-2 --profile ${{ env.AWS_PROFILE }} - - - name: Install Cloud Nuke - run: | - wget https://github.com/gruntwork-io/cloud-nuke/releases/download/v0.33.0/cloud-nuke_linux_amd64 - chmod +x cloud-nuke_linux_amd64 - - # This is likely to fail, therefore we ignore the error - # We're ignoring ec2_dhcp_option as they couldn't be deleted - # cloudtrail is managed by IT and can't be deleted either - - name: Run Cloud Nuke - timeout-minutes: 45 - env: - DISABLE_TELEMETRY: "true" - run: | - ./cloud-nuke_linux_amd64 aws \ - --region eu-west-2 \ - --region eu-west-3 \ - --force \ - --newer-than 4h \ - --exclude-resource-type ec2_dhcp_option \ - --exclude-resource-type cloudtrail || true - - # Following will delete global resources and things that cloud-nuke does not support - - name: Delete additional AWS resources - timeout-minutes: 15 - run: .github/workflows/scripts/aws_cleanup.sh - - # The second run should remove the remaining resources (VPCs) and fail if there's anything left - - name: Run Cloud Nuke - timeout-minutes: 45 - env: - DISABLE_TELEMETRY: "true" - run: | - ./cloud-nuke_linux_amd64 aws \ - --region eu-west-2 \ - --region eu-west-3 \ - --force \ - --newer-than 4h \ - --exclude-resource-type ec2_dhcp_option \ - --exclude-resource-type cloudtrail diff --git a/test/src/.tool-versions b/test/src/.tool-versions new file mode 100644 index 00000000..6bc74c51 --- /dev/null +++ b/test/src/.tool-versions @@ -0,0 +1 @@ +golang 1.22.1 diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go new file mode 100644 index 00000000..be7c71ca --- /dev/null +++ b/test/src/custom_eks_rds_test.go @@ -0,0 +1,283 @@ +package test + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/eks" + "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/camunda/camunda-tf-eks-module/utils" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "os" + "strings" + "testing" + "time" +) + +type CustomEKSRDSTestSuite struct { + suite.Suite + logger *zap.Logger + sugaredLogger *zap.SugaredLogger + clusterName string + expectedNodes int + kubeConfigPath string + region string + tfDataDir string + varTf map[string]interface{} +} + +func (suite *CustomEKSRDSTestSuite) SetupTest() { + suite.logger = zaptest.NewLogger(suite.T()) + suite.sugaredLogger = suite.logger.Sugar() + + clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) + suite.clusterName = fmt.Sprintf("cluster-rds-%s", clusterSuffix) + suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") + suite.expectedNodes = 3 + suite.kubeConfigPath = "kubeconfig-rds-eks" + suite.tfDataDir = fmt.Sprintf("tf-data-%s", clusterSuffix) +} + +func (suite *CustomEKSRDSTestSuite) TearUpTest() { + err := os.Setenv("TF_DATA_DIR", suite.tfDataDir) + suite.Require().NoError(err) +} + +func (suite *CustomEKSRDSTestSuite) TearDownTest() { + suite.T().Log("Cleaning up resources...") + + err := os.Remove(suite.kubeConfigPath) + if err != nil && !os.IsNotExist(err) { + suite.T().Errorf("Failed to remove kubeConfigPath: %v", err) + } +} + +// TestCustomEKSAndRDS spawns a custom EKS cluster with custom parameters, and spawns a +// pg client pod that will test connection to AuroraDB +func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { + terraformOptions := suite.createEKS() + + // Wait for the worker nodes to join the cluster + sess, err := utils.GetAwsClient() + suite.Require().NoErrorf(err, "Failed to get aws client") + + // list your services here + eksSvc := eks.NewFromConfig(sess) + rdsSvc := rds.NewFromConfig(sess) + + inputEKS := &eks.DescribeClusterInput{ + Name: aws.String(suite.clusterName), + } + + result, err := eksSvc.DescribeCluster(context.Background(), inputEKS) + suite.Assert().NoError(err) + + suite.sugaredLogger.Infow("Waiting for worker nodes to join the EKS cluster") + errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(suite.expectedNodes)) + suite.Require().NoError(errClusterReady) + + // Spawn RDS within the EKS VPC/subnet + publicBlocks := strings.Fields(strings.Trim(terraform.Output(suite.T(), terraformOptions, "public_vpc_cidr_blocks"), "[]")) + privateBlocks := strings.Fields(strings.Trim(terraform.Output(suite.T(), terraformOptions, "private_vpc_cidr_blocks"), "[]")) + + auroraUsername := "myuser" + auroraPassword := "mypassword123secure" + auroraDatabase := "camunda" + + varsConfigAurora := map[string]interface{}{ + "username": auroraUsername, + "password": auroraPassword, + "default_database_name": auroraDatabase, + "cluster_name": fmt.Sprintf("postgres-%s", suite.clusterName), + "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, + "vpc_id": *result.Cluster.ResourcesVpcConfig.VpcId, + "cidr_blocks": append(publicBlocks, privateBlocks...), + "iam_auth_enabled": true, + } + + terraformOptionsRDS := &terraform.Options{ + TerraformDir: "../../modules/aurora", + Upgrade: false, + VarFiles: []string{"../../test/src/fixtures/fixtures.default.aurora.tfvars"}, + Vars: suite.varTf, + } + + terraformOptionsRDS = utils.ApplyTfAndCleanup(suite.T(), terraformOptionsRDS) + auroraEndpoint := terraform.Output(suite.T(), terraformOptionsRDS, "aurora_endpoint") + suite.Assert().NotEmpty(auroraEndpoint) + + // Test of the RDS connection is performed by launching a pod on the cluster and test the pg connection + kubeClient, err := utils.NewKubeClientSet(result.Cluster) + suite.Require().NoError(err) + + kubeConfigPath := "kubeconfig-eks-rds" + utils.GenerateKubeConfigFromAWS(suite.T(), suite.region, suite.clusterName, utils.GetAwsProfile(), kubeConfigPath) + + namespace := "postgres-client" + pgKubeCtlOptions := k8s.NewKubectlOptions("", kubeConfigPath, namespace) + utils.CreateIfNotExistsNamespace(suite.T(), pgKubeCtlOptions, namespace) + + // deploy the postgres-client ConfigMap + configMapPostgres := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aurora-config", + Namespace: namespace, + }, + Data: map[string]string{ + "aurora_endpoint": auroraEndpoint, + "aurora_username": auroraUsername, + "aurora_username_irsa": fmt.Sprintf("%s-irsa", auroraUsername), + "aurora_port": "5432", + "aws_region": suite.region, + "aurora_db_name": auroraDatabase, + }, + } + + err = kubeClient.CoreV1().ConfigMaps(namespace).Delete(context.Background(), configMapPostgres.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + suite.Require().NoError(err) + } + _, err = kubeClient.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMapPostgres, metav1.CreateOptions{}) + k8s.WaitUntilConfigMapAvailable(suite.T(), pgKubeCtlOptions, configMapPostgres.Name, 6, 10*time.Second) + + // create the secret for aurora pg password + secretPostgres := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aurora-secret", + Namespace: namespace, + }, + StringData: map[string]string{ + "aurora_password": auroraPassword, + }, + } + err = kubeClient.CoreV1().Secrets(namespace).Delete(context.Background(), configMapPostgres.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + suite.Require().NoError(err) + } + _, err = kubeClient.CoreV1().Secrets(namespace).Create(context.Background(), secretPostgres, metav1.CreateOptions{}) + k8s.WaitUntilSecretAvailable(suite.T(), pgKubeCtlOptions, secretPostgres.Name, 6, 10*time.Second) + + // add the scripts as a ConfigMap + scriptPath := "../../test/src/fixtures/scripts/create_aurora_pg_db.sh" + scriptContent, err := os.ReadFile(scriptPath) + suite.Require().NoError(err) + + configMapScript := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-scripts", + Namespace: namespace, + }, + Data: map[string]string{ + "create_aurora_pg_db.sh": string(scriptContent), + }, + } + + err = kubeClient.CoreV1().ConfigMaps(namespace).Delete(context.Background(), configMapScript.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + suite.Require().NoError(err) + } + _, err = kubeClient.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMapScript, metav1.CreateOptions{}) + k8s.WaitUntilConfigMapAvailable(suite.T(), pgKubeCtlOptions, configMapScript.Name, 6, 10*time.Second) + + // cleanup existing jobs + jobListOptions := metav1.ListOptions{LabelSelector: "app=postgres-client"} + existingJobs := k8s.ListJobs(suite.T(), pgKubeCtlOptions, jobListOptions) + for _, job := range existingJobs { + err := kubeClient.BatchV1().Jobs(namespace).Delete(context.Background(), job.Name, metav1.DeleteOptions{}) + suite.Assert().NoError(err) + } + + // deploy the postgres-client Job to test the connection + k8s.KubectlApply(suite.T(), pgKubeCtlOptions, "../../test/src/fixtures/postgres-client.yml") + errJob := utils.WaitForJobCompletion(kubeClient, namespace, "postgres-client", 5*time.Minute, jobListOptions) + suite.Require().NoError(errJob) + + // TODO: test IRSA apply https://kubedemy.io/aws-eks-part-13-setup-iam-roles-for-service-accounts-irsa to setup iam + + // Retrieve RDS information + describeDBClusterInput := &rds.DescribeDBClustersInput{ + DBClusterIdentifier: aws.String(varsConfigAurora["cluster_name"].(string)), + } + describeDBClusterOutput, err := rdsSvc.DescribeDBClusters(context.Background(), describeDBClusterInput) + suite.Require().NoError(err) + + expectedRDSAZ := []string{fmt.Sprintf("%sa", suite.region), fmt.Sprintf("%sb", suite.region), fmt.Sprintf("%sc", suite.region)} + suite.Assert().Equal(varsConfigAurora["iam_auth_enabled"].(bool), *describeDBClusterOutput.DBClusters[0].IAMDatabaseAuthenticationEnabled) + suite.Assert().Equal(varsConfigAurora["username"].(string), *describeDBClusterOutput.DBClusters[0].MasterUsername) + suite.Assert().Equal(auroraDatabase, *describeDBClusterOutput.DBClusters[0].DatabaseName) + suite.Assert().Equal(int32(5432), *describeDBClusterOutput.DBClusters[0].Port) + suite.Assert().Equal("15.4", *describeDBClusterOutput.DBClusters[0].EngineVersion) + suite.Assert().ElementsMatch(expectedRDSAZ, describeDBClusterOutput.DBClusters[0].AvailabilityZones) + suite.Assert().Equal(varsConfigAurora["cluster_name"].(string), *describeDBClusterOutput.DBClusters[0].DBClusterIdentifier) + + // Some of the tests are performed on the first instance of the cluster + describeDBInstanceInput := &rds.DescribeDBInstancesInput{ + DBInstanceIdentifier: describeDBClusterOutput.DBClusters[0].DBClusterMembers[0].DBInstanceIdentifier, + } + describeDBInstanceOutput, err := rdsSvc.DescribeDBInstances(context.Background(), describeDBInstanceInput) + suite.Require().NoError(err) + + suite.Assert().Equal("db.t3.medium", *describeDBInstanceOutput.DBInstances[0].DBInstanceClass) + suite.Assert().Equal(true, *describeDBInstanceOutput.DBInstances[0].AutoMinorVersionUpgrade) + suite.Assert().Equal("aurora-postgresql", *describeDBInstanceOutput.DBInstances[0].Engine) + suite.Assert().Equal("rds-ca-2019", *describeDBInstanceOutput.DBInstances[0].CertificateDetails.CAIdentifier) + suite.Assert().Equal(varsConfigAurora["vpc_id"].(string), *describeDBInstanceOutput.DBInstances[0].DBSubnetGroup.VpcId) + suite.Assert().Contains(*describeDBInstanceOutput.DBInstances[0].AvailabilityZone, suite.region) + + // construct the subnet ids + actualSubnetIds := make([]string, len(describeDBInstanceOutput.DBInstances[0].DBSubnetGroup.Subnets)) + for id, subnet := range describeDBInstanceOutput.DBInstances[0].DBSubnetGroup.Subnets { + actualSubnetIds[id] = *subnet.SubnetIdentifier + } + suite.Assert().ElementsMatch(varsConfigAurora["subnet_ids"].([]string), actualSubnetIds) + + // EKS test that custom cluster parameters are applied as expected + + // count nb of nodes + nodes, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + suite.Require().NoError(err) + suite.Assert().Equal(suite.expectedNodes, len(nodes.Items)) + + // verifies for each node, the flavor and the region + expectedInstanceType := "t2.medium" + for _, node := range nodes.Items { + regionNode, _ := node.Labels["failure-domain.beta.kubernetes.io/region"] + instanceType, _ := node.Labels["node.kubernetes.io/instance-type"] + for _, addr := range node.Status.Addresses { + if addr.Type == "InternalIP" { + suite.Assert().Equal(suite.region, regionNode) + suite.Assert().Equal(expectedInstanceType, instanceType) + } + } + } +} + +func (suite *CustomEKSRDSTestSuite) createEKS() *terraform.Options { + suite.varTf = map[string]interface{}{ + "name": suite.clusterName, + "region": suite.region, + "np_desired_node_count": suite.expectedNodes, + } + + suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) + + terraformOptions := &terraform.Options{ + TerraformDir: "../../modules/eks-cluster", + Upgrade: false, + VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, + } + return utils.ApplyTfAndCleanup(suite.T(), terraformOptions) +} + +func TestCustomEKSRDSTestSuite(t *testing.T) { + suite.Run(t, new(CustomEKSRDSTestSuite)) +} diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go new file mode 100644 index 00000000..e432fe7a --- /dev/null +++ b/test/src/default_eks_test.go @@ -0,0 +1,210 @@ +package test + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/eks" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/camunda/camunda-tf-eks-module/utils" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" + "os" + "strings" + "testing" + "time" +) + +type DefaultEKSTestSuite struct { + suite.Suite + logger *zap.Logger + sugaredLogger *zap.SugaredLogger + clusterName string + expectedNodes int + kubeConfigPath string + region string + tfDataDir string + varTf map[string]interface{} +} + +func (suite *DefaultEKSTestSuite) SetupTest() { + suite.logger = zaptest.NewLogger(suite.T()) + suite.sugaredLogger = suite.logger.Sugar() + + clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) + suite.clusterName = fmt.Sprintf("cluster-test-%s", clusterSuffix) + suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") + suite.kubeConfigPath = "kubeconfig-default-eks" + suite.expectedNodes = 4 + suite.tfDataDir = fmt.Sprintf("tf-data-%s", clusterSuffix) +} + +func (suite *DefaultEKSTestSuite) TearUpTest() { + err := os.Setenv("TF_DATA_DIR", suite.tfDataDir) + suite.Require().NoError(err) +} + +func (suite *DefaultEKSTestSuite) TearDownTest() { + suite.T().Log("Cleaning up resources...") + + err := os.Remove(suite.kubeConfigPath) + if err != nil && !os.IsNotExist(err) { + suite.T().Errorf("Failed to remove kubeConfigPath: %v", err) + } +} + +// TestDefaultEKS spawns an EKS cluster with the default parameters and checks the parameters +func (suite *DefaultEKSTestSuite) TestDefaultEKS() { + + suite.varTf = map[string]interface{}{ + "name": suite.clusterName, + "region": suite.region, + "np_desired_node_count": suite.expectedNodes, + } + + terraformOptions := &terraform.Options{ + TerraformDir: "../../modules/eks-cluster", + Upgrade: false, + VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, + } + + terraformOptions = utils.ApplyTfAndCleanup(suite.T(), terraformOptions) + suite.baseChecksEKS(terraformOptions) +} + +// baseChecksEKS checks the defaults of an EKS cluster +func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Options) { + clusterName := terraformOptions.Vars["name"].(string) + suite.sugaredLogger.Infow("Testing status of the EKS cluster", "clusterName", clusterName) + + // Do some basic not empty tests on outputs + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "cluster_endpoint")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "cluster_security_group_id")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "cluster_security_group_arn")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "cluster_primary_security_group_id")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "cluster_iam_role_arn")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "ebs_cs_arn")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "external_dns_arn")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "vpc_id")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "private_vpc_cidr_blocks")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "private_subnet_ids")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "default_security_group_id")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "vpc_main_route_table_id")) + suite.Assert().NotEmpty(terraform.Output(suite.T(), terraformOptions, "private_route_table_ids")) + + // test IAM roles + suite.Assert().Equal(fmt.Sprintf("%s-eks-iam-role", clusterName), terraform.Output(suite.T(), terraformOptions, "cluster_iam_role_name")) + + // this is a split(6)[0..2] of the base cluster_node_ipv4_cidr = "10.192.0.0/16" + expectedPrivateVpcCidrBlocks := "[10.192.0.0/19 10.192.32.0/19 10.192.64.0/19]" + suite.Assert().Equal(expectedPrivateVpcCidrBlocks, terraform.Output(suite.T(), terraformOptions, "private_vpc_cidr_blocks")) + + // this is a split(6)[3..5] of the base cluster_node_ipv4_cidr = "10.192.0.0/16" + expectedPublicVpcCidrBlocks := "[10.192.96.0/19 10.192.128.0/19 10.192.160.0/19]" + suite.Assert().Equal(expectedPublicVpcCidrBlocks, terraform.Output(suite.T(), terraformOptions, "public_vpc_cidr_blocks")) + + sess, err := utils.GetAwsClient() + suite.Require().NoErrorf(err, "Failed to get aws client") + + // list your services here + eksSvc := eks.NewFromConfig(sess) + iamSvc := iam.NewFromConfig(sess) + ec2Svc := ec2.NewFromConfig(sess) + kmsSvc := kms.NewFromConfig(sess) + + inputEKS := &eks.DescribeClusterInput{ + Name: aws.String(clusterName), + } + + result, err := eksSvc.DescribeCluster(context.Background(), inputEKS) + suite.Assert().NoError(err) + + // Wait for the worker nodes to join the cluster + suite.sugaredLogger.Infow("Waiting for worker nodes to join the EKS cluster") + errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(suite.expectedNodes)) + suite.Require().NoError(errClusterReady) + + // Verify list of addons installed on the EKS + expectedEKSAddons := []string{"coredns", "kube-proxy", "vpc-cni", "aws-ebs-csi-driver"} + inputDescribeAddons := &eks.ListAddonsInput{ + ClusterName: aws.String(clusterName), + } + outputEKSAddons, errEKSAddons := eksSvc.ListAddons(context.Background(), inputDescribeAddons) + suite.Require().NoError(errEKSAddons) + + // perform the diff + presenceAddonsMap := make(map[string]bool) + for _, addon := range outputEKSAddons.Addons { + presenceAddonsMap[addon] = true + } + for _, addonName := range expectedEKSAddons { + suite.Assert().Truef(presenceAddonsMap[addonName], "Addon %s not installed on the EKS cluster", addonName) + } + + // Verifies EKS roles + roleNames := []string{ + fmt.Sprintf("%s-cert-manager-role", clusterName), + fmt.Sprintf("%s-external-dns-role", clusterName), + fmt.Sprintf("%s-ebs-cs-role", clusterName), + fmt.Sprintf("%s-eks-iam-role", clusterName), + } + + for _, roleName := range roleNames { + input := &iam.GetRoleInput{ + RoleName: aws.String(roleName), + } + + _, err := iamSvc.GetRole(context.Background(), input) + suite.Assert().NoErrorf(err, "Failed to get IAM EKS role %s", roleName) + } + + // verifies the VPC + + vpcName := fmt.Sprintf("%s-vpc", clusterName) + + inputVPC := &ec2.DescribeVpcsInput{ + Filters: []types.Filter{ + { + Name: aws.String("tag:Name"), + Values: []string{vpcName}, + }, + }, + } + + outputVPC, errVPC := ec2Svc.DescribeVpcs(context.Background(), inputVPC) + suite.Require().NoError(errVPC) + + suite.Assert().Equal(len(outputVPC.Vpcs), 1) + + // key + keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) + inputKMS := &kms.ListKeysInput{} + outputKMSList, errKMSList := kmsSvc.ListKeys(context.Background(), inputKMS) + suite.Assert().NoError(errKMSList) + + // Check if the key corresponding to the description exists + keyFound := false + for _, key := range outputKMSList.Keys { + keyDetails, errKey := kmsSvc.DescribeKey(context.Background(), &kms.DescribeKeyInput{ + KeyId: key.KeyId, + }) + suite.Require().NoErrorf(errKey, "Failed to describe key %s", *key.KeyId) + + keyFound = *keyDetails.KeyMetadata.Description == keyDescription + if keyFound { + break + } + } + suite.Assert().Truef(keyFound, "Failed to find key %s", keyDescription) +} + +func TestDefaultEKSTestSuite(t *testing.T) { + suite.Run(t, new(DefaultEKSTestSuite)) +} diff --git a/test/src/modules_eks_cluster_test.go b/test/src/modules_eks_cluster_test.go deleted file mode 100644 index 51a18991..00000000 --- a/test/src/modules_eks_cluster_test.go +++ /dev/null @@ -1,575 +0,0 @@ -package test - -import ( - "context" - "fmt" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/ec2" - "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/aws/aws-sdk-go-v2/service/eks" - "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/aws/aws-sdk-go-v2/service/kms" - "github.com/aws/aws-sdk-go-v2/service/rds" - "github.com/camunda/camunda-tf-eks-module/utils" - http_helper "github.com/gruntwork-io/terratest/modules/http-helper" - "github.com/gruntwork-io/terratest/modules/k8s" - "github.com/gruntwork-io/terratest/modules/random" - "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - "go.uber.org/zap/zaptest" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/runtime" - "os" - "strings" - "testing" - "time" -) - -// TestDefaultEKS spawns an EKS cluster with the default parameters and checks the parameters -func TestDefaultEKS(t *testing.T) { - // log - logger := zaptest.NewLogger(t) - sugar := logger.Sugar() - - clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) - clusterName := fmt.Sprintf("cluster-%s", clusterSuffix) - region := utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") - sugar.Infow("Creating EKS cluster...") - expectedCapacity := 4 - - varsConfig := map[string]interface{}{ - "name": clusterName, - "region": region, - "np_desired_node_count": expectedCapacity, - } - - terraformOptions := SpawnEKS(t, sugar, varsConfig) - - // test suite - baseChecksEKS(t, sugar, terraformOptions, uint64(expectedCapacity)) - - TearsDown(t, sugar) -} - -// TestCustomEKSAndRDS spawns a custom EKS cluster with custom parameters, and spawns a -// pg client pod that will test connection to AuroraDB -func TestCustomEKSAndRDS(t *testing.T) { - // log - logger := zaptest.NewLogger(t) - sugar := logger.Sugar() - - clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) - clusterName := fmt.Sprintf("cluster-rds-%s", clusterSuffix) - region := utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") - sugar.Infow("Creating EKS cluster...") - expectedCapacity := 3 - - varsConfigEKS := map[string]interface{}{ - "name": clusterName, - "region": region, - "np_desired_node_count": expectedCapacity, - } - - terraformOptions := SpawnEKS(t, sugar, varsConfigEKS) - - // Wait for the worker nodes to join the cluster - sess, err := utils.GetAwsClient() - require.NoErrorf(t, err, "Failed to get aws client") - - // list your services here - eksSvc := eks.NewFromConfig(sess) - rdsSvc := rds.NewFromConfig(sess) - - inputEKS := &eks.DescribeClusterInput{ - Name: aws.String(clusterName), - } - - result, err := eksSvc.DescribeCluster(context.Background(), inputEKS) - assert.NoError(t, err) - - sugar.Infow("Waiting for worker nodes to join the EKS cluster") - errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) - require.NoError(t, errClusterReady) - - // Spawn RDS within the EKS VPC/subnet - publicBlocks := strings.Fields(strings.Trim(terraform.Output(t, terraformOptions, "public_vpc_cidr_blocks"), "[]")) - privateBlocks := strings.Fields(strings.Trim(terraform.Output(t, terraformOptions, "private_vpc_cidr_blocks"), "[]")) - - auroraUsername := "myuser" - auroraPassword := "mypassword123secure" - auroraDatabase := "camunda" - - varsConfigAurora := map[string]interface{}{ - "username": auroraUsername, - "password": auroraPassword, - "default_database_name": auroraDatabase, - "cluster_name": fmt.Sprintf("postgres-%s", clusterSuffix), - "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, - "vpc_id": *result.Cluster.ResourcesVpcConfig.VpcId, - "cidr_blocks": append(publicBlocks, privateBlocks...), - "iam_auth_enabled": true, - } - - terraformOptionsRDS := SpawnAurora(t, sugar, varsConfigAurora) - auroraEndpoint := terraform.Output(t, terraformOptionsRDS, "aurora_endpoint") - assert.NotEmpty(t, auroraEndpoint) - - // Test of the RDS connection is performed by launching a pod on the cluster and test the pg connection - kubeClient, err := utils.NewKubeClientSet(result.Cluster) - require.NoError(t, err) - - kubeConfigPath := "kubeconfig-eks-rds" - utils.GenerateKubeConfigFromAWS(t, region, clusterName, utils.GetAwsProfile(), kubeConfigPath) - - namespace := "postgres-client" - pgKubeCtlOptions := k8s.NewKubectlOptions("", kubeConfigPath, namespace) - utils.CreateIfNotExistsNamespace(t, pgKubeCtlOptions, namespace) - - // deploy the postgres-client ConfigMap - configMapPostgres := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "aurora-config", - Namespace: namespace, - }, - Data: map[string]string{ - "aurora_endpoint": auroraEndpoint, - "aurora_username": auroraUsername, - "aurora_username_irsa": fmt.Sprintf("%s-irsa", auroraUsername), - "aurora_port": "5432", - "aws_region": region, - "aurora_db_name": auroraDatabase, - }, - } - - err = kubeClient.CoreV1().ConfigMaps(namespace).Delete(context.Background(), configMapPostgres.Name, metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - require.NoError(t, err) - } - _, err = kubeClient.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMapPostgres, metav1.CreateOptions{}) - k8s.WaitUntilConfigMapAvailable(t, pgKubeCtlOptions, configMapPostgres.Name, 6, 10*time.Second) - - // create the secret for aurora pg password - secretPostgres := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "aurora-secret", - Namespace: namespace, - }, - StringData: map[string]string{ - "aurora_password": auroraPassword, - }, - } - err = kubeClient.CoreV1().Secrets(namespace).Delete(context.Background(), configMapPostgres.Name, metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - require.NoError(t, err) - } - _, err = kubeClient.CoreV1().Secrets(namespace).Create(context.Background(), secretPostgres, metav1.CreateOptions{}) - k8s.WaitUntilSecretAvailable(t, pgKubeCtlOptions, secretPostgres.Name, 6, 10*time.Second) - - // add the scripts as a ConfigMap - scriptPath := "../../test/src/fixtures/scripts/create_aurora_pg_db.sh" - scriptContent, err := os.ReadFile(scriptPath) - require.NoError(t, err) - - configMapScript := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgres-scripts", - Namespace: namespace, - }, - Data: map[string]string{ - "create_aurora_pg_db.sh": string(scriptContent), - }, - } - - err = kubeClient.CoreV1().ConfigMaps(namespace).Delete(context.Background(), configMapScript.Name, metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - require.NoError(t, err) - } - _, err = kubeClient.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMapScript, metav1.CreateOptions{}) - k8s.WaitUntilConfigMapAvailable(t, pgKubeCtlOptions, configMapScript.Name, 6, 10*time.Second) - - // cleanup existing jobs - jobListOptions := metav1.ListOptions{LabelSelector: "app=postgres-client"} - existingJobs := k8s.ListJobs(t, pgKubeCtlOptions, jobListOptions) - for _, job := range existingJobs { - err := kubeClient.BatchV1().Jobs(namespace).Delete(context.Background(), job.Name, metav1.DeleteOptions{}) - assert.NoError(t, err) - } - - // deploy the postgres-client Job to test the connection - k8s.KubectlApply(t, pgKubeCtlOptions, "../../test/src/fixtures/postgres-client.yml") - errJob := utils.WaitForJobCompletion(kubeClient, namespace, "postgres-client", 5*time.Minute, jobListOptions) - require.NoError(t, errJob) - - // TODO: test IRSA apply https://kubedemy.io/aws-eks-part-13-setup-iam-roles-for-service-accounts-irsa to setup iam - - // Retrieve RDS information - describeDBClusterInput := &rds.DescribeDBClustersInput{ - DBClusterIdentifier: aws.String(varsConfigAurora["cluster_name"].(string)), - } - describeDBClusterOutput, err := rdsSvc.DescribeDBClusters(context.Background(), describeDBClusterInput) - require.NoError(t, err) - - expectedRDSAZ := []string{fmt.Sprintf("%sa", region), fmt.Sprintf("%sb", region), fmt.Sprintf("%sc", region)} - assert.Equal(t, varsConfigAurora["iam_auth_enabled"].(bool), *describeDBClusterOutput.DBClusters[0].IAMDatabaseAuthenticationEnabled) - assert.Equal(t, varsConfigAurora["username"].(string), *describeDBClusterOutput.DBClusters[0].MasterUsername) - assert.Equal(t, auroraDatabase, *describeDBClusterOutput.DBClusters[0].DatabaseName) - assert.Equal(t, int32(5432), *describeDBClusterOutput.DBClusters[0].Port) - assert.Equal(t, "15.4", *describeDBClusterOutput.DBClusters[0].EngineVersion) - assert.ElementsMatch(t, expectedRDSAZ, describeDBClusterOutput.DBClusters[0].AvailabilityZones) - assert.Equal(t, varsConfigAurora["cluster_name"].(string), *describeDBClusterOutput.DBClusters[0].DBClusterIdentifier) - - // Some of the tests are performed on the first instance of the cluster - describeDBInstanceInput := &rds.DescribeDBInstancesInput{ - DBInstanceIdentifier: describeDBClusterOutput.DBClusters[0].DBClusterMembers[0].DBInstanceIdentifier, - } - describeDBInstanceOutput, err := rdsSvc.DescribeDBInstances(context.Background(), describeDBInstanceInput) - require.NoError(t, err) - - assert.Equal(t, "db.t3.medium", *describeDBInstanceOutput.DBInstances[0].DBInstanceClass) - assert.Equal(t, true, *describeDBInstanceOutput.DBInstances[0].AutoMinorVersionUpgrade) - assert.Equal(t, "aurora-postgresql", *describeDBInstanceOutput.DBInstances[0].Engine) - assert.Equal(t, "rds-ca-2019", *describeDBInstanceOutput.DBInstances[0].CertificateDetails.CAIdentifier) - assert.Equal(t, varsConfigAurora["vpc_id"].(string), *describeDBInstanceOutput.DBInstances[0].DBSubnetGroup.VpcId) - assert.Contains(t, *describeDBInstanceOutput.DBInstances[0].AvailabilityZone, region) - - // construct the subnet ids - actualSubnetIds := make([]string, len(describeDBInstanceOutput.DBInstances[0].DBSubnetGroup.Subnets)) - for id, subnet := range describeDBInstanceOutput.DBInstances[0].DBSubnetGroup.Subnets { - actualSubnetIds[id] = *subnet.SubnetIdentifier - } - assert.ElementsMatch(t, varsConfigAurora["subnet_ids"].([]string), actualSubnetIds) - - // EKS test that custom cluster parameters are applied as expected - - // count nb of nodes - nodes, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) - require.NoError(t, err) - assert.Equal(t, expectedCapacity, len(nodes.Items)) - - // verifies for each node, the flavor and the region - expectedInstanceType := "t2.medium" - for _, node := range nodes.Items { - regionNode, _ := node.Labels["failure-domain.beta.kubernetes.io/region"] - instanceType, _ := node.Labels["node.kubernetes.io/instance-type"] - for _, addr := range node.Status.Addresses { - if addr.Type == "InternalIP" { - assert.Equal(t, region, regionNode) - assert.Equal(t, expectedInstanceType, instanceType) - } - } - } -} - -// TestUpgradeEKS starts from a version of EKS, deploy a simple chart, upgrade the cluster -// and check that everything is working as expected -func TestUpgradeEKS(t *testing.T) { - // log - logger := zaptest.NewLogger(t) - sugar := logger.Sugar() - - clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) - clusterName := fmt.Sprintf("cluster-upgrade-%s", clusterSuffix) - region := utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") - sugar.Infow("Creating EKS cluster...") - expectedCapacity := 3 - - varsConfig := map[string]interface{}{ - "name": clusterName, - "region": region, - "np_desired_node_count": expectedCapacity, - "kubernetes_version": "1.27", - } - - SpawnEKS(t, sugar, varsConfig) - - // Wait for the worker nodes to join the cluster - sess, err := utils.GetAwsClient() - require.NoErrorf(t, err, "Failed to get aws client") - - // list your services here - eksSvc := eks.NewFromConfig(sess) - - inputEKS := &eks.DescribeClusterInput{ - Name: aws.String(clusterName), - } - - result, err := eksSvc.DescribeCluster(context.Background(), inputEKS) - assert.NoError(t, err) - - sugar.Infow("Waiting for worker nodes to join the EKS cluster") - errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) - require.NoError(t, errClusterReady) - - assert.Equal(t, varsConfig["kubernetes_version"], *result.Cluster.Version) - - kubeConfigPath := "kubeconfig-upgrade-eks" - utils.GenerateKubeConfigFromAWS(t, region, clusterName, utils.GetAwsProfile(), kubeConfigPath) - - // test suite: deploy a pod and check it is healthy - namespace := "example" - kubeCtlOptions := k8s.NewKubectlOptions("", kubeConfigPath, namespace) - utils.CreateIfNotExistsNamespace(t, kubeCtlOptions, namespace) - - // deploy the postgres-client Job to test the connection - k8s.KubectlApply(t, kubeCtlOptions, "../../test/src/fixtures/whoami-deployment.yml") - - k8s.WaitUntilServiceAvailable(t, kubeCtlOptions, "whoami-service", 10, 1*time.Second) - - // Now we verify that the service will successfully boot and start serving requests - localPort1 := 8883 - - service := k8s.GetService(t, kubeCtlOptions, "whoami-service") - portForwardProc1 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort1, 80) - defer portForwardProc1.Close() - portForwardProc1.ForwardPort(t) - - // wait for the port forward to be ready - time.Sleep(5 * time.Second) - - http_helper.HttpGetWithRetryWithCustomValidation( - t, - fmt.Sprintf("http://%s", portForwardProc1.Endpoint()), - nil, - 30, - 10*time.Second, - func(statusCode int, body string) bool { - return statusCode == 200 - }, - ) - - // upgrade the cluster - varsConfig["kubernetes_version"] = "1.28" - SpawnEKS(t, sugar, varsConfig) - - sugar.Infow("Waiting for worker nodes to join the EKS cluster after the upgrade") - errClusterReady = utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(expectedCapacity)) - require.NoError(t, errClusterReady) - - // check version of the upgraded cluster - result, err = eksSvc.DescribeCluster(context.Background(), inputEKS) - assert.NoError(t, err) - assert.Equal(t, varsConfig["kubernetes_version"], *result.Cluster.Version) - - // check everything works as expected - k8s.WaitUntilServiceAvailable(t, kubeCtlOptions, "whoami-service", 10, 1*time.Second) - - // Now we verify that the service will successfully boot and start serving requests - localPort2 := 8887 - - service = k8s.GetService(t, kubeCtlOptions, "whoami-service") - portForwardProc2 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort2, 80) - defer portForwardProc2.Close() - portForwardProc2.ForwardPort(t) - - // wait for the port forward to be ready - time.Sleep(5 * time.Second) - - http_helper.HttpGetWithRetryWithCustomValidation( - t, - fmt.Sprintf("http://%s", portForwardProc2.Endpoint()), - nil, - 30, - 10*time.Second, - func(statusCode int, body string) bool { - return statusCode == 200 - }, - ) - - TearsDown(t, sugar) -} - -// SpawnEKS spawns a new EKS Cluster from a default fixture file -func SpawnEKS(t *testing.T, sugar *zap.SugaredLogger, varsConfig map[string]interface{}) *terraform.Options { - sugar.Infow("TF vars", "vars", varsConfig) - - terraformOptions := &terraform.Options{ - // The path to where our Terraform code is located - TerraformDir: "../../modules/eks-cluster", - Upgrade: false, - // Variables to pass to our Terraform code using -var-file options - VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, - Vars: varsConfig, - } - - cleanClusterAtTheEnd := utils.GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") - - if cleanClusterAtTheEnd == "true" { - // At the end of the test, run `terraform destroy` to clean up any resources that were created - defer terraform.Destroy(t, terraformOptions) - - // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created - defer runtime.HandleCrash(func(i interface{}) { - terraform.Destroy(t, terraformOptions) - }) - } - - // This will run `terraform init` and `terraform apply` and fail the test if there are any errors - // then it will re-run apply to make sure that out tf is idempotent - terraform.InitAndApplyAndIdempotent(t, terraformOptions) - return terraformOptions -} - -// SpawnAurora spawns a new Aurora RDS from a default fixture file -func SpawnAurora(t *testing.T, sugar *zap.SugaredLogger, varsConfig map[string]interface{}) *terraform.Options { - sugar.Infow("TF vars", "vars", varsConfig) - - terraformOptions := &terraform.Options{ - // The path to where our Terraform code is located - TerraformDir: "../../modules/aurora", - Upgrade: false, - // Variables to pass to our Terraform code using -var-file options - VarFiles: []string{"../../test/src/fixtures/fixtures.default.aurora.tfvars"}, - Vars: varsConfig, - } - - cleanClusterAtTheEnd := utils.GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") - - if cleanClusterAtTheEnd == "true" { - // At the end of the test, run `terraform destroy` to clean up any resources that were created - defer terraform.Destroy(t, terraformOptions) - - // If Go runtime crushes, run `terraform destroy` to clean up any resources that were created - defer runtime.HandleCrash(func(i interface{}) { - terraform.Destroy(t, terraformOptions) - }) - } - - // This will run `terraform init` and `terraform apply` and fail the test if there are any errors - // then it will re-run apply to make sure that out tf is idempotent - terraform.InitAndApplyAndIdempotent(t, terraformOptions) - return terraformOptions -} - -func TearsDown(t *testing.T, sugar *zap.SugaredLogger) { - sugar.Infow("All tests completed") -} - -// baseChecksEKS checks the defaults of an EKS cluster -func baseChecksEKS(t *testing.T, sugar *zap.SugaredLogger, terraformOptions *terraform.Options, expectedNodesCount uint64) { - clusterName := terraformOptions.Vars["name"].(string) - sugar.Infow("Testing status of the EKS cluster", "clusterName", clusterName) - - // Do some basic not empty tests on outputs - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_endpoint")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_security_group_id")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_security_group_arn")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_primary_security_group_id")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "cluster_iam_role_arn")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "ebs_cs_arn")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "external_dns_arn")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "vpc_id")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_vpc_cidr_blocks")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_subnet_ids")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "default_security_group_id")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "vpc_main_route_table_id")) - assert.NotEmpty(t, terraform.Output(t, terraformOptions, "private_route_table_ids")) - - // test IAM roles - assert.Equal(t, fmt.Sprintf("%s-eks-iam-role", clusterName), terraform.Output(t, terraformOptions, "cluster_iam_role_name")) - - // this is a split(6)[0..2] of the base cluster_node_ipv4_cidr = "10.192.0.0/16" - expectedPrivateVpcCidrBlocks := "[10.192.0.0/19 10.192.32.0/19 10.192.64.0/19]" - assert.Equal(t, expectedPrivateVpcCidrBlocks, terraform.Output(t, terraformOptions, "private_vpc_cidr_blocks")) - - // this is a split(6)[3..5] of the base cluster_node_ipv4_cidr = "10.192.0.0/16" - expectedPublicVpcCidrBlocks := "[10.192.96.0/19 10.192.128.0/19 10.192.160.0/19]" - assert.Equal(t, expectedPublicVpcCidrBlocks, terraform.Output(t, terraformOptions, "public_vpc_cidr_blocks")) - - sess, err := utils.GetAwsClient() - require.NoErrorf(t, err, "Failed to get aws client") - - // list your services here - eksSvc := eks.NewFromConfig(sess) - iamSvc := iam.NewFromConfig(sess) - ec2Svc := ec2.NewFromConfig(sess) - kmsSvc := kms.NewFromConfig(sess) - - inputEKS := &eks.DescribeClusterInput{ - Name: aws.String(clusterName), - } - - result, err := eksSvc.DescribeCluster(context.Background(), inputEKS) - assert.NoError(t, err) - - // Wait for the worker nodes to join the cluster - sugar.Infow("Waiting for worker nodes to join the EKS cluster") - errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, expectedNodesCount) - require.NoError(t, errClusterReady) - - // Verify list of addons installed on the EKS - expectedEKSAddons := []string{"coredns", "kube-proxy", "vpc-cni", "aws-ebs-csi-driver"} - inputDescribeAddons := &eks.ListAddonsInput{ - ClusterName: aws.String(clusterName), - } - outputEKSAddons, errEKSAddons := eksSvc.ListAddons(context.Background(), inputDescribeAddons) - require.NoError(t, errEKSAddons) - - // perform the diff - presenceAddonsMap := make(map[string]bool) - for _, addon := range outputEKSAddons.Addons { - presenceAddonsMap[addon] = true - } - for _, addonName := range expectedEKSAddons { - assert.Truef(t, presenceAddonsMap[addonName], "Addon %s not installed on the EKS cluster", addonName) - } - - // Verifies EKS roles - roleNames := []string{ - fmt.Sprintf("%s-cert-manager-role", clusterName), - fmt.Sprintf("%s-external-dns-role", clusterName), - fmt.Sprintf("%s-ebs-cs-role", clusterName), - fmt.Sprintf("%s-eks-iam-role", clusterName), - } - - for _, roleName := range roleNames { - input := &iam.GetRoleInput{ - RoleName: aws.String(roleName), - } - - _, err := iamSvc.GetRole(context.Background(), input) - assert.NoErrorf(t, err, "Failed to get IAM EKS role %s", roleName) - } - - // verifies the VPC - - vpcName := fmt.Sprintf("%s-vpc", clusterName) - - inputVPC := &ec2.DescribeVpcsInput{ - Filters: []types.Filter{ - { - Name: aws.String("tag:Name"), - Values: []string{vpcName}, - }, - }, - } - - outputVPC, errVPC := ec2Svc.DescribeVpcs(context.Background(), inputVPC) - require.NoError(t, errVPC) - - assert.Equal(t, len(outputVPC.Vpcs), 1) - - // key - keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) - inputKMS := &kms.ListKeysInput{} - outputKMSList, errKMSList := kmsSvc.ListKeys(context.Background(), inputKMS) - assert.NoError(t, errKMSList) - - // Check if the key corresponding to the description exists - keyFound := false - for _, key := range outputKMSList.Keys { - keyDetails, errKey := kmsSvc.DescribeKey(context.Background(), &kms.DescribeKeyInput{ - KeyId: key.KeyId, - }) - require.NoErrorf(t, errKey, "Failed to describe key %s", *key.KeyId) - - keyFound = *keyDetails.KeyMetadata.Description == keyDescription - if keyFound { - break - } - } - assert.Truef(t, keyFound, "Failed to find key %s", keyDescription) -} diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go new file mode 100644 index 00000000..896865f8 --- /dev/null +++ b/test/src/upgrade_eks_test.go @@ -0,0 +1,183 @@ +package test + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/eks" + "github.com/camunda/camunda-tf-eks-module/utils" + http_helper "github.com/gruntwork-io/terratest/modules/http-helper" + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" + "os" + "strings" + "testing" + "time" +) + +type UpgradeEKSTestSuite struct { + suite.Suite + logger *zap.Logger + sugaredLogger *zap.SugaredLogger + clusterName string + expectedNodes int + kubeConfigPath string + kubeVersion string + tfDataDir string + region string + varTf map[string]interface{} +} + +func (suite *UpgradeEKSTestSuite) SetupTest() { + suite.logger = zaptest.NewLogger(suite.T()) + suite.sugaredLogger = suite.logger.Sugar() + + clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) + suite.clusterName = fmt.Sprintf("cluster-upgrade-%s", clusterSuffix) + suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") + suite.expectedNodes = 3 + suite.kubeConfigPath = "kubeconfig-upgrade-eks" + suite.kubeVersion = "1.28" + suite.tfDataDir = fmt.Sprintf("tf-data-%s", clusterSuffix) + + // Create EKS cluster + suite.createEKS() +} + +func (suite *UpgradeEKSTestSuite) TearUpTest() { + err := os.Setenv("TF_DATA_DIR", suite.tfDataDir) + suite.Require().NoError(err) +} + +func (suite *UpgradeEKSTestSuite) TearDownTest() { + suite.T().Log("Cleaning up resources...") + + err := os.Remove(suite.kubeConfigPath) + if err != nil && !os.IsNotExist(err) { + suite.T().Errorf("Failed to remove kubeConfigPath: %v", err) + } +} + +// TestUpgradeEKS starts from a version of EKS, deploy a simple chart, upgrade the cluster +// and check that everything is working as expected +func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { + // Wait for the worker nodes to join the cluster + sess, err := utils.GetAwsClient() + suite.Require().NoErrorf(err, "Failed to get aws client") + + // list your services here + eksSvc := eks.NewFromConfig(sess) + + inputEKS := &eks.DescribeClusterInput{ + Name: aws.String(suite.clusterName), + } + + result, err := eksSvc.DescribeCluster(context.Background(), inputEKS) + suite.Assert().NoError(err) + + suite.sugaredLogger.Infow("Waiting for worker nodes to join the EKS cluster") + errClusterReady := utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(suite.expectedNodes)) + suite.Require().NoError(errClusterReady) + + suite.Assert().Equal(suite.kubeVersion, *result.Cluster.Version) + + utils.GenerateKubeConfigFromAWS(suite.T(), suite.region, suite.clusterName, utils.GetAwsProfile(), suite.kubeConfigPath) + + // test suite: deploy a pod and check it is healthy + namespace := "example" + kubeCtlOptions := k8s.NewKubectlOptions("", suite.kubeConfigPath, namespace) + utils.CreateIfNotExistsNamespace(suite.T(), kubeCtlOptions, namespace) + + // deploy the postgres-client Job to test the connection + k8s.KubectlApply(suite.T(), kubeCtlOptions, "../../test/src/fixtures/whoami-deployment.yml") + + k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 60, 1*time.Second) + + // Now we verify that the service will successfully boot and start serving requests + localPort1 := 8883 + + service := k8s.GetService(suite.T(), kubeCtlOptions, "whoami-service") + portForwardProc1 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort1, 80) + defer portForwardProc1.Close() + portForwardProc1.ForwardPort(suite.T()) + + // wait for the port forward to be ready + time.Sleep(5 * time.Second) + + http_helper.HttpGetWithRetryWithCustomValidation( + suite.T(), + fmt.Sprintf("http://%s", portForwardProc1.Endpoint()), + nil, + 30, + 10*time.Second, + func(statusCode int, body string) bool { + return statusCode == 200 + }, + ) + + // upgrade the cluster + suite.varTf["kubernetes_version"] = "1.29" + suite.sugaredLogger.Infow(fmt.Sprintf("Upgrading the EKS cluster to v%s using aws sdk", suite.varTf["kubernetes_version"]), "extraVars", suite.varTf) + errUpdate := utils.UpgradeEKS(context.Background(), eksSvc, suite.clusterName, suite.varTf["kubernetes_version"].(string)) + suite.Require().NoError(errUpdate) + + suite.sugaredLogger.Infow("Waiting for worker nodes to join the EKS cluster after the upgrade") + errClusterReady = utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(suite.expectedNodes)) + suite.Require().NoError(errClusterReady) + + // Check version of the upgraded cluster + result, err = eksSvc.DescribeCluster(context.Background(), inputEKS) + suite.Assert().NoError(err) + suite.Assert().Equal(suite.varTf["kubernetes_version"], *result.Cluster.Version) + + // check everything works as expected + k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 10, 1*time.Second) + + // Forward port again + localPort2 := 8887 + service = k8s.GetService(suite.T(), kubeCtlOptions, "whoami-service") + portForwardProc2 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort2, 80) + defer portForwardProc2.Close() + portForwardProc2.ForwardPort(suite.T()) + + // Wait for port forward to be ready + time.Sleep(5 * time.Second) + + http_helper.HttpGetWithRetryWithCustomValidation( + suite.T(), + fmt.Sprintf("http://%s", portForwardProc2.Endpoint()), + nil, + 30, + 10*time.Second, + func(statusCode int, body string) bool { + return statusCode == 200 + }, + ) +} + +func (suite *UpgradeEKSTestSuite) createEKS() *terraform.Options { + suite.varTf = map[string]interface{}{ + "name": suite.clusterName, + "region": suite.region, + "np_desired_node_count": suite.expectedNodes, + "kubernetes_version": suite.kubeVersion, + } + + terraformOptions := &terraform.Options{ + TerraformDir: "../../modules/eks-cluster", + Upgrade: false, + VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, + } + + suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) + return utils.ApplyTfAndCleanup(suite.T(), terraformOptions) +} + +func TestUpgradeEKSTestSuite(t *testing.T) { + suite.Run(t, new(UpgradeEKSTestSuite)) +} diff --git a/test/src/utils/aws.go b/test/src/utils/aws.go index f29c6741..0eec6cb7 100644 --- a/test/src/utils/aws.go +++ b/test/src/utils/aws.go @@ -2,8 +2,15 @@ package utils import ( "context" + "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/eks" + "github.com/aws/aws-sdk-go-v2/service/eks/types" + "github.com/gruntwork-io/terratest/modules/terraform" + "k8s.io/apimachinery/pkg/util/runtime" + "testing" + "time" ) func GetAwsProfile() string { @@ -24,3 +31,73 @@ func GetAwsClient() (aws.Config, error) { config.WithSharedConfigProfile(awsProfile), ) } + +// ApplyTfAndCleanup applies a Tf resource and cleanup at the end +func ApplyTfAndCleanup(t *testing.T, terraformOptions *terraform.Options) *terraform.Options { + cleanClusterAtTheEnd := GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") + + if cleanClusterAtTheEnd == "true" { + defer terraform.Destroy(t, terraformOptions) + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(t, terraformOptions) + }) + } + + terraform.InitAndApplyAndIdempotent(t, terraformOptions) + return terraformOptions +} + +func WaitForUpdateEKS(ctx context.Context, client *eks.Client, clusterName, updateID string) error { + describeUpdateInput := &eks.DescribeUpdateInput{ + Name: &clusterName, + UpdateId: &updateID, + } + +L: + for { + updateOutput, err := client.DescribeUpdate(ctx, describeUpdateInput) + if err != nil { + return err + } + + status := updateOutput.Update.Status + fmt.Printf("Update status: %s\n", status) + + switch status { + case types.UpdateStatusFailed: + return fmt.Errorf("update failed") + case types.UpdateStatusCancelled: + return fmt.Errorf("update cancelled") + case types.UpdateStatusSuccessful: + break L + case types.UpdateStatusInProgress: + time.Sleep(5 * time.Second) + default: + return fmt.Errorf("update status unknown: %s", status) + } + } + + return nil +} + +func UpgradeEKS(ctx context.Context, client *eks.Client, clusterName, version string) error { + input := &eks.UpdateClusterVersionInput{ + Name: &clusterName, + Version: &version, + } + + output, err := client.UpdateClusterVersion(ctx, input) + if err != nil { + return err + } + + fmt.Printf("Update initiated, update ID: %s\n", *output.Update.Id) + + err = WaitForUpdateEKS(ctx, client, clusterName, *output.Update.Id) + if err != nil { + return err + } + + fmt.Println("Update completed successfully") + return nil +} diff --git a/test/src/utils/helper.go b/test/src/utils/helper.go index 8216c370..c5daf3fb 100644 --- a/test/src/utils/helper.go +++ b/test/src/utils/helper.go @@ -1,6 +1,8 @@ package utils -import "os" +import ( + "os" +) func GetEnv(key, fallback string) string { value, exists := os.LookupEnv(key) From f61245407576ff83cd0e74d98786562bbb379598 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 9 Apr 2024 23:56:34 +0200 Subject: [PATCH 15/93] refactor tests to be run in parallel --- modules/fixtures/README.md | 3 + .../fixtures/fixtures.default.aurora.tfvars | 0 .../fixtures/fixtures.default.eks.tfvars | 0 .../fixtures/postgres-client.bak.yml | 0 .../fixtures/postgres-client.yml | 0 .../fixtures/scripts/create_aurora_pg_db.sh | 0 .../fixtures/whoami-deployment.yml | 0 test/.gitignore | 1 + test/README.md | 40 +++++---- test/src/.tool-versions | 2 +- test/src/custom_eks_rds_test.go | 87 +++++++++++++------ test/src/default_eks_test.go | 50 +++++++++-- test/src/go.mod | 8 +- test/src/go.sum | 16 ++-- test/src/upgrade_eks_test.go | 70 +++++++++------ test/src/utils/aws.go | 18 ---- 16 files changed, 189 insertions(+), 106 deletions(-) create mode 100644 modules/fixtures/README.md rename {test/src => modules}/fixtures/fixtures.default.aurora.tfvars (100%) rename {test/src => modules}/fixtures/fixtures.default.eks.tfvars (100%) rename {test/src => modules}/fixtures/postgres-client.bak.yml (100%) rename {test/src => modules}/fixtures/postgres-client.yml (100%) rename {test/src => modules}/fixtures/scripts/create_aurora_pg_db.sh (100%) rename {test/src => modules}/fixtures/whoami-deployment.yml (100%) create mode 100644 test/.gitignore diff --git a/modules/fixtures/README.md b/modules/fixtures/README.md new file mode 100644 index 00000000..6db98598 --- /dev/null +++ b/modules/fixtures/README.md @@ -0,0 +1,3 @@ +# Fixtures + +This directory is used by tests, due to a limitation of the terratest, it needs to be located in the modules directory to be copied at tests time. diff --git a/test/src/fixtures/fixtures.default.aurora.tfvars b/modules/fixtures/fixtures.default.aurora.tfvars similarity index 100% rename from test/src/fixtures/fixtures.default.aurora.tfvars rename to modules/fixtures/fixtures.default.aurora.tfvars diff --git a/test/src/fixtures/fixtures.default.eks.tfvars b/modules/fixtures/fixtures.default.eks.tfvars similarity index 100% rename from test/src/fixtures/fixtures.default.eks.tfvars rename to modules/fixtures/fixtures.default.eks.tfvars diff --git a/test/src/fixtures/postgres-client.bak.yml b/modules/fixtures/postgres-client.bak.yml similarity index 100% rename from test/src/fixtures/postgres-client.bak.yml rename to modules/fixtures/postgres-client.bak.yml diff --git a/test/src/fixtures/postgres-client.yml b/modules/fixtures/postgres-client.yml similarity index 100% rename from test/src/fixtures/postgres-client.yml rename to modules/fixtures/postgres-client.yml diff --git a/test/src/fixtures/scripts/create_aurora_pg_db.sh b/modules/fixtures/scripts/create_aurora_pg_db.sh similarity index 100% rename from test/src/fixtures/scripts/create_aurora_pg_db.sh rename to modules/fixtures/scripts/create_aurora_pg_db.sh diff --git a/test/src/fixtures/whoami-deployment.yml b/modules/fixtures/whoami-deployment.yml similarity index 100% rename from test/src/fixtures/whoami-deployment.yml rename to modules/fixtures/whoami-deployment.yml diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..2dc04f43 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +states diff --git a/test/README.md b/test/README.md index 4fbab2fd..6446a05b 100644 --- a/test/README.md +++ b/test/README.md @@ -1,12 +1,16 @@ +# Testing + ## Requirements +Make sure you have `opentofu` installed: + ```bash brew update brew install opentofu -```[README.md](..%2FREADME.md) +``` + +Ensure you have `awscli` installed and configured with a proper AWS profile and region: -Make sure you have awscli installed and configured -Make sure you have an AWS profile setup and a region: ```bash # install aws cli brew install awscli @@ -18,37 +22,43 @@ export AWS_DEFAULT_PROFILE=SystemAdministrator-**** export AWS_REGION=eu-central-1 ``` -If you want to have a non-random cluster uid: -``` +If you want to specify a non-random cluster UID: + +```bash export TEST_CLUSTER_ID="myTest" ``` -If you don't want to delete the ressources at the end: -``` -CLEAN_CLUSTER_AT_THE_END=false +If you don't want to delete the resources at the end of the test: + +```bash +export CLEAN_CLUSTER_AT_THE_END=false ``` -test with +Test with: + ```bash make test # or just test one case -go test -v -timeout 120m -run TestDefaultEKS +go test -v -timeout 120m -run TestDefaultEKSTestSuite ``` -### Troubleshooting +When you run the test, terratest will create a copy of the module to be tested in the `tests/states` directory. You can later navigate to the directory and use its content to manipulate the cluster. You can set the `SKIP_XXX` variable to prevent unique IDs of tests from being generated each time, thus using the same resources instead of deploying new resources with terraform. -```bash -# make sure you don't have test clusters running since a while +## Troubleshooting + +Ensure you don't have test clusters running for a while: +```bash eksctl get clusters ``` - You can change the default deployment region: + ```bash -export TESTS_CLUSTER_REGION "eu-west-1" +export TESTS_CLUSTER_REGION="eu-west-1" ``` + # TODO : tests weekly # see https://github.com/camunda/c8-multi-region/blob/main/.github/workflows/nightly_aws_region_cleanup.yml diff --git a/test/src/.tool-versions b/test/src/.tool-versions index 6bc74c51..3960504e 100644 --- a/test/src/.tool-versions +++ b/test/src/.tool-versions @@ -1 +1 @@ -golang 1.22.1 +golang 1.22.2 diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index be7c71ca..317d85ae 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -10,13 +10,16 @@ import ( "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" + test_structure "github.com/gruntwork-io/terratest/modules/test-structure" "github.com/stretchr/testify/suite" "go.uber.org/zap" "go.uber.org/zap/zaptest" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/runtime" "os" + "path/filepath" "strings" "testing" "time" @@ -42,12 +45,17 @@ func (suite *CustomEKSRDSTestSuite) SetupTest() { suite.clusterName = fmt.Sprintf("cluster-rds-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") suite.expectedNodes = 3 - suite.kubeConfigPath = "kubeconfig-rds-eks" - suite.tfDataDir = fmt.Sprintf("tf-data-%s", clusterSuffix) + var errAbsPath error + suite.tfDataDir, errAbsPath = filepath.Abs(fmt.Sprintf("../../test/states/tf-data-%s", suite.clusterName)) + suite.Require().NoError(errAbsPath) + suite.kubeConfigPath = fmt.Sprintf("%s/kubeconfig-rds-eks", suite.tfDataDir) } func (suite *CustomEKSRDSTestSuite) TearUpTest() { - err := os.Setenv("TF_DATA_DIR", suite.tfDataDir) + // create tf state + absPath, err := filepath.Abs(suite.tfDataDir) + suite.Require().NoError(err) + err = os.MkdirAll(absPath, os.ModePerm) suite.Require().NoError(err) } @@ -63,7 +71,36 @@ func (suite *CustomEKSRDSTestSuite) TearDownTest() { // TestCustomEKSAndRDS spawns a custom EKS cluster with custom parameters, and spawns a // pg client pod that will test connection to AuroraDB func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { - terraformOptions := suite.createEKS() + suite.varTf = map[string]interface{}{ + "name": suite.clusterName, + "region": suite.region, + "np_desired_node_count": suite.expectedNodes, + } + + suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) + + fullDir := fmt.Sprintf("%seks-cluster/", suite.tfDataDir) + errTfDir := os.MkdirAll(fullDir, os.ModePerm) + suite.Require().NoError(errTfDir) + tfDir := test_structure.CopyTerraformFolderToDest(suite.T(), "../../", "modules/eks-cluster", fullDir) + + terraformOptions := &terraform.Options{ + TerraformDir: tfDir, + Upgrade: false, + VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, + } + + cleanClusterAtTheEnd := utils.GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") + + if cleanClusterAtTheEnd == "true" { + defer terraform.Destroy(suite.T(), terraformOptions) + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(suite.T(), terraformOptions) + }) + } + + terraform.InitAndApplyAndIdempotent(suite.T(), terraformOptions) // Wait for the worker nodes to join the cluster sess, err := utils.GetAwsClient() @@ -103,14 +140,27 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { "iam_auth_enabled": true, } + fullDirAurora := fmt.Sprintf("%s/aurora/", suite.tfDataDir) + errTfDirAurora := os.MkdirAll(fullDir, os.ModePerm) + suite.Require().NoError(errTfDirAurora) + + tfDirAurora := test_structure.CopyTerraformFolderToDest(suite.T(), "../../modules/", "aurora/", fullDirAurora) + terraformOptionsRDS := &terraform.Options{ - TerraformDir: "../../modules/aurora", + TerraformDir: tfDirAurora, Upgrade: false, - VarFiles: []string{"../../test/src/fixtures/fixtures.default.aurora.tfvars"}, + VarFiles: []string{"../fixtures/fixtures.default.aurora.tfvars"}, Vars: suite.varTf, } - terraformOptionsRDS = utils.ApplyTfAndCleanup(suite.T(), terraformOptionsRDS) + if cleanClusterAtTheEnd == "true" { + defer terraform.Destroy(suite.T(), terraformOptionsRDS) + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(suite.T(), terraformOptionsRDS) + }) + } + + terraform.InitAndApplyAndIdempotent(suite.T(), terraformOptionsRDS) auroraEndpoint := terraform.Output(suite.T(), terraformOptionsRDS, "aurora_endpoint") suite.Assert().NotEmpty(auroraEndpoint) @@ -118,11 +168,10 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { kubeClient, err := utils.NewKubeClientSet(result.Cluster) suite.Require().NoError(err) - kubeConfigPath := "kubeconfig-eks-rds" - utils.GenerateKubeConfigFromAWS(suite.T(), suite.region, suite.clusterName, utils.GetAwsProfile(), kubeConfigPath) + utils.GenerateKubeConfigFromAWS(suite.T(), suite.region, suite.clusterName, utils.GetAwsProfile(), suite.kubeConfigPath) namespace := "postgres-client" - pgKubeCtlOptions := k8s.NewKubectlOptions("", kubeConfigPath, namespace) + pgKubeCtlOptions := k8s.NewKubectlOptions("", suite.kubeConfigPath, namespace) utils.CreateIfNotExistsNamespace(suite.T(), pgKubeCtlOptions, namespace) // deploy the postgres-client ConfigMap @@ -260,24 +309,6 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { } } -func (suite *CustomEKSRDSTestSuite) createEKS() *terraform.Options { - suite.varTf = map[string]interface{}{ - "name": suite.clusterName, - "region": suite.region, - "np_desired_node_count": suite.expectedNodes, - } - - suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) - - terraformOptions := &terraform.Options{ - TerraformDir: "../../modules/eks-cluster", - Upgrade: false, - VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, - Vars: suite.varTf, - } - return utils.ApplyTfAndCleanup(suite.T(), terraformOptions) -} - func TestCustomEKSRDSTestSuite(t *testing.T) { suite.Run(t, new(CustomEKSRDSTestSuite)) } diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go index e432fe7a..82aa2930 100644 --- a/test/src/default_eks_test.go +++ b/test/src/default_eks_test.go @@ -2,8 +2,10 @@ package test import ( "context" + "errors" "fmt" "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/eks" @@ -12,10 +14,13 @@ import ( "github.com/camunda/camunda-tf-eks-module/utils" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" + test_structure "github.com/gruntwork-io/terratest/modules/test-structure" "github.com/stretchr/testify/suite" "go.uber.org/zap" "go.uber.org/zap/zaptest" + "k8s.io/apimachinery/pkg/util/runtime" "os" + "path/filepath" "strings" "testing" "time" @@ -40,13 +45,18 @@ func (suite *DefaultEKSTestSuite) SetupTest() { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) suite.clusterName = fmt.Sprintf("cluster-test-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") - suite.kubeConfigPath = "kubeconfig-default-eks" suite.expectedNodes = 4 - suite.tfDataDir = fmt.Sprintf("tf-data-%s", clusterSuffix) + var errAbsPath error + suite.tfDataDir, errAbsPath = filepath.Abs(fmt.Sprintf("../../test/states/tf-data-%s", suite.clusterName)) + suite.Require().NoError(errAbsPath) + suite.kubeConfigPath = fmt.Sprintf("%s/kubeconfig-default-eks", suite.tfDataDir) } func (suite *DefaultEKSTestSuite) TearUpTest() { - err := os.Setenv("TF_DATA_DIR", suite.tfDataDir) + // create tf state + absPath, err := filepath.Abs(suite.tfDataDir) + suite.Require().NoError(err) + err = os.MkdirAll(absPath, os.ModePerm) suite.Require().NoError(err) } @@ -68,14 +78,29 @@ func (suite *DefaultEKSTestSuite) TestDefaultEKS() { "np_desired_node_count": suite.expectedNodes, } + fullDir := fmt.Sprintf("%s/eks-cluster/", suite.tfDataDir) + errTfDir := os.MkdirAll(fullDir, os.ModePerm) + suite.Require().NoError(errTfDir) + + tfDir := test_structure.CopyTerraformFolderToDest(suite.T(), "../../modules/", "eks-cluster/", fullDir) + terraformOptions := &terraform.Options{ - TerraformDir: "../../modules/eks-cluster", + TerraformDir: tfDir, Upgrade: false, - VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, + VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, Vars: suite.varTf, } - terraformOptions = utils.ApplyTfAndCleanup(suite.T(), terraformOptions) + cleanClusterAtTheEnd := utils.GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") + + if cleanClusterAtTheEnd == "true" { + defer terraform.Destroy(suite.T(), terraformOptions) + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(suite.T(), terraformOptions) + }) + } + + terraform.InitAndApplyAndIdempotent(suite.T(), terraformOptions) suite.baseChecksEKS(terraformOptions) } @@ -195,7 +220,18 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti keyDetails, errKey := kmsSvc.DescribeKey(context.Background(), &kms.DescribeKeyInput{ KeyId: key.KeyId, }) - suite.Require().NoErrorf(errKey, "Failed to describe key %s", *key.KeyId) + + if errKey != nil { + // ignore AccessDenied + var re *awshttp.ResponseError + if errors.As(err, &re) { + if re.HTTPStatusCode() == 400 { + continue + } + } + + suite.Require().NoErrorf(errKey, "Failed to describe key %s", *key.KeyId) + } keyFound = *keyDetails.KeyMetadata.Description == keyDescription if keyFound { diff --git a/test/src/go.mod b/test/src/go.mod index 41ef451c..429b10e4 100644 --- a/test/src/go.mod +++ b/test/src/go.mod @@ -9,7 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/eks v1.41.2 github.com/aws/aws-sdk-go-v2/service/iam v1.31.3 github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 - github.com/aws/aws-sdk-go-v2/service/rds v1.76.1 + github.com/aws/aws-sdk-go-v2/service/rds v1.76.0 github.com/gruntwork-io/terratest v0.46.13 github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.27.0 @@ -30,11 +30,11 @@ require ( github.com/aws/aws-sdk-go v1.44.332 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect diff --git a/test/src/go.sum b/test/src/go.sum index 0a115fbb..5c13ce7c 100644 --- a/test/src/go.sum +++ b/test/src/go.sum @@ -214,10 +214,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMN github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/service/ec2 v1.154.0 h1:+OJ9EhHaqjtA4YTTbxxLxMffrWuGWh0qMaBmGJTLSSg= @@ -228,12 +228,12 @@ github.com/aws/aws-sdk-go-v2/service/iam v1.31.3 h1:cJn9Snros9WmDA7/qCCN7jSkowcu github.com/aws/aws-sdk-go-v2/service/iam v1.31.3/go.mod h1:+nAQlxsBxPFf6GrL93lvCuv5PxSTX3GO0RYrURyzl/Q= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 h1:b+E7zIUHMmcB4Dckjpkapoy47W6C9QBv/zoUP+Hn8Kc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6/go.mod h1:S2fNV0rxrP78NhPbCZeQgY8H9jdDMeGtwcfZIRxzBqU= github.com/aws/aws-sdk-go-v2/service/kms v1.30.0 h1:yS0JkEdV6h9JOo8sy2JSpjX+i7vsKifU8SIeHrqiDhU= github.com/aws/aws-sdk-go-v2/service/kms v1.30.0/go.mod h1:+I8VUUSVD4p5ISQtzpgSva4I8cJ4SQ4b1dcBcof7O+g= -github.com/aws/aws-sdk-go-v2/service/rds v1.76.1 h1:6NeDO4UyHun2N5Lmkv2yW1sNblRLRjq3j6Azv7kUyuM= -github.com/aws/aws-sdk-go-v2/service/rds v1.76.1/go.mod h1:Rw15qGaGWu3jO0dOz7JyvdOEjgae//YrJxVWLYGynvg= +github.com/aws/aws-sdk-go-v2/service/rds v1.76.0 h1:cQUdm2sU/71O1vCCV627GrQz5b9RmfuxViYDiLsAdZg= +github.com/aws/aws-sdk-go-v2/service/rds v1.76.0/go.mod h1:TsRoxafRyxgt1c1JWQXmxj/dCEwOkBapTwskET8vgFo= github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 h1:mnbuWHOcM70/OFUlZZ5rcdfA8PflGXXiefU/O+1S3+8= github.com/aws/aws-sdk-go-v2/service/sso v1.20.3/go.mod h1:5HFu51Elk+4oRBZVxmHrSds5jFXmFj8C3w7DVF2gnrs= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 h1:uLq0BKatTmDzWa/Nu4WO0M1AaQDaPpwTKAeByEc6WFM= diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index 896865f8..f713e8bf 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -10,10 +10,13 @@ import ( "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" + test_structure "github.com/gruntwork-io/terratest/modules/test-structure" "github.com/stretchr/testify/suite" "go.uber.org/zap" "go.uber.org/zap/zaptest" + "k8s.io/apimachinery/pkg/util/runtime" "os" + "path/filepath" "strings" "testing" "time" @@ -40,16 +43,18 @@ func (suite *UpgradeEKSTestSuite) SetupTest() { suite.clusterName = fmt.Sprintf("cluster-upgrade-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") suite.expectedNodes = 3 - suite.kubeConfigPath = "kubeconfig-upgrade-eks" suite.kubeVersion = "1.28" - suite.tfDataDir = fmt.Sprintf("tf-data-%s", clusterSuffix) - - // Create EKS cluster - suite.createEKS() + var errAbsPath error + suite.tfDataDir, errAbsPath = filepath.Abs(fmt.Sprintf("../../test/states/tf-data-%s", suite.clusterName)) + suite.Require().NoError(errAbsPath) + suite.kubeConfigPath = fmt.Sprintf("%s/kubeconfig-upgrade-eks", suite.tfDataDir) } func (suite *UpgradeEKSTestSuite) TearUpTest() { - err := os.Setenv("TF_DATA_DIR", suite.tfDataDir) + // create tf state + absPath, err := filepath.Abs(suite.tfDataDir) + suite.Require().NoError(err) + err = os.MkdirAll(absPath, os.ModePerm) suite.Require().NoError(err) } @@ -65,6 +70,40 @@ func (suite *UpgradeEKSTestSuite) TearDownTest() { // TestUpgradeEKS starts from a version of EKS, deploy a simple chart, upgrade the cluster // and check that everything is working as expected func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { + // create the eks cluster + suite.varTf = map[string]interface{}{ + "name": suite.clusterName, + "region": suite.region, + "np_desired_node_count": suite.expectedNodes, + "kubernetes_version": suite.kubeVersion, + } + + fullDir := fmt.Sprintf("%s/eks-cluster/", suite.tfDataDir) + errTfDir := os.MkdirAll(fullDir, os.ModePerm) + suite.Require().NoError(errTfDir) + + tfDir := test_structure.CopyTerraformFolderToDest(suite.T(), "../../modules/", "eks-cluster/", fullDir) + + terraformOptions := &terraform.Options{ + TerraformDir: tfDir, + Upgrade: false, + VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, + } + + suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) + + cleanClusterAtTheEnd := utils.GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") + + if cleanClusterAtTheEnd == "true" { + defer terraform.Destroy(suite.T(), terraformOptions) + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(suite.T(), terraformOptions) + }) + } + + terraform.InitAndApplyAndIdempotent(suite.T(), terraformOptions) + // Wait for the worker nodes to join the cluster sess, err := utils.GetAwsClient() suite.Require().NoErrorf(err, "Failed to get aws client") @@ -159,25 +198,6 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { ) } -func (suite *UpgradeEKSTestSuite) createEKS() *terraform.Options { - suite.varTf = map[string]interface{}{ - "name": suite.clusterName, - "region": suite.region, - "np_desired_node_count": suite.expectedNodes, - "kubernetes_version": suite.kubeVersion, - } - - terraformOptions := &terraform.Options{ - TerraformDir: "../../modules/eks-cluster", - Upgrade: false, - VarFiles: []string{"../../test/src/fixtures/fixtures.default.eks.tfvars"}, - Vars: suite.varTf, - } - - suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) - return utils.ApplyTfAndCleanup(suite.T(), terraformOptions) -} - func TestUpgradeEKSTestSuite(t *testing.T) { suite.Run(t, new(UpgradeEKSTestSuite)) } diff --git a/test/src/utils/aws.go b/test/src/utils/aws.go index 0eec6cb7..45d2f51f 100644 --- a/test/src/utils/aws.go +++ b/test/src/utils/aws.go @@ -7,9 +7,6 @@ import ( "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/eks/types" - "github.com/gruntwork-io/terratest/modules/terraform" - "k8s.io/apimachinery/pkg/util/runtime" - "testing" "time" ) @@ -32,21 +29,6 @@ func GetAwsClient() (aws.Config, error) { ) } -// ApplyTfAndCleanup applies a Tf resource and cleanup at the end -func ApplyTfAndCleanup(t *testing.T, terraformOptions *terraform.Options) *terraform.Options { - cleanClusterAtTheEnd := GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") - - if cleanClusterAtTheEnd == "true" { - defer terraform.Destroy(t, terraformOptions) - defer runtime.HandleCrash(func(i interface{}) { - terraform.Destroy(t, terraformOptions) - }) - } - - terraform.InitAndApplyAndIdempotent(t, terraformOptions) - return terraformOptions -} - func WaitForUpdateEKS(ctx context.Context, client *eks.Client, clusterName, updateID string) error { describeUpdateInput := &eks.DescribeUpdateInput{ Name: &clusterName, From 254e385fa0ba75670d162080744f6dfeb0292458 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 08:09:19 +0200 Subject: [PATCH 16/93] enable parallel and reduce nb nodes --- test/src/custom_eks_rds_test.go | 3 ++- test/src/default_eks_test.go | 1 + test/src/upgrade_eks_test.go | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index 317d85ae..742e962c 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -44,7 +44,7 @@ func (suite *CustomEKSRDSTestSuite) SetupTest() { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) suite.clusterName = fmt.Sprintf("cluster-rds-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") - suite.expectedNodes = 3 + suite.expectedNodes = 1 var errAbsPath error suite.tfDataDir, errAbsPath = filepath.Abs(fmt.Sprintf("../../test/states/tf-data-%s", suite.clusterName)) suite.Require().NoError(errAbsPath) @@ -310,5 +310,6 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { } func TestCustomEKSRDSTestSuite(t *testing.T) { + t.Parallel() suite.Run(t, new(CustomEKSRDSTestSuite)) } diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go index 82aa2930..b50c0a26 100644 --- a/test/src/default_eks_test.go +++ b/test/src/default_eks_test.go @@ -242,5 +242,6 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti } func TestDefaultEKSTestSuite(t *testing.T) { + t.Parallel() suite.Run(t, new(DefaultEKSTestSuite)) } diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index f713e8bf..01ab7b67 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -199,5 +199,6 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { } func TestUpgradeEKSTestSuite(t *testing.T) { + t.Parallel() suite.Run(t, new(UpgradeEKSTestSuite)) } From 146dc181cace8321fcfae09df1370063deafec28 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 09:59:48 +0200 Subject: [PATCH 17/93] improve CI and fix rds --- .github/labeler.yml | 19 ++++++++ .github/markdown-links.json | 17 +++++++ .github/workflows/labeler.yml | 13 ++++++ .github/workflows/lint.yml | 16 +++++++ .github/workflows/tests.yml | 83 +++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 34 +++++++++++++- test/src/custom_eks_rds_test.go | 10 ++-- 7 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 .github/labeler.yml create mode 100644 .github/markdown-links.json create mode 100644 .github/workflows/labeler.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..e908ab44 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,19 @@ +# Add 'feature' label to any PR where the head branch name starts with `feature` or has a `feature` section in the name +feature: + - head-branch: ['^feature', 'feature'] + +# Add 'test' label to any changes within 'test' folder or any subfolders +test: + - changed-files: + - any-glob-to-any-file: test/** + +doc: + - changed-files: + - any-glob-to-any-file: + - docs/** + - "**.md" + +terraform: + - changed-files: + - any-glob-to-any-file: + - modules/**.tf diff --git a/.github/markdown-links.json b/.github/markdown-links.json new file mode 100644 index 00000000..74ff705e --- /dev/null +++ b/.github/markdown-links.json @@ -0,0 +1,17 @@ +{ + "projectBaseUrl":"${workspaceFolder}", + "ignorePatterns": [ + { + "pattern": "^http(s?)://localhost" + } + ], + "replacementPatterns": [ + ], + "httpHeaders": [ + ], + "timeout": "20s", + "retryOn429": true, + "retryCount": 5, + "fallbackRetryDelay": "30s", + "aliveStatusCodes": [200, 206] +} diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 00000000..e7847016 --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,13 @@ +name: "Pull Request Labeler" +on: + pull_request_target: + schedule: + - cron: "0 1 * * 1" +jobs: + labeler: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v5 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..a6c7bd49 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,16 @@ +--- +# desc: lint sanity check +name: lint + +on: + push: + workflow_dispatch: + +jobs: + lint: + name: pre-commit + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..2abba58d --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,83 @@ +--- +name: Terraform modules tests + +on: + schedule: + - cron: '0 0 * * 1-5' + workflow_dispatch: + pull_request_review: + types: [submitted] + +env: + AWS_PROFILE: "infex" + AWS_REGION: "eu-central-1" + +jobs: + integration-test: + runs-on: ubuntu-22.04 + if: (github.event.label.name == 'test' || github.event.label.name == 'terraform') && github.event.review.state == 'approved' + # Other dependencies from hosted runner + # AWS CLI + # indirectly node used by actions + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get go.mod details + uses: Eun/go-mod-details@v1 + id: go-mod-details + with: + modfile: ${{ github.workspace }}/test/src/go.mod + + - name: Set up Go environment + uses: actions/setup-go@v5 + with: + go-version: ${{ steps.go-mod-details.outputs.go_version }} + + - name: Set up Terraform (OpenTofu) + uses: opentofu/setup-opentofu@v1 + with: + tofu_version: 1.6.2 + + - name: Import Secrets + id: secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_ADDR }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + exportEnv: false + secrets: | + secret/data/products/infrastructure-experience/ci/common AWS_ACCESS_KEY; + secret/data/products/infrastructure-experience/ci/common AWS_SECRET_KEY; + + # Official action does not support profiles + - name: Add profile credentials to ~/.aws/credentials + run: | + aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }} + + - name: Get Short GitHub SHA and Save in Environment Variable + run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" + + - name: Configure tests + id: configure-tests + run: | + echo "TEST_CLUSTER_ID=${{ env.SHORT_GITHUB_SHA }}" >> "$GITHUB_ENV" + echo "CLEAN_CLUSTER_AT_THE_END=true" >> "$GITHUB_ENV" + + - name: Launch tests + working-directory: ./test/src/ + timeout-minutes: 400 # todo: replace all single tests with a global one + run: | + go test -v -timeout 120m -run TestDefaultEKSTestSuite + go test -v -timeout 120m -run TestUpgradeEKSTestSuite + go test -v -timeout 120m -run TestCustomEKSRDSTestSuite + + - name: Remove profile credentials from ~/.aws/credentials + if: always() + run: | + rm -rf ~/.aws/credentials diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d09dcc8a..ae771e49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,9 +6,39 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - id: check-added-large-files + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + - id: check-toml + - id: check-json + - id: check-symlinks + - id: check-shebang-scripts-are-executable + - id: detect-private-key + +- repo: https://github.com/rhysd/actionlint + rev: v1.6.27 + hooks: + - id: actionlint-docker + +- repo: https://github.com/renovatebot/pre-commit-hooks + rev: 37.278.0 + hooks: + - id: renovate-config-validator + args: ["--strict"] + +- repo: https://github.com/compilerla/conventional-pre-commit + rev: v3.2.0 + hooks: + - id: conventional-pre-commit + stages: [commit-msg] + args: ["--strict" , "--force-scope"] + +- repo: https://github.com/tcort/markdown-link-check + rev: v3.12.1 # use tags until renovate supports sha: https://github.com/renovatebot/renovate/issues/22567 + hooks: + - id: markdown-link-check + args: [-q, -c .github/markdown-links.json] - repo: https://github.com/antonbabenko/pre-commit-terraform rev: v1.88.3 diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index 742e962c..244da7b4 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -79,10 +79,10 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) - fullDir := fmt.Sprintf("%seks-cluster/", suite.tfDataDir) - errTfDir := os.MkdirAll(fullDir, os.ModePerm) - suite.Require().NoError(errTfDir) - tfDir := test_structure.CopyTerraformFolderToDest(suite.T(), "../../", "modules/eks-cluster", fullDir) + fullDirEKS := fmt.Sprintf("%seks-cluster/", suite.tfDataDir) + errTfDirEKS := os.MkdirAll(fullDirEKS, os.ModePerm) + suite.Require().NoError(errTfDirEKS) + tfDir := test_structure.CopyTerraformFolderToDest(suite.T(), "../../modules/", "eks-cluster/", fullDirEKS) terraformOptions := &terraform.Options{ TerraformDir: tfDir, @@ -141,7 +141,7 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { } fullDirAurora := fmt.Sprintf("%s/aurora/", suite.tfDataDir) - errTfDirAurora := os.MkdirAll(fullDir, os.ModePerm) + errTfDirAurora := os.MkdirAll(fullDirAurora, os.ModePerm) suite.Require().NoError(errTfDirAurora) tfDirAurora := test_structure.CopyTerraformFolderToDest(suite.T(), "../../modules/", "aurora/", fullDirAurora) From 5f10d550ca97dd025fa987825510b810957ea09a Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 10:10:50 +0200 Subject: [PATCH 18/93] fix pre-commit --- .github/markdown-links.json | 3 +++ .pre-commit-config.yaml | 1 + README.md | 4 ++-- modules/fixtures/scripts/create_aurora_pg_db.sh | 0 modules/fixtures/whoami-deployment.yml | 1 + test/src/Makefile | 0 6 files changed, 7 insertions(+), 2 deletions(-) mode change 100644 => 100755 modules/fixtures/scripts/create_aurora_pg_db.sh mode change 100644 => 100755 test/src/Makefile diff --git a/.github/markdown-links.json b/.github/markdown-links.json index 74ff705e..6d173512 100644 --- a/.github/markdown-links.json +++ b/.github/markdown-links.json @@ -3,6 +3,9 @@ "ignorePatterns": [ { "pattern": "^http(s?)://localhost" + }, + { + "pattern": "^#" } ], "replacementPatterns": [ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ae771e49..5c39e7ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - id: check-yaml + args: [--allow-multiple-documents] - id: check-toml - id: check-json - id: check-symlinks diff --git a/README.md b/README.md index 3d4554f9..6fb1cd8e 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ Terraform module which creates AWS EKS (Kubernetes) resources with an opinionate ## Documentation -The related [guide](https://docs.camunda.io/docs/next/self-managed/platform-deployment/helm-kubernetes/platforms/amazon-eks/eks-terraform/) describing a more detailed usage. -Consider installing Camunda 8 via [following guide](https://docs.camunda.io/docs/next/self-managed/platform-deployment/helm-kubernetes/platforms/amazon-eks/eks-helm/) after having deployed the AWS EKS cluster. +The related [guide](https://docs.camunda.io/docs/next/self-managed/setup/deploy/amazon/amazon-eks/eks-terraform/) describing a more detailed usage. +Consider installing Camunda 8 via [following guide](https://docs.camunda.io/docs/next/self-managed/setup/deploy/amazon/amazon-eks/eks-helm/) after having deployed the AWS EKS cluster. ## Usage diff --git a/modules/fixtures/scripts/create_aurora_pg_db.sh b/modules/fixtures/scripts/create_aurora_pg_db.sh old mode 100644 new mode 100755 diff --git a/modules/fixtures/whoami-deployment.yml b/modules/fixtures/whoami-deployment.yml index 4ff4d643..2eb1abf7 100644 --- a/modules/fixtures/whoami-deployment.yml +++ b/modules/fixtures/whoami-deployment.yml @@ -1,3 +1,4 @@ +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/test/src/Makefile b/test/src/Makefile old mode 100644 new mode 100755 From b9a18c4235641bbffffa917f09f133230bdd46d3 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 11:00:46 +0200 Subject: [PATCH 19/93] fix some tests --- .github/workflows/tests.yml | 6 ++---- test/src/custom_eks_rds_test.go | 2 +- test/src/upgrade_eks_test.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2abba58d..1e38d293 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -71,11 +71,9 @@ jobs: - name: Launch tests working-directory: ./test/src/ - timeout-minutes: 400 # todo: replace all single tests with a global one + timeout-minutes: 125 # due to a limit on elastic ips, tests suites are ran 1 by 1 run: | - go test -v -timeout 120m -run TestDefaultEKSTestSuite - go test -v -timeout 120m -run TestUpgradeEKSTestSuite - go test -v -timeout 120m -run TestCustomEKSRDSTestSuite + go test -v -timeout 120m -p 1 ./... - name: Remove profile credentials from ~/.aws/credentials if: always() diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index 244da7b4..e563b28b 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -150,7 +150,7 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { TerraformDir: tfDirAurora, Upgrade: false, VarFiles: []string{"../fixtures/fixtures.default.aurora.tfvars"}, - Vars: suite.varTf, + Vars: varsConfigAurora, } if cleanClusterAtTheEnd == "true" { diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index 01ab7b67..20fd7e5f 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -132,7 +132,7 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { utils.CreateIfNotExistsNamespace(suite.T(), kubeCtlOptions, namespace) // deploy the postgres-client Job to test the connection - k8s.KubectlApply(suite.T(), kubeCtlOptions, "../../test/src/fixtures/whoami-deployment.yml") + k8s.KubectlApply(suite.T(), kubeCtlOptions, "../fixtures/whoami-deployment.yml") k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 60, 1*time.Second) From 6d01a09bfba496607f1d340d25ae69439618e570 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 11:11:28 +0200 Subject: [PATCH 20/93] fix wrong paths for rds --- test/src/custom_eks_rds_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index e563b28b..61a84750 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -215,7 +215,7 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { k8s.WaitUntilSecretAvailable(suite.T(), pgKubeCtlOptions, secretPostgres.Name, 6, 10*time.Second) // add the scripts as a ConfigMap - scriptPath := "../../test/src/fixtures/scripts/create_aurora_pg_db.sh" + scriptPath := "../fixtures/scripts/create_aurora_pg_db.sh" scriptContent, err := os.ReadFile(scriptPath) suite.Require().NoError(err) @@ -245,7 +245,7 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { } // deploy the postgres-client Job to test the connection - k8s.KubectlApply(suite.T(), pgKubeCtlOptions, "../../test/src/fixtures/postgres-client.yml") + k8s.KubectlApply(suite.T(), pgKubeCtlOptions, "../fixtures/postgres-client.yml") errJob := utils.WaitForJobCompletion(kubeClient, namespace, "postgres-client", 5*time.Minute, jobListOptions) suite.Require().NoError(errJob) From f1d9b2cd747e0c3426e3dea6983f14ab6ec77a26 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 11:17:56 +0200 Subject: [PATCH 21/93] setup go --- .github/workflows/lint.yml | 9 +++++++++ test/src/Makefile | 6 +----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a6c7bd49..c9e6cd3b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,4 +13,13 @@ jobs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 + + - uses: terraform-linters/setup-tflint@v4 + name: Setup TFLint + with: + tflint_version: v0.50.3 + + - uses: actions/setup-go@v5 + - run: go install github.com/terraform-docs/terraform-docs@v0.17.0 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/test/src/Makefile b/test/src/Makefile index 4d944493..765e24e3 100755 --- a/test/src/Makefile +++ b/test/src/Makefile @@ -6,14 +6,10 @@ init: ## Initialize tests @exit 0 .PHONY : test -test: init clean ## Run tests +test: init ## Run tests go mod download go test -v -timeout 120m -.PHONY : clean -clean: ## Clean up files - rm -rf ../../*/*/*.tfstate* - # see https://suva.sh/posts/well-documented-makefiles/ .PHONY : help help: ## Show this help prompt. From 18cb1e196087fd33bf11fdfad03bad84ad4f118a Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 11:20:49 +0200 Subject: [PATCH 22/93] fix path --- test/src/custom_eks_rds_test.go | 4 ++-- test/src/upgrade_eks_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index 61a84750..30e4c3f8 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -215,7 +215,7 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { k8s.WaitUntilSecretAvailable(suite.T(), pgKubeCtlOptions, secretPostgres.Name, 6, 10*time.Second) // add the scripts as a ConfigMap - scriptPath := "../fixtures/scripts/create_aurora_pg_db.sh" + scriptPath := "../../modules/fixtures/scripts/create_aurora_pg_db.sh" scriptContent, err := os.ReadFile(scriptPath) suite.Require().NoError(err) @@ -245,7 +245,7 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { } // deploy the postgres-client Job to test the connection - k8s.KubectlApply(suite.T(), pgKubeCtlOptions, "../fixtures/postgres-client.yml") + k8s.KubectlApply(suite.T(), pgKubeCtlOptions, "../../modules/fixtures/postgres-client.yml") errJob := utils.WaitForJobCompletion(kubeClient, namespace, "postgres-client", 5*time.Minute, jobListOptions) suite.Require().NoError(errJob) diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index 20fd7e5f..fe7474fc 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -132,7 +132,7 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { utils.CreateIfNotExistsNamespace(suite.T(), kubeCtlOptions, namespace) // deploy the postgres-client Job to test the connection - k8s.KubectlApply(suite.T(), kubeCtlOptions, "../fixtures/whoami-deployment.yml") + k8s.KubectlApply(suite.T(), kubeCtlOptions, "../../modules/fixtures/whoami-deployment.yml") k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 60, 1*time.Second) From 9209dbeb1967c7edf234ec580bfa4d95bf3f559e Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 14:42:41 +0200 Subject: [PATCH 23/93] add DEVELOPER.md and MAINTENANCE.md --- DEVELOPER.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ MAINTENANCE.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 DEVELOPER.md create mode 100644 MAINTENANCE.md diff --git a/DEVELOPER.md b/DEVELOPER.md new file mode 100644 index 00000000..eae27d83 --- /dev/null +++ b/DEVELOPER.md @@ -0,0 +1,48 @@ +# Developer's Guide + +Welcome to the development reference for Camunda's Terraform EKS module! +This document provides guidance on setting up a testing environment, running tests, and managing releases. + +## Setting up Development Environment + +To start developing or testing the EKS module, follow these steps: + +1. **Clone the Repository:** + - Clone the repository from [camunda/camunda-tf-eks-module](https://github.com/camunda/camunda-tf-eks-module) to your local machine. + +2. **Navigate to Test Suite:** + - Go to the `test/src` directory to access the test suite. + +3. **Test-Driven Development (TDD):** + - Use the Test-Driven Development approach to iterate on the module. + - Add or modify test cases in the test suite to match the desired functionality. + - Run tests frequently to ensure changes meet requirements. + +4. **Local Development:** + - Utilize environment variables like `SKIP_XXX` to control certain behaviors during local development. + - Ensure to use a unique identifier for the cluster to avoid conflicts with existing resources. + +5. **Testing Tools:** + - Refer to `test/README.md` for instructions on setting up and using testing tools. + - Add fixtures and test cases using Terratest and Testify to validate module functionality. + +6. **Cluster Cleanup:** + - Set `CLEAN_CLUSTER_AT_THE_END=false` to prevent automatic cluster deletion in case of errors. + - Optionally, manually clean up the cluster after testing by reversing this setting. + +## Releasing a New Version + +We follow Semantic Versioning (SemVer) guidelines for versioning. Follow these steps to release a new version: + +1. **Commit History:** + - Maintain a clear commit history with explicit messages detailing additions and deletions. + +2. **Versioning:** + - Determine the appropriate version number based on the changes made since the last release. + - Follow the format `MAJOR.MINOR.PATCH` as per Semantic Versioning guidelines. + +3. **GitHub Releases:** + - Publish the new version on GitHub Releases. + - Tag the release with the version number and include release notes summarizing changes. + +By following these guidelines, we ensure smooth development iterations, robust testing practices, and clear version management for the Terraform EKS module. Happy coding! diff --git a/MAINTENANCE.md b/MAINTENANCE.md new file mode 100644 index 00000000..bea84c69 --- /dev/null +++ b/MAINTENANCE.md @@ -0,0 +1,42 @@ +# MAINTENANCE.md + +_This file serves as a reference for the maintenance procedures and guidelines for the EKS modules in this project._ +_Note: Please keep this document updated with any changes in maintenance procedures, dependencies, actions, or restrictions._ + +## Maintenance Procedures + +### Before New Releases + +- Update documentation related to new features or changes. + - `README.md` + - Official Camunda documentation: + - [C8SM: Amazon EKS](https://github.com/camunda/camunda-docs/blob/main/docs/self-managed/setup/deploy/amazon/amazon-eks/amazon-eks.md) + - When releasing an update containing breaking changes, it should be accompanied by a migration guide in this repository to guide the user. + +- Make internal announcements on Slack regarding upcoming releases. + - `#infex-internal` + - `#engineering` if relevant + +- Refer to `DEVELOPER.md` to see the release process. + +### After New Releases + +_Nothing referenced yet._ + +## Dependencies + +### Upstream Dependencies: dependencies of this project + +- **terraform-aws-modules**: This project relies on the official AWS modules available at [terraform-aws-modules](https://github.com/terraform-aws-modules). + +### Downstream Dependencies: things that depend on this project + +- **c8-multi-region**: This project utilizes the EKS modules for multi-region deployment, available at [c8-multi-region](https://github.com/camunda/c8-multi-region). + +## Actions + +- Notify the **Product Management Team** of any new releases, especially if there are breaking changes or critical updates. + +## Restrictions + +- Never remove modules in the history of this repository, even if the sources are deprecated or removed. From a7f2e928ac2a9d92d2e627c90edc877a8ba47792 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:38:09 +0200 Subject: [PATCH 24/93] add cloud cleanup --- .github/workflows/nightly_cleanup.yml | 79 ++++++++++++++++++++ .github/workflows/scripts/aws_cleanup.sh | 94 ++++++++++++++++++++++++ test/src/upgrade_eks_test.go | 2 +- 3 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/nightly_cleanup.yml create mode 100755 .github/workflows/scripts/aws_cleanup.sh diff --git a/.github/workflows/nightly_cleanup.yml b/.github/workflows/nightly_cleanup.yml new file mode 100644 index 00000000..8580813b --- /dev/null +++ b/.github/workflows/nightly_cleanup.yml @@ -0,0 +1,79 @@ +--- +name: Nightly Cleanup + +on: + schedule: + - cron: '0 5 * * 1-5' + workflow_dispatch: + +env: + AWS_PROFILE: "infex" + AWS_REGION: "eu-central-1" + +jobs: + aws-nightly-cleanup: + runs-on: ubuntu-latest + # Other dependencies from hosted runner + # AWS CLI + # indirectly node used by actions + + steps: + - uses: actions/checkout@v4 + + - name: Import Secrets + id: secrets + uses: hashicorp/vault-action@v3 + with: + url: ${{ secrets.VAULT_ADDR }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + exportEnv: false + secrets: | + secret/data/products/infrastructure-experience/ci/common AWS_ACCESS_KEY; + secret/data/products/infrastructure-experience/ci/common AWS_SECRET_KEY; + + # Official action does not support profiles + - name: Add profile credentials to ~/.aws/credentials + run: | + aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }} + + - name: Install Cloud Nuke + run: | + wget https://github.com/gruntwork-io/cloud-nuke/releases/download/v0.33.0/cloud-nuke_linux_amd64 + chmod +x cloud-nuke_linux_amd64 + + # This is likely to fail, therefore we ignore the error + # We're ignoring ec2_dhcp_option as they couldn't be deleted + # cloudtrail is managed by IT and can't be deleted either + - name: Run Cloud Nuke + timeout-minutes: 45 + env: + DISABLE_TELEMETRY: "true" + run: | + ./cloud-nuke_linux_amd64 aws \ + --region ${{ env.AWS_REGION }} \ + --force \ + --newer-than 4h \ + --exclude-resource-type ec2_dhcp_option \ + --exclude-resource-type cloudtrail || true + + # Following will delete global resources and things that cloud-nuke does not support + - name: Delete additional AWS resources + timeout-minutes: 15 + run: .github/workflows/scripts/aws_cleanup.sh "${{ env.AWS_REGION }}" + + # The second run should remove the remaining resources (VPCs) and fail if there's anything left + - name: Run Cloud Nuke + timeout-minutes: 45 + env: + DISABLE_TELEMETRY: "true" + run: | + ./cloud-nuke_linux_amd64 aws \ + --region ${{ env.AWS_REGION }} \ + --force \ + --newer-than 4h \ + --exclude-resource-type ec2_dhcp_option \ + --exclude-resource-type cloudtrail diff --git a/.github/workflows/scripts/aws_cleanup.sh b/.github/workflows/scripts/aws_cleanup.sh new file mode 100755 index 00000000..6cd4fb2d --- /dev/null +++ b/.github/workflows/scripts/aws_cleanup.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# This script deletes additional AWS resources based on specified criteria. + +# Check if the region argument is provided +if [ -z "$1" ]; then + echo "Please provide the AWS region as the first argument." + exit 1 +fi + +region="$1" + +echo "Deleting additional resources in the $region region..." + + +echo "Deleting additional resources..." +# KMS keys can't be deleted due to resource policies, requires manual intervention + +echo "Deleting IAM Roles" +# Detach permissions and profile instances and delete IAM roles +role_arns=$(aws iam list-roles --query "Roles[?contains(RoleName, 'nightly')].RoleName" --output text) + +read -r -a role_arns_array <<< "$role_arns" + +for role_arn in "${role_arns_array[@]}" +do + echo "Removing instance profiles and policies of role: $role_arn" + attached_policy_arns=$(aws iam list-attached-role-policies --role-name "$role_arn" --query 'AttachedPolicies[].PolicyArn' --output text) + read -r -a attached_policy_arns_array <<< "$attached_policy_arns" + + for policy_arn in "${attached_policy_arns_array[@]}" + do + echo "Removing attached policy: $policy_arn" + aws iam detach-role-policy --role-name "$role_arn" --policy-arn "$policy_arn" + done + + policy_arns=$(aws iam list-role-policies --role-name "$role_arn" --query 'PolicyNames' --output text) + read -r -a policy_arns_array <<< "$policy_arns" + + for policy_arn in "${policy_arns_array[@]}" + do + echo "Deleting policy: $policy_arn" + aws iam delete-role-policy --role-name "$role_arn" --policy-name "$policy_arn" + done + + instance_profile_arns=$(aws iam list-instance-profiles-for-role --role-name "$role_arn" --query 'InstanceProfiles[].InstanceProfileName' --output text) + read -r -a instance_profile_arns_array <<< "$instance_profile_arns" + + for instance_profile_arn in "${instance_profile_arns_array[@]}" + do + echo "Removing instance profile: $instance_profile_arn" + aws iam remove-role-from-instance-profile --instance-profile-name "$instance_profile_arn" --role-name "$role_arn" + done + + echo "Deleting role: $role_arn" + aws iam delete-role --role-name "$role_arn" + +done + +echo "Deleting IAM Policies" +# Delete Policies +iam_policies=$(aws iam list-policies --query "Policies[?contains(PolicyName, 'nightly')].Arn" --output text) + +read -r -a iam_policies_array <<< "$iam_policies" + +for iam_policy in "${iam_policies_array[@]}" +do + echo "Deleting policy: $iam_policy" + aws iam delete-policy --policy-arn "$iam_policy" +done + +echo "Deleting OIDC Providers" +# Delete OIDC Provider +oidc_providers=$(aws iam list-open-id-connect-providers --query "OpenIDConnectProviderList[?contains(Arn, 'eu-west-2') || contains(Arn, 'eu-west-3')].Arn" --output text) + +read -r -a oidc_providers_array <<< "$oidc_providers" + +for oidc_provider in "${oidc_providers_array[@]}" +do + echo "Deleting OIDC Provider: $oidc_provider" + aws iam delete-open-id-connect-provider --open-id-connect-provider-arn "$oidc_provider" +done + +echo "Deleting VPC Peering Connections" +# Delete VPC Peering Connection +peering_connection_ids=$(aws ec2 describe-vpc-peering-connections --region "$region" --query "VpcPeeringConnections[?Status.Code == 'active' && Tags[?contains(Value, 'nightly')]]".VpcPeeringConnectionId --output text) + +read -r -a peering_connection_ids_array <<< "$peering_connection_ids" + +for peering_connection_id in "${peering_connection_ids_array[@]}" +do + echo "Deleting VPC Peering Connection: $peering_connection_id" + aws ec2 delete-vpc-peering-connection --region "$region" --vpc-peering-connection-id "$peering_connection_id" +done diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index fe7474fc..02e63044 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -174,7 +174,7 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { suite.Assert().Equal(suite.varTf["kubernetes_version"], *result.Cluster.Version) // check everything works as expected - k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 10, 1*time.Second) + k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 60, 1*time.Second) // Forward port again localPort2 := 8887 From fae5e6eaff1bc976502b35e11924a2e89ae08b3a Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:24:57 +0200 Subject: [PATCH 25/93] fix tests, update workflows --- .github/workflows/nightly_cleanup.yml | 2 +- .github/workflows/tests.yml | 2 +- test/README.md | 7 ------- test/src/upgrade_eks_test.go | 4 ++++ 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly_cleanup.yml b/.github/workflows/nightly_cleanup.yml index 8580813b..65e72692 100644 --- a/.github/workflows/nightly_cleanup.yml +++ b/.github/workflows/nightly_cleanup.yml @@ -3,7 +3,7 @@ name: Nightly Cleanup on: schedule: - - cron: '0 5 * * 1-5' + - cron: '0 5 * * *' workflow_dispatch: env: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1e38d293..3434f8f6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,7 +3,7 @@ name: Terraform modules tests on: schedule: - - cron: '0 0 * * 1-5' + - cron: '0 1 * * 2' workflow_dispatch: pull_request_review: types: [submitted] diff --git a/test/README.md b/test/README.md index 6446a05b..af909df8 100644 --- a/test/README.md +++ b/test/README.md @@ -58,10 +58,3 @@ You can change the default deployment region: ```bash export TESTS_CLUSTER_REGION="eu-west-1" ``` - -# TODO : tests weekly -# see https://github.com/camunda/c8-multi-region/blob/main/.github/workflows/nightly_aws_region_cleanup.yml - -# TODO : https://github.com/gruntwork-io/cloud-nuke every weekend -# => we should have a dedicated tenant for CI -# => sometimes, EKS deletion fails with error: DeleteCluster ResourceInUseException: Cluster has nodegroups attached terraform diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index 02e63044..a23bc331 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -135,6 +135,8 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { k8s.KubectlApply(suite.T(), kubeCtlOptions, "../../modules/fixtures/whoami-deployment.yml") k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 60, 1*time.Second) + // wait to ensure service available + time.Sleep(30 * time.Second) // Now we verify that the service will successfully boot and start serving requests localPort1 := 8883 @@ -175,6 +177,8 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { // check everything works as expected k8s.WaitUntilServiceAvailable(suite.T(), kubeCtlOptions, "whoami-service", 60, 1*time.Second) + // wait to ensure service available + time.Sleep(30 * time.Second) // Forward port again localPort2 := 8887 From 14ab3c7024aad21b6c2eb773020de4a9a992e1a0 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:34:47 +0200 Subject: [PATCH 26/93] add labels logic --- .github/workflows/check-labels.yml | 21 +++++++++++++++++ .github/workflows/tests.yml | 2 +- DEVELOPER.md | 37 +++++++++++++++--------------- 3 files changed, 41 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/check-labels.yml diff --git a/.github/workflows/check-labels.yml b/.github/workflows/check-labels.yml new file mode 100644 index 00000000..c6ac3c12 --- /dev/null +++ b/.github/workflows/check-labels.yml @@ -0,0 +1,21 @@ +name: Check Pull Request Labels + +on: + pull_request: + types: [opened, synchronize] + +jobs: + check_labels: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Check labels + run: | + labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") + if [[ ! $labels =~ (^|\s)(testing-allowed|testing-ci-not-necessary)(\s|$) ]]; then + echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label." + exit 1 + fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3434f8f6..2785a194 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ env: jobs: integration-test: runs-on: ubuntu-22.04 - if: (github.event.label.name == 'test' || github.event.label.name == 'terraform') && github.event.review.state == 'approved' + if: (github.event.label.name == 'test' || github.event.label.name == 'terraform') && github.event.label.name == 'testing-allowed' # Other dependencies from hosted runner # AWS CLI # indirectly node used by actions diff --git a/DEVELOPER.md b/DEVELOPER.md index eae27d83..fb019f92 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -1,48 +1,49 @@ # Developer's Guide -Welcome to the development reference for Camunda's Terraform EKS module! -This document provides guidance on setting up a testing environment, running tests, and managing releases. +Welcome to the development reference for Camunda's Terraform EKS module! This document provides guidance on setting up a testing environment, running tests, and managing releases. ## Setting up Development Environment To start developing or testing the EKS module, follow these steps: 1. **Clone the Repository:** - - Clone the repository from [camunda/camunda-tf-eks-module](https://github.com/camunda/camunda-tf-eks-module) to your local machine. + - Clone the repository from [camunda/camunda-tf-eks-module](https://github.com/camunda/camunda-tf-eks-module) to your local machine. 2. **Navigate to Test Suite:** - - Go to the `test/src` directory to access the test suite. + - Go to the `test/src` directory to access the test suite. 3. **Test-Driven Development (TDD):** - - Use the Test-Driven Development approach to iterate on the module. - - Add or modify test cases in the test suite to match the desired functionality. - - Run tests frequently to ensure changes meet requirements. + - Use the Test-Driven Development approach to iterate on the module. + - Add or modify test cases in the test suite to match the desired functionality. + - Run tests frequently to ensure changes meet requirements. 4. **Local Development:** - - Utilize environment variables like `SKIP_XXX` to control certain behaviors during local development. - - Ensure to use a unique identifier for the cluster to avoid conflicts with existing resources. + - Utilize environment variables like `SKIP_XXX` to control certain behaviors during local development. + - Ensure to use a unique identifier for the cluster to avoid conflicts with existing resources. 5. **Testing Tools:** - - Refer to `test/README.md` for instructions on setting up and using testing tools. - - Add fixtures and test cases using Terratest and Testify to validate module functionality. + - Refer to `test/README.md` for instructions on setting up and using testing tools. + - Add fixtures and test cases using Terratest and Testify to validate module functionality. 6. **Cluster Cleanup:** - - Set `CLEAN_CLUSTER_AT_THE_END=false` to prevent automatic cluster deletion in case of errors. - - Optionally, manually clean up the cluster after testing by reversing this setting. + - Set `CLEAN_CLUSTER_AT_THE_END=false` to prevent automatic cluster deletion in case of errors. + - Optionally, manually clean up the cluster after testing by reversing this setting. + +**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. ## Releasing a New Version We follow Semantic Versioning (SemVer) guidelines for versioning. Follow these steps to release a new version: 1. **Commit History:** - - Maintain a clear commit history with explicit messages detailing additions and deletions. + - Maintain a clear commit history with explicit messages detailing additions and deletions. 2. **Versioning:** - - Determine the appropriate version number based on the changes made since the last release. - - Follow the format `MAJOR.MINOR.PATCH` as per Semantic Versioning guidelines. + - Determine the appropriate version number based on the changes made since the last release. + - Follow the format `MAJOR.MINOR.PATCH` as per Semantic Versioning guidelines. 3. **GitHub Releases:** - - Publish the new version on GitHub Releases. - - Tag the release with the version number and include release notes summarizing changes. + - Publish the new version on GitHub Releases. + - Tag the release with the version number and include release notes summarizing changes. By following these guidelines, we ensure smooth development iterations, robust testing practices, and clear version management for the Terraform EKS module. Happy coding! From 9b21b3394e33205cb430a4f5580be0de40ada913 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:40:09 +0200 Subject: [PATCH 27/93] fix --- .github/workflows/check-labels.yml | 2 +- DEVELOPER.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-labels.yml b/.github/workflows/check-labels.yml index c6ac3c12..dabb9bde 100644 --- a/.github/workflows/check-labels.yml +++ b/.github/workflows/check-labels.yml @@ -16,6 +16,6 @@ jobs: run: | labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") if [[ ! $labels =~ (^|\s)(testing-allowed|testing-ci-not-necessary)(\s|$) ]]; then - echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label." + echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label. Then push a new dummy commit." exit 1 fi diff --git a/DEVELOPER.md b/DEVELOPER.md index fb019f92..33ee7767 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -29,7 +29,7 @@ To start developing or testing the EKS module, follow these steps: - Set `CLEAN_CLUSTER_AT_THE_END=false` to prevent automatic cluster deletion in case of errors. - Optionally, manually clean up the cluster after testing by reversing this setting. -**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. +**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. Then push a new dummy commit. ## Releasing a New Version From 4bfa8400bef4792986536e7b97bfa45b6c646f13 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:43:16 +0200 Subject: [PATCH 28/93] fix --- .github/workflows/check-labels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-labels.yml b/.github/workflows/check-labels.yml index dabb9bde..f0c99d60 100644 --- a/.github/workflows/check-labels.yml +++ b/.github/workflows/check-labels.yml @@ -10,11 +10,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Check labels run: | labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") + echo "Labels: $labels" if [[ ! $labels =~ (^|\s)(testing-allowed|testing-ci-not-necessary)(\s|$) ]]; then echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label. Then push a new dummy commit." exit 1 From e14ceb7b89effb42d4f4888a57004e995f4714aa Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:45:38 +0200 Subject: [PATCH 29/93] fix check labels --- .github/workflows/check-labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-labels.yml b/.github/workflows/check-labels.yml index f0c99d60..fc2f2121 100644 --- a/.github/workflows/check-labels.yml +++ b/.github/workflows/check-labels.yml @@ -16,7 +16,7 @@ jobs: run: | labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") echo "Labels: $labels" - if [[ ! $labels =~ (^|\s)(testing-allowed|testing-ci-not-necessary)(\s|$) ]]; then + if ! echo "$labels" | grep -qE "^testing-allowed$|^testing-ci-not-necessary$"; then echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label. Then push a new dummy commit." exit 1 fi From ff8213cd2de342c2ca450089aa82788b29b4cd5f Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:50:26 +0200 Subject: [PATCH 30/93] update checks --- .github/workflows/check-labels.yml | 22 ---------------------- .github/workflows/tests.yml | 14 +++++++++++++- DEVELOPER.md | 2 +- 3 files changed, 14 insertions(+), 24 deletions(-) delete mode 100644 .github/workflows/check-labels.yml diff --git a/.github/workflows/check-labels.yml b/.github/workflows/check-labels.yml deleted file mode 100644 index fc2f2121..00000000 --- a/.github/workflows/check-labels.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Check Pull Request Labels - -on: - pull_request: - types: [opened, synchronize] - -jobs: - check_labels: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Check labels - run: | - labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") - echo "Labels: $labels" - if ! echo "$labels" | grep -qE "^testing-allowed$|^testing-ci-not-necessary$"; then - echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label. Then push a new dummy commit." - exit 1 - fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2785a194..53118ff3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,6 @@ env: jobs: integration-test: runs-on: ubuntu-22.04 - if: (github.event.label.name == 'test' || github.event.label.name == 'terraform') && github.event.label.name == 'testing-allowed' # Other dependencies from hosted runner # AWS CLI # indirectly node used by actions @@ -24,6 +23,19 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Check labels + run: | + labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") + echo "Labels: $labels" + if ! echo "$labels" | grep -qE "^testing-allowed$|^testing-ci-not-necessary$"; then + echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label. Then re-run this workflow." + exit 1 + fi + + - name: Stop if testing is not necessary + if: ${{ github.event.label.name == 'testing-ci-not-necessary' }} + run: exit 78 # Exit code 78 marks the job as skipped + - name: Get go.mod details uses: Eun/go-mod-details@v1 id: go-mod-details diff --git a/DEVELOPER.md b/DEVELOPER.md index 33ee7767..c056d4c6 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -29,7 +29,7 @@ To start developing or testing the EKS module, follow these steps: - Set `CLEAN_CLUSTER_AT_THE_END=false` to prevent automatic cluster deletion in case of errors. - Optionally, manually clean up the cluster after testing by reversing this setting. -**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. Then push a new dummy commit. +**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. Then re-run this workflow. ## Releasing a New Version From 107870ae4527b219e7a200f888ce913548e90fda Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:53:10 +0200 Subject: [PATCH 31/93] fix conditions --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 53118ff3..3ba7e61d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,8 +5,7 @@ on: schedule: - cron: '0 1 * * 2' workflow_dispatch: - pull_request_review: - types: [submitted] + pull_request: env: AWS_PROFILE: "infex" From 20a0bf3f4c8c33cb53d68a93a346ff88eff12eb1 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:55:46 +0200 Subject: [PATCH 32/93] fix --- .github/workflows/tests.yml | 2 +- DEVELOPER.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3ba7e61d..06fada55 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") echo "Labels: $labels" if ! echo "$labels" | grep -qE "^testing-allowed$|^testing-ci-not-necessary$"; then - echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label. Then re-run this workflow." + echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label. Then re-run this workflow by pushing a dummy commit." exit 1 fi diff --git a/DEVELOPER.md b/DEVELOPER.md index c056d4c6..f7a54375 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -29,7 +29,7 @@ To start developing or testing the EKS module, follow these steps: - Set `CLEAN_CLUSTER_AT_THE_END=false` to prevent automatic cluster deletion in case of errors. - Optionally, manually clean up the cluster after testing by reversing this setting. -**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. Then re-run this workflow. +**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. Then re-run this workflow by pushing a dummy commit. ## Releasing a New Version From 59d3bb9cef9d6a90020b9b3c05b3c0818f5ce4ee Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:59:24 +0200 Subject: [PATCH 33/93] fix --- .github/workflows/tests.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 06fada55..0fc13987 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,27 +31,27 @@ jobs: exit 1 fi - - name: Stop if testing is not necessary - if: ${{ github.event.label.name == 'testing-ci-not-necessary' }} - run: exit 78 # Exit code 78 marks the job as skipped - - name: Get go.mod details + if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} uses: Eun/go-mod-details@v1 id: go-mod-details with: modfile: ${{ github.workspace }}/test/src/go.mod - name: Set up Go environment + if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} uses: actions/setup-go@v5 with: go-version: ${{ steps.go-mod-details.outputs.go_version }} - name: Set up Terraform (OpenTofu) + if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} uses: opentofu/setup-opentofu@v1 with: tofu_version: 1.6.2 - name: Import Secrets + if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} id: secrets uses: hashicorp/vault-action@v3 with: @@ -66,21 +66,25 @@ jobs: # Official action does not support profiles - name: Add profile credentials to ~/.aws/credentials + if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} run: | aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }} - name: Get Short GitHub SHA and Save in Environment Variable + if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - name: Configure tests + if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} id: configure-tests run: | echo "TEST_CLUSTER_ID=${{ env.SHORT_GITHUB_SHA }}" >> "$GITHUB_ENV" echo "CLEAN_CLUSTER_AT_THE_END=true" >> "$GITHUB_ENV" - name: Launch tests + if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} working-directory: ./test/src/ timeout-minutes: 125 # due to a limit on elastic ips, tests suites are ran 1 by 1 run: | From 50cc17a960020dfac86c92d99354ce154e433e12 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 18:07:05 +0200 Subject: [PATCH 34/93] reapply after upgrade --- .github/workflows/tests.yml | 16 ++++++++-------- test/src/upgrade_eks_test.go | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0fc13987..0bbceba8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,26 +32,26 @@ jobs: fi - name: Get go.mod details - if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} + if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" uses: Eun/go-mod-details@v1 id: go-mod-details with: modfile: ${{ github.workspace }}/test/src/go.mod - name: Set up Go environment - if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} + if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" uses: actions/setup-go@v5 with: go-version: ${{ steps.go-mod-details.outputs.go_version }} - name: Set up Terraform (OpenTofu) - if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} + if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" uses: opentofu/setup-opentofu@v1 with: tofu_version: 1.6.2 - name: Import Secrets - if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} + if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" id: secrets uses: hashicorp/vault-action@v3 with: @@ -66,25 +66,25 @@ jobs: # Official action does not support profiles - name: Add profile credentials to ~/.aws/credentials - if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} + if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" run: | aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }} - name: Get Short GitHub SHA and Save in Environment Variable - if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} + if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - name: Configure tests - if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} + if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" id: configure-tests run: | echo "TEST_CLUSTER_ID=${{ env.SHORT_GITHUB_SHA }}" >> "$GITHUB_ENV" echo "CLEAN_CLUSTER_AT_THE_END=true" >> "$GITHUB_ENV" - name: Launch tests - if: ${{ github.event.label.name != 'testing-ci-not-necessary' }} + if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" working-directory: ./test/src/ timeout-minutes: 125 # due to a limit on elastic ips, tests suites are ran 1 by 1 run: | diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index a23bc331..1ed36080 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -170,6 +170,28 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { errClusterReady = utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(suite.expectedNodes)) suite.Require().NoError(errClusterReady) + // perform update with terraform + terraformOptions = &terraform.Options{ + TerraformDir: tfDir, + Upgrade: false, + VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, + } + + suite.sugaredLogger.Infow("Reapply terraform after EKS cluster upgrade...", "extraVars", suite.varTf) + + if cleanClusterAtTheEnd == "true" { + defer terraform.Destroy(suite.T(), terraformOptions) + defer runtime.HandleCrash(func(i interface{}) { + terraform.Destroy(suite.T(), terraformOptions) + }) + } + + terraform.InitAndApplyAndIdempotent(suite.T(), terraformOptions) + + errClusterReady = utils.WaitUntilKubeClusterIsReady(result.Cluster, 5*time.Minute, uint64(suite.expectedNodes)) + suite.Require().NoError(errClusterReady) + // Check version of the upgraded cluster result, err = eksSvc.DescribeCluster(context.Background(), inputEKS) suite.Assert().NoError(err) From 294d42a7a4e1a8e30cf13c96f31ee6c864f0cb9b Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 18:10:03 +0200 Subject: [PATCH 35/93] dummy commit --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0bbceba8..96d9019a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,6 +18,7 @@ jobs: # AWS CLI # indirectly node used by actions + steps: - name: Checkout repository uses: actions/checkout@v4 From 5b8c0ab9b85b8332cdd8d793b21f4de0a1866bf9 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:06:24 +0200 Subject: [PATCH 36/93] improve go tests summary --- .github/workflows/tests.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 96d9019a..342b4fe5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -82,14 +82,22 @@ jobs: id: configure-tests run: | echo "TEST_CLUSTER_ID=${{ env.SHORT_GITHUB_SHA }}" >> "$GITHUB_ENV" - echo "CLEAN_CLUSTER_AT_THE_END=true" >> "$GITHUB_ENV" - - name: Launch tests + - name: Launch tests in parallel if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" working-directory: ./test/src/ - timeout-minutes: 125 # due to a limit on elastic ips, tests suites are ran 1 by 1 + timeout-minutes: 125 run: | - go test -v -timeout 120m -p 1 ./... + export TEST_CLUSTER_ID="${{ env.TEST_CLUSTER_ID }}" + export CLEAN_CLUSTER_AT_THE_END=true + go install gotest.tools/gotestsum@v1.1.0 + gotestsum --junitfile unit-tests.xml --format pkgname -- -v --timeout=120m -p 1 ./... + + - name: Test Summary + uses: test-summary/action@v2 + with: + paths: "unit-tests.xml" + if: always() - name: Remove profile credentials from ~/.aws/credentials if: always() From 28c2f3c230271af4cec82c8d3551753a65ad6132 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:11:00 +0200 Subject: [PATCH 37/93] fix tests ci --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 342b4fe5..68e09710 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -90,14 +90,14 @@ jobs: run: | export TEST_CLUSTER_ID="${{ env.TEST_CLUSTER_ID }}" export CLEAN_CLUSTER_AT_THE_END=true - go install gotest.tools/gotestsum@v1.1.0 + go install gotest.tools/gotestsum@v1.11.0 gotestsum --junitfile unit-tests.xml --format pkgname -- -v --timeout=120m -p 1 ./... - name: Test Summary uses: test-summary/action@v2 with: paths: "unit-tests.xml" - if: always() + if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') && always()" - name: Remove profile credentials from ~/.aws/credentials if: always() From 25dcb6a1308db069b72ad3127986d7080c55eb31 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:22:36 +0200 Subject: [PATCH 38/93] fix path --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 68e09710..86d7fe12 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -96,7 +96,7 @@ jobs: - name: Test Summary uses: test-summary/action@v2 with: - paths: "unit-tests.xml" + paths: "./test/src/unit-tests.xml" if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') && always()" - name: Remove profile credentials from ~/.aws/credentials From 47e8bd971147b97e2d7193b79bd23998f738e82c Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:34:05 +0200 Subject: [PATCH 39/93] update report (may fail) --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 86d7fe12..576afc00 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -94,7 +94,7 @@ jobs: gotestsum --junitfile unit-tests.xml --format pkgname -- -v --timeout=120m -p 1 ./... - name: Test Summary - uses: test-summary/action@v2 + uses: test-summary/action@v2 # may not work due to https://github.com/test-summary/action/issues/5 with: paths: "./test/src/unit-tests.xml" if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') && always()" From 9ce7ebc5cc1bd2b3f5fe52a3a5b28c44571119ef Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Thu, 11 Apr 2024 09:40:07 +0200 Subject: [PATCH 40/93] fix typo --- .github/workflows/tests.yml | 8 +------- test/README.md | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 576afc00..b0bbbd09 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,18 +77,12 @@ jobs: if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - - name: Configure tests - if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" - id: configure-tests - run: | - echo "TEST_CLUSTER_ID=${{ env.SHORT_GITHUB_SHA }}" >> "$GITHUB_ENV" - - name: Launch tests in parallel if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" working-directory: ./test/src/ timeout-minutes: 125 run: | - export TEST_CLUSTER_ID="${{ env.TEST_CLUSTER_ID }}" + export TESTS_CLUSTER_ID="${{ env.SHORT_GITHUB_SHA }}" export CLEAN_CLUSTER_AT_THE_END=true go install gotest.tools/gotestsum@v1.11.0 gotestsum --junitfile unit-tests.xml --format pkgname -- -v --timeout=120m -p 1 ./... diff --git a/test/README.md b/test/README.md index af909df8..4de90f1b 100644 --- a/test/README.md +++ b/test/README.md @@ -25,7 +25,7 @@ export AWS_REGION=eu-central-1 If you want to specify a non-random cluster UID: ```bash -export TEST_CLUSTER_ID="myTest" +export TESTS_CLUSTER_ID="myTest" ``` If you don't want to delete the resources at the end of the test: From 7c4cc68f1425671b9d570aba26fb9359874895a4 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Thu, 11 Apr 2024 10:17:56 +0200 Subject: [PATCH 41/93] ping gh actions --- .github/workflows/labeler.yml | 2 +- .github/workflows/lint.yml | 5 +++-- .github/workflows/nightly_cleanup.yml | 5 +++-- .github/workflows/tests.yml | 14 ++++++++------ DEVELOPER.md | 6 ++++++ 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index e7847016..907942e4 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -10,4 +10,4 @@ jobs: pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/labeler@v5 + - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c9e6cd3b..9fd10a31 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,6 +6,7 @@ on: push: workflow_dispatch: + jobs: lint: name: pre-commit @@ -14,12 +15,12 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 - - uses: terraform-linters/setup-tflint@v4 + - uses: terraform-linters/setup-tflint@19a52fbac37dacb22a09518e4ef6ee234f2d4987 # v4 name: Setup TFLint with: tflint_version: v0.50.3 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 - run: go install github.com/terraform-docs/terraform-docs@v0.17.0 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/nightly_cleanup.yml b/.github/workflows/nightly_cleanup.yml index 65e72692..4f9a5689 100644 --- a/.github/workflows/nightly_cleanup.yml +++ b/.github/workflows/nightly_cleanup.yml @@ -6,6 +6,7 @@ on: - cron: '0 5 * * *' workflow_dispatch: + env: AWS_PROFILE: "infex" AWS_REGION: "eu-central-1" @@ -18,11 +19,11 @@ jobs: # indirectly node used by actions steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Import Secrets id: secrets - uses: hashicorp/vault-action@v3 + uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3 with: url: ${{ secrets.VAULT_ADDR }} method: approle diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b0bbbd09..a035474f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,6 +7,7 @@ on: workflow_dispatch: pull_request: + env: AWS_PROFILE: "infex" AWS_REGION: "eu-central-1" @@ -21,7 +22,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Check labels run: | @@ -34,27 +35,27 @@ jobs: - name: Get go.mod details if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" - uses: Eun/go-mod-details@v1 + uses: Eun/go-mod-details@b719cd324463e2037cf3a0dd1dd6091bdc2730f4 # v1 id: go-mod-details with: modfile: ${{ github.workspace }}/test/src/go.mod - name: Set up Go environment if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" - uses: actions/setup-go@v5 + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 with: go-version: ${{ steps.go-mod-details.outputs.go_version }} - name: Set up Terraform (OpenTofu) if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" - uses: opentofu/setup-opentofu@v1 + uses: opentofu/setup-opentofu@ae80d4ecaab946d8f5ff18397fbf6d0686c6d46a # v1 with: tofu_version: 1.6.2 - name: Import Secrets if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" id: secrets - uses: hashicorp/vault-action@v3 + uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3 with: url: ${{ secrets.VAULT_ADDR }} method: approle @@ -88,7 +89,8 @@ jobs: gotestsum --junitfile unit-tests.xml --format pkgname -- -v --timeout=120m -p 1 ./... - name: Test Summary - uses: test-summary/action@v2 # may not work due to https://github.com/test-summary/action/issues/5 + # may not work due to https://github.com/test-summary/action/issues/5 + uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336 # v2 with: paths: "./test/src/unit-tests.xml" if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') && always()" diff --git a/DEVELOPER.md b/DEVELOPER.md index f7a54375..b2328145 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -46,4 +46,10 @@ We follow Semantic Versioning (SemVer) guidelines for versioning. Follow these s - Publish the new version on GitHub Releases. - Tag the release with the version number and include release notes summarizing changes. +## Adding new GH actions + +Please pin GitHub action, if you need you can use [pin-github-action](https://github.com/mheap/pin-github-action) cli tool. + +--- + By following these guidelines, we ensure smooth development iterations, robust testing practices, and clear version management for the Terraform EKS module. Happy coding! From c76f27d79f160e391076a9bb34f649707485e92a Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Thu, 11 Apr 2024 13:36:32 +0200 Subject: [PATCH 42/93] only run tests if necessary --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a035474f..9ddcc84b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,6 +14,7 @@ env: jobs: integration-test: + if: "contains(github.event.pull_request.labels.*.name, 'test') || contains(github.event.pull_request.labels.*.name, 'terraform')" runs-on: ubuntu-22.04 # Other dependencies from hosted runner # AWS CLI From 3ddf0a9f2c057d713003fda97c9091b11e18dfac Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:03:13 +0200 Subject: [PATCH 43/93] improve tests condition --- .github/workflows/nightly_cleanup.yml | 5 +++-- .github/workflows/tests.yml | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nightly_cleanup.yml b/.github/workflows/nightly_cleanup.yml index 4f9a5689..8aa1ce53 100644 --- a/.github/workflows/nightly_cleanup.yml +++ b/.github/workflows/nightly_cleanup.yml @@ -10,6 +10,7 @@ on: env: AWS_PROFILE: "infex" AWS_REGION: "eu-central-1" + CLEANUP_NEWER_THAN: "6 hours" jobs: aws-nightly-cleanup: @@ -57,7 +58,7 @@ jobs: ./cloud-nuke_linux_amd64 aws \ --region ${{ env.AWS_REGION }} \ --force \ - --newer-than 4h \ + --newer-than ${{ env.CLEANUP_NEWER_THAN }} \ --exclude-resource-type ec2_dhcp_option \ --exclude-resource-type cloudtrail || true @@ -75,6 +76,6 @@ jobs: ./cloud-nuke_linux_amd64 aws \ --region ${{ env.AWS_REGION }} \ --force \ - --newer-than 4h \ + --newer-than ${{ env.CLEANUP_NEWER_THAN }} \ --exclude-resource-type ec2_dhcp_option \ --exclude-resource-type cloudtrail diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9ddcc84b..50e2a4b6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,16 @@ env: jobs: integration-test: - if: "contains(github.event.pull_request.labels.*.name, 'test') || contains(github.event.pull_request.labels.*.name, 'terraform')" + # we allow integration tests for schedules, and for PR only if the content is tests or terraform related + if: >- + github.event_name == 'schedule' || ( + github.event_name == 'pull_request' && + ( + contains(github.event.pull_request.labels.*.name, 'test') || + contains(github.event.pull_request.labels.*.name, 'terraform') + ) && + !contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') + ) runs-on: ubuntu-22.04 # Other dependencies from hosted runner # AWS CLI @@ -29,32 +38,28 @@ jobs: run: | labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") echo "Labels: $labels" - if ! echo "$labels" | grep -qE "^testing-allowed$|^testing-ci-not-necessary$"; then + if ! echo "$labels" | grep -qE "^testing-allowed$"; then echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label. Then re-run this workflow by pushing a dummy commit." exit 1 fi - name: Get go.mod details - if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" uses: Eun/go-mod-details@b719cd324463e2037cf3a0dd1dd6091bdc2730f4 # v1 id: go-mod-details with: modfile: ${{ github.workspace }}/test/src/go.mod - name: Set up Go environment - if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 with: go-version: ${{ steps.go-mod-details.outputs.go_version }} - name: Set up Terraform (OpenTofu) - if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" uses: opentofu/setup-opentofu@ae80d4ecaab946d8f5ff18397fbf6d0686c6d46a # v1 with: tofu_version: 1.6.2 - name: Import Secrets - if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" id: secrets uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3 with: @@ -69,18 +74,15 @@ jobs: # Official action does not support profiles - name: Add profile credentials to ~/.aws/credentials - if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" run: | aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }} - name: Get Short GitHub SHA and Save in Environment Variable - if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - name: Launch tests in parallel - if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary')" working-directory: ./test/src/ timeout-minutes: 125 run: | @@ -94,7 +96,6 @@ jobs: uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336 # v2 with: paths: "./test/src/unit-tests.xml" - if: "!contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') && always()" - name: Remove profile credentials from ~/.aws/credentials if: always() From 25d0cc058cba8c4e4607d0e730e747789b7ae014 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:09:21 +0200 Subject: [PATCH 44/93] rename file --- .../{postgres-client.bak.yml => postgres-client-irsa.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename modules/fixtures/{postgres-client.bak.yml => postgres-client-irsa.yml} (96%) diff --git a/modules/fixtures/postgres-client.bak.yml b/modules/fixtures/postgres-client-irsa.yml similarity index 96% rename from modules/fixtures/postgres-client.bak.yml rename to modules/fixtures/postgres-client-irsa.yml index 6cb35533..5fabb4bc 100644 --- a/modules/fixtures/postgres-client.bak.yml +++ b/modules/fixtures/postgres-client-irsa.yml @@ -1,5 +1,5 @@ # this manifest contains a version with the IRSA connection check, it is currently listed as a TODO -# it may be implemented or dropped depending if it's relevant or not to test IRSA connection for the db +# it may be implemented or dropped depending on if it's relevant or not to test IRSA connection for the db apiVersion: batch/v1 kind: Job metadata: From 625724370cdfd0a122993a8d3972649da1b9d47c Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:22:32 +0200 Subject: [PATCH 45/93] dynamic increase version --- test/src/upgrade_eks_test.go | 5 ++++- test/src/utils/helper.go | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index 1ed36080..d7d84198 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -161,7 +161,10 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { ) // upgrade the cluster - suite.varTf["kubernetes_version"] = "1.29" + var errIncVersion error + suite.varTf["kubernetes_version"], errIncVersion = utils.IncrementMinorVersionTwoParts(suite.kubeVersion) + suite.Require().NoError(errIncVersion) + suite.sugaredLogger.Infow(fmt.Sprintf("Upgrading the EKS cluster to v%s using aws sdk", suite.varTf["kubernetes_version"]), "extraVars", suite.varTf) errUpdate := utils.UpgradeEKS(context.Background(), eksSvc, suite.clusterName, suite.varTf["kubernetes_version"].(string)) suite.Require().NoError(errUpdate) diff --git a/test/src/utils/helper.go b/test/src/utils/helper.go index c5daf3fb..08814611 100644 --- a/test/src/utils/helper.go +++ b/test/src/utils/helper.go @@ -1,7 +1,10 @@ package utils import ( + "fmt" "os" + "strconv" + "strings" ) func GetEnv(key, fallback string) string { @@ -11,3 +14,20 @@ func GetEnv(key, fallback string) string { } return value } + +func IncrementMinorVersionTwoParts(version string) (string, error) { + parts := strings.Split(version, ".") + + if len(parts) != 2 { + return "", fmt.Errorf("invalid version format, expected 2 parts got %d", len(parts)) + } + + minor, err := strconv.Atoi(parts[1]) + if err != nil { + return "", err + } + + newVersion := fmt.Sprintf("%s.%d", parts[0], minor+1) + + return newVersion, nil +} From 81fd65ef5b369c5dca24fd7b185dabb16dbcedd1 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:45:34 +0200 Subject: [PATCH 46/93] auto port selection --- test/src/upgrade_eks_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index d7d84198..c323a902 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -139,10 +139,8 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { time.Sleep(30 * time.Second) // Now we verify that the service will successfully boot and start serving requests - localPort1 := 8883 - service := k8s.GetService(suite.T(), kubeCtlOptions, "whoami-service") - portForwardProc1 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort1, 80) + portForwardProc1 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, 0, 80) defer portForwardProc1.Close() portForwardProc1.ForwardPort(suite.T()) @@ -206,9 +204,8 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { time.Sleep(30 * time.Second) // Forward port again - localPort2 := 8887 service = k8s.GetService(suite.T(), kubeCtlOptions, "whoami-service") - portForwardProc2 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, localPort2, 80) + portForwardProc2 := k8s.NewTunnel(kubeCtlOptions, k8s.ResourceTypeService, service.ObjectMeta.Name, 0, 80) defer portForwardProc2.Close() portForwardProc2.ForwardPort(suite.T()) From e151eb446b7b87fbbd935d6add2305bc2a13fdb0 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:48:15 +0200 Subject: [PATCH 47/93] remove toml --- .pre-commit-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5c39e7ed..ddc9b4f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,6 @@ repos: - id: trailing-whitespace - id: check-yaml args: [--allow-multiple-documents] - - id: check-toml - id: check-json - id: check-symlinks - id: check-shebang-scripts-are-executable @@ -59,7 +58,6 @@ repos: rev: v0.5.1 hooks: - id: go-fmt - - id: validate-toml - id: no-go-testing - id: go-mod-tidy ... From 418aa6c0d7bfa871c8804889360be4ba38b7ba40 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:50:50 +0200 Subject: [PATCH 48/93] use latest runs-on --- .github/workflows/lint.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9fd10a31..20419677 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ on: jobs: lint: name: pre-commit - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 50e2a4b6..a0bf48c0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: ) && !contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') ) - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest # Other dependencies from hosted runner # AWS CLI # indirectly node used by actions From aa37e00a7a36ba15e71e717e41f500be6aa289be Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 12 Apr 2024 16:26:33 +0200 Subject: [PATCH 49/93] implement just, update tests doc --- .github/workflows/nightly_cleanup.yml | 3 +- .github/workflows/tests.yml | 28 +++------ .tool-versions | 10 ++++ README.md | 4 ++ justfile | 41 +++++++++++++ test/README.md | 85 ++++++++++++++++++++------- test/src/.tool-versions | 1 - 7 files changed, 130 insertions(+), 42 deletions(-) create mode 100644 justfile delete mode 100644 test/src/.tool-versions diff --git a/.github/workflows/nightly_cleanup.yml b/.github/workflows/nightly_cleanup.yml index 8aa1ce53..6eeb50db 100644 --- a/.github/workflows/nightly_cleanup.yml +++ b/.github/workflows/nightly_cleanup.yml @@ -11,6 +11,7 @@ env: AWS_PROFILE: "infex" AWS_REGION: "eu-central-1" CLEANUP_NEWER_THAN: "6 hours" + CLOUD_NUKE_VERSION: "v0.35.0" jobs: aws-nightly-cleanup: @@ -44,7 +45,7 @@ jobs: - name: Install Cloud Nuke run: | - wget https://github.com/gruntwork-io/cloud-nuke/releases/download/v0.33.0/cloud-nuke_linux_amd64 + wget https://github.com/gruntwork-io/cloud-nuke/releases/download/${{ env.CLOUD_NUKE_VERSION }}/cloud-nuke_linux_amd64 chmod +x cloud-nuke_linux_amd64 # This is likely to fail, therefore we ignore the error diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a0bf48c0..57fd9322 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,21 +43,8 @@ jobs: exit 1 fi - - name: Get go.mod details - uses: Eun/go-mod-details@b719cd324463e2037cf3a0dd1dd6091bdc2730f4 # v1 - id: go-mod-details - with: - modfile: ${{ github.workspace }}/test/src/go.mod - - - name: Set up Go environment - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 - with: - go-version: ${{ steps.go-mod-details.outputs.go_version }} - - - name: Set up Terraform (OpenTofu) - uses: opentofu/setup-opentofu@ae80d4ecaab946d8f5ff18397fbf6d0686c6d46a # v1 - with: - tofu_version: 1.6.2 + - name: Install tooling using asdf + uses: asdf-vm/actions/install@v3 - name: Import Secrets id: secrets @@ -79,17 +66,20 @@ jobs: aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }} + - name: Get go.mod details + uses: Eun/go-mod-details@b719cd324463e2037cf3a0dd1dd6091bdc2730f4 # v1 + id: go-mod-details + with: + modfile: ${{ github.workspace }}/test/src/go.mod + - name: Get Short GitHub SHA and Save in Environment Variable run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - name: Launch tests in parallel - working-directory: ./test/src/ timeout-minutes: 125 run: | export TESTS_CLUSTER_ID="${{ env.SHORT_GITHUB_SHA }}" - export CLEAN_CLUSTER_AT_THE_END=true - go install gotest.tools/gotestsum@v1.11.0 - gotestsum --junitfile unit-tests.xml --format pkgname -- -v --timeout=120m -p 1 ./... + just tests "--junitfile unit-tests.xml" - name: Test Summary # may not work due to https://github.com/test-summary/action/issues/5 diff --git a/.tool-versions b/.tool-versions index b3f10a7a..c7da0742 100644 --- a/.tool-versions +++ b/.tool-versions @@ -7,3 +7,13 @@ terraform-docs 0.17.0 tflint 0.50.3 tfsec 1.28.5 + +opentofu 1.6.2 + +golang 1.22.2 + +awscli 2.15.20 + +eksctl 0.175.0 + +just 1.25.2 diff --git a/README.md b/README.md index 6fb1cd8e..079f5089 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,7 @@ module "postgresql" { depends_on = [module.eks_cluster] } ``` + +## Support + +Please note that the modules have been tested with **[OpenTofu](https://opentofu.org/)** in the version described in the [.tool-versions](./.tool-versions) of this project. diff --git a/justfile b/justfile new file mode 100644 index 00000000..9a4b703f --- /dev/null +++ b/justfile @@ -0,0 +1,41 @@ +# this file is a recipe file for the project + +gotestsum_version := "v1.11.0" + +# Launch a single test using go test in verbose mode +test-verbose testname: install-tests-go-mod + cd test/src/ && go test -v --timeout=120m -p 1 -run {{testname}} + +# Launch a single test using gotestsum +test testname gts_options="": install-tests-go-mod + cd test/src/ && go run gotest.tools/gotestsum@{{gotestsum_version}} {{gts_options}} -- --timeout=120m -p 1 -run {{testname}} + +# Launch the tests in parallel using go test in verbose mode +tests-verbose: install-tests-go-mod + cd test/src/ && go test -v --timeout=120m -p 1 . + +# Launch the tests in parallel using gotestsum +tests gts_options="": install-tests-go-mod + cd test/src/ && go run gotest.tools/gotestsum@{{gotestsum_version}} {{gts_options}} -- --timeout=120m -p 1 . + +# Install go dependencies from test/src/go.mod +install-tests-go-mod: + cd test/src/ && go mod download + +# Install all the tooling +install-tooling: asdf-install + +# Install asdf plugins +asdf-plugins: + #!/bin/sh + echo "Installing asdf plugins" + for plugin in $(awk '{print $1}' .tool-versions); do \ + asdf plugin add ${plugin} 2>&1 | (grep "already added" && exit 0); \ + done + + echo "Update all asdf plugins" + asdf plugin update --all + +# Install tools using asdf +asdf-install: asdf-plugins + asdf install diff --git a/test/README.md b/test/README.md index 4de90f1b..8c31f50f 100644 --- a/test/README.md +++ b/test/README.md @@ -2,48 +2,97 @@ ## Requirements -Make sure you have `opentofu` installed: +To gather all specifics versions of this project, we use: +- [asdf](https://asdf-vm.com/) version manager (see [installation](https://asdf-vm.com/guide/getting-started.html)). +- [just](https://github.com/casey/just) as a command runner + - install it using asdf: `asdf plugin add just && asdf install just` +Then we will install all the tooling listed in the `.tool-versions` of this root project using just: ```bash -brew update -brew install opentofu +just install-tooling + +# list available recipes +just --list ``` -Ensure you have `awscli` installed and configured with a proper AWS profile and region: +## Configure AWSCli -```bash -# install aws cli -brew install awscli +You should now have `awscli` installed, verify it with: `aws --version` -# sso login -aws sso login --profile SystemAdministrator-*** +Make sure that your aws cli is configured with a proper AWS Profile and a region: + +**Aws Cli Auth**: https://docs.aws.amazon.com/cli/latest/userguide/cli-authentication-user.html#cli-authentication-user-configure.title + +```bash +export AWS_DEFAULT_PROFILE= +# you can also use +# export AWS_PROFILE= -export AWS_DEFAULT_PROFILE=SystemAdministrator-**** export AWS_REGION=eu-central-1 + +# if you are using sso +aws sso login --profile "$AWS_DEFAULT_PROFILE" + +# verify that you are correctly authenticated +aws eks list-clusters ``` -If you want to specify a non-random cluster UID: +## Configure the tests + +The tests will create resources on AWS using the provided account + +By default, a random uuid is assigned and prepended to the end of each object created. +If you want to specify a non-random cluster UID: ```bash export TESTS_CLUSTER_ID="myTest" ``` If you don't want to delete the resources at the end of the test: - ```bash export CLEAN_CLUSTER_AT_THE_END=false ``` +You can change the default deployment region: +```bash +export TESTS_CLUSTER_REGION="eu-west-1" +``` + +### Run the tests + Test with: ```bash -make test +# Launch all the tests +just test + +# if you want the live output +just tests-verbose # or just test one case -go test -v -timeout 120m -run TestDefaultEKSTestSuite +just test-verbose TestUpgradeEKSTestSuite ``` -When you run the test, terratest will create a copy of the module to be tested in the `tests/states` directory. You can later navigate to the directory and use its content to manipulate the cluster. You can set the `SKIP_XXX` variable to prevent unique IDs of tests from being generated each time, thus using the same resources instead of deploying new resources with terraform. +When you run the test, terratest will create a copy of the module to be tested in the `tests/states` directory. +You can later navigate to the directory and use its content to manipulate the cluster. + +**Local development note:** +You can set the `SKIP_XXX` variable to prevent unique IDs of tests from being generated each time, thus using the same resources instead of deploying new resources with terraform. + +### Just Reference + +May not be up-to-date, please verify with `just --list`: +```text +╰─λ just --list 130 (10.968s) < 15:28:36 +Available recipes: + asdf-install # Install tools using asdf + asdf-plugins # Install asdf plugins + install-tooling # Install all the tooling + test TEST # Launch a single test using gotestsum + test-verbose TEST # Launch a single test using go test in verbose mode + tests # Launch the tests in parallel using gotestsum + tests-verbose # Launch the tests in parallel using go test in verbose mode +``` ## Troubleshooting @@ -52,9 +101,3 @@ Ensure you don't have test clusters running for a while: ```bash eksctl get clusters ``` - -You can change the default deployment region: - -```bash -export TESTS_CLUSTER_REGION="eu-west-1" -``` diff --git a/test/src/.tool-versions b/test/src/.tool-versions deleted file mode 100644 index 3960504e..00000000 --- a/test/src/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -golang 1.22.2 From ad8c962e20674c45897a3deea3f750367d4df01b Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 12 Apr 2024 16:31:52 +0200 Subject: [PATCH 50/93] document dummy commit --- DEVELOPER.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DEVELOPER.md b/DEVELOPER.md index b2328145..a48eedbd 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -29,7 +29,8 @@ To start developing or testing the EKS module, follow these steps: - Set `CLEAN_CLUSTER_AT_THE_END=false` to prevent automatic cluster deletion in case of errors. - Optionally, manually clean up the cluster after testing by reversing this setting. -**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. Then re-run this workflow by pushing a dummy commit. +**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. +Then re-run this workflow by pushing a dummy commit: `git commit --allow-empty -m "trigger workflow"`. ## Releasing a New Version From 43cabc571f9077c506607843cc7fecc88c5d8166 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 12 Apr 2024 16:50:08 +0200 Subject: [PATCH 51/93] pin action --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 57fd9322..8b700d27 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,7 +44,7 @@ jobs: fi - name: Install tooling using asdf - uses: asdf-vm/actions/install@v3 + uses: asdf-vm/actions/install@05e0d2ed97b598bfce82fd30daf324ae0c4570e6 # v3 - name: Import Secrets id: secrets From 750c31b1b6cc0ae25500a1344bb7aeccea5122f5 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 08:29:25 +0200 Subject: [PATCH 52/93] update labeler --- .github/labeler.yml | 10 ++++------ test/src/Makefile | 16 ---------------- 2 files changed, 4 insertions(+), 22 deletions(-) delete mode 100755 test/src/Makefile diff --git a/.github/labeler.yml b/.github/labeler.yml index e908ab44..c0978e34 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -4,16 +4,14 @@ feature: # Add 'test' label to any changes within 'test' folder or any subfolders test: - - changed-files: - - any-glob-to-any-file: test/** - -doc: - changed-files: - any-glob-to-any-file: - - docs/** - - "**.md" + - test/**.go + - test/**/go.mod + - modules/fixtures/** terraform: - changed-files: - any-glob-to-any-file: - modules/**.tf + - .tool-versions diff --git a/test/src/Makefile b/test/src/Makefile deleted file mode 100755 index 765e24e3..00000000 --- a/test/src/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -#!make -.DEFAULT_GOAL= help - -.PHONY : init -init: ## Initialize tests - @exit 0 - -.PHONY : test -test: init ## Run tests - go mod download - go test -v -timeout 120m - -# see https://suva.sh/posts/well-documented-makefiles/ -.PHONY : help -help: ## Show this help prompt. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) From e82b89533fe4049f321de463f08fa89028432a7f Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 08:52:52 +0200 Subject: [PATCH 53/93] fix region --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8b700d27..83726890 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ on: env: AWS_PROFILE: "infex" - AWS_REGION: "eu-central-1" + AWS_REGION: "eu-west-2" jobs: integration-test: From be3798402f3b651b4d4c722a9e4e20f930bad65c Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 10:49:49 +0200 Subject: [PATCH 54/93] add tests in parallel --- .github/workflows/tests.yml | 99 ++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 83726890..79818cc6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,8 @@ env: AWS_REGION: "eu-west-2" jobs: - integration-test: + check-run-tests: + runs-on: ubuntu-latest # we allow integration tests for schedules, and for PR only if the content is tests or terraform related if: >- github.event_name == 'schedule' || ( @@ -24,16 +25,7 @@ jobs: ) && !contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') ) - runs-on: ubuntu-latest - # Other dependencies from hosted runner - # AWS CLI - # indirectly node used by actions - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Check labels run: | labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") @@ -43,8 +35,18 @@ jobs: exit 1 fi - - name: Install tooling using asdf - uses: asdf-vm/actions/install@05e0d2ed97b598bfce82fd30daf324ae0c4570e6 # v3 + configure-tests: + runs-on: ubuntu-latest + # Other dependencies from hosted runner + # AWS CLI + # indirectly node used by actions + needs: + - check-run-tests + outputs: + test_functions: ${{ steps.extract_test_functions.outputs.test_functions }} + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Import Secrets id: secrets @@ -54,16 +56,52 @@ jobs: method: approle roleId: ${{ secrets.VAULT_ROLE_ID }} secretId: ${{ secrets.VAULT_SECRET_ID }} - exportEnv: false + exportEnv: true secrets: | secret/data/products/infrastructure-experience/ci/common AWS_ACCESS_KEY; secret/data/products/infrastructure-experience/ci/common AWS_SECRET_KEY; + - name: Get Short GitHub SHA and Save in Environment Variable + run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" + + - name: Extract Test Functions + id: extract_test_functions + run: | + test_functions=$(grep -rho 'func \(Test[^ ]*\)' ./test/src/ | sed 's/func \(Test[^ ]*\)(t/\1/') + + : # Extract test names marked to be skipped from the commit message description + skipped_tests=$(echo "$GITHUB_EVENT_PATH" | xargs cat | jq -r '.commits[].message' | grep -oP '(?<=skip: )[^\,]+' | tr '\n' ',') + + : # If any tests are marked to be skipped, remove them from the list of test functions + if [ -n "$skipped_tests" ]; then + for test in $(echo "$skipped_tests" | tr ',' '\n'); do + test_functions=$(echo "$test_functions" | grep -v "$test") + done + fi + + echo "test_functions=${test_functions}" >> "$GITHUB_OUTPUT" + echo "test_functions=${test_functions}" + + integration-tests: + runs-on: ubuntu-latest + needs: + - configure-tests + strategy: + fail-fast: false # don't propagate failing jobs + matrix: + test_function: ${{ fromJson(needs.configure-tests.outputs.test_functions) }} + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + + - name: Install tooling using asdf + uses: asdf-vm/actions/install@05e0d2ed97b598bfce82fd30daf324ae0c4570e6 # v3 + # Official action does not support profiles - name: Add profile credentials to ~/.aws/credentials run: | - aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} - aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set aws_access_key_id ${{ env.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set aws_secret_access_key ${{ env.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }} - name: Get go.mod details @@ -72,22 +110,37 @@ jobs: with: modfile: ${{ github.workspace }}/test/src/go.mod - - name: Get Short GitHub SHA and Save in Environment Variable - run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" - - name: Launch tests in parallel timeout-minutes: 125 run: | export TESTS_CLUSTER_ID="${{ env.SHORT_GITHUB_SHA }}" - just tests "--junitfile unit-tests.xml" + just test ${{ matrix.test_function }} "--junitfile ${{ matrix.test_function }}_unit-tests.xml" - - name: Test Summary - # may not work due to https://github.com/test-summary/action/issues/5 - uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336 # v2 + - name: Upload test reports + if: always() + uses: actions/upload-artifact@v3 with: - paths: "./test/src/unit-tests.xml" + name: test-reports + path: "./test/src/${{ matrix.test_function }}_unit-tests.xml" - name: Remove profile credentials from ~/.aws/credentials if: always() run: | rm -rf ~/.aws/credentials + + test-report: + runs-on: ubuntu-latest + if: always() + needs: + - integration-tests + steps: + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + name: test-reports + path: /tmp/testreports + + - name: Run test-summary + uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336 # v2 + with: + paths: /tmp/testreports/**/*.xml From 44f05732a81954e58cd4255111625a43a3633761 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 10:53:20 +0200 Subject: [PATCH 55/93] fix report --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 79818cc6..dd689255 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -130,8 +130,9 @@ jobs: test-report: runs-on: ubuntu-latest - if: always() + if: ${{ always() && needs.check-run-tests.result == 'success' }} needs: + - check-run-tests - integration-tests steps: - name: Download artifacts From b9d63503b7aa65d7e4a7454df26b77f5393d682f Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 10:54:35 +0200 Subject: [PATCH 56/93] trigger workflow From ca3660f9151bc97aeb31ce298283f37c9998191d Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 11:00:29 +0200 Subject: [PATCH 57/93] skip tests can be empty --- .github/workflows/tests.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dd689255..3150a551 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -70,11 +70,17 @@ jobs: test_functions=$(grep -rho 'func \(Test[^ ]*\)' ./test/src/ | sed 's/func \(Test[^ ]*\)(t/\1/') : # Extract test names marked to be skipped from the commit message description - skipped_tests=$(echo "$GITHUB_EVENT_PATH" | xargs cat | jq -r '.commits[].message' | grep -oP '(?<=skip: )[^\,]+' | tr '\n' ',') + commit_message=$(jq -r '.commits[].message' "$GITHUB_EVENT_PATH") + if [[ $commit_message == *skip-tests:* ]]; then + skipped_tests=$(echo "$commit_message" | grep -oP '(?<=skip-tests: )[^\,]+' | tr '\n' ',') + else + skipped_tests="" + fi : # If any tests are marked to be skipped, remove them from the list of test functions if [ -n "$skipped_tests" ]; then for test in $(echo "$skipped_tests" | tr ',' '\n'); do + echo "Skipping test: $test" test_functions=$(echo "$test_functions" | grep -v "$test") done fi From f7d096b8e5a61129f87230351391bb64bfd58f94 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 11:07:00 +0200 Subject: [PATCH 58/93] debug --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3150a551..a5bcf128 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,6 +69,8 @@ jobs: run: | test_functions=$(grep -rho 'func \(Test[^ ]*\)' ./test/src/ | sed 's/func \(Test[^ ]*\)(t/\1/') + cat "$GITHUB_EVENT_PATH" + : # Extract test names marked to be skipped from the commit message description commit_message=$(jq -r '.commits[].message' "$GITHUB_EVENT_PATH") if [[ $commit_message == *skip-tests:* ]]; then From 1b65994bad0683253baccb07194f4a26f1eca031 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 11:12:50 +0200 Subject: [PATCH 59/93] fix --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a5bcf128..41641767 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -72,7 +72,7 @@ jobs: cat "$GITHUB_EVENT_PATH" : # Extract test names marked to be skipped from the commit message description - commit_message=$(jq -r '.commits[].message' "$GITHUB_EVENT_PATH") + commit_message=$(git log -1 --pretty=format:"%s") if [[ $commit_message == *skip-tests:* ]]; then skipped_tests=$(echo "$commit_message" | grep -oP '(?<=skip-tests: )[^\,]+' | tr '\n' ',') else From 4e9aeca4ef756e7c962c5ecf47391aad6c3de670 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 11:40:30 +0200 Subject: [PATCH 60/93] json format --- .github/workflows/tests.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 41641767..ac8f3662 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,9 +67,7 @@ jobs: - name: Extract Test Functions id: extract_test_functions run: | - test_functions=$(grep -rho 'func \(Test[^ ]*\)' ./test/src/ | sed 's/func \(Test[^ ]*\)(t/\1/') - - cat "$GITHUB_EVENT_PATH" + test_functions=$(grep -rho 'func \(Test[^ ]*\)' ./test/src/ | sed 's/func \(Test[^ ]*\)(t/\1/' | tr '\n' ',' | sed 's/,$//') : # Extract test names marked to be skipped from the commit message description commit_message=$(git log -1 --pretty=format:"%s") @@ -83,10 +81,19 @@ jobs: if [ -n "$skipped_tests" ]; then for test in $(echo "$skipped_tests" | tr ',' '\n'); do echo "Skipping test: $test" - test_functions=$(echo "$test_functions" | grep -v "$test") + test_functions=$(echo "$test_functions" | sed -E "s/([,]+)$test//") done fi + : # to json array + IFS=',' read -ra array <<< "$test_functions" + json_array="[" + for element in "${array[@]}" + do + json_array+="\"$element\"," + done + test_functions="${json_array%,}]" + echo "test_functions=${test_functions}" >> "$GITHUB_OUTPUT" echo "test_functions=${test_functions}" From 94702bc73ac1f162235288c93a0f80f3ee9e92de Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 11:44:22 +0200 Subject: [PATCH 61/93] fix secret --- .github/workflows/tests.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ac8f3662..24150006 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,19 +48,6 @@ jobs: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Import Secrets - id: secrets - uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3 - with: - url: ${{ secrets.VAULT_ADDR }} - method: approle - roleId: ${{ secrets.VAULT_ROLE_ID }} - secretId: ${{ secrets.VAULT_SECRET_ID }} - exportEnv: true - secrets: | - secret/data/products/infrastructure-experience/ci/common AWS_ACCESS_KEY; - secret/data/products/infrastructure-experience/ci/common AWS_SECRET_KEY; - - name: Get Short GitHub SHA and Save in Environment Variable run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" @@ -112,11 +99,24 @@ jobs: - name: Install tooling using asdf uses: asdf-vm/actions/install@05e0d2ed97b598bfce82fd30daf324ae0c4570e6 # v3 + - name: Import Secrets + id: secrets + uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3 + with: + url: ${{ secrets.VAULT_ADDR }} + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + exportEnv: false + secrets: | + secret/data/products/infrastructure-experience/ci/common AWS_ACCESS_KEY; + secret/data/products/infrastructure-experience/ci/common AWS_SECRET_KEY; + # Official action does not support profiles - name: Add profile credentials to ~/.aws/credentials run: | - aws configure set aws_access_key_id ${{ env.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} - aws configure set aws_secret_access_key ${{ env.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }} + aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }} aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }} - name: Get go.mod details From e21a399fc038f4afaa026cc8fe35d19a8bb7763b Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 12:18:41 +0200 Subject: [PATCH 62/93] update test report --- .github/workflows/tests.yml | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 24150006..adf89e0b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -125,38 +125,19 @@ jobs: with: modfile: ${{ github.workspace }}/test/src/go.mod - - name: Launch tests in parallel + - name: Launch test timeout-minutes: 125 run: | export TESTS_CLUSTER_ID="${{ env.SHORT_GITHUB_SHA }}" just test ${{ matrix.test_function }} "--junitfile ${{ matrix.test_function }}_unit-tests.xml" - - name: Upload test reports + - name: Run test-summary if: always() - uses: actions/upload-artifact@v3 + uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336 # v2 with: - name: test-reports path: "./test/src/${{ matrix.test_function }}_unit-tests.xml" - name: Remove profile credentials from ~/.aws/credentials if: always() run: | rm -rf ~/.aws/credentials - - test-report: - runs-on: ubuntu-latest - if: ${{ always() && needs.check-run-tests.result == 'success' }} - needs: - - check-run-tests - - integration-tests - steps: - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - name: test-reports - path: /tmp/testreports - - - name: Run test-summary - uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336 # v2 - with: - paths: /tmp/testreports/**/*.xml From ce7e83018b07f8ed988c8ec687c97b8c42959233 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 13:17:05 +0200 Subject: [PATCH 63/93] debug tests --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index adf89e0b..7841a780 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -130,12 +130,13 @@ jobs: run: | export TESTS_CLUSTER_ID="${{ env.SHORT_GITHUB_SHA }}" just test ${{ matrix.test_function }} "--junitfile ${{ matrix.test_function }}_unit-tests.xml" + cat ./test/src/${{ matrix.test_function }}_unit-tests.xml - name: Run test-summary if: always() uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336 # v2 with: - path: "./test/src/${{ matrix.test_function }}_unit-tests.xml" + paths: "./test/src/${{ matrix.test_function }}_unit-tests.xml" - name: Remove profile credentials from ~/.aws/credentials if: always() From 59ecc22ded45b55f31257e3f26c08997bbc15758 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:01:38 +0200 Subject: [PATCH 64/93] debug tests --- .github/workflows/lint.yml | 10 ++-------- .github/workflows/tests.yml | 5 +++++ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 20419677..d75bd94f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,14 +13,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 + - name: Install tooling using asdf + uses: asdf-vm/actions/install@05e0d2ed97b598bfce82fd30daf324ae0c4570e6 # v3 - - uses: terraform-linters/setup-tflint@19a52fbac37dacb22a09518e4ef6ee234f2d4987 # v4 - name: Setup TFLint - with: - tflint_version: v0.50.3 - - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5 - - run: go install github.com/terraform-docs/terraform-docs@v0.17.0 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7841a780..204d0d3e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -129,7 +129,12 @@ jobs: timeout-minutes: 125 run: | export TESTS_CLUSTER_ID="${{ env.SHORT_GITHUB_SHA }}" + export TESTS_CLUSTER_REGION="${{ env.AWS_REGION }}" just test ${{ matrix.test_function }} "--junitfile ${{ matrix.test_function }}_unit-tests.xml" + + - name: Show test-summary + if: always() + run: | cat ./test/src/${{ matrix.test_function }}_unit-tests.xml - name: Run test-summary From 7f06dc3ed87bdcf7efd844af6829ff8584ae5a56 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 17:06:17 +0200 Subject: [PATCH 65/93] improve tests debug status --- .github/workflows/tests.yml | 40 +++++++++++++++++++++++++++++-------- DEVELOPER.md | 2 ++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 204d0d3e..d1959e84 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,12 +44,14 @@ jobs: - check-run-tests outputs: test_functions: ${{ steps.extract_test_functions.outputs.test_functions }} + cluster_id: ${{ steps.short_git_sha.outputs.short_git_sha }} steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Get Short GitHub SHA and Save in Environment Variable - run: echo "SHORT_GITHUB_SHA=$(git rev-parse --short HEAD)" >> "$GITHUB_ENV" + - name: Get Short GitHub SHA + id: short_git_sha + run: echo "short_git_sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" - name: Extract Test Functions id: extract_test_functions @@ -128,22 +130,44 @@ jobs: - name: Launch test timeout-minutes: 125 run: | - export TESTS_CLUSTER_ID="${{ env.SHORT_GITHUB_SHA }}" + export TESTS_CLUSTER_ID="${{ needs.configure-tests.outputs.cluster_id }}" export TESTS_CLUSTER_REGION="${{ env.AWS_REGION }}" just test ${{ matrix.test_function }} "--junitfile ${{ matrix.test_function }}_unit-tests.xml" - - name: Show test-summary + # The test report is currently not working as expected due to https://github.com/test-summary/action/issues/5 + # TODO: filter logger.go from the xml to make it work + + - name: Filter logger.go from the test report (too large) if: always() run: | - cat ./test/src/${{ matrix.test_function }}_unit-tests.xml + sed 's/ /\n/g' < "./test/src/${{ matrix.test_function }}_unit-tests.xml" | grep -E -v '^.*logger\.go.*$' | sed 's/\n/ /g' > "./test/src/${{ matrix.test_function }}_unit-tests_filtered.xml" - - name: Run test-summary + - name: Upload test reports if: always() - uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336 # v2 + uses: actions/upload-artifact@v3 with: - paths: "./test/src/${{ matrix.test_function }}_unit-tests.xml" + name: test-reports + path: "./test/src/${{ matrix.test_function }}_unit-tests_filtered.xml" - name: Remove profile credentials from ~/.aws/credentials if: always() run: | rm -rf ~/.aws/credentials + + test-report: + runs-on: ubuntu-latest + if: ${{ always() && needs.check-run-tests.result == 'success' }} + needs: + - check-run-tests + - integration-tests + steps: + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + name: test-reports + path: /tmp/testreports + + - name: Run test-summary + uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336 # v2 + with: + paths: /tmp/testreports/**/*.xml diff --git a/DEVELOPER.md b/DEVELOPER.md index a48eedbd..a7ff21d6 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -32,6 +32,8 @@ To start developing or testing the EKS module, follow these steps: **Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. Then re-run this workflow by pushing a dummy commit: `git commit --allow-empty -m "trigger workflow"`. +You can skip specific tests in the CI by listing them in the commit description with the prefix `skip-tests:` (e.g.: `skip-tests: Test1,Test2`). + ## Releasing a New Version We follow Semantic Versioning (SemVer) guidelines for versioning. Follow these steps to release a new version: From 2df9a8f7376206ac0a490ea243ece767e108ab45 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 18:41:33 +0200 Subject: [PATCH 66/93] make kms optional and fix rds zones --- .github/workflows/tests.yml | 5 ++- test/src/custom_eks_rds_test.go | 1 + test/src/default_eks_test.go | 58 ++++++++++++++++++--------------- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d1959e84..27634618 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -132,11 +132,10 @@ jobs: run: | export TESTS_CLUSTER_ID="${{ needs.configure-tests.outputs.cluster_id }}" export TESTS_CLUSTER_REGION="${{ env.AWS_REGION }}" + export TESTS_DISABLE_KMS_CHECKS=true just test ${{ matrix.test_function }} "--junitfile ${{ matrix.test_function }}_unit-tests.xml" - # The test report is currently not working as expected due to https://github.com/test-summary/action/issues/5 - # TODO: filter logger.go from the xml to make it work - + # this is a workaround for test report not working as expected due to https://github.com/test-summary/action/issues/5 - name: Filter logger.go from the test report (too large) if: always() run: | diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index 30e4c3f8..0560c829 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -136,6 +136,7 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { "cluster_name": fmt.Sprintf("postgres-%s", suite.clusterName), "subnet_ids": result.Cluster.ResourcesVpcConfig.SubnetIds, "vpc_id": *result.Cluster.ResourcesVpcConfig.VpcId, + "availability_zones": []string{fmt.Sprintf("%sa", suite.region), fmt.Sprintf("%sb", suite.region), fmt.Sprintf("%sc", suite.region)}, "cidr_blocks": append(publicBlocks, privateBlocks...), "iam_auth_enabled": true, } diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go index b50c0a26..d5a9b422 100644 --- a/test/src/default_eks_test.go +++ b/test/src/default_eks_test.go @@ -208,37 +208,43 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti suite.Assert().Equal(len(outputVPC.Vpcs), 1) - // key - keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) - inputKMS := &kms.ListKeysInput{} - outputKMSList, errKMSList := kmsSvc.ListKeys(context.Background(), inputKMS) - suite.Assert().NoError(errKMSList) - - // Check if the key corresponding to the description exists - keyFound := false - for _, key := range outputKMSList.Keys { - keyDetails, errKey := kmsSvc.DescribeKey(context.Background(), &kms.DescribeKeyInput{ - KeyId: key.KeyId, - }) - - if errKey != nil { - // ignore AccessDenied - var re *awshttp.ResponseError - if errors.As(err, &re) { - if re.HTTPStatusCode() == 400 { - continue + // KMS checks are not working as expected in CI due to permission + skipTestKMS := utils.GetEnv("TESTS_DISABLE_KMS_CHECKS", "false") + if skipTestKMS == "true" { + suite.sugaredLogger.Infow("Skipping tests as KMS check (TESTS_DISABLE_KMS_CHECKS) is disabled") + } else { + // key + keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) + inputKMS := &kms.ListKeysInput{} + outputKMSList, errKMSList := kmsSvc.ListKeys(context.Background(), inputKMS) + suite.Assert().NoError(errKMSList) + + // Check if the key corresponding to the description exists + keyFound := false + for _, key := range outputKMSList.Keys { + keyDetails, errKey := kmsSvc.DescribeKey(context.Background(), &kms.DescribeKeyInput{ + KeyId: key.KeyId, + }) + + if errKey != nil { + // ignore AccessDenied + var re *awshttp.ResponseError + if errors.As(err, &re) { + if re.HTTPStatusCode() == 400 { + continue + } } - } - suite.Require().NoErrorf(errKey, "Failed to describe key %s", *key.KeyId) - } + suite.Require().NoErrorf(errKey, "Failed to describe key %s", *key.KeyId) + } - keyFound = *keyDetails.KeyMetadata.Description == keyDescription - if keyFound { - break + keyFound = *keyDetails.KeyMetadata.Description == keyDescription + if keyFound { + break + } } + suite.Assert().Truef(keyFound, "Failed to find key %s", keyDescription) } - suite.Assert().Truef(keyFound, "Failed to find key %s", keyDescription) } func TestDefaultEKSTestSuite(t *testing.T) { From 8afcf0656acc51f4885c4fcaf02f3e64484867a7 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:55:04 +0200 Subject: [PATCH 67/93] pin actions skip-tests:TestCustomEKSRDSTestSuite --- .github/workflows/lint.yml | 1 - .github/workflows/tests.yml | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d75bd94f..3241ced1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,5 +16,4 @@ jobs: - name: Install tooling using asdf uses: asdf-vm/actions/install@05e0d2ed97b598bfce82fd30daf324ae0c4570e6 # v3 - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 27634618..d32f8fc6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -143,7 +143,7 @@ jobs: - name: Upload test reports if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3 with: name: test-reports path: "./test/src/${{ matrix.test_function }}_unit-tests_filtered.xml" @@ -161,7 +161,7 @@ jobs: - integration-tests steps: - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3 with: name: test-reports path: /tmp/testreports From d1846c7cd0c8ef76645d7e5ed136ad857c2b2dd9 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:05:48 +0200 Subject: [PATCH 68/93] add license, fix skip skip-tests:TestCustomEKSRDSTestSuite,TestUpgradeEKSTestSuite,TestDefaultEKSTestSuite --- .github/workflows/tests.yml | 2 +- LICENSE | 176 ++++++++++++++++++++++++++++++++++++ README.md | 4 + 3 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 LICENSE diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d32f8fc6..2a354d79 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: : # Extract test names marked to be skipped from the commit message description commit_message=$(git log -1 --pretty=format:"%s") if [[ $commit_message == *skip-tests:* ]]; then - skipped_tests=$(echo "$commit_message" | grep -oP '(?<=skip-tests: )[^\,]+' | tr '\n' ',') + skipped_tests=$(echo "$commit_message" | grep -oP '(?<=skip-tests:)[^\,]+' | tr '\n' ',') else skipped_tests="" fi diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d9a10c0d --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md index 079f5089..4e83f486 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Camunda Terraform EKS Modules +[![Camunda](https://img.shields.io/badge/Camunda-FC5D0D)](https://www.camunda.com/) +[![tests](https://github.com/camunda/camunda-tf-eks-module/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/camunda/camunda-tf-eks-module/actions/workflows/tests.yml) +[![License](https://img.shields.io/github/license/camunda/camunda-tf-eks-module)](LICENSE) + Terraform module which creates AWS EKS (Kubernetes) resources with an opinionated configuration targeting Camunda 8. ## Documentation From 9963b572b16bedb711644a9e8f2bc7bcc5ea6d56 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:18:48 +0200 Subject: [PATCH 69/93] implement custom tf binary skip-tests:TestCustomEKSRDSTestSuite,TestUpgradeEKSTestSuite,TestDefaultEKSTestSuite --- .github/workflows/tests.yml | 4 +++- test/README.md | 5 +++++ test/src/custom_eks_rds_test.go | 22 ++++++++++++++-------- test/src/default_eks_test.go | 13 +++++++++---- test/src/upgrade_eks_test.go | 21 +++++++++++++-------- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a354d79..d5c62489 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,7 @@ on: env: AWS_PROFILE: "infex" AWS_REGION: "eu-west-2" + TESTS_TF_BINARY_NAME: "terraform" jobs: check-run-tests: @@ -59,7 +60,7 @@ jobs: test_functions=$(grep -rho 'func \(Test[^ ]*\)' ./test/src/ | sed 's/func \(Test[^ ]*\)(t/\1/' | tr '\n' ',' | sed 's/,$//') : # Extract test names marked to be skipped from the commit message description - commit_message=$(git log -1 --pretty=format:"%s") + commit_message=$(git log -1 --pretty=format:"%B") if [[ $commit_message == *skip-tests:* ]]; then skipped_tests=$(echo "$commit_message" | grep -oP '(?<=skip-tests:)[^\,]+' | tr '\n' ',') else @@ -133,6 +134,7 @@ jobs: export TESTS_CLUSTER_ID="${{ needs.configure-tests.outputs.cluster_id }}" export TESTS_CLUSTER_REGION="${{ env.AWS_REGION }}" export TESTS_DISABLE_KMS_CHECKS=true + export TESTS_TF_BINARY_NAME="${{ env.TESTS_TF_BINARY_NAME }}" just test ${{ matrix.test_function }} "--junitfile ${{ matrix.test_function }}_unit-tests.xml" # this is a workaround for test report not working as expected due to https://github.com/test-summary/action/issues/5 diff --git a/test/README.md b/test/README.md index 8c31f50f..c840063e 100644 --- a/test/README.md +++ b/test/README.md @@ -58,6 +58,11 @@ You can change the default deployment region: export TESTS_CLUSTER_REGION="eu-west-1" ``` +You can change the terraform binary (default is `tofu`): +```bash +export TESTS_TF_BINARY_NAME="terraform" +``` + ### Run the tests Test with: diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index 0560c829..eeb90942 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -34,6 +34,7 @@ type CustomEKSRDSTestSuite struct { kubeConfigPath string region string tfDataDir string + tfBinaryName string varTf map[string]interface{} } @@ -44,6 +45,9 @@ func (suite *CustomEKSRDSTestSuite) SetupTest() { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) suite.clusterName = fmt.Sprintf("cluster-rds-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") + suite.tfBinaryName = utils.GetEnv("TESTS_TF_BINARY_NAME", "tofu") + suite.sugaredLogger.Infow("Terraform binary for the suite", "binary", suite.tfBinaryName) + suite.expectedNodes = 1 var errAbsPath error suite.tfDataDir, errAbsPath = filepath.Abs(fmt.Sprintf("../../test/states/tf-data-%s", suite.clusterName)) @@ -85,10 +89,11 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { tfDir := test_structure.CopyTerraformFolderToDest(suite.T(), "../../modules/", "eks-cluster/", fullDirEKS) terraformOptions := &terraform.Options{ - TerraformDir: tfDir, - Upgrade: false, - VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, - Vars: suite.varTf, + TerraformBinary: suite.tfBinaryName, + TerraformDir: tfDir, + Upgrade: false, + VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, } cleanClusterAtTheEnd := utils.GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") @@ -148,10 +153,11 @@ func (suite *CustomEKSRDSTestSuite) TestCustomEKSAndRDS() { tfDirAurora := test_structure.CopyTerraformFolderToDest(suite.T(), "../../modules/", "aurora/", fullDirAurora) terraformOptionsRDS := &terraform.Options{ - TerraformDir: tfDirAurora, - Upgrade: false, - VarFiles: []string{"../fixtures/fixtures.default.aurora.tfvars"}, - Vars: varsConfigAurora, + TerraformBinary: suite.tfBinaryName, + TerraformDir: tfDirAurora, + Upgrade: false, + VarFiles: []string{"../fixtures/fixtures.default.aurora.tfvars"}, + Vars: varsConfigAurora, } if cleanClusterAtTheEnd == "true" { diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go index d5a9b422..a1c8c563 100644 --- a/test/src/default_eks_test.go +++ b/test/src/default_eks_test.go @@ -35,6 +35,7 @@ type DefaultEKSTestSuite struct { kubeConfigPath string region string tfDataDir string + tfBinaryName string varTf map[string]interface{} } @@ -45,6 +46,9 @@ func (suite *DefaultEKSTestSuite) SetupTest() { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) suite.clusterName = fmt.Sprintf("cluster-test-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") + suite.tfBinaryName = utils.GetEnv("TESTS_TF_BINARY_NAME", "tofu") + suite.sugaredLogger.Infow("Terraform binary for the suite", "binary", suite.tfBinaryName) + suite.expectedNodes = 4 var errAbsPath error suite.tfDataDir, errAbsPath = filepath.Abs(fmt.Sprintf("../../test/states/tf-data-%s", suite.clusterName)) @@ -85,10 +89,11 @@ func (suite *DefaultEKSTestSuite) TestDefaultEKS() { tfDir := test_structure.CopyTerraformFolderToDest(suite.T(), "../../modules/", "eks-cluster/", fullDir) terraformOptions := &terraform.Options{ - TerraformDir: tfDir, - Upgrade: false, - VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, - Vars: suite.varTf, + TerraformBinary: suite.tfBinaryName, + TerraformDir: tfDir, + Upgrade: false, + VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, } cleanClusterAtTheEnd := utils.GetEnv("CLEAN_CLUSTER_AT_THE_END", "true") diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index c323a902..4f2c2e53 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -31,6 +31,7 @@ type UpgradeEKSTestSuite struct { kubeConfigPath string kubeVersion string tfDataDir string + tfBinaryName string region string varTf map[string]interface{} } @@ -42,6 +43,8 @@ func (suite *UpgradeEKSTestSuite) SetupTest() { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) suite.clusterName = fmt.Sprintf("cluster-upgrade-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") + suite.tfBinaryName = utils.GetEnv("TESTS_TF_BINARY_NAME", "tofu") + suite.sugaredLogger.Infow("Terraform binary for the suite", "binary", suite.tfBinaryName) suite.expectedNodes = 3 suite.kubeVersion = "1.28" var errAbsPath error @@ -85,10 +88,11 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { tfDir := test_structure.CopyTerraformFolderToDest(suite.T(), "../../modules/", "eks-cluster/", fullDir) terraformOptions := &terraform.Options{ - TerraformDir: tfDir, - Upgrade: false, - VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, - Vars: suite.varTf, + TerraformBinary: suite.tfBinaryName, + TerraformDir: tfDir, + Upgrade: false, + VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, } suite.sugaredLogger.Infow("Creating EKS cluster...", "extraVars", suite.varTf) @@ -173,10 +177,11 @@ func (suite *UpgradeEKSTestSuite) TestUpgradeEKS() { // perform update with terraform terraformOptions = &terraform.Options{ - TerraformDir: tfDir, - Upgrade: false, - VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, - Vars: suite.varTf, + TerraformBinary: suite.tfBinaryName, + TerraformDir: tfDir, + Upgrade: false, + VarFiles: []string{"../fixtures/fixtures.default.eks.tfvars"}, + Vars: suite.varTf, } suite.sugaredLogger.Infow("Reapply terraform after EKS cluster upgrade...", "extraVars", suite.varTf) From 281b81fd977d498bd3951229d4b82d70d38990e9 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:29:53 +0200 Subject: [PATCH 70/93] fix skip skip-tests:TestCustomEKSRDSTestSuite,TestUpgradeEKSTestSuite,TestDefaultEKSTestSuite --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d5c62489..7f872eb9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,7 +62,7 @@ jobs: : # Extract test names marked to be skipped from the commit message description commit_message=$(git log -1 --pretty=format:"%B") if [[ $commit_message == *skip-tests:* ]]; then - skipped_tests=$(echo "$commit_message" | grep -oP '(?<=skip-tests:)[^\,]+' | tr '\n' ',') + skipped_tests=$(echo "$commit_message" | grep 'skip-tests' | sed 's/skip-tests://') else skipped_tests="" fi From fd9e2dc74e53e26dbed40c135b90a3f03af2c666 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:32:11 +0200 Subject: [PATCH 71/93] fix skip skip-tests:TestCustomEKSRDSTestSuite,TestUpgradeEKSTestSuite,TestDefaultEKSTestSuite --- .github/workflows/tests.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7f872eb9..f27324da 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,11 +61,9 @@ jobs: : # Extract test names marked to be skipped from the commit message description commit_message=$(git log -1 --pretty=format:"%B") - if [[ $commit_message == *skip-tests:* ]]; then - skipped_tests=$(echo "$commit_message" | grep 'skip-tests' | sed 's/skip-tests://') - else - skipped_tests="" - fi + skipped_tests=$(echo "$commit_message" | grep 'skip-tests' | sed 's/skip-tests://') + + echo "skipped_tests=$skipped_tests" : # If any tests are marked to be skipped, remove them from the list of test functions if [ -n "$skipped_tests" ]; then From 5d22e60ffdfe4bd44adad72163bf1dcab00bb2f3 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:35:57 +0200 Subject: [PATCH 72/93] fix skip skip-tests:TestCustomEKSRDSTestSuite,TestUpgradeEKSTestSuite,TestDefaultEKSTestSuite --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f27324da..36522e3e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,12 +57,14 @@ jobs: - name: Extract Test Functions id: extract_test_functions run: | + sudo apt install git -y test_functions=$(grep -rho 'func \(Test[^ ]*\)' ./test/src/ | sed 's/func \(Test[^ ]*\)(t/\1/' | tr '\n' ',' | sed 's/,$//') : # Extract test names marked to be skipped from the commit message description commit_message=$(git log -1 --pretty=format:"%B") - skipped_tests=$(echo "$commit_message" | grep 'skip-tests' | sed 's/skip-tests://') + echo "commit_message=$commit_message" + skipped_tests=$(echo "$commit_message" | grep 'skip-tests' | sed 's/skip-tests://') echo "skipped_tests=$skipped_tests" : # If any tests are marked to be skipped, remove them from the list of test functions From ec0dfc93bafa0828f7338d34f25902d908f48415 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:38:38 +0200 Subject: [PATCH 73/93] fix skip skip-tests:TestCustomEKSRDSTestSuite,TestUpgradeEKSTestSuite,TestDefaultEKSTestSuite --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36522e3e..bce96275 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,6 +49,9 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 - name: Get Short GitHub SHA id: short_git_sha @@ -57,7 +60,6 @@ jobs: - name: Extract Test Functions id: extract_test_functions run: | - sudo apt install git -y test_functions=$(grep -rho 'func \(Test[^ ]*\)' ./test/src/ | sed 's/func \(Test[^ ]*\)(t/\1/' | tr '\n' ',' | sed 's/,$//') : # Extract test names marked to be skipped from the commit message description From 1341a990890fb2eba5174d18f0e043a9f0fbe2b6 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:53:38 +0200 Subject: [PATCH 74/93] fix skip skip-tests:TestCustomEKSRDSTestSuite,TestUpgradeEKSTestSuite,TestDefaultEKSTestSuite --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bce96275..e0a984c1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -73,7 +73,8 @@ jobs: if [ -n "$skipped_tests" ]; then for test in $(echo "$skipped_tests" | tr ',' '\n'); do echo "Skipping test: $test" - test_functions=$(echo "$test_functions" | sed -E "s/([,]+)$test//") + test_functions=$(echo "$test_functions" | sed -E "s/(^|,)${test}(,|$)//g") + echo "test_functions=$test_functions" done fi From 2bafa735619ef1e86d00fd7248135b709b073445 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:59:55 +0200 Subject: [PATCH 75/93] trigger workflow From 87752ea149f02875f22bb01ae2f5ea8d8cb5c8f2 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Mon, 15 Apr 2024 21:02:27 +0200 Subject: [PATCH 76/93] fix cleanup aws region skip-tests:TestCustomEKSRDSTestSuite,TestUpgradeEKSTestSuite,TestDefaultEKSTestSuite --- .github/workflows/nightly_cleanup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly_cleanup.yml b/.github/workflows/nightly_cleanup.yml index 6eeb50db..d3f019aa 100644 --- a/.github/workflows/nightly_cleanup.yml +++ b/.github/workflows/nightly_cleanup.yml @@ -9,7 +9,7 @@ on: env: AWS_PROFILE: "infex" - AWS_REGION: "eu-central-1" + AWS_REGION: "eu-west-2" CLEANUP_NEWER_THAN: "6 hours" CLOUD_NUKE_VERSION: "v0.35.0" From e50341f8de052850d26ea6bc2530885455d439e5 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 10:28:21 +0200 Subject: [PATCH 77/93] implement skip-test strategy skip-tests:all --- .github/workflows/tests.yml | 48 ++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e0a984c1..0b9e0a50 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,9 +14,12 @@ env: TESTS_TF_BINARY_NAME: "terraform" jobs: - check-run-tests: + + # We can skip some tests (skip-tests:NameOfTest1,NameOfTest2) or all tests (skip-tests:all) + # If all tests are skipped, the result of this workflow will be `failed` on purpose + # If you want to skip tests and have no error, you need to use `testing-ci-not-necessary` as a label on the PR + configure-tests: runs-on: ubuntu-latest - # we allow integration tests for schedules, and for PR only if the content is tests or terraform related if: >- github.event_name == 'schedule' || ( github.event_name == 'pull_request' && @@ -26,23 +29,6 @@ jobs: ) && !contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') ) - steps: - - name: Check labels - run: | - labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") - echo "Labels: $labels" - if ! echo "$labels" | grep -qE "^testing-allowed$"; then - echo "ERROR: Pull request must have either 'testing-allowed' or 'testing-ci-not-necessary' label. Then re-run this workflow by pushing a dummy commit." - exit 1 - fi - - configure-tests: - runs-on: ubuntu-latest - # Other dependencies from hosted runner - # AWS CLI - # indirectly node used by actions - needs: - - check-run-tests outputs: test_functions: ${{ steps.extract_test_functions.outputs.test_functions }} cluster_id: ${{ steps.short_git_sha.outputs.short_git_sha }} @@ -69,13 +55,19 @@ jobs: skipped_tests=$(echo "$commit_message" | grep 'skip-tests' | sed 's/skip-tests://') echo "skipped_tests=$skipped_tests" - : # If any tests are marked to be skipped, remove them from the list of test functions - if [ -n "$skipped_tests" ]; then - for test in $(echo "$skipped_tests" | tr ',' '\n'); do - echo "Skipping test: $test" - test_functions=$(echo "$test_functions" | sed -E "s/(^|,)${test}(,|$)//g") - echo "test_functions=$test_functions" - done + : # If all tests are marked to be skipped, then clear the test_functions list completely + if [ "$skipped_tests" == "all" ]; then + test_functions="" + echo "Skipping all tests (skip-tests:all found)" + else + : # Otherwise, remove the tests marked to be skipped from the test_functions list + if [ -n "$skipped_tests" ]; then + for test in $(echo "$skipped_tests" | tr ',' '\n'); do + echo "Skipping test: $test" + test_functions=$(echo "$test_functions" | sed -E "s/(^|,)${test}(,|$)//g") + echo "test_functions=$test_functions" + done + fi fi : # to json array @@ -160,9 +152,9 @@ jobs: test-report: runs-on: ubuntu-latest - if: ${{ always() && needs.check-run-tests.result == 'success' }} + if: ${{ always() && needs.configure-tests.result == 'success' }} needs: - - check-run-tests + - configure-tests - integration-tests steps: - name: Download artifacts From 40e2a5e8b81e2709eeda232543cd0de3bab11688 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 10:40:40 +0200 Subject: [PATCH 78/93] implement skip-test strategy skip-tests:all --- .github/workflows/tests.yml | 2 +- DEVELOPER.md | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0b9e0a50..a990d9b8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,7 +58,7 @@ jobs: : # If all tests are marked to be skipped, then clear the test_functions list completely if [ "$skipped_tests" == "all" ]; then test_functions="" - echo "Skipping all tests (skip-tests:all found)" + echo "Skipping all tests (skip-tests:all found), this workflow will fail. If you want to skip-tests for a PR, please use the label 'testing-ci-not-necessary'" else : # Otherwise, remove the tests marked to be skipped from the test_functions list if [ -n "$skipped_tests" ]; then diff --git a/DEVELOPER.md b/DEVELOPER.md index a7ff21d6..30731a7c 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -29,10 +29,16 @@ To start developing or testing the EKS module, follow these steps: - Set `CLEAN_CLUSTER_AT_THE_END=false` to prevent automatic cluster deletion in case of errors. - Optionally, manually clean up the cluster after testing by reversing this setting. -**Note**: Ensure that the "testing-allowed" label is added to a pull request to trigger the tests. -Then re-run this workflow by pushing a dummy commit: `git commit --allow-empty -m "trigger workflow"`. +Understood, here's the updated documentation: -You can skip specific tests in the CI by listing them in the commit description with the prefix `skip-tests:` (e.g.: `skip-tests: Test1,Test2`). +## Tests in the CI + +The tests in the CI can be triggered automatically by labeling the pull request appropriately. It will be labeled either `test` or `terraform` automatically by the labeler. + +You can skip specific tests in the CI by listing them in the commit description with the prefix `skip-tests:` (e.g.: `skip-tests:Test1,Test2`). +If you want to skip all tests, use `skip-tests:all`. +Remember, if all tests are skipped, the workflow will intentionally result in a `failed` status. +To skip tests without triggering an error, add the label `testing-ci-not-necessary` to the PR. ## Releasing a New Version From 3eea7ca5a83a9a9bb717d96daa2fc01d5679d168 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 10:46:57 +0200 Subject: [PATCH 79/93] fix retention days --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a990d9b8..0c4ca0a2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -144,6 +144,7 @@ jobs: with: name: test-reports path: "./test/src/${{ matrix.test_function }}_unit-tests_filtered.xml" + retention-days: 1 - name: Remove profile credentials from ~/.aws/credentials if: always() From df9f3f0f486a8b18fc531becc3601db9fe550fe6 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 10:49:29 +0200 Subject: [PATCH 80/93] update doc --- .github/workflows/tests.yml | 2 +- DEVELOPER.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0c4ca0a2..35593705 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ env: jobs: - # We can skip some tests (skip-tests:NameOfTest1,NameOfTest2) or all tests (skip-tests:all) + # We can skip some tests using the commit description (skip-tests:NameOfTest1,NameOfTest2) or all tests (skip-tests:all) (see `DEVELOPER.md`) # If all tests are skipped, the result of this workflow will be `failed` on purpose # If you want to skip tests and have no error, you need to use `testing-ci-not-necessary` as a label on the PR configure-tests: diff --git a/DEVELOPER.md b/DEVELOPER.md index 30731a7c..5da453d0 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -33,7 +33,8 @@ Understood, here's the updated documentation: ## Tests in the CI -The tests in the CI can be triggered automatically by labeling the pull request appropriately. It will be labeled either `test` or `terraform` automatically by the labeler. +The tests in the CI can be triggered automatically by labeling the pull request appropriately. +It will be labeled either `test` or `terraform` automatically by the labeler. You can skip specific tests in the CI by listing them in the commit description with the prefix `skip-tests:` (e.g.: `skip-tests:Test1,Test2`). If you want to skip all tests, use `skip-tests:all`. From 43130478839c8ce68584a08834754353fbc2247d Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 11:03:36 +0200 Subject: [PATCH 81/93] trigger workflow From 188874abe97078ca50cb7604fb23eb193079ce39 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:26:06 +0200 Subject: [PATCH 82/93] group all renovate in a single group, disable automerge --- .github/renovate.json5 | 81 +++++++----------------------------------- 1 file changed, 12 insertions(+), 69 deletions(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index d5a02f68..8c920212 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -8,6 +8,7 @@ ":enablePreCommit", ], schedule: ["every weekend"], + groupName: "mono-update-renovate", // we keep all updates in a single renovate branch in order to save CI tests platformAutomerge: false, prHourlyLimit: 6, prConcurrentLimit: 20, @@ -31,7 +32,7 @@ }, vulnerabilityAlerts: { addLabels: ["security"], - enabled: true + enabled: true, }, packageRules: [ // limit the PR creation for the Renovate pre-commit hook (it's released very frequently) @@ -53,7 +54,6 @@ matchDatasources: ["terraform-module"], matchPackagePatterns: ["terraform-aws-modules.*"], addLabels: ["group:terraform"], - groupName: "Terraform AWS modules", schedule: [ "every 2 weeks on Saturday and Sunday", ], @@ -63,72 +63,8 @@ matchDatasources: ["terraform-provider"], addLabels: ["group:terraform"], schedule: [ - "every 2 weeks on Saturday and Sunday", - ] - }, - // Terraform patch provider updates - { - matchUpdateTypes: ["patch"], - matchDatasources: ["terraform-provider"], - addLabels: ["group:terraform", "automerge"], - groupName: "Terraform providers", - automerge: true, - schedule: [ - "every 2 weeks on Saturday and Sunday", - ] - }, - // GitHub Actions - { - matchUpdateTypes: ["minor", "patch"], - matchManagers: ["github-actions"], - addLabels: ["automerge"], - automerge: true, - }, - // Patches - // Those are tested packages, and we know that they follow the semver convention, - // but it's fine to have candidate packages to test before move them to minor section. - { - matchUpdateTypes: ["patch"], - matchPackagePatterns: [ - "aquasecurity/tfsec", - "pre-commit", - "^terraform$", - "terraform-aws-modules/.+", - "terraform-docs", - "terraform-linters/tflint", - ], - addLabels: ["automerge"], - automerge: true, - }, - // Minor versions - // Those are tested packages, and we know that they follow the semver convention, - // but it's fine to have candidate packages to test before move them to minor section. - { - matchUpdateTypes: ["minor"], - matchPackagePatterns: [ - "pre-commit", - "terraform-docs", - "terraform-linters/tflint", - ], - addLabels: ["automerge"], - automerge: true, - }, - // Create PRs and automerge mentioned components afterward - // Those are tested packages - { - matchUpdateTypes: ["major"], - matchPackagePatterns: [ - "pre-commit/.+", + "every 2 weeks on Saturday and Sunday" ], - addLabels: ["automerge"], - automerge: true, - }, - { - matchUpdateTypes: ["major", "minor", "patch"], - matchManagers: ["pre-commit"], - groupName: "pre-commit hooks", - addLabels: ["automerge"], - automerge: true }, // For known GitHub repositories that use GitHub tags/releases of format // "v1.2.3" and where the asdf plugin ignores the "v" prefix, we also tell @@ -138,8 +74,15 @@ matchPackageNames: [ "hashicorp/terraform", "pre-commit/pre-commit", + "casey/just", + "eksctl-io/eksctl", + "opentofu/opentofu", + "aquasecurity/tfsec", + "terraform-linters/tflint", + "terraform-docs/terraform-docs" + // TODO: add awscli ], extractVersion: "^v(?.*)$", - }, - ], + } + ] } From 70b978dee27e76b3cd71235727c0fabb06eec2c7 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:28:57 +0200 Subject: [PATCH 83/93] replace labeling ci with path trigger --- .github/workflows/nightly_cleanup.yml | 5 +---- .github/workflows/tests.yml | 17 +++++++++++------ DEVELOPER.md | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/nightly_cleanup.yml b/.github/workflows/nightly_cleanup.yml index d3f019aa..948a5c72 100644 --- a/.github/workflows/nightly_cleanup.yml +++ b/.github/workflows/nightly_cleanup.yml @@ -9,6 +9,7 @@ on: env: AWS_PROFILE: "infex" + # remember to also update tests.yml! AWS_REGION: "eu-west-2" CLEANUP_NEWER_THAN: "6 hours" CLOUD_NUKE_VERSION: "v0.35.0" @@ -16,10 +17,6 @@ env: jobs: aws-nightly-cleanup: runs-on: ubuntu-latest - # Other dependencies from hosted runner - # AWS CLI - # indirectly node used by actions - steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 35593705..5b5af6e0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,10 +6,19 @@ on: - cron: '0 1 * * 2' workflow_dispatch: pull_request: - + # the paths should be synced with ../labeler.yml + paths: + - test/**.go + - test/**/go.mod + - modules/fixtures/** + - modules/**.tf + - .tool-versions + - .github/workflows/tests.yml + - justfile env: AWS_PROFILE: "infex" + # remember to also update nightly_cleanup.yml! AWS_REGION: "eu-west-2" TESTS_TF_BINARY_NAME: "terraform" @@ -21,12 +30,8 @@ jobs: configure-tests: runs-on: ubuntu-latest if: >- - github.event_name == 'schedule' || ( + github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || ( github.event_name == 'pull_request' && - ( - contains(github.event.pull_request.labels.*.name, 'test') || - contains(github.event.pull_request.labels.*.name, 'terraform') - ) && !contains(github.event.pull_request.labels.*.name, 'testing-ci-not-necessary') ) outputs: diff --git a/DEVELOPER.md b/DEVELOPER.md index 5da453d0..d3e208fc 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -33,7 +33,7 @@ Understood, here's the updated documentation: ## Tests in the CI -The tests in the CI can be triggered automatically by labeling the pull request appropriately. +The tests in the CI can be triggered automatically by modifying terraform or test files. It will be labeled either `test` or `terraform` automatically by the labeler. You can skip specific tests in the CI by listing them in the commit description with the prefix `skip-tests:` (e.g.: `skip-tests:Test1,Test2`). From 3b7a09b719fc02054284f37be77dcb4b3a6d1727 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:36:23 +0200 Subject: [PATCH 84/93] re-enable kms check --- test/src/default_eks_test.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go index a1c8c563..394cb2da 100644 --- a/test/src/default_eks_test.go +++ b/test/src/default_eks_test.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/smithy-go" "github.com/camunda/camunda-tf-eks-module/utils" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" @@ -19,6 +20,7 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zaptest" "k8s.io/apimachinery/pkg/util/runtime" + "net/http" "os" "path/filepath" "strings" @@ -232,11 +234,18 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti }) if errKey != nil { - // ignore AccessDenied - var re *awshttp.ResponseError - if errors.As(err, &re) { - if re.HTTPStatusCode() == 400 { - continue + // ignore AccessDenied, the user may not have the permission to describe the key + // operation error KMS: DescribeKey, https response error StatusCode: 400,... + + suite.sugaredLogger.Infow("Failing operation: DescribeKey", "keyId", key.KeyId, "err", errKey) + + var oe *smithy.OperationError + if errors.As(err, &oe) { + var opErrHttp *awshttp.ResponseError + if errors.As(oe.Err, &opErrHttp) { + if opErrHttp.HTTPStatusCode() == http.StatusBadRequest { + continue + } } } From a0d429a296e9380d6abc8de7f48ec0879b881d09 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:36:34 +0200 Subject: [PATCH 85/93] re-enable kms check skip-tests:TestUpgradeEKSTestSuite,TestCustomEKSRDSTestSuite --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5b5af6e0..a54342a3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -133,7 +133,7 @@ jobs: run: | export TESTS_CLUSTER_ID="${{ needs.configure-tests.outputs.cluster_id }}" export TESTS_CLUSTER_REGION="${{ env.AWS_REGION }}" - export TESTS_DISABLE_KMS_CHECKS=true + export TESTS_DISABLE_KMS_CHECKS=false export TESTS_TF_BINARY_NAME="${{ env.TESTS_TF_BINARY_NAME }}" just test ${{ matrix.test_function }} "--junitfile ${{ matrix.test_function }}_unit-tests.xml" From d2d018d06af52db29a6baa10de53e6c2ab5c95bc Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:54:24 +0200 Subject: [PATCH 86/93] fix tests skip-tests:TestUpgradeEKSTestSuite,TestCustomEKSRDSTestSuite --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a54342a3..f5ee49ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,6 +52,7 @@ jobs: id: extract_test_functions run: | test_functions=$(grep -rho 'func \(Test[^ ]*\)' ./test/src/ | sed 's/func \(Test[^ ]*\)(t/\1/' | tr '\n' ',' | sed 's/,$//') + echo "test_functions=$test_functions" : # Extract test names marked to be skipped from the commit message description commit_message=$(git log -1 --pretty=format:"%B") @@ -69,7 +70,7 @@ jobs: if [ -n "$skipped_tests" ]; then for test in $(echo "$skipped_tests" | tr ',' '\n'); do echo "Skipping test: $test" - test_functions=$(echo "$test_functions" | sed -E "s/(^|,)${test}(,|$)//g") + test_functions=$(echo "$test_functions" | sed "s/$test//g" | sed 's/,,/,/g' | sed 's/^,//' | sed 's/,$//') echo "test_functions=$test_functions" done fi From 873f6bc4bf325c694e9f08ac23c7c8dd91b5bbde Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:36:36 +0200 Subject: [PATCH 87/93] fix tests skip-tests:TestUpgradeEKSTestSuite,TestCustomEKSRDSTestSuite --- test/src/default_eks_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go index 394cb2da..a86ce794 100644 --- a/test/src/default_eks_test.go +++ b/test/src/default_eks_test.go @@ -237,13 +237,14 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti // ignore AccessDenied, the user may not have the permission to describe the key // operation error KMS: DescribeKey, https response error StatusCode: 400,... - suite.sugaredLogger.Infow("Failing operation: DescribeKey", "keyId", key.KeyId, "err", errKey) - var oe *smithy.OperationError - if errors.As(err, &oe) { + suite.sugaredLogger.Debugw("Failing (non-fatal) operation: DescribeKey", "keyId", key.KeyId, "err", errKey, "errType", fmt.Sprintf("%T", errKey)) + if errors.As(errKey, &oe) { var opErrHttp *awshttp.ResponseError + suite.sugaredLogger.Debugw("Failing (non-fatal) operation: DescribeKey", "keyId", key.KeyId, "err", oe.Err, "errType", fmt.Sprintf("%T", oe.Err)) if errors.As(oe.Err, &opErrHttp) { if opErrHttp.HTTPStatusCode() == http.StatusBadRequest { + suite.sugaredLogger.Infow("Skipping not authorized describing key...", "keyId", key.KeyId) continue } } @@ -254,6 +255,7 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti keyFound = *keyDetails.KeyMetadata.Description == keyDescription if keyFound { + suite.sugaredLogger.Infow("Successfully described key", "keyId", key.KeyId) break } } From ceaf0b78e7f6e078717c7f453f90fd52580d9393 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Tue, 16 Apr 2024 22:06:10 +0200 Subject: [PATCH 88/93] fix tests kms with pagination --- .github/workflows/tests.yml | 1 - test/src/default_eks_test.go | 25 +++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f5ee49ca..9a81f48b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -134,7 +134,6 @@ jobs: run: | export TESTS_CLUSTER_ID="${{ needs.configure-tests.outputs.cluster_id }}" export TESTS_CLUSTER_REGION="${{ env.AWS_REGION }}" - export TESTS_DISABLE_KMS_CHECKS=false export TESTS_TF_BINARY_NAME="${{ env.TESTS_TF_BINARY_NAME }}" just test ${{ matrix.test_function }} "--junitfile ${{ matrix.test_function }}_unit-tests.xml" diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go index a86ce794..6f1bafe4 100644 --- a/test/src/default_eks_test.go +++ b/test/src/default_eks_test.go @@ -215,19 +215,19 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti suite.Assert().Equal(len(outputVPC.Vpcs), 1) - // KMS checks are not working as expected in CI due to permission - skipTestKMS := utils.GetEnv("TESTS_DISABLE_KMS_CHECKS", "false") - if skipTestKMS == "true" { - suite.sugaredLogger.Infow("Skipping tests as KMS check (TESTS_DISABLE_KMS_CHECKS) is disabled") - } else { - // key - keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) - inputKMS := &kms.ListKeysInput{} - outputKMSList, errKMSList := kmsSvc.ListKeys(context.Background(), inputKMS) + // key + keyDescription := fmt.Sprintf("%s - EKS Secret Encryption Key", clusterName) + inputKMS := &kms.ListKeysInput{} + paginatorKms := kms.NewListKeysPaginator(kmsSvc, inputKMS, func(o *kms.ListKeysPaginatorOptions) { + o.Limit = 50 + }) + + // Check if the key corresponding to the description exists + keyFound := false + for !keyFound && paginatorKms.HasMorePages() { + outputKMSList, errKMSList := paginatorKms.NextPage(context.TODO()) suite.Assert().NoError(errKMSList) - // Check if the key corresponding to the description exists - keyFound := false for _, key := range outputKMSList.Keys { keyDetails, errKey := kmsSvc.DescribeKey(context.Background(), &kms.DescribeKeyInput{ KeyId: key.KeyId, @@ -259,8 +259,9 @@ func (suite *DefaultEKSTestSuite) baseChecksEKS(terraformOptions *terraform.Opti break } } - suite.Assert().Truef(keyFound, "Failed to find key %s", keyDescription) } + + suite.Assert().Truef(keyFound, "Failed to find key %s", keyDescription) } func TestDefaultEKSTestSuite(t *testing.T) { From 8276f7a3c0b095d7a5a8b3ea8df6591779740da0 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 17 Apr 2024 08:02:18 +0200 Subject: [PATCH 89/93] re-enable automerge of renovate --- .github/renovate.json5 | 61 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 8c920212..6bdf7223 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -53,11 +53,70 @@ { matchDatasources: ["terraform-module"], matchPackagePatterns: ["terraform-aws-modules.*"], - addLabels: ["group:terraform"], + addLabels: ["group:terraform", "automerge"], + automerge: true, + groupName: "mono-update-renovate-automerge", schedule: [ "every 2 weeks on Saturday and Sunday", ], }, + // GitHub Actions + { + matchUpdateTypes: ["minor", "patch"], + matchManagers: ["github-actions"], + addLabels: ["automerge"], + groupName: "mono-update-renovate-automerge", + automerge: true, + }, + // Patches + // Those are tested packages, and we know that they follow the semver convention, + // but it's fine to have candidate packages to test before move them to minor section. + { + matchUpdateTypes: ["patch"], + matchPackagePatterns: [ + "aquasecurity/tfsec", + "pre-commit", + "^terraform$", + "terraform-aws-modules/.+", + "terraform-docs", + "terraform-linters/tflint", + ], + addLabels: ["automerge"], + groupName: "mono-update-renovate-automerge", + automerge: true, + }, + // Minor versions + // Those are tested packages, and we know that they follow the semver convention, + // but it's fine to have candidate packages to test before move them to minor section. + { + matchUpdateTypes: ["minor"], + matchPackagePatterns: [ + "pre-commit", + "terraform-docs", + "terraform-linters/tflint", + ], + addLabels: ["automerge"], + groupName: "mono-update-renovate-automerge", + automerge: true, + }, + // Create PRs and automerge mentioned components afterward + // Those are tested packages + { + matchUpdateTypes: ["major"], + matchPackagePatterns: [ + "pre-commit/.+", + ], + addLabels: ["automerge"], + groupName: "mono-update-renovate-automerge", + automerge: true, + }, + { + matchUpdateTypes: ["major", "minor", "patch"], + matchManagers: ["pre-commit"], + groupName: "mono-update-renovate-automerge", + addLabels: ["automerge"], + automerge: true + }, // Terraform major provider updates { matchDatasources: ["terraform-provider"], From 3a5060f7c5ea720ff3606e0933e6fcdfce4b7d19 Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:41:37 +0200 Subject: [PATCH 90/93] extend renovation --- .github/renovate.json5 | 52 +++++++++++++++++++++++++++++------------- .tool-versions | 10 ++++++++ justfile | 1 + 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 6bdf7223..3f9ae5d5 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -34,6 +34,22 @@ addLabels: ["security"], enabled: true, }, + customManagers: [ + { + "customType": "regex", + "fileMatch": ["^.tool-versions$"], + "matchStrings": [ + "datasource=(?.*?) depName=(?.*?)( versioning=(?.*?))?( extractVersion=(?.*?))?\\s.*? (?.*)\\s" + ], + }, + { + "customType": "regex", + "fileMatch": ["^justfile$"], + "matchStrings": [ + "datasource=(?.*?) depName=(?.*?)( versioning=(?.*?))?( extractVersion=(?.*?))?\\s(?.*) := \"(?.*?)\"\\s" + ], + }, + ], packageRules: [ // limit the PR creation for the Renovate pre-commit hook (it's released very frequently) { @@ -60,6 +76,25 @@ "every 2 weeks on Saturday and Sunday", ], }, + // Terraform major provider updates + { + matchDatasources: ["terraform-provider"], + addLabels: ["group:terraform"], + schedule: [ + "every 2 weeks on Saturday and Sunday" + ], + }, + // Terraform patch provider updates + { + matchUpdateTypes: ["patch"], + matchDatasources: ["terraform-provider"], + addLabels: ["group:terraform", "automerge"], + groupName: "mono-update-renovate-automerge", + automerge: true, + schedule: [ + "every 2 weeks on Saturday and Sunday", + ] + }, // GitHub Actions { matchUpdateTypes: ["minor", "patch"], @@ -117,29 +152,14 @@ addLabels: ["automerge"], automerge: true }, - // Terraform major provider updates - { - matchDatasources: ["terraform-provider"], - addLabels: ["group:terraform"], - schedule: [ - "every 2 weeks on Saturday and Sunday" - ], - }, // For known GitHub repositories that use GitHub tags/releases of format // "v1.2.3" and where the asdf plugin ignores the "v" prefix, we also tell // Renovate to ignore it via extractVersion when updating .tool-version file { - matchFileNames: ["**/.tool-versions", "**/*.tf"], + matchFileNames: ["**/*.tf"], matchPackageNames: [ "hashicorp/terraform", "pre-commit/pre-commit", - "casey/just", - "eksctl-io/eksctl", - "opentofu/opentofu", - "aquasecurity/tfsec", - "terraform-linters/tflint", - "terraform-docs/terraform-docs" - // TODO: add awscli ], extractVersion: "^v(?.*)$", } diff --git a/.tool-versions b/.tool-versions index c7da0742..e4efa760 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,19 +1,29 @@ +# renovate: datasource=github-releases depName=pre-commit/pre-commit pre-commit 3.7.0 +# renovate: datasource=github-releases depName=hashicorp/terraform terraform 1.7.5 +# renovate: datasource=github-releases depName=terraform-docs/terraform-docs terraform-docs 0.17.0 +# renovate: datasource=github-releases depName=terraform-linters/tflint tflint 0.50.3 +# renovate: datasource=github-releases depName=aquasecurity/tfsec tfsec 1.28.5 +# renovate: datasource=github-releases depName=opentofu/opentofu opentofu 1.6.2 +# renovate: datasource=golang-version depName=golang golang 1.22.2 +# renovate: datasource=github-releases depName=aws/aws-cli awscli 2.15.20 +# renovate: datasource=github-releases depName=eksctl-io/eksctl eksctl 0.175.0 +# renovate: datasource=github-releases depName=casey/just just 1.25.2 diff --git a/justfile b/justfile index 9a4b703f..43926ca9 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,6 @@ # this file is a recipe file for the project +# renovate: datasource=github-releases depName=gotestyourself/gotestsum gotestsum_version := "v1.11.0" # Launch a single test using go test in verbose mode From 907230a7bdcd27b6baf202a2c432ac292da9e96d Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:03:18 +0200 Subject: [PATCH 91/93] switch back from tofu to terraform by default globally --- README.md | 2 +- test/README.md | 4 ++-- test/src/custom_eks_rds_test.go | 2 +- test/src/default_eks_test.go | 2 +- test/src/upgrade_eks_test.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4e83f486..a9356637 100644 --- a/README.md +++ b/README.md @@ -50,4 +50,4 @@ module "postgresql" { ## Support -Please note that the modules have been tested with **[OpenTofu](https://opentofu.org/)** in the version described in the [.tool-versions](./.tool-versions) of this project. +Please note that the modules have been tested with **[Terraform](https://github.com/hashicorp/terraform)** in the version described in the [.tool-versions](./.tool-versions) of this project. diff --git a/test/README.md b/test/README.md index c840063e..62f7f853 100644 --- a/test/README.md +++ b/test/README.md @@ -58,9 +58,9 @@ You can change the default deployment region: export TESTS_CLUSTER_REGION="eu-west-1" ``` -You can change the terraform binary (default is `tofu`): +You can change the terraform binary (default is `terraform`): ```bash -export TESTS_TF_BINARY_NAME="terraform" +export TESTS_TF_BINARY_NAME="tofu" ``` ### Run the tests diff --git a/test/src/custom_eks_rds_test.go b/test/src/custom_eks_rds_test.go index eeb90942..7f51457c 100644 --- a/test/src/custom_eks_rds_test.go +++ b/test/src/custom_eks_rds_test.go @@ -45,7 +45,7 @@ func (suite *CustomEKSRDSTestSuite) SetupTest() { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) suite.clusterName = fmt.Sprintf("cluster-rds-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") - suite.tfBinaryName = utils.GetEnv("TESTS_TF_BINARY_NAME", "tofu") + suite.tfBinaryName = utils.GetEnv("TESTS_TF_BINARY_NAME", "terraform") suite.sugaredLogger.Infow("Terraform binary for the suite", "binary", suite.tfBinaryName) suite.expectedNodes = 1 diff --git a/test/src/default_eks_test.go b/test/src/default_eks_test.go index 6f1bafe4..004cdd8c 100644 --- a/test/src/default_eks_test.go +++ b/test/src/default_eks_test.go @@ -48,7 +48,7 @@ func (suite *DefaultEKSTestSuite) SetupTest() { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) suite.clusterName = fmt.Sprintf("cluster-test-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") - suite.tfBinaryName = utils.GetEnv("TESTS_TF_BINARY_NAME", "tofu") + suite.tfBinaryName = utils.GetEnv("TESTS_TF_BINARY_NAME", "terraform") suite.sugaredLogger.Infow("Terraform binary for the suite", "binary", suite.tfBinaryName) suite.expectedNodes = 4 diff --git a/test/src/upgrade_eks_test.go b/test/src/upgrade_eks_test.go index 4f2c2e53..eddfe83d 100644 --- a/test/src/upgrade_eks_test.go +++ b/test/src/upgrade_eks_test.go @@ -43,7 +43,7 @@ func (suite *UpgradeEKSTestSuite) SetupTest() { clusterSuffix := utils.GetEnv("TESTS_CLUSTER_ID", strings.ToLower(random.UniqueId())) suite.clusterName = fmt.Sprintf("cluster-upgrade-%s", clusterSuffix) suite.region = utils.GetEnv("TESTS_CLUSTER_REGION", "eu-central-1") - suite.tfBinaryName = utils.GetEnv("TESTS_TF_BINARY_NAME", "tofu") + suite.tfBinaryName = utils.GetEnv("TESTS_TF_BINARY_NAME", "terraform") suite.sugaredLogger.Infow("Terraform binary for the suite", "binary", suite.tfBinaryName) suite.expectedNodes = 3 suite.kubeVersion = "1.28" From dd2b1052b5abc3947f7bca862188695b9066056e Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 19 Apr 2024 12:08:18 +0200 Subject: [PATCH 92/93] sort tool-versions and associated pre-commit-hook --- .pre-commit-config.yaml | 8 +++ .pre-commit-hooks/sort_tool_versions.py | 83 +++++++++++++++++++++++++ .tool-versions | 30 ++++----- 3 files changed, 106 insertions(+), 15 deletions(-) create mode 100755 .pre-commit-hooks/sort_tool_versions.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ddc9b4f0..13edb96a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,4 +60,12 @@ repos: - id: go-fmt - id: no-go-testing - id: go-mod-tidy + +- repo: local + hooks: + - id: sort_tool_versions + name: sort_tool_versions + language: python + files: ^.tool-versions$ + entry: .pre-commit-hooks/sort_tool_versions.py ... diff --git a/.pre-commit-hooks/sort_tool_versions.py b/.pre-commit-hooks/sort_tool_versions.py new file mode 100755 index 00000000..a694970e --- /dev/null +++ b/.pre-commit-hooks/sort_tool_versions.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +""" +Pre-commit hook to sort dependencies in a file. + +This script sorts the dependencies listed in a file. It expects the file +to contain dependencies in the format 'package_name version', with one +dependency per line. Optional comment lines starting with '#' are allowed. +The script sorts the dependencies alphabetically by package name and +rewrites the file with the sorted dependencies. + +Usage: pre-commit hook + +Arguments: + file_path: Path to the file containing dependencies to be sorted. +""" + +import re +import sys +import argparse +from typing import IO +from typing import Sequence + +PASS = 0 +FAIL = 1 + +# captures package version +PACKAGE_REGEX = r"^([\w|\_|\-|\.]+)\s([\w|\_|\-|\.]+)$" + + +def sort_dependencies( + f: IO[bytes], +) -> int: + # Read the content of the file and decode it into a string + content = f.read().decode() + + package_dict = {} + + # capture an optional comment line followed by the package and its version + groups = re.findall(f"((^#.*\n)?{PACKAGE_REGEX})", content, re.MULTILINE) + + for group in groups: + package_name = group[2] + package_dict[package_name] = group[0] + + sorted_packages = sorted(package_dict.items()) + + sorted_content = "\n\n".join([package[1] for package in sorted_packages]) + sorted_content += "\n" + # Write sorted content back to the file + f.seek(0) + f.write(sorted_content.encode()) + f.truncate() # Truncate any extra content beyond what's written + + # Compare sorted content with original content + if sorted_content.encode() == content.encode(): + return PASS + else: + return FAIL + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("filenames", nargs="+", help="Files to sort") + args = parser.parse_args(argv) + + retv = PASS + + for arg in args.filenames: + with open(arg, "rb+") as file_obj: + ret_for_file = sort_dependencies( + file_obj, + ) + + if ret_for_file: + print(f"Sorting {arg}") + + retv |= ret_for_file + + return retv + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.tool-versions b/.tool-versions index e4efa760..b788f22b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,18 @@ +# renovate: datasource=github-releases depName=aws/aws-cli +awscli 2.15.20 + +# renovate: datasource=github-releases depName=eksctl-io/eksctl +eksctl 0.175.0 + +# renovate: datasource=golang-version depName=golang +golang 1.22.2 + +# renovate: datasource=github-releases depName=casey/just +just 1.25.2 + +# renovate: datasource=github-releases depName=opentofu/opentofu +opentofu 1.6.2 + # renovate: datasource=github-releases depName=pre-commit/pre-commit pre-commit 3.7.0 @@ -12,18 +27,3 @@ tflint 0.50.3 # renovate: datasource=github-releases depName=aquasecurity/tfsec tfsec 1.28.5 - -# renovate: datasource=github-releases depName=opentofu/opentofu -opentofu 1.6.2 - -# renovate: datasource=golang-version depName=golang -golang 1.22.2 - -# renovate: datasource=github-releases depName=aws/aws-cli -awscli 2.15.20 - -# renovate: datasource=github-releases depName=eksctl-io/eksctl -eksctl 0.175.0 - -# renovate: datasource=github-releases depName=casey/just -just 1.25.2 From 267a8dc611381bd7df2f3ec5a69ef68e62eca77d Mon Sep 17 00:00:00 2001 From: leiicamundi <153937047+leiicamundi@users.noreply.github.com> Date: Fri, 19 Apr 2024 16:29:05 +0200 Subject: [PATCH 93/93] replace the autosort by a comment --- .pre-commit-config.yaml | 8 --- .pre-commit-hooks/sort_tool_versions.py | 83 ------------------------- .tool-versions | 4 ++ 3 files changed, 4 insertions(+), 91 deletions(-) delete mode 100755 .pre-commit-hooks/sort_tool_versions.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 13edb96a..ddc9b4f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,12 +60,4 @@ repos: - id: go-fmt - id: no-go-testing - id: go-mod-tidy - -- repo: local - hooks: - - id: sort_tool_versions - name: sort_tool_versions - language: python - files: ^.tool-versions$ - entry: .pre-commit-hooks/sort_tool_versions.py ... diff --git a/.pre-commit-hooks/sort_tool_versions.py b/.pre-commit-hooks/sort_tool_versions.py deleted file mode 100755 index a694970e..00000000 --- a/.pre-commit-hooks/sort_tool_versions.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -""" -Pre-commit hook to sort dependencies in a file. - -This script sorts the dependencies listed in a file. It expects the file -to contain dependencies in the format 'package_name version', with one -dependency per line. Optional comment lines starting with '#' are allowed. -The script sorts the dependencies alphabetically by package name and -rewrites the file with the sorted dependencies. - -Usage: pre-commit hook - -Arguments: - file_path: Path to the file containing dependencies to be sorted. -""" - -import re -import sys -import argparse -from typing import IO -from typing import Sequence - -PASS = 0 -FAIL = 1 - -# captures package version -PACKAGE_REGEX = r"^([\w|\_|\-|\.]+)\s([\w|\_|\-|\.]+)$" - - -def sort_dependencies( - f: IO[bytes], -) -> int: - # Read the content of the file and decode it into a string - content = f.read().decode() - - package_dict = {} - - # capture an optional comment line followed by the package and its version - groups = re.findall(f"((^#.*\n)?{PACKAGE_REGEX})", content, re.MULTILINE) - - for group in groups: - package_name = group[2] - package_dict[package_name] = group[0] - - sorted_packages = sorted(package_dict.items()) - - sorted_content = "\n\n".join([package[1] for package in sorted_packages]) - sorted_content += "\n" - # Write sorted content back to the file - f.seek(0) - f.write(sorted_content.encode()) - f.truncate() # Truncate any extra content beyond what's written - - # Compare sorted content with original content - if sorted_content.encode() == content.encode(): - return PASS - else: - return FAIL - - -def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("filenames", nargs="+", help="Files to sort") - args = parser.parse_args(argv) - - retv = PASS - - for arg in args.filenames: - with open(arg, "rb+") as file_obj: - ret_for_file = sort_dependencies( - file_obj, - ) - - if ret_for_file: - print(f"Sorting {arg}") - - retv |= ret_for_file - - return retv - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.tool-versions b/.tool-versions index b788f22b..9191aa1b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,7 @@ +# /!\ Please maintain this file sorted alphabetically. +# check it with +# diff <(sed '/^#/d; /^$/d' .tool-versions | sort) <(sed '/^#/d; /^$/d' .tool-versions) && echo ".tool-versions is sorted correctly" || echo ".tool-versions is not sorted correctly" + # renovate: datasource=github-releases depName=aws/aws-cli awscli 2.15.20