From 42d1953c520cbc4cedfd616a601c6d5718615816 Mon Sep 17 00:00:00 2001 From: Marina Shustova Date: Thu, 23 Jun 2022 20:08:13 +0700 Subject: [PATCH 1/6] add simple Consul example on a single cluster Signed-off-by: Marina Shustova --- examples/nsm_consul_single_cluster/.gitignore | 2 + examples/nsm_consul_single_cluster/README.md | 84 +++++++++++++++++++ .../client/client.yaml | 24 ++++++ .../helm-consul-values.yaml | 9 ++ .../kind-cluster-config.yaml | 6 ++ .../networkservice.yaml | 18 ++++ .../nse-auto-scale/iptables-map | 1 + .../nse-auto-scale/kustomization.yaml | 20 +++++ .../nse-auto-scale/patch-supplier.yaml | 32 +++++++ .../nse-auto-scale/pod-template.yaml | 76 +++++++++++++++++ .../server/static-server.yaml | 54 ++++++++++++ .../nsm_consul_single_cluster/service.yaml | 16 ++++ 12 files changed, 342 insertions(+) create mode 100644 examples/nsm_consul_single_cluster/.gitignore create mode 100644 examples/nsm_consul_single_cluster/README.md create mode 100644 examples/nsm_consul_single_cluster/client/client.yaml create mode 100644 examples/nsm_consul_single_cluster/helm-consul-values.yaml create mode 100644 examples/nsm_consul_single_cluster/kind-cluster-config.yaml create mode 100644 examples/nsm_consul_single_cluster/networkservice.yaml create mode 100644 examples/nsm_consul_single_cluster/nse-auto-scale/iptables-map create mode 100644 examples/nsm_consul_single_cluster/nse-auto-scale/kustomization.yaml create mode 100644 examples/nsm_consul_single_cluster/nse-auto-scale/patch-supplier.yaml create mode 100644 examples/nsm_consul_single_cluster/nse-auto-scale/pod-template.yaml create mode 100644 examples/nsm_consul_single_cluster/server/static-server.yaml create mode 100644 examples/nsm_consul_single_cluster/service.yaml diff --git a/examples/nsm_consul_single_cluster/.gitignore b/examples/nsm_consul_single_cluster/.gitignore new file mode 100644 index 000000000000..bc946ca4e91d --- /dev/null +++ b/examples/nsm_consul_single_cluster/.gitignore @@ -0,0 +1,2 @@ +!**/kustomization.yaml +!**/patch-*.yaml \ No newline at end of file diff --git a/examples/nsm_consul_single_cluster/README.md b/examples/nsm_consul_single_cluster/README.md new file mode 100644 index 000000000000..30b1e3511520 --- /dev/null +++ b/examples/nsm_consul_single_cluster/README.md @@ -0,0 +1,84 @@ +# NSM + Istio interdomain example over kind cluster + +## Setup Cluster + +### KIND +Setup + +```bash +go install sigs.k8s.io/kind@v0.13.0 + +kind create cluster --config kind-cluster-config.yaml +``` + + +## SPIRE & NSM + +Use instructions from [Basic](../basic/README.md) + + +## CONSUL + +Install Consul for a cluster: +```bash +brew tap hashicorp/tap +brew install hashicorp/tap/consul-k8s +consul-k8s install -config-file=helm-consul-values.yaml -set global.image=hashicorp/consul:1.12.0 +``` + +### Verify NSM+CONSUL + +Install networkservice: +```bash +kubectl apply -f networkservice.yaml +``` + +Start `alpine` networkservicemesh client: + +```bash +kubectl apply -f client/client.yaml +``` + +Create kubernetes service for networkservicemesh endpoint: +```bash +kubectl apply -f service.yaml +``` + +Start `auto-scale` networkservicemesh endpoint: +```bash +kubectl apply -k nse-auto-scale +``` + +Install `static-server` Consul workload: +```bash +kubectl apply -f server/static-server.yaml +``` + +Verify connection from networkservicemesh client to consul server: +```bash +kubectl exec -it alpine-nsc -- apk add curl +kubectl exec -it alpine-nsc -- curl 172.16.1.2:8080 +``` + +You should see "hello world" answer. + +## Cleanup + + +```bash +WH=$(kubectl get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') +kubectl delete mutatingwebhookconfiguration ${WH} + +kubectl delete crd spiffeids.spiffeid.spiffe.io +kubectl delete ns spire + +kubectl delete -k nse-auto-scale + +kubectl delete -f client.yaml + +consul-k8s uninstall -auto-approve=true -wipe-data=true + +kubectl delete pods --all + +kind delete cluster +``` diff --git a/examples/nsm_consul_single_cluster/client/client.yaml b/examples/nsm_consul_single_cluster/client/client.yaml new file mode 100644 index 000000000000..a3e56030bd56 --- /dev/null +++ b/examples/nsm_consul_single_cluster/client/client.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: alpine-nsc + labels: + app: alpine-nsc + annotations: + networkservicemesh.io: kernel://autoscale-consul-proxy/nsm-1?app=alpine-nsc +spec: + containers: + - name: alpine-nsc + image: alpine:3.15.0 + imagePullPolicy: IfNotPresent + stdin: true + tty: true + - name: static-client + image: hashicorp/http-echo:latest + args: + - -text="hello world from nsc" + - -listen=:9090 + ports: + - containerPort: 9090 + name: http \ No newline at end of file diff --git a/examples/nsm_consul_single_cluster/helm-consul-values.yaml b/examples/nsm_consul_single_cluster/helm-consul-values.yaml new file mode 100644 index 000000000000..7118ad7629f2 --- /dev/null +++ b/examples/nsm_consul_single_cluster/helm-consul-values.yaml @@ -0,0 +1,9 @@ +global: + name: consul + datacenter: dc1 +server: + replicas: 1 +connectInject: + enabled: true + transparentProxy: + defaultEnabled: false diff --git a/examples/nsm_consul_single_cluster/kind-cluster-config.yaml b/examples/nsm_consul_single_cluster/kind-cluster-config.yaml new file mode 100644 index 000000000000..f57c67e5b02b --- /dev/null +++ b/examples/nsm_consul_single_cluster/kind-cluster-config.yaml @@ -0,0 +1,6 @@ +--- +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker diff --git a/examples/nsm_consul_single_cluster/networkservice.yaml b/examples/nsm_consul_single_cluster/networkservice.yaml new file mode 100644 index 000000000000..72f6d769eb30 --- /dev/null +++ b/examples/nsm_consul_single_cluster/networkservice.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: networkservicemesh.io/v1 +kind: NetworkService +metadata: + name: autoscale-consul-proxy + namespace: nsm-system +spec: + payload: IP + matches: + - source_selector: + fallthrough: true + routes: + - destination_selector: + podName: "{{ .podName }}" + - source_selector: + routes: + - destination_selector: + any: "true" diff --git a/examples/nsm_consul_single_cluster/nse-auto-scale/iptables-map b/examples/nsm_consul_single_cluster/nse-auto-scale/iptables-map new file mode 100644 index 000000000000..7b3b7c6de92a --- /dev/null +++ b/examples/nsm_consul_single_cluster/nse-auto-scale/iptables-map @@ -0,0 +1 @@ +-I PREROUTING 1 -p tcp -i {{ .NsmInterfaceName }} -j DNAT --to-destination 127.0.0.1 \ No newline at end of file diff --git a/examples/nsm_consul_single_cluster/nse-auto-scale/kustomization.yaml b/examples/nsm_consul_single_cluster/nse-auto-scale/kustomization.yaml new file mode 100644 index 000000000000..87c75206a1f7 --- /dev/null +++ b/examples/nsm_consul_single_cluster/nse-auto-scale/kustomization.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: +- https://github.com/networkservicemesh/deployments-k8s/apps/nse-supplier-k8s?ref=b4bddacfa45fafb7c15a769a1fc0f319e63d6a8d + +patchesStrategicMerge: +- patch-supplier.yaml + +configMapGenerator: + - name: supplier-pod-template-configmap + files: + - pod-template.yaml + - name: iptables-map + files: + - iptables-map + +generatorOptions: + disableNameSuffixHash: true diff --git a/examples/nsm_consul_single_cluster/nse-auto-scale/patch-supplier.yaml b/examples/nsm_consul_single_cluster/nse-auto-scale/patch-supplier.yaml new file mode 100644 index 000000000000..eab4bae2a9bf --- /dev/null +++ b/examples/nsm_consul_single_cluster/nse-auto-scale/patch-supplier.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nse-supplier-k8s +spec: + template: + metadata: + annotations: + 'consul.hashicorp.com/connect-inject': 'false' + spec: + containers: + - name: nse-supplier + env: + - name: NSM_SERVICE_NAME + value: autoscale-consul-proxy + - name: NSM_LABELS + value: any:true + - name: NSM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NSM_POD_DESCRIPTION_FILE + value: /run/supplier/pod-template.yaml + volumeMounts: + - name: pod-file + mountPath: /run/supplier + readOnly: true + volumes: + - name: pod-file + configMap: + name: supplier-pod-template-configmap diff --git a/examples/nsm_consul_single_cluster/nse-auto-scale/pod-template.yaml b/examples/nsm_consul_single_cluster/nse-auto-scale/pod-template.yaml new file mode 100644 index 000000000000..7a6fcc088e4b --- /dev/null +++ b/examples/nsm_consul_single_cluster/nse-auto-scale/pod-template.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: apps/v1 +kind: Pod +metadata: + name: proxy-{{ index .Labels "podName" }} + labels: + app: proxy-{{ index .Labels "podName" }} + "spiffe.io/spiffe-id": "true" + annotations: + 'consul.hashicorp.com/connect-inject': 'true' + 'consul.hashicorp.com/connect-service-upstreams': 'static-server:8080' +spec: + restartPolicy: Never + containers: + - name: nse + image: ghcr.io/networkservicemesh/cmd-nse-icmp-responder:v1.4.0-rc.1 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: SPIFFE_ENDPOINT_SOCKET + value: unix:///run/spire/sockets/agent.sock + - name: NSM_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAME + value: {{ index .Labels "podName" }} + - name: NSM_CONNECT_TO + value: unix:///var/lib/networkservicemesh/nsm.io.sock + - name: NSM_CIDR_PREFIX + value: 172.16.1.2/31 + - name: NSM_SERVICE_NAMES + value: autoscale-consul-proxy + - name: NSM_LABELS + value: app:{{ index .Labels "app" }} + - name: NSM_IDLE_TIMEOUT + value: 240s + - name: NSM_LOG_LEVEL + value: TRACE + - name: NSM_RULES_CONFIG + value: iptables-map + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/sockets + readOnly: true + - name: nsm-socket + mountPath: /var/lib/networkservicemesh + readOnly: true + - name: iptables-config-map + mountPath: /iptables-map + resources: + limits: + memory: 40Mi + cpu: 150m + - name: proxy-alpine-nsc + image: hashicorp/http-echo:latest + args: + - -text="hello world from nse" + - -listen=:9090 + ports: + - containerPort: 9090 + name: http + serviceAccountName: proxy-alpine-nsc + volumes: + - name: spire-agent-socket + hostPath: + path: /run/spire/sockets + type: Directory + - name: nsm-socket + hostPath: + path: /var/lib/networkservicemesh + type: DirectoryOrCreate + - name: iptables-config-map + configMap: + name: iptables-map diff --git a/examples/nsm_consul_single_cluster/server/static-server.yaml b/examples/nsm_consul_single_cluster/server/static-server.yaml new file mode 100644 index 000000000000..6d270ec43526 --- /dev/null +++ b/examples/nsm_consul_single_cluster/server/static-server.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: Service +metadata: + # This name will be the service name in Consul. + name: static-server +spec: + selector: + app: static-server + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: static-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-server +spec: + replicas: 1 + selector: + matchLabels: + app: static-server + template: + metadata: + name: static-server + labels: + app: static-server + annotations: + 'consul.hashicorp.com/connect-inject': 'true' + 'consul.hashicorp.com/connect-service-upstreams': 'proxy-alpine-nsc:9090' + spec: + containers: + - name: static-server + image: hashicorp/http-echo:latest + args: + - -text="hello world" + - -listen=:8080 + ports: + - containerPort: 8080 + name: http + - name: alpine + securityContext: + privileged: true + image: alpine:3.15.0 + imagePullPolicy: IfNotPresent + stdin: true + tty: true + # If ACLs are enabled, the serviceAccountName must match the Consul service name. + serviceAccountName: static-server \ No newline at end of file diff --git a/examples/nsm_consul_single_cluster/service.yaml b/examples/nsm_consul_single_cluster/service.yaml new file mode 100644 index 000000000000..ad8f187b4964 --- /dev/null +++ b/examples/nsm_consul_single_cluster/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxy-alpine-nsc +--- +apiVersion: v1 +kind: Service +metadata: + name: proxy-alpine-nsc +spec: + selector: + app: proxy-alpine-nsc + ports: + - protocol: TCP + port: 9090 + targetPort: 9090 \ No newline at end of file From 617f05f52a69b0b555f9146876d4a753065c4639 Mon Sep 17 00:00:00 2001 From: Marina Shustova Date: Fri, 24 Jun 2022 16:23:55 +0700 Subject: [PATCH 2/6] change single cluster example to an interdomain one Signed-off-by: Marina Shustova --- .../.gitignore | 0 examples/nsm_consul/README.md | 72 ++++++++++++++++ examples/nsm_consul/client/client.yaml | 16 ++++ .../helm-consul-values.yaml | 1 + .../kind-cluster-config.yaml | 0 .../networkservice.yaml | 0 .../nse-auto-scale/iptables-map | 0 .../nse-auto-scale/kustomization.yaml | 0 .../nse-auto-scale/patch-supplier.yaml | 0 .../nse-auto-scale/pod-template.yaml | 2 +- .../server/static-server.yaml | 10 +-- .../service.yaml | 3 +- examples/nsm_consul_single_cluster/README.md | 84 ------------------- .../client/client.yaml | 24 ------ 14 files changed, 94 insertions(+), 118 deletions(-) rename examples/{nsm_consul_single_cluster => nsm_consul}/.gitignore (100%) create mode 100644 examples/nsm_consul/README.md create mode 100644 examples/nsm_consul/client/client.yaml rename examples/{nsm_consul_single_cluster => nsm_consul}/helm-consul-values.yaml (97%) rename examples/{nsm_consul_single_cluster => nsm_consul}/kind-cluster-config.yaml (100%) rename examples/{nsm_consul_single_cluster => nsm_consul}/networkservice.yaml (100%) rename examples/{nsm_consul_single_cluster => nsm_consul}/nse-auto-scale/iptables-map (100%) rename examples/{nsm_consul_single_cluster => nsm_consul}/nse-auto-scale/kustomization.yaml (100%) rename examples/{nsm_consul_single_cluster => nsm_consul}/nse-auto-scale/patch-supplier.yaml (100%) rename examples/{nsm_consul_single_cluster => nsm_consul}/nse-auto-scale/pod-template.yaml (96%) rename examples/{nsm_consul_single_cluster => nsm_consul}/server/static-server.yaml (81%) rename examples/{nsm_consul_single_cluster => nsm_consul}/service.yaml (89%) delete mode 100644 examples/nsm_consul_single_cluster/README.md delete mode 100644 examples/nsm_consul_single_cluster/client/client.yaml diff --git a/examples/nsm_consul_single_cluster/.gitignore b/examples/nsm_consul/.gitignore similarity index 100% rename from examples/nsm_consul_single_cluster/.gitignore rename to examples/nsm_consul/.gitignore diff --git a/examples/nsm_consul/README.md b/examples/nsm_consul/README.md new file mode 100644 index 000000000000..6d1674f45814 --- /dev/null +++ b/examples/nsm_consul/README.md @@ -0,0 +1,72 @@ +# NSM + Consul interdomain example over kind clusters + +This example show how can be used nsm over + +![NSM interdomain Scheme](./NSM+Istio_Datapath.svg "NSM Basic floating interdomain Scheme") + + +## Requires + +- [Load balancer](./loadbalancer) +- [Interdomain DNS](./dns) +- [Interdomain spire](./spire) +- [Interdomain nsm](./nsm) + + +## Run + +Install Consul for the second cluster: +```bash +brew tap hashicorp/tap +brew install hashicorp/tap/consul-k8s +consul-k8s install -config-file=helm-consul-values.yaml -set global.image=hashicorp/consul:1.12.0 --kubeconfig=$KUBECONFIG2 +``` + +### Verify NSM+CONSUL + +Install networkservice for the second cluster:: +```bash +kubectl --kubeconfig=$KUBECONFIG2 apply -f networkservice.yaml +``` + +Start `alpine` networkservicemesh client for the first cluster: + +```bash +kubectl --kubeconfig=$KUBECONFIG1 apply -f client/client.yaml +``` + +Create kubernetes service for the networkservicemesh endpoint: +```bash +kubectl --kubeconfig=$KUBECONFIG2 apply -f service.yaml +``` + +Start `auto-scale` networkservicemesh endpoint: +```bash +kubectl --kubeconfig=$KUBECONFIG2 apply -k nse-auto-scale +``` + +Install `static-server` Consul workload on the second cluster: +```bash +kubectl --kubeconfig=$KUBECONFIG2 apply -f server/static-server.yaml +``` + +Verify connection from networkservicemesh client to consul server: +```bash +kubectl --kubeconfig=$KUBECONFIG1 exec -it alpine-nsc -- apk add curl +kubectl --kubeconfig=$KUBECONFIG1 exec -it alpine-nsc -- curl 172.16.1.2:8080 +``` + +You should see "hello world" answer. + +## Cleanup + + +```bash +kubectl --kubeconfig=$KUBECONFIG2 delete deployment static-server +kubectl --kubeconfig=$KUBECONFIG2 delete -k nse-auto-scale +kubectl --kubeconfig=$KUBECONFIG1 delete -f client/client.yaml +kubectl --kubeconfig=$KUBECONFIG2 delete -f networkservice.yaml +consul-k8s uninstall --kubeconfig=$KUBECONFIG2 -auto-approve=true -wipe-data=true +kubectl --kubeconfig=$KUBECONFIG2 delete pods --all +kind delete clusters cluster-1 cluster-2 +``` diff --git a/examples/nsm_consul/client/client.yaml b/examples/nsm_consul/client/client.yaml new file mode 100644 index 000000000000..bcfbfa248e52 --- /dev/null +++ b/examples/nsm_consul/client/client.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: alpine-nsc + labels: + app: alpine-nsc + annotations: + networkservicemesh.io: kernel://autoscale-consul-proxy@my.cluster2/nsm-1?app=alpine-nsc +spec: + containers: + - name: alpine-nsc + image: alpine:3.15.0 + imagePullPolicy: IfNotPresent + stdin: true + tty: true diff --git a/examples/nsm_consul_single_cluster/helm-consul-values.yaml b/examples/nsm_consul/helm-consul-values.yaml similarity index 97% rename from examples/nsm_consul_single_cluster/helm-consul-values.yaml rename to examples/nsm_consul/helm-consul-values.yaml index 7118ad7629f2..b0b8be1b6db3 100644 --- a/examples/nsm_consul_single_cluster/helm-consul-values.yaml +++ b/examples/nsm_consul/helm-consul-values.yaml @@ -1,3 +1,4 @@ +--- global: name: consul datacenter: dc1 diff --git a/examples/nsm_consul_single_cluster/kind-cluster-config.yaml b/examples/nsm_consul/kind-cluster-config.yaml similarity index 100% rename from examples/nsm_consul_single_cluster/kind-cluster-config.yaml rename to examples/nsm_consul/kind-cluster-config.yaml diff --git a/examples/nsm_consul_single_cluster/networkservice.yaml b/examples/nsm_consul/networkservice.yaml similarity index 100% rename from examples/nsm_consul_single_cluster/networkservice.yaml rename to examples/nsm_consul/networkservice.yaml diff --git a/examples/nsm_consul_single_cluster/nse-auto-scale/iptables-map b/examples/nsm_consul/nse-auto-scale/iptables-map similarity index 100% rename from examples/nsm_consul_single_cluster/nse-auto-scale/iptables-map rename to examples/nsm_consul/nse-auto-scale/iptables-map diff --git a/examples/nsm_consul_single_cluster/nse-auto-scale/kustomization.yaml b/examples/nsm_consul/nse-auto-scale/kustomization.yaml similarity index 100% rename from examples/nsm_consul_single_cluster/nse-auto-scale/kustomization.yaml rename to examples/nsm_consul/nse-auto-scale/kustomization.yaml diff --git a/examples/nsm_consul_single_cluster/nse-auto-scale/patch-supplier.yaml b/examples/nsm_consul/nse-auto-scale/patch-supplier.yaml similarity index 100% rename from examples/nsm_consul_single_cluster/nse-auto-scale/patch-supplier.yaml rename to examples/nsm_consul/nse-auto-scale/patch-supplier.yaml diff --git a/examples/nsm_consul_single_cluster/nse-auto-scale/pod-template.yaml b/examples/nsm_consul/nse-auto-scale/pod-template.yaml similarity index 96% rename from examples/nsm_consul_single_cluster/nse-auto-scale/pod-template.yaml rename to examples/nsm_consul/nse-auto-scale/pod-template.yaml index 7a6fcc088e4b..f0d6712ac25f 100644 --- a/examples/nsm_consul_single_cluster/nse-auto-scale/pod-template.yaml +++ b/examples/nsm_consul/nse-auto-scale/pod-template.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: Never containers: - name: nse - image: ghcr.io/networkservicemesh/cmd-nse-icmp-responder:v1.4.0-rc.1 + image: ghcr.io/networkservicemesh/cmd-nse-l7-proxy:v1.4.0 imagePullPolicy: IfNotPresent securityContext: privileged: true diff --git a/examples/nsm_consul_single_cluster/server/static-server.yaml b/examples/nsm_consul/server/static-server.yaml similarity index 81% rename from examples/nsm_consul_single_cluster/server/static-server.yaml rename to examples/nsm_consul/server/static-server.yaml index 6d270ec43526..efa7878ff8c7 100644 --- a/examples/nsm_consul_single_cluster/server/static-server.yaml +++ b/examples/nsm_consul/server/static-server.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: @@ -43,12 +44,5 @@ spec: ports: - containerPort: 8080 name: http - - name: alpine - securityContext: - privileged: true - image: alpine:3.15.0 - imagePullPolicy: IfNotPresent - stdin: true - tty: true # If ACLs are enabled, the serviceAccountName must match the Consul service name. - serviceAccountName: static-server \ No newline at end of file + serviceAccountName: static-server diff --git a/examples/nsm_consul_single_cluster/service.yaml b/examples/nsm_consul/service.yaml similarity index 89% rename from examples/nsm_consul_single_cluster/service.yaml rename to examples/nsm_consul/service.yaml index ad8f187b4964..51d89d1d3c7e 100644 --- a/examples/nsm_consul_single_cluster/service.yaml +++ b/examples/nsm_consul/service.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -13,4 +14,4 @@ spec: ports: - protocol: TCP port: 9090 - targetPort: 9090 \ No newline at end of file + targetPort: 9090 diff --git a/examples/nsm_consul_single_cluster/README.md b/examples/nsm_consul_single_cluster/README.md deleted file mode 100644 index 30b1e3511520..000000000000 --- a/examples/nsm_consul_single_cluster/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# NSM + Istio interdomain example over kind cluster - -## Setup Cluster - -### KIND -Setup - -```bash -go install sigs.k8s.io/kind@v0.13.0 - -kind create cluster --config kind-cluster-config.yaml -``` - - -## SPIRE & NSM - -Use instructions from [Basic](../basic/README.md) - - -## CONSUL - -Install Consul for a cluster: -```bash -brew tap hashicorp/tap -brew install hashicorp/tap/consul-k8s -consul-k8s install -config-file=helm-consul-values.yaml -set global.image=hashicorp/consul:1.12.0 -``` - -### Verify NSM+CONSUL - -Install networkservice: -```bash -kubectl apply -f networkservice.yaml -``` - -Start `alpine` networkservicemesh client: - -```bash -kubectl apply -f client/client.yaml -``` - -Create kubernetes service for networkservicemesh endpoint: -```bash -kubectl apply -f service.yaml -``` - -Start `auto-scale` networkservicemesh endpoint: -```bash -kubectl apply -k nse-auto-scale -``` - -Install `static-server` Consul workload: -```bash -kubectl apply -f server/static-server.yaml -``` - -Verify connection from networkservicemesh client to consul server: -```bash -kubectl exec -it alpine-nsc -- apk add curl -kubectl exec -it alpine-nsc -- curl 172.16.1.2:8080 -``` - -You should see "hello world" answer. - -## Cleanup - - -```bash -WH=$(kubectl get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') -kubectl delete mutatingwebhookconfiguration ${WH} - -kubectl delete crd spiffeids.spiffeid.spiffe.io -kubectl delete ns spire - -kubectl delete -k nse-auto-scale - -kubectl delete -f client.yaml - -consul-k8s uninstall -auto-approve=true -wipe-data=true - -kubectl delete pods --all - -kind delete cluster -``` diff --git a/examples/nsm_consul_single_cluster/client/client.yaml b/examples/nsm_consul_single_cluster/client/client.yaml deleted file mode 100644 index a3e56030bd56..000000000000 --- a/examples/nsm_consul_single_cluster/client/client.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - name: alpine-nsc - labels: - app: alpine-nsc - annotations: - networkservicemesh.io: kernel://autoscale-consul-proxy/nsm-1?app=alpine-nsc -spec: - containers: - - name: alpine-nsc - image: alpine:3.15.0 - imagePullPolicy: IfNotPresent - stdin: true - tty: true - - name: static-client - image: hashicorp/http-echo:latest - args: - - -text="hello world from nsc" - - -listen=:9090 - ports: - - containerPort: 9090 - name: http \ No newline at end of file From 2997464ea1e6dbaa68768cd185e15c7dd6c8c5bc Mon Sep 17 00:00:00 2001 From: Marina Shustova Date: Fri, 24 Jun 2022 16:31:01 +0700 Subject: [PATCH 3/6] Fix lint errors. Add required links Signed-off-by: Marina Shustova --- examples/nsm_consul/README.md | 4 +- examples/nsm_consul/dns/README.md | 157 ++++++++++++++++++ examples/nsm_consul/loadbalancer/README.md | 73 ++++++++ examples/nsm_consul/nsm/README.md | 26 +++ .../nsm/cluster1/kustomization.yaml | 21 +++ .../nsm_consul/nsm/cluster1/namespace.yaml | 5 + .../nsm/cluster1/patch-nsmgr-proxy.yaml | 10 ++ .../cluster1/patch-registry-proxy-dns.yaml | 10 ++ .../nsm/cluster1/patch-registry.yaml | 10 ++ .../nsm/cluster2/kustomization.yaml | 21 +++ .../nsm_consul/nsm/cluster2/namespace.yaml | 5 + .../nsm/cluster2/patch-nsmgr-proxy.yaml | 10 ++ .../cluster2/patch-registry-proxy-dns.yaml | 10 ++ .../nsm/cluster2/patch-registry.yaml | 10 ++ examples/nsm_consul/spire/README.md | 39 +++++ examples/nsm_consul/spire/cluster1/agent.conf | 32 ++++ .../cluster1/k8s-workload-registrar.conf | 11 ++ .../spire/cluster1/kustomization.yaml | 26 +++ .../nsm_consul/spire/cluster1/server.conf | 58 +++++++ examples/nsm_consul/spire/cluster2/agent.conf | 32 ++++ .../cluster2/k8s-workload-registrar.conf | 11 ++ .../spire/cluster2/kustomization.yaml | 25 +++ .../nsm_consul/spire/cluster2/server.conf | 58 +++++++ 23 files changed, 661 insertions(+), 3 deletions(-) create mode 100644 examples/nsm_consul/dns/README.md create mode 100644 examples/nsm_consul/loadbalancer/README.md create mode 100644 examples/nsm_consul/nsm/README.md create mode 100644 examples/nsm_consul/nsm/cluster1/kustomization.yaml create mode 100644 examples/nsm_consul/nsm/cluster1/namespace.yaml create mode 100644 examples/nsm_consul/nsm/cluster1/patch-nsmgr-proxy.yaml create mode 100644 examples/nsm_consul/nsm/cluster1/patch-registry-proxy-dns.yaml create mode 100644 examples/nsm_consul/nsm/cluster1/patch-registry.yaml create mode 100644 examples/nsm_consul/nsm/cluster2/kustomization.yaml create mode 100644 examples/nsm_consul/nsm/cluster2/namespace.yaml create mode 100644 examples/nsm_consul/nsm/cluster2/patch-nsmgr-proxy.yaml create mode 100644 examples/nsm_consul/nsm/cluster2/patch-registry-proxy-dns.yaml create mode 100644 examples/nsm_consul/nsm/cluster2/patch-registry.yaml create mode 100644 examples/nsm_consul/spire/README.md create mode 100644 examples/nsm_consul/spire/cluster1/agent.conf create mode 100644 examples/nsm_consul/spire/cluster1/k8s-workload-registrar.conf create mode 100644 examples/nsm_consul/spire/cluster1/kustomization.yaml create mode 100644 examples/nsm_consul/spire/cluster1/server.conf create mode 100644 examples/nsm_consul/spire/cluster2/agent.conf create mode 100644 examples/nsm_consul/spire/cluster2/k8s-workload-registrar.conf create mode 100644 examples/nsm_consul/spire/cluster2/kustomization.yaml create mode 100644 examples/nsm_consul/spire/cluster2/server.conf diff --git a/examples/nsm_consul/README.md b/examples/nsm_consul/README.md index 6d1674f45814..6d6bfd8830e4 100644 --- a/examples/nsm_consul/README.md +++ b/examples/nsm_consul/README.md @@ -1,8 +1,6 @@ # NSM + Consul interdomain example over kind clusters -This example show how can be used nsm over - -![NSM interdomain Scheme](./NSM+Istio_Datapath.svg "NSM Basic floating interdomain Scheme") +This example show how Consul can be used over nsm ## Requires diff --git a/examples/nsm_consul/dns/README.md b/examples/nsm_consul/dns/README.md new file mode 100644 index 000000000000..fa531822411e --- /dev/null +++ b/examples/nsm_consul/dns/README.md @@ -0,0 +1,157 @@ +## Setup DNS for two clusters + +This example shows how to simply configure three k8s clusters to know each other. +Can be skipped if clusters setupped with external DNS. + +## Run + +Expose dns service for first cluster +```bash +kubectl --kubeconfig=$KUBECONFIG1 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer +``` + +Wait for assigning IP address (note: you should see IP address in logs. If you dont see repeat this): +```bash +kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' +ip1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') +if [[ $ip1 == *"no value"* ]]; then + ip1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') + ip1=$(dig +short $ip1 | head -1) +fi +echo Selected externalIP: $ip1 for cluster1 +``` + +Expose dns service for the second cluster: +```bash +kubectl --kubeconfig=$KUBECONFIG2 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer +``` + +Wait for assigning IP address (note: you should see IP address in logs. If you dont see repeat this): +```bash +kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' +ip2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') +if [[ $ip2 == *"no value"* ]]; then + ip2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') + ip2=$(dig +short $ip2 | head -1) +fi +echo Selected externalIP: $ip2 for cluster2 +``` + +Add DNS forwarding from cluster1 to cluster2: +```bash +cat > configmap.yaml < custom-configmap.yaml < configmap.yaml < custom-configmap.yaml < metallb-config.yaml < metallb-config.yaml < Date: Fri, 24 Jun 2022 16:33:17 +0700 Subject: [PATCH 4/6] Fix lint errors. Delete spaces Signed-off-by: Marina Shustova --- examples/nsm_consul/client/client.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/nsm_consul/client/client.yaml b/examples/nsm_consul/client/client.yaml index bcfbfa248e52..06723b6a6093 100644 --- a/examples/nsm_consul/client/client.yaml +++ b/examples/nsm_consul/client/client.yaml @@ -4,7 +4,7 @@ kind: Pod metadata: name: alpine-nsc labels: - app: alpine-nsc + app: alpine-nsc annotations: networkservicemesh.io: kernel://autoscale-consul-proxy@my.cluster2/nsm-1?app=alpine-nsc spec: From 123ddb2aa98ce75c76b171b393b4d95098c09a48 Mon Sep 17 00:00:00 2001 From: Marina Shustova Date: Fri, 24 Jun 2022 18:17:40 +0700 Subject: [PATCH 5/6] Update Istio example Signed-off-by: Marina Shustova --- examples/nsm_consul/nse-auto-scale/iptables-map | 2 +- examples/nsm_consul/nse-auto-scale/pod-template.yaml | 2 +- examples/nsm_istio/nse-auto-scale/iptables-map | 9 +++++++++ examples/nsm_istio/nse-auto-scale/kustomization.yaml | 3 +++ examples/nsm_istio/nse-auto-scale/pod-template.yaml | 9 ++++++++- 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 examples/nsm_istio/nse-auto-scale/iptables-map diff --git a/examples/nsm_consul/nse-auto-scale/iptables-map b/examples/nsm_consul/nse-auto-scale/iptables-map index 7b3b7c6de92a..9c3e1a008384 100644 --- a/examples/nsm_consul/nse-auto-scale/iptables-map +++ b/examples/nsm_consul/nse-auto-scale/iptables-map @@ -1 +1 @@ --I PREROUTING 1 -p tcp -i {{ .NsmInterfaceName }} -j DNAT --to-destination 127.0.0.1 \ No newline at end of file +-I PREROUTING 1 -p tcp -i {{ .NsmInterfaceName }} -j DNAT --to-destination 127.0.0.1 \ No newline at end of file diff --git a/examples/nsm_consul/nse-auto-scale/pod-template.yaml b/examples/nsm_consul/nse-auto-scale/pod-template.yaml index f0d6712ac25f..a3f75b9ceda3 100644 --- a/examples/nsm_consul/nse-auto-scale/pod-template.yaml +++ b/examples/nsm_consul/nse-auto-scale/pod-template.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: Never containers: - name: nse - image: ghcr.io/networkservicemesh/cmd-nse-l7-proxy:v1.4.0 + image: ghcr.io/networkservicemesh/ci/cmd-nse-l7-proxy:32fbf26 imagePullPolicy: IfNotPresent securityContext: privileged: true diff --git a/examples/nsm_istio/nse-auto-scale/iptables-map b/examples/nsm_istio/nse-auto-scale/iptables-map new file mode 100644 index 000000000000..3951201dbfc7 --- /dev/null +++ b/examples/nsm_istio/nse-auto-scale/iptables-map @@ -0,0 +1,9 @@ +-N NSM_PREROUTE, +-A NSM_PREROUTE -j ISTIO_REDIRECT, +-I PREROUTING 1 -p tcp -i {{ .NsmInterfaceName }} -j NSM_PREROUTE, +-N NSM_OUTPUT, +-A NSM_OUTPUT -j DNAT --to-destination {{ index .NsmSrcIPs 0 }}, +-A OUTPUT -p tcp -s 127.0.0.6 -j NSM_OUTPUT, +-N NSM_POSTROUTING, +-A NSM_POSTROUTING -j SNAT --to-source {{ index .NsmDstIPs 0 }}, +-A POSTROUTING -p tcp -o {{ .NsmInterfaceName }} -j NSM_POSTROUTING \ No newline at end of file diff --git a/examples/nsm_istio/nse-auto-scale/kustomization.yaml b/examples/nsm_istio/nse-auto-scale/kustomization.yaml index 3799a388804f..87c75206a1f7 100644 --- a/examples/nsm_istio/nse-auto-scale/kustomization.yaml +++ b/examples/nsm_istio/nse-auto-scale/kustomization.yaml @@ -12,6 +12,9 @@ configMapGenerator: - name: supplier-pod-template-configmap files: - pod-template.yaml + - name: iptables-map + files: + - iptables-map generatorOptions: disableNameSuffixHash: true diff --git a/examples/nsm_istio/nse-auto-scale/pod-template.yaml b/examples/nsm_istio/nse-auto-scale/pod-template.yaml index 63ade4e9e55d..8cfac0c7e6aa 100644 --- a/examples/nsm_istio/nse-auto-scale/pod-template.yaml +++ b/examples/nsm_istio/nse-auto-scale/pod-template.yaml @@ -11,7 +11,7 @@ spec: restartPolicy: Never containers: - name: nse - image: ghcr.io/networkservicemesh/ci/cmd-nse-istio-proxy:c26db55 + image: ghcr.io/networkservicemesh/ci/cmd-nse-l7-proxy:32fbf26 imagePullPolicy: IfNotPresent env: - name: SPIFFE_ENDPOINT_SOCKET @@ -34,6 +34,8 @@ spec: value: 240s - name: NSM_LOG_LEVEL value: TRACE + - name: NSM_RULES_CONFIG + value: iptables-map volumeMounts: - name: spire-agent-socket mountPath: /run/spire/sockets @@ -41,6 +43,8 @@ spec: - name: nsm-socket mountPath: /var/lib/networkservicemesh readOnly: true + - name: iptables-config-map + mountPath: /iptables-map resources: limits: memory: 40Mi @@ -54,3 +58,6 @@ spec: hostPath: path: /var/lib/networkservicemesh type: DirectoryOrCreate + - name: iptables-config-map + configMap: + name: iptables-map From 7c24cb81635d7f8a34695a0e952eabe490672e27 Mon Sep 17 00:00:00 2001 From: Marina Shustova Date: Fri, 24 Jun 2022 21:11:33 +0700 Subject: [PATCH 6/6] Use istio dns, nsm, etc. configs to setup consul Signed-off-by: Marina Shustova --- examples/nsm_consul/README.md | 8 +- examples/nsm_consul/dns/README.md | 157 ------------------ examples/nsm_consul/loadbalancer/README.md | 73 -------- examples/nsm_consul/nsm/README.md | 26 --- .../nsm/cluster1/kustomization.yaml | 21 --- .../nsm_consul/nsm/cluster1/namespace.yaml | 5 - .../nsm/cluster1/patch-nsmgr-proxy.yaml | 10 -- .../cluster1/patch-registry-proxy-dns.yaml | 10 -- .../nsm/cluster1/patch-registry.yaml | 10 -- .../nsm/cluster2/kustomization.yaml | 21 --- .../nsm_consul/nsm/cluster2/namespace.yaml | 5 - .../nsm/cluster2/patch-nsmgr-proxy.yaml | 10 -- .../cluster2/patch-registry-proxy-dns.yaml | 10 -- .../nsm/cluster2/patch-registry.yaml | 10 -- examples/nsm_consul/spire/README.md | 39 ----- examples/nsm_consul/spire/cluster1/agent.conf | 32 ---- .../cluster1/k8s-workload-registrar.conf | 11 -- .../spire/cluster1/kustomization.yaml | 26 --- .../nsm_consul/spire/cluster1/server.conf | 58 ------- examples/nsm_consul/spire/cluster2/agent.conf | 32 ---- .../cluster2/k8s-workload-registrar.conf | 11 -- .../spire/cluster2/kustomization.yaml | 25 --- .../nsm_consul/spire/cluster2/server.conf | 58 ------- 23 files changed, 4 insertions(+), 664 deletions(-) delete mode 100644 examples/nsm_consul/dns/README.md delete mode 100644 examples/nsm_consul/loadbalancer/README.md delete mode 100644 examples/nsm_consul/nsm/README.md delete mode 100644 examples/nsm_consul/nsm/cluster1/kustomization.yaml delete mode 100644 examples/nsm_consul/nsm/cluster1/namespace.yaml delete mode 100644 examples/nsm_consul/nsm/cluster1/patch-nsmgr-proxy.yaml delete mode 100644 examples/nsm_consul/nsm/cluster1/patch-registry-proxy-dns.yaml delete mode 100644 examples/nsm_consul/nsm/cluster1/patch-registry.yaml delete mode 100644 examples/nsm_consul/nsm/cluster2/kustomization.yaml delete mode 100644 examples/nsm_consul/nsm/cluster2/namespace.yaml delete mode 100644 examples/nsm_consul/nsm/cluster2/patch-nsmgr-proxy.yaml delete mode 100644 examples/nsm_consul/nsm/cluster2/patch-registry-proxy-dns.yaml delete mode 100644 examples/nsm_consul/nsm/cluster2/patch-registry.yaml delete mode 100644 examples/nsm_consul/spire/README.md delete mode 100644 examples/nsm_consul/spire/cluster1/agent.conf delete mode 100644 examples/nsm_consul/spire/cluster1/k8s-workload-registrar.conf delete mode 100644 examples/nsm_consul/spire/cluster1/kustomization.yaml delete mode 100644 examples/nsm_consul/spire/cluster1/server.conf delete mode 100644 examples/nsm_consul/spire/cluster2/agent.conf delete mode 100644 examples/nsm_consul/spire/cluster2/k8s-workload-registrar.conf delete mode 100644 examples/nsm_consul/spire/cluster2/kustomization.yaml delete mode 100644 examples/nsm_consul/spire/cluster2/server.conf diff --git a/examples/nsm_consul/README.md b/examples/nsm_consul/README.md index 6d6bfd8830e4..a60259e1b1cf 100644 --- a/examples/nsm_consul/README.md +++ b/examples/nsm_consul/README.md @@ -5,10 +5,10 @@ This example show how Consul can be used over nsm ## Requires -- [Load balancer](./loadbalancer) -- [Interdomain DNS](./dns) -- [Interdomain spire](./spire) -- [Interdomain nsm](./nsm) +- [Load balancer](../nsm_istio/loadbalancer) +- [Interdomain DNS](../nsm_istio/dns) +- [Interdomain spire](../nsm_istio/spire) +- [Interdomain nsm](../nsm_istio/nsm) ## Run diff --git a/examples/nsm_consul/dns/README.md b/examples/nsm_consul/dns/README.md deleted file mode 100644 index fa531822411e..000000000000 --- a/examples/nsm_consul/dns/README.md +++ /dev/null @@ -1,157 +0,0 @@ -## Setup DNS for two clusters - -This example shows how to simply configure three k8s clusters to know each other. -Can be skipped if clusters setupped with external DNS. - -## Run - -Expose dns service for first cluster -```bash -kubectl --kubeconfig=$KUBECONFIG1 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer -``` - -Wait for assigning IP address (note: you should see IP address in logs. If you dont see repeat this): -```bash -kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' -ip1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') -if [[ $ip1 == *"no value"* ]]; then - ip1=$(kubectl --kubeconfig=$KUBECONFIG1 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') - ip1=$(dig +short $ip1 | head -1) -fi -echo Selected externalIP: $ip1 for cluster1 -``` - -Expose dns service for the second cluster: -```bash -kubectl --kubeconfig=$KUBECONFIG2 expose service kube-dns -n kube-system --port=53 --target-port=53 --protocol=TCP --name=exposed-kube-dns --type=LoadBalancer -``` - -Wait for assigning IP address (note: you should see IP address in logs. If you dont see repeat this): -```bash -kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}' -ip2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "ip"}}') -if [[ $ip2 == *"no value"* ]]; then - ip2=$(kubectl --kubeconfig=$KUBECONFIG2 get services exposed-kube-dns -n kube-system -o go-template='{{index (index (index (index .status "loadBalancer") "ingress") 0) "hostname"}}') - ip2=$(dig +short $ip2 | head -1) -fi -echo Selected externalIP: $ip2 for cluster2 -``` - -Add DNS forwarding from cluster1 to cluster2: -```bash -cat > configmap.yaml < custom-configmap.yaml < configmap.yaml < custom-configmap.yaml < metallb-config.yaml < metallb-config.yaml <