Merge pull request #109 from bcgov/feat/106-elk-certs

feat/106 elk certs

helm/cas-efk/README.md

@@ -1,3 +1,37 @@
 # CAS ElasticSearch, Fluent Bit, and Kibana
 
 This chart deploys ElasticSearch and Kibana to a namespace, ready to be used alongside the CAS-logging-sidecar chart which comprises the Fluent Bit part of the EFK stack.
+
+## TLS Internode Certificates
+
+In order to enable inter-node security, and therefore basic auth, a certificate authority needs to be generated to be used by ElasticSearch. This is currently done manually, but only needs to be done if the certificate authority isn't already in the OpenShift Namespace's secret store. These directions can also be used to rotate/update an existing certificate authority.
+
+### Directions
+
+1. Get your login command from the OpenShift cluster and login in your terminal. _Ensure you are logged in to the correct project_!
+1. In your `values.yaml` file, change `elasticsearch.security` to `false` and `elastic.replicas` to `1`.
+1. Use `helm install` to temporarily deploy Elastic to the cluster.
+1. In your console, use `oc get pods` in your namespace to find the deployed Elastic pod name. It should be named something like `es-cluster-0`. Use this wherever you see `<pod-name>` in the directions below.
+1. Use `oc exec -it <pod-name> -- bash` to get a shell in the Elastic pod.
+1. Run `bin/elasticsearch-certutil ca --pem --pass <PASSWORD> --silent --out ./certs/elastic-stack-ca.zip` to generate the certificate authority. *Ensure that you store the password in 1pass and as a secret in the namespace*.
+1. Exit the shell.
+1. Run `mkdir certs && oc cp es-cluster-0:/usr/share/elasticsearch/certs/elastic-stack-ca.zip ./certs/elastic-stack-ca.zip` to copy the certificate authority to a local directory.
+1. Unzip the CA with `unzip ./certs/elastic-stack-ca.zip -d ./certs`.
+1. Run `oc create secret generic <secret-name> --from-file=certs/ca/ca.crt --from-file=certs/ca/ca.key -n <namespace>` to create a secret with the certificate authority.
+
+## Updating the Kibana ElasticSearch password
+
+In order for Kibana to be able to connect with Elasticsearch, the password for the `elastic` user needs to be acquired. This can be done by running the following:
+
+1. Get your login command from the OpenShift cluster and login in your terminal. _Ensure you are logged in to the correct project_!
+1. In your console, use `oc get pods` in your namespace to find the deployed Elastic pod name. It should be named something like `es-cluster-0`. Use this wherever you see `<pod-name>` in the directions below.
+1. After the ElasticSearch pods have been deployed, run `oc exec -it es-cluster-0 -- bin/elasticsearch-reset-password -bs -u kibana_system` to reset the password for the `kibana_system` user. The output below will be the new password. Copy this into the `password` field in the secret `kibana` in the `es-password` key.
+1. Restart the Kibana pod with `oc rollout restart deployment/kibana -n <namespace>`.
+
+## Accessing Kibana
+
+Kibana does not currently have a secure route to access it. You will need to use `oc port-forward` to access it. In your console, run `oc port-forward service/kibana 5601:5601 -n <namespace>` to forward port 5601 to your local machine. You can then access Kibana at `http://localhost:5601`.
+
+### Updating the HTTPS certificate for Kibana
+
+Follow the directions [in the elasticseach documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-basic-setup-https.html#encrypt-kibana-browser) to acquire the cert and key for Kibana. Then get the certificate signed by a CA and then update the Kibana Route with new certificates in OpenShift.

helm/cas-efk/templates/elasticsearch-configmap.yaml

@@ -0,0 +1,23 @@
+{{ if .Values.elasticsearch.security }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elasticsearch-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    "app.kubernetes.io/part-of": efk-stack
+data:
+  elasticsearch.yml: |
+    network.host: 0.0.0.0
+    xpack.security.enabled: true
+    xpack.security.transport.ssl.enabled: true
+    xpack.security.transport.ssl.verification_mode: full
+    xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/${HOSTNAME}/${HOSTNAME}.key
+    xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/${HOSTNAME}/${HOSTNAME}.crt
+    xpack.security.transport.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+
+    # Kibana security config
+    xpack.security.authc.realms.pki.realm1.order: 1
+    xpack.security.authc.realms.pki.realm1.certificate_authorities: "/usr/share/elasticsearch/config/certs/ca.crt"
+    xpack.security.authc.realms.native.realm2.order: 2
+{{ end }}

helm/cas-efk/templates/elasticsearch-statefulset.yaml

@@ -33,7 +33,24 @@ spec:
           volumeMounts:
             - name: {{ .Values.elasticsearch.volume }}
               mountPath: /usr/share/elasticsearch/data
+            {{- if .Values.elasticsearch.security }}
+            - name: cert-store
+              mountPath: /usr/share/elasticsearch/config/certs
+            - name: cert-authority
+              mountPath: /usr/share/elasticsearch/config/certs/ca.crt
+              subPath: ca.crt
+            - name: cert-authority
+              mountPath: /usr/share/elasticsearch/config/certs/ca.key
+              subPath: ca.key
+            - name: config
+              mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
+              subPath: elasticsearch.yml
+            {{ end }}
           env:
+            {{- if not .Values.elasticsearch.security }}
+            - name: xpack.security.enabled
+              value: "false"
+            {{ end }}
             - name: cluster.name
               value: elasticsearch
             - name: node.name
@@ -51,12 +68,51 @@ spec:
                 {{- end }}"
             - name: ES_JAVA_OPTS
               value: "-Xms512m -Xmx512m"
-            - name: xpack.security.enabled
-              value: "false"
+            - name: CERT_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.elasticsearch.caSecret }}
+                  key: pass
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: KIBANA_ADMIN_USER
+              valueFrom:
+                secretKeyRef:
+                  name: cas-kibana-admin
+                  key: "username"
+            - name: KIBANA_ADMIN_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: cas-kibana-admin
+                  key: "password"
+          command:
+            - sh
+            - -c
+            - |
+              bin/elasticsearch-certutil cert -pem \
+                --ca-cert ./config/certs/ca.crt --ca-key ./config/certs/ca.key --ca-pass ${CERT_PASS} \
+                --name ${HOSTNAME} --dns ${HOSTNAME},${HOSTNAME}.elasticsearch --ip ${POD_IP} \
+                --out ./config/certs/${HOSTNAME}.zip --silent
+              unzip -q -o ./config/certs/${HOSTNAME}.zip -d ./config/certs
+              rm ./config/certs/${HOSTNAME}.zip
+              bin/elasticsearch-users useradd ${KIBANA_ADMIN_USER} -p ${KIBANA_ADMIN_PASS} -r kibana_admin
+              exec /usr/local/bin/docker-entrypoint.sh
       volumes:
         - name: {{ .Values.elasticsearch.volume }}
           persistentVolumeClaim:
             claimName: {{ .Values.elasticsearch.volume }}
+        {{- if .Values.elasticsearch.security }}
+        - name: cert-authority
+          secret:
+            secretName: {{ .Values.elasticsearch.caSecret }}
+        - name: cert-store
+          emptyDir: {}
+        - name: config
+          configMap:
+            name: elasticsearch-config
+        {{ end }}
   volumeClaimTemplates:
     - metadata:
         name: {{ .Values.elasticsearch.volume }}

helm/cas-efk/templates/kibana-configmap

@@ -0,0 +1,15 @@
+{{ if .Values.elasticsearch.security }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kibana-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    "app.kubernetes.io/part-of": efk-stack
+data:
+  kibana.yml: |
+    elasticsearch.username: "kibana_system"
+    elasticsearch.password: ${ES_PASSWORD}
+    elasticsearch.hosts: ["http://elasticsearch:9200"]
+    elasticsearch.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca.crt"]
+{{ end }}

helm/cas-efk/templates/kibana-deployment.yaml

@@ -25,5 +25,44 @@ spec:
           env:
             - name: ELASTICSEARCH_URL
               value: {{ .Values.elasticsearch.host }}:{{ .Values.elasticsearch.port.rest }}
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: ES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: kibana
+                  key: "es-password"
           ports:
             - containerPort: {{ .Values.kibana.port }}
+          volumeMounts:
+            {{- if .Values.elasticsearch.security }}
+            - name: kibana-cert
+              mountPath: /usr/share/kibana/config/certs/kibana.crt
+              subPath: kibana.crt
+            - name: kibana-cert
+              mountPath: /usr/share/kibana/config/certs/kibana.key
+              subPath: kibana.key
+            - name: cert-authority
+              mountPath: /usr/share/kibana/config/certs/ca.crt
+              subPath: ca.crt
+            - name: cert-authority
+              mountPath: /usr/share/kibana/config/certs/ca.key
+              subPath: ca.key
+            - name: config
+              mountPath: /usr/share/kibana/config/kibana.yml
+              subPath: kibana.yml
+            {{ end }}
+      volumes:
+        {{- if .Values.elasticsearch.security }}
+        - name: cert-authority
+          secret:
+            secretName: {{ .Values.elasticsearch.caSecret }}
+        - name: kibana-cert
+          secret:
+            secretName: elastic-certs
+        - name: config
+          configMap:
+            name: kibana-config
+        {{ end }}

helm/cas-efk/values-tools.yaml

@@ -0,0 +1,2 @@
+kibana:
+  route: cas-kibana.apps.silver.devops.gov.bc.ca

helm/cas-efk/values.yaml

@@ -15,6 +15,9 @@ elasticsearch:
   cpuRequest: 500m
   memoryRequest: 1Gi
 
+  security: true
+  caSecret: elastic-certificate-authority
+
 kibana:
   image: docker.elastic.co/kibana/kibana
   version: 8.4.3