Test with "external" elasticsearch

* Strip all ES config except "enable" (for forwarding)
* Create ECK subscription no matter the observability_strategy
* Deploy ES from CI for events testing (Code copied/trimmed from STO)
* Default to use_redhat for CI

build/stf-run-ci/defaults/main.yml

@@ -27,7 +27,7 @@ __service_telemetry_snmptraps_trap_oid_prefix: "1.3.6.1.4.1.50495.15"
 __service_telemetry_snmptraps_trap_default_oid: "1.3.6.1.4.1.50495.15.1.2.1"
 __service_telemetry_snmptraps_trap_default_severity: ""
 __service_telemetry_logs_enabled: false
-__service_telemetry_observability_strategy: use_hybrid
+__service_telemetry_observability_strategy: use_redhat
 __service_telemetry_transports_certificates_endpoint_cert_duration: 70080h
 __service_telemetry_transports_certificates_ca_cert_duration: 70080h
 __internal_registry_path: image-registry.openshift-image-registry.svc:5000
@@ -37,6 +37,8 @@ __smart_gateway_bundle_image_path:
 default_operator_registry_image_base: registry.redhat.io/openshift4/ose-operator-registry
 default_operator_registry_image_tag: v4.12
 
+elastic_version: 7.16.1
+
 sgo_image_tag: latest
 sto_image_tag: latest
 sg_core_image_tag: latest

build/stf-run-ci/tasks/deploy_stf.yml

@@ -34,15 +34,6 @@
           events:
             elasticsearch:
               enabled: {{ __service_telemetry_events_enabled }}
-              storage:
-                strategy: {{ "ephemeral" if __service_telemetry_storage_ephemeral_enabled else "persistent" }}
-      {% if __service_telemetry_storage_persistent_storage_class is defined %}
-                persistent:
-                  storageClass: {{ __service_telemetry_storage_persistent_storage_class }}
-      {% endif %}
-              certificates:
-                endpointCertDuration: {{ __service_telemetry_events_certificates_endpoint_cert_duration }}
-                caCertDuration: {{ __service_telemetry_events_certificates_ca_cert_duration }}
           metrics:
             prometheus:
               enabled: {{ __service_telemetry_metrics_enabled }}

build/stf-run-ci/tasks/main.yml

@@ -60,6 +60,9 @@
   tags:
     - deploy
 
+- name: Deploy ES for events testing
+  include_tasks: setup_elasticsearch.yml
+
 - name: Set default base dir if not provided
   set_fact:
     base_dir: "{{ playbook_dir }}"
@@ -89,7 +92,7 @@
         - { name: prometheus-webhook-snmp, dockerfile_path: Dockerfile, image_reference_name: prometheus_webhook_snmp_image_path, working_build_dir: ./working/prometheus-webhook-snmp }
 
   - debug:
-     var: build_list
+      var: build_list
 
   - name: Create builds and artifacts
     include_tasks: create_builds.yml

build/stf-run-ci/tasks/pre-clean.yml

@@ -119,3 +119,14 @@
       kind: Project
       metadata:
         name: openshift-cert-manager-operator
+
+- name: Remove elasticsearch
+  k8s:
+    state: absent
+    wait: yes
+    definition:
+      apiVersion: elasticsearch.k8s.elastic.co/v1
+      kind: Elasticsearch
+      metadata:
+        name: elasticsearch
+        namespace: "{{ namespace }}"

build/stf-run-ci/tasks/setup_base.yml

@@ -117,8 +117,6 @@
         name: elasticsearch-eck-operator-certified
         source: certified-operators
         sourceNamespace: openshift-marketplace
-  when:
-    - __service_telemetry_observability_strategy in ['use_community', 'use_hybrid']
 
 - block:
   # Upstream Source + Sub from https://github.com/rhobs/observability-operator/tree/main/hack/olm

build/stf-run-ci/tasks/setup_elasticsearch.yml

@@ -0,0 +1,31 @@
+- name: Set default ElasticSearch manifest
+  set_fact:
+    elasticsearch_manifest: "{{ lookup('template', './manifest_elasticsearch.j2') | from_yaml }}"
+  when: elasticsearch_manifest is not defined
+
+- name: Create an instance of ElasticSearch
+  k8s:
+    state: present
+    definition:
+      '{{ elasticsearch_manifest }}'
+
+- name: Look up the newly generated ES Certs
+  k8s_info:
+    api_version: v1
+    kind: Secret
+    name: elasticsearch-es-http-certs-public
+  register: elasticsearch_certs
+  until: elasticsearch_certs[0].data[ca.crt] is defined
+  retries: 5
+  delay: 10
+
+- name: Copy the ES CA cert to our TLS secret
+  k8s:
+    definition:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: elasticsearch-es-cert
+        namespace: '{{ namespace }}'
+      stringData:
+        ca.crt: '{{ elasticsearch_certs[0].data[ca.crt] }}'

build/stf-run-ci/templates/manifest_elasticsearch.j2

@@ -0,0 +1,49 @@
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: elasticsearch
+  namespace: {{ namespace }}
+spec:
+  auth: {}
+  http:
+    service:
+      metadata: {}
+      spec: {}
+    tls:
+      certificate: {}
+  monitoring:
+    logs: {}
+    metrics: {}
+  nodeSets:
+  - count: 1
+    name: default
+    config:
+      node.roles:
+      - master
+      - data
+      - ingest
+      node.store.allow_mmap: true
+    podTemplate:
+      metadata:
+        labels:
+          tuned.openshift.io/elasticsearch: elasticsearch
+      spec:
+        containers:
+        - name: elasticsearch
+          resources:
+            limits:
+              cpu: "2"
+              memory: 4Gi
+            requests:
+              cpu: "1"
+              memory: 4Gi
+  transport:
+    service:
+      metadata: {}
+      spec: {}
+    tls:
+      certificate: {}
+      certificateAuthorities: {}
+  updateStrategy:
+    changeBudget: {}
+  version: {{ elastic_version }}

roles/servicetelemetry/tasks/component_elasticsearch.yml

@@ -1,3 +1,10 @@
+# DEPRECATED
+#
+# This code in the servicetelemetry role is deprecated as of STF 1.5.3, after
+# which only forwarding to an external elasticsearch is supported.
+#
+# The code lives on in the stf-run-ci role for CI testing of the forwarding
+# feature.
 - name: Lookup template
   debug:
     msg: "{{ lookup('template', './manifest_elasticsearch.j2') | from_yaml }}"