MAINT: yaml file

Signed-off-by: qchea <[email protected]>

examples/gitpod-synthetic-load-generator/demo/demo-data-prepper.crt

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICqjCCAZICCQDBYJxuuzsbczANBgkqhkiG9w0BAQsFADAXMRUwEwYDVQQDDAxk
+YXRhLXByZXBwZXIwHhcNMjAxMTMwMTc0NDM4WhcNMzAxMTI4MTc0NDM4WjAXMRUw
+EwYDVQQDDAxkYXRhLXByZXBwZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDBuSTI3oz9ZJRiVQCdTfbQlG9TWCXwuIjjJcVwvgVjhIuj5VLEOWUpC8nr
+be4/3ttGLuqOBViLlw2aelN7ZANgFy33ONF7kDh4xFMp4rZdFKxCOsmUpbSD7hRf
+8mVRFv30bx9//bVjLP8qi2QKuHc8WOevMsdNENFI6BgiI52EoVPyYJIIMaE2ripG
+qkQVn04Tk+rWz3zQ+NyOcExow98+pxMIjAanMUELTpMjQCWq2HtV2nnqcfqqNCae
+ig7NSDRvQtPL84Fjy7y4B1pS558hBmfOyZcGHo/gxLxQmvb1UKy4bcL7Kd8H3aNr
+LZ8zsr+he3cM9a7SZbiilALdcv0NAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAKvw
+fLiflX0Wd54ExsQvLvwZbrVU0z110h2N+E7SXt9K4NhHsANsX2mWxbqU/0YMYQpo
+fPO0XJSqmo28YD/eyKEazJg7tFISEqQ5gfWZM8BkfHcjbHMixtRDm6SpNzfeQT97
+C0zZo5kAD4d0Y3FwVjQ5FkvlmM6rCaJ0NyvZBUJseCsWpZsV2vF0QbMQcQ0lxjqY
+Vxs8Cnwp9bfF7ZP3i5KD5JxtVm80g6MIQdoYDQqtMjhZ8rCg5e1S6DbPnLRlmXkz
+4cFfD4V/7+ldSpEt38QBtCFWalfem+4dtbEX8FoTciNZfCkeUotpO2jPS1IzzghA
+yy6bLJd2MHl8AnGRdQg=
+-----END CERTIFICATE-----

examples/gitpod-synthetic-load-generator/demo/demo-data-prepper.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDBuSTI3oz9ZJRi
+VQCdTfbQlG9TWCXwuIjjJcVwvgVjhIuj5VLEOWUpC8nrbe4/3ttGLuqOBViLlw2a
+elN7ZANgFy33ONF7kDh4xFMp4rZdFKxCOsmUpbSD7hRf8mVRFv30bx9//bVjLP8q
+i2QKuHc8WOevMsdNENFI6BgiI52EoVPyYJIIMaE2ripGqkQVn04Tk+rWz3zQ+NyO
+cExow98+pxMIjAanMUELTpMjQCWq2HtV2nnqcfqqNCaeig7NSDRvQtPL84Fjy7y4
+B1pS558hBmfOyZcGHo/gxLxQmvb1UKy4bcL7Kd8H3aNrLZ8zsr+he3cM9a7SZbii
+lALdcv0NAgMBAAECggEAd0rwmPVa8usN6OOZRkTtWRIe377nVi38tYvb4opRPTAw
+4VNlPUpuvfDxfZWrYg91Den0LFKkbQkuqlLrt48l+PUtLBEEBsgPzmZvI9rlG7qQ
+PRSAb5X+7EMertrQgzfaWLxZNHLiUNkzpFpD18oyOqowuZVWX2Gczv2FZvhxlmrB
+fCajh2k4kYfFfedA8xpmUr1ZXRFsVCioTZswovN7CjNqPdNgjTho62mt62XdVTn1
+KpGwCquJLHfttG71qkRhhAaOqgPv+egbxAxbRx5dy2PHnKB3jl68LLXDAog/deEs
+Px39CoGiNT6v0TDPmxKY3Btns0Mf97GW1JxKJz/GIQKBgQD9ZimNv03tVE9Hfg9a
+TwgBzDmwQeAc+70g6hdtxTMp0CWWkgx/Pdb9tN9tfCTEA7/TeRYV2YwxMRgAChPK
+8NfPrB8ctq6IlsjANtiJ0OqCARFsuMc2oA0ecosA9Dt5IRz/9r+mjpaDvCWv2s7D
+D6yK7WlcY9mIOAdYPeVMvwTXhQKBgQDDtiz1SeSvn4Jboi2zNspkJ4K/7gXdj73b
+tmQVT8mVZKuxOCOFqC8e69/KcnOu3sOWowFVkbCmQswYmMiwvm2d2APfk1wC1+i4
+cGkBspzvl/N9dyBCU2QwaHW0Op/jva3HIWm6dCh0D6W5+r7BEWE3AGfENW0/4E6q
+v8qv7j8R6QKBgG7Id2pNNNiJf4+WAUhOPQXsL1xrsHt7Y5g8P7nx+QhSw9wsReJr
+Os74gKXxHHHF4dummI086xmmiQObZEPZP8C/U2rOQ0fcBDBkyg6Wk6/6n4+UySOM
+na+uDC+Cl23BmFgfnHJI7GzzmNZlJvgIpjqmyOVnkfcH584gYVtIN+mxAoGAddFM
+3Vlfly5CTqGiO1oHmyWB+8LxuwDWfdceBwpiQ1I4XdZ64/Jsc4Px1DD0XVhzjFKS
+8g1jt6NMtJ39axEocjZhlo1ifb+2tJnPTRtWiJIEHgrB28vR66Zm/VmxeWttmwwN
+VxaoZ74nuX8uqJGq4W6YCTFUlHiXQwl0nlahl0kCgYEAjkKqOzYDr/1/HalLdyct
+AnkTGqPRA+ml888VMlgmGss1EpFvxHTxWNpSfBoF9jRY2eO0N2yMwzvRqFjiA4WV
+bLKUTD6E3t12XEZPIpZA5UamWvZDaN3pXkONMZN+4+F1r+j9MB/626dPqBUDpq3V
+G+yZxerQdvFvNby4HJxkXSM=
+-----END PRIVATE KEY-----

examples/gitpod-synthetic-load-generator/demo/root-ca.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID/jCCAuagAwIBAgIBATANBgkqhkiG9w0BAQsFADCBjzETMBEGCgmSJomT8ixk
+ARkWA2NvbTEXMBUGCgmSJomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1w
+bGUgQ29tIEluYy4xITAfBgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEh
+MB8GA1UEAwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMB4XDTE4MDQyMjAzNDM0
+NloXDTI4MDQxOTAzNDM0NlowgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJ
+kiaJk/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEw
+HwYDVQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1w
+bGUgQ29tIEluYy4gUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBAK/u+GARP5innhpXK0c0q7s1Su1VTEaIgmZr8VWI6S8amf5cU3ktV7WT9SuV
+TsAm2i2A5P+Ctw7iZkfnHWlsC3HhPUcd6mvzGZ4moxnamM7r+a9otRp3owYoGStX
+ylVTQusAjbq9do8CMV4hcBTepCd+0w0v4h6UlXU8xjhj1xeUIz4DKbRgf36q0rv4
+VIX46X72rMJSETKOSxuwLkov1ZOVbfSlPaygXIxqsHVlj1iMkYRbQmaTib6XWHKf
+MibDaqDejOhukkCjzpptGZOPFQ8002UtTTNv1TiaKxkjMQJNwz6jfZ53ws3fh1I0
+RWT6WfM4oeFRFnyFRmc4uYTUgAkCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAf
+BgNVHSMEGDAWgBSSNQzgDx4rRfZNOfN7X6LmEpdAczAdBgNVHQ4EFgQUkjUM4A8e
+K0X2TTnze1+i5hKXQHMwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4IB
+AQBoQHvwsR34hGO2m8qVR9nQ5Klo5HYPyd6ySKNcT36OZ4AQfaCGsk+SecTi35QF
+RHL3g2qffED4tKR0RBNGQSgiLavmHGCh3YpDupKq2xhhEeS9oBmQzxanFwWFod4T
+nnsG2cCejyR9WXoRzHisw0KJWeuNlwjUdJY0xnn16srm1zL/M/f0PvCyh9HU1mF1
+ivnOSqbDD2Z7JSGyckgKad1Omsg/rr5XYtCeyJeXUPcmpeX6erWJJNTUh6yWC/hY
+G/dFC4xrJhfXwz6Z0ytUygJO32bJG4Np2iGAwvvgI9EfxzEv/KP+FGrJOvQJAq4/
+BU36ZAa80W/8TBnqZTkNnqZV
+-----END CERTIFICATE-----

examples/gitpod-synthetic-load-generator/docker-compose.yaml

@@ -17,8 +17,9 @@ services:
 
   # Collector
   otel-collector:
-    image: ${OTELCOL_IMG}
-    command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"]
+    restart: unless-stopped
+    image: otel/opentelemetry-collector:0.24.0
+    command: ["--config=/etc/otel-collector-config.yaml"]
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
     ports:
@@ -31,11 +32,12 @@ services:
     depends_on:
       - jaeger-all-in-one
       - zipkin-all-in-one
+      - data-prepper
 
   # Agent
   otel-agent:
-    image: ${OTELCOL_IMG}
-    command: ["--config=/etc/otel-agent-config.yaml", "${OTELCOL_ARGS}"]
+    image: otel/opentelemetry-collector:0.24.0
+    command: ["--config=/etc/otel-agent-config.yaml"]
     volumes:
       - ./otel-agent-config.yaml:/etc/otel-agent-config.yaml
     ports:
@@ -51,6 +53,47 @@ services:
     depends_on:
       - otel-collector
 
+  data-prepper:
+    restart: unless-stopped
+    image: opensearchproject/data-prepper:latest
+    container_name: data-prepper
+    working_dir: /usr/share/data-prepper/
+    command: sh data-prepper-wait-for-os-and-start.sh
+    volumes:
+      - ./resources/data-prepper-wait-for-os-and-start.sh:/usr/share/data-prepper/data-prepper-wait-for-os-and-start.sh
+      - ./resources/pipelines.yaml:/usr/share/data-prepper/pipelines.yaml
+      - ./resources/data-prepper-config.yaml:/usr/share/data-prepper/data-prepper-config.yaml
+      - ./shared-config/log4j2.properties:/usr/share/data-prepper/log4j.properties
+      - ./demo/root-ca.pem:/usr/share/data-prepper/root-ca.pem
+      - ./demo/test_keystore.p12:/usr/share/data-prepper/test_keystore.p12
+      - ./demo/demo-data-prepper.crt:/usr/share/data-prepper/demo-data-prepper.crt
+      - ./demo/demo-data-prepper.key:/usr/share/data-prepper/demo-data-prepper.key
+    ports:
+      - "4900:4900" # DataPrepperServer port
+      - "21890:21890" # OTelTraceSource port
+    depends_on:
+      - opensearch
+
+  opensearch:
+    container_name: node-0.example.com
+    image: opensearchproject/opensearch:latest
+    ports:
+      - "9200:9200"
+      - "9600:9600"
+    environment:
+      - discovery.type=single-node
+
+  opensearch-dashboard:
+    container_name: opensearch-dashboard
+    image: opensearchproject/opensearch-dashboards:latest
+    ports:
+      - 5601:5601
+    expose:
+      - "5601"
+    environment:
+      OPENSEARCH_URL: https://node-0.example.com:9200
+      OPENSEARCH_HOSTS: https://node-0.example.com:9200
+
   # Synthetic load generators
   jaeger-emitter:
     image: omnition/synthetic-load-generator:1.0.25

examples/gitpod-synthetic-load-generator/otel-collector-config.yaml

@@ -4,6 +4,11 @@ receivers:
       grpc:
 
 exporters:
+  otlp/2:
+    endpoint: data-prepper:21890
+    insecure: false
+    insecure_skip_verify: true
+
   prometheus:
     endpoint: "0.0.0.0:8889"
     namespace: promexample
@@ -35,7 +40,7 @@ service:
     traces:
       receivers: [otlp]
       processors: [batch]
-      exporters: [logging, zipkin, jaeger]
+      exporters: [logging, zipkin, jaeger, otlp/2]
     metrics:
       receivers: [otlp]
       processors: [batch]

examples/gitpod-synthetic-load-generator/resources/data-prepper-config.yaml

@@ -0,0 +1 @@
+ssl: false

examples/gitpod-synthetic-load-generator/resources/data-prepper-wait-for-os-and-start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+until [[ $(curl --write-out %{http_code} --output /dev/null --silent --head --fail https://node-0.example.com:9200 -u admin:admin --insecure) == 200 ]]; do
+  echo "Waiting for OpenSearch to be ready"
+  sleep 1
+done
+
+java -Dlog4j.configurationFile=log4j.properties -Xms128m -Xmx128m -jar data-prepper.jar pipelines.yaml data-prepper-config.yaml

examples/gitpod-synthetic-load-generator/resources/pipelines.yaml

@@ -0,0 +1,39 @@
+entry-pipeline:
+  delay: "100"
+  source:
+    otel_trace_source:
+      ssl: true
+      sslKeyFile: /usr/share/data-prepper/demo-data-prepper.key
+      sslKeyCertChainFile: /usr/share/data-prepper/demo-data-prepper.crt
+  sink:
+    - pipeline:
+        name: "raw-pipeline"
+    - pipeline:
+        name: "service-map-pipeline"
+raw-pipeline:
+  source:
+    pipeline:
+      name: "entry-pipeline"
+  prepper:
+    - otel_trace_raw_prepper:
+  sink:
+    - opensearch:
+        hosts: [ "https://node-0.example.com:9200" ]
+        cert: "/usr/share/data-prepper/root-ca.pem"
+        username: "admin"
+        password: "admin"
+        trace_analytics_raw: true
+service-map-pipeline:
+  delay: "100"
+  source:
+    pipeline:
+      name: "entry-pipeline"
+  prepper:
+    - service_map_stateful:
+  sink:
+    - opensearch:
+        hosts: ["https://node-0.example.com:9200"]
+        cert: "/usr/share/data-prepper/root-ca.pem"
+        username: "admin"
+        password: "admin"
+        trace_analytics_service_map: true

examples/gitpod-synthetic-load-generator/shared-config/log4j2.properties

@@ -0,0 +1,36 @@
+#
+#  SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+#
+
+status = error
+dest = err
+name = PropertiesConfig
+
+property.filename = log/data-prepper/data-prepper.log
+
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n
+
+rootLogger.level = warn
+rootLogger.appenderRef.stdout.ref = STDOUT
+
+logger.core.name = com.amazon.dataprepper
+logger.core.level = info
+
+logger.pipeline.name = com.amazon.dataprepper.pipeline
+logger.pipeline.level = info
+
+logger.parser.name = com.amazon.dataprepper.parser
+logger.parser.level = info
+
+logger.plugins.name = com.amazon.dataprepper.plugins
+logger.plugins.level = info