From 1b13ee7a1307b449c78da2c6b34ffd58ea16e76d Mon Sep 17 00:00:00 2001 From: yiyuanh Date: Fri, 28 Feb 2025 16:09:42 -0800 Subject: [PATCH 1/6] set up release udp exporter gh workflow --- .github/workflows/release-udp-exporter.yml | 117 ++++++++++++++++++ .../validation-app/app.py | 62 ++++++++++ 2 files changed, 179 insertions(+) create mode 100644 .github/workflows/release-udp-exporter.yml create mode 100644 exporters/aws-otel-otlp-udp-exporter/validation-app/app.py diff --git a/.github/workflows/release-udp-exporter.yml b/.github/workflows/release-udp-exporter.yml new file mode 100644 index 00000000..c74ed718 --- /dev/null +++ b/.github/workflows/release-udp-exporter.yml @@ -0,0 +1,117 @@ +name: Build, Test, and Publish ADOT OTLP UDP Exporter + +on: + push: + branches: + - "udp-*" + workflow_dispatch: + inputs: + version: + description: 'Version number for deployment e.g. 0.1.0' + required: true + type: string + +jobs: + build-test-publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install hatch pytest + + - name: Build package + working-directory: exporters/aws-otel-otlp-udp-exporter + run: hatch build + + - name: Setup X-Ray daemon + run: | + # Download X-Ray daemon + wget https://s3.us-east-2.amazonaws.com/aws-xray-assets.us-east-2/xray-daemon/aws-xray-daemon-linux-3.x.zip + unzip -o aws-xray-daemon-linux-3.x.zip + + # Create config file + echo '{ + "Version": 2, + "TotalBufferSizeMB": 10, + "Logging": { + "LogLevel": "debug" + }, + }' > xray-daemon-config.json + + # Make sure xray is executable + chmod +x ./xray + + # Create logs directory + mkdir -p daemon-logs + + # Start X-Ray daemon + ./xray -o -n us-west-2 -c xray-daemon-config.json > daemon-logs/xray-daemon.log 2>&1 & + XRAY_PID=$! + echo "X-Ray daemon started with PID $XRAY_PID" + + # Wait for daemon to be ready + echo "Waiting for X-Ray daemon to start..." + sleep 5 + + # Check if process is still running + if ps -p $XRAY_PID > /dev/null; then + echo "✅ X-Ray daemon process is running" + else + echo "❌ X-Ray daemon process is not running" + echo "Log contents:" + cat daemon-logs/xray-daemon.log + exit 1 + fi + + # Try to connect to the daemon + if nc -zv 127.0.0.1 2000 2>&1; then + echo "✅ Successfully connected to X-Ray daemon on port 2000" + else + echo "❌ Cannot connect to X-Ray daemon on port 2000" + echo "Log contents:" + cat daemon-logs/xray-daemon.log + exit 1 + fi + + # Extra verification with curl (might not work depending on daemon setup) + if curl -s http://localhost:2000/GetDaemonVersion; then + echo "✅ X-Ray daemon API responded" + else + echo "ℹ️ X-Ray daemon doesn't support API or not ready yet" + # Don't exit with error as this might not be reliable + fi + + echo "X-Ray daemon setup completed" + + - name: Setup validation app + run: | + pip install ./exporters/aws-otel-otlp-udp-exporter/dist/*.whl + + - name: Run validation test + working-directory: exporters/aws-otel-otlp-udp-exporter/validation-app + run: python app.py + + - name: Verify X-Ray daemon received traces + run: | + echo "X-Ray daemon logs:" + cat daemon-logs/xray-daemon.log + + # Check if the daemon received and processed some data + if grep -q "sending.*batch" daemon-logs/xray-daemon.log; then + echo "✅ X-Ray daemon processed trace data (AWS upload errors are expected)" + exit 0 + elif grep -q "processor:.*segment" daemon-logs/xray-daemon.log; then + echo "✅ X-Ray daemon processed segment data (AWS upload errors are expected)" + exit 0 + else + echo "❌ No evidence of traces being received by X-Ray daemon" + exit 1 + fi diff --git a/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py b/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py new file mode 100644 index 00000000..b1cd0efd --- /dev/null +++ b/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py @@ -0,0 +1,62 @@ +import socket +import threading +import time +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from amazon.opentelemetry.exporters.otlp.udp import OTLPUdpSpanExporter + +# Set up a UDP server to verify data is sent +def udp_server(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(('127.0.0.1', 2000)) + sock.settimeout(5) + print("UDP server listening on 127.0.0.1:2000") + try: + data, addr = sock.recvfrom(4096) + print(f"Received data from {addr}") + if data: + print("✅ Successfully received exported span data") + return True + except socket.timeout: + print("❌ No data received within timeout period") + return False + finally: + sock.close() + +# Start UDP server in a separate thread +server_thread = threading.Thread(target=udp_server) +server_thread.daemon = True +server_thread.start() + +# Set up tracer provider +tracer_provider = TracerProvider() +trace.set_tracer_provider(tracer_provider) + +# Set up UDP exporter with batch processor (more realistic usage) +exporter = OTLPUdpSpanExporter(endpoint="127.0.0.1:2000") +span_processor = BatchSpanProcessor(exporter) +tracer_provider.add_span_processor(span_processor) + +# Create a span for testing with various attributes +tracer = trace.get_tracer(__name__) +with tracer.start_as_current_span("test_parent_span") as parent: + parent.set_attribute("service.name", "validation-app") + parent.set_attribute("test.attribute", "test_value") + parent.add_event("test-event", {"event.data": "some data"}) + + # Add a child span + with tracer.start_as_current_span("test_child_span") as child: + child.set_attribute("child.attribute", "child_value") + print("Created spans with attributes and events") + +# Force flush to ensure spans are exported immediately +success = tracer_provider.force_flush() +print(f"Force flush {'succeeded' if success else 'failed'}") + +# Give some time for the UDP packet to be processed +time.sleep(2) + +# Shutdown +tracer_provider.shutdown() +print("Test completed") From ab18595fda4a05069aca6a2736d70655ed1fc830 Mon Sep 17 00:00:00 2001 From: yiyuanh Date: Mon, 3 Mar 2025 15:09:21 -0800 Subject: [PATCH 2/6] remove on push dispatch for udp validation workflow --- .github/workflows/release-udp-exporter.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release-udp-exporter.yml b/.github/workflows/release-udp-exporter.yml index c74ed718..4dbce0dc 100644 --- a/.github/workflows/release-udp-exporter.yml +++ b/.github/workflows/release-udp-exporter.yml @@ -1,9 +1,6 @@ name: Build, Test, and Publish ADOT OTLP UDP Exporter on: - push: - branches: - - "udp-*" workflow_dispatch: inputs: version: From 94cf9ced5351947f670dc8a3d0f10360328fe87d Mon Sep 17 00:00:00 2001 From: yiyuanh Date: Mon, 3 Mar 2025 16:16:30 -0800 Subject: [PATCH 3/6] apply black formatting --- .../aws-otel-otlp-udp-exporter/validation-app/app.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py b/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py index b1cd0efd..5d49e9f1 100644 --- a/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py +++ b/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py @@ -1,15 +1,17 @@ import socket import threading import time + +from amazon.opentelemetry.exporters.otlp.udp import OTLPUdpSpanExporter from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from amazon.opentelemetry.exporters.otlp.udp import OTLPUdpSpanExporter + # Set up a UDP server to verify data is sent def udp_server(): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(('127.0.0.1', 2000)) + sock.bind(("127.0.0.1", 2000)) sock.settimeout(5) print("UDP server listening on 127.0.0.1:2000") try: @@ -24,6 +26,7 @@ def udp_server(): finally: sock.close() + # Start UDP server in a separate thread server_thread = threading.Thread(target=udp_server) server_thread.daemon = True @@ -44,7 +47,7 @@ def udp_server(): parent.set_attribute("service.name", "validation-app") parent.set_attribute("test.attribute", "test_value") parent.add_event("test-event", {"event.data": "some data"}) - + # Add a child span with tracer.start_as_current_span("test_child_span") as child: child.set_attribute("child.attribute", "child_value") From bf11d7f1a1d427e5e6e28198c04f29ca1ab4ceb4 Mon Sep 17 00:00:00 2001 From: yiyuanh Date: Tue, 4 Mar 2025 11:59:39 -0800 Subject: [PATCH 4/6] clean up validation app workflow and move lambda env check --- .github/workflows/release-udp-exporter.yml | 97 ++++++------------- .../exporters/otlp/udp/exporter.py | 8 +- .../validation-app/app.py | 65 ------------- .../udp_exporter_validation_app.py | 48 +++++++++ 4 files changed, 84 insertions(+), 134 deletions(-) delete mode 100644 exporters/aws-otel-otlp-udp-exporter/validation-app/app.py create mode 100644 sample-applications/integ-test-app/udp_exporter_validation_app.py diff --git a/.github/workflows/release-udp-exporter.yml b/.github/workflows/release-udp-exporter.yml index 4dbce0dc..466f1b06 100644 --- a/.github/workflows/release-udp-exporter.yml +++ b/.github/workflows/release-udp-exporter.yml @@ -22,93 +22,60 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install hatch pytest + pip install hatch pytest flask - name: Build package working-directory: exporters/aws-otel-otlp-udp-exporter run: hatch build - - name: Setup X-Ray daemon + - name: Download and run X-Ray Daemon run: | - # Download X-Ray daemon - wget https://s3.us-east-2.amazonaws.com/aws-xray-assets.us-east-2/xray-daemon/aws-xray-daemon-linux-3.x.zip - unzip -o aws-xray-daemon-linux-3.x.zip - - # Create config file - echo '{ - "Version": 2, - "TotalBufferSizeMB": 10, - "Logging": { - "LogLevel": "debug" - }, - }' > xray-daemon-config.json + mkdir xray-daemon + cd xray-daemon + wget https://s3.us-west-2.amazonaws.com/aws-xray-assets.us-west-2/xray-daemon/aws-xray-daemon-linux-3.x.zip + unzip aws-xray-daemon-linux-3.x.zip + ./xray -o -n us-west-2 -f ./daemon-logs.log --log-level debug & - # Make sure xray is executable - chmod +x ./xray - - # Create logs directory - mkdir -p daemon-logs - - # Start X-Ray daemon - ./xray -o -n us-west-2 -c xray-daemon-config.json > daemon-logs/xray-daemon.log 2>&1 & - XRAY_PID=$! - echo "X-Ray daemon started with PID $XRAY_PID" - - # Wait for daemon to be ready - echo "Waiting for X-Ray daemon to start..." + - name: Install UDP Exporter + run: | + pip install ./exporters/aws-otel-otlp-udp-exporter/dist/*.whl + + - name: Ensure Unit Tests are passing + run: | + pytest exporters/aws-otel-otlp-udp-exporter/tests/test_exporter.py + + - name: Run Sample App in Background + working-directory: sample-applications/integ-test-app + run: | + # Start validation app + python udp_exporter_validation_app.py & + # Wait for validation app to initialize sleep 5 - - # Check if process is still running - if ps -p $XRAY_PID > /dev/null; then - echo "✅ X-Ray daemon process is running" - else - echo "❌ X-Ray daemon process is not running" - echo "Log contents:" - cat daemon-logs/xray-daemon.log - exit 1 - fi - - # Try to connect to the daemon - if nc -zv 127.0.0.1 2000 2>&1; then - echo "✅ Successfully connected to X-Ray daemon on port 2000" - else - echo "❌ Cannot connect to X-Ray daemon on port 2000" - echo "Log contents:" - cat daemon-logs/xray-daemon.log - exit 1 - fi - - # Extra verification with curl (might not work depending on daemon setup) - if curl -s http://localhost:2000/GetDaemonVersion; then - echo "✅ X-Ray daemon API responded" - else - echo "ℹ️ X-Ray daemon doesn't support API or not ready yet" - # Don't exit with error as this might not be reliable - fi - - echo "X-Ray daemon setup completed" - - name: Setup validation app + - name: Call Sample App Endpoint run: | - pip install ./exporters/aws-otel-otlp-udp-exporter/dist/*.whl + echo "traceId=$(curl localhost:8080/test)" >> $GITHUB_OUTPUT - - name: Run validation test - working-directory: exporters/aws-otel-otlp-udp-exporter/validation-app - run: python app.py + - name: Print Daemon Logs + run: | + sleep 20 + cat xray-daemon/daemon-logs.log - name: Verify X-Ray daemon received traces run: | echo "X-Ray daemon logs:" - cat daemon-logs/xray-daemon.log + cat xray-daemon/daemon-logs.log # Check if the daemon received and processed some data - if grep -q "sending.*batch" daemon-logs/xray-daemon.log; then + if grep -q "sending.*batch" xray-daemon/daemon-logs.log; then echo "✅ X-Ray daemon processed trace data (AWS upload errors are expected)" exit 0 - elif grep -q "processor:.*segment" daemon-logs/xray-daemon.log; then + elif grep -q "processor:.*segment" xray-daemon/daemon-logs.log; then echo "✅ X-Ray daemon processed segment data (AWS upload errors are expected)" exit 0 else echo "❌ No evidence of traces being received by X-Ray daemon" exit 1 fi + + # TODO: Steps to publish to PyPI diff --git a/exporters/aws-otel-otlp-udp-exporter/src/amazon/opentelemetry/exporters/otlp/udp/exporter.py b/exporters/aws-otel-otlp-udp-exporter/src/amazon/opentelemetry/exporters/otlp/udp/exporter.py index 095d4090..81b94980 100644 --- a/exporters/aws-otel-otlp-udp-exporter/src/amazon/opentelemetry/exporters/otlp/udp/exporter.py +++ b/exporters/aws-otel-otlp-udp-exporter/src/amazon/opentelemetry/exporters/otlp/udp/exporter.py @@ -23,10 +23,6 @@ class UdpExporter: def __init__(self, endpoint: Optional[str] = None): - if endpoint is None and "AWS_LAMBDA_FUNCTION_NAME" in os.environ: - # If in an AWS Lambda Environment, `AWS_XRAY_DAEMON_ADDRESS` will be defined - endpoint = os.environ.get("AWS_XRAY_DAEMON_ADDRESS", DEFAULT_ENDPOINT) - self._endpoint = endpoint or DEFAULT_ENDPOINT self._host, self._port = self._parse_endpoint(self._endpoint) self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -61,6 +57,10 @@ def _parse_endpoint(self, endpoint: str) -> Tuple[str, int]: class OTLPUdpSpanExporter(SpanExporter): def __init__(self, endpoint: Optional[str] = None, sampled: bool = True): + if endpoint is None and "AWS_LAMBDA_FUNCTION_NAME" in os.environ: + # If in an AWS Lambda Environment, `AWS_XRAY_DAEMON_ADDRESS` will be defined + endpoint = os.environ.get("AWS_XRAY_DAEMON_ADDRESS", DEFAULT_ENDPOINT) + self._udp_exporter = UdpExporter(endpoint=endpoint) self._sampled = sampled diff --git a/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py b/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py deleted file mode 100644 index 5d49e9f1..00000000 --- a/exporters/aws-otel-otlp-udp-exporter/validation-app/app.py +++ /dev/null @@ -1,65 +0,0 @@ -import socket -import threading -import time - -from amazon.opentelemetry.exporters.otlp.udp import OTLPUdpSpanExporter -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor - - -# Set up a UDP server to verify data is sent -def udp_server(): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(("127.0.0.1", 2000)) - sock.settimeout(5) - print("UDP server listening on 127.0.0.1:2000") - try: - data, addr = sock.recvfrom(4096) - print(f"Received data from {addr}") - if data: - print("✅ Successfully received exported span data") - return True - except socket.timeout: - print("❌ No data received within timeout period") - return False - finally: - sock.close() - - -# Start UDP server in a separate thread -server_thread = threading.Thread(target=udp_server) -server_thread.daemon = True -server_thread.start() - -# Set up tracer provider -tracer_provider = TracerProvider() -trace.set_tracer_provider(tracer_provider) - -# Set up UDP exporter with batch processor (more realistic usage) -exporter = OTLPUdpSpanExporter(endpoint="127.0.0.1:2000") -span_processor = BatchSpanProcessor(exporter) -tracer_provider.add_span_processor(span_processor) - -# Create a span for testing with various attributes -tracer = trace.get_tracer(__name__) -with tracer.start_as_current_span("test_parent_span") as parent: - parent.set_attribute("service.name", "validation-app") - parent.set_attribute("test.attribute", "test_value") - parent.add_event("test-event", {"event.data": "some data"}) - - # Add a child span - with tracer.start_as_current_span("test_child_span") as child: - child.set_attribute("child.attribute", "child_value") - print("Created spans with attributes and events") - -# Force flush to ensure spans are exported immediately -success = tracer_provider.force_flush() -print(f"Force flush {'succeeded' if success else 'failed'}") - -# Give some time for the UDP packet to be processed -time.sleep(2) - -# Shutdown -tracer_provider.shutdown() -print("Test completed") diff --git a/sample-applications/integ-test-app/udp_exporter_validation_app.py b/sample-applications/integ-test-app/udp_exporter_validation_app.py new file mode 100644 index 00000000..286f42d4 --- /dev/null +++ b/sample-applications/integ-test-app/udp_exporter_validation_app.py @@ -0,0 +1,48 @@ +from flask import Flask, jsonify + +from amazon.opentelemetry.exporters.otlp.udp import OTLPUdpSpanExporter +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +app = Flask(__name__) + +# Set up tracer provider +tracer_provider = TracerProvider() +trace.set_tracer_provider(tracer_provider) + +# Set up UDP exporter with batch processor +exporter = OTLPUdpSpanExporter(endpoint="127.0.0.1:2000") +span_processor = BatchSpanProcessor(exporter) +tracer_provider.add_span_processor(span_processor) + +# Get tracer +tracer = trace.get_tracer(__name__) + + +@app.route("/test", methods=["GET"]) +def create_trace(): + # Create a span for testing with various attributes + tracer = trace.get_tracer(__name__) + with tracer.start_as_current_span("test_parent_span") as parent: + parent.set_attribute("service.name", "validation-app") + parent.set_attribute("test.attribute", "test_value") + parent.add_event("test-event", {"event.data": "some data"}) + + # Get the trace ID + trace_id = format(parent.get_span_context().trace_id, "032x") + + # Add a child span + with tracer.start_as_current_span("test_child_span") as child: + child.set_attribute("child.attribute", "child_value") + print("Created spans with attributes and events") + + # Force flush to ensure spans are exported immediately + success = tracer_provider.force_flush() + print(f"Force flush {'succeeded' if success else 'failed'}") + + return jsonify({"trace_id": trace_id}) + + +if __name__ == "__main__": + app.run(port=8080) From bbb24984928990a554539ef9e33c159af8f5edc6 Mon Sep 17 00:00:00 2001 From: yiyuanh Date: Tue, 4 Mar 2025 15:10:23 -0800 Subject: [PATCH 5/6] some small updates --- .github/workflows/release-udp-exporter.yml | 8 ++------ .../integ-test-app/udp_exporter_validation_app.py | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-udp-exporter.yml b/.github/workflows/release-udp-exporter.yml index 466f1b06..81280c1a 100644 --- a/.github/workflows/release-udp-exporter.yml +++ b/.github/workflows/release-udp-exporter.yml @@ -42,7 +42,7 @@ jobs: - name: Ensure Unit Tests are passing run: | - pytest exporters/aws-otel-otlp-udp-exporter/tests/test_exporter.py + pytest exporters/aws-otel-otlp-udp-exporter/tests/ - name: Run Sample App in Background working-directory: sample-applications/integ-test-app @@ -56,13 +56,9 @@ jobs: run: | echo "traceId=$(curl localhost:8080/test)" >> $GITHUB_OUTPUT - - name: Print Daemon Logs - run: | - sleep 20 - cat xray-daemon/daemon-logs.log - - name: Verify X-Ray daemon received traces run: | + sleep 10 echo "X-Ray daemon logs:" cat xray-daemon/daemon-logs.log diff --git a/sample-applications/integ-test-app/udp_exporter_validation_app.py b/sample-applications/integ-test-app/udp_exporter_validation_app.py index 286f42d4..23b3b751 100644 --- a/sample-applications/integ-test-app/udp_exporter_validation_app.py +++ b/sample-applications/integ-test-app/udp_exporter_validation_app.py @@ -1,4 +1,4 @@ -from flask import Flask, jsonify +from flask import Flask from amazon.opentelemetry.exporters.otlp.udp import OTLPUdpSpanExporter from opentelemetry import trace @@ -41,7 +41,7 @@ def create_trace(): success = tracer_provider.force_flush() print(f"Force flush {'succeeded' if success else 'failed'}") - return jsonify({"trace_id": trace_id}) + return trace_id if __name__ == "__main__": From d0ea21d87261b15c829a6304dee04397fba4b02b Mon Sep 17 00:00:00 2001 From: yiyuanh Date: Tue, 4 Mar 2025 15:43:19 -0800 Subject: [PATCH 6/6] rename udp release workflow --- .github/workflows/release-udp-exporter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-udp-exporter.yml b/.github/workflows/release-udp-exporter.yml index 81280c1a..314582bf 100644 --- a/.github/workflows/release-udp-exporter.yml +++ b/.github/workflows/release-udp-exporter.yml @@ -1,4 +1,4 @@ -name: Build, Test, and Publish ADOT OTLP UDP Exporter +name: Release ADOT OTLP UDP Exporter on: workflow_dispatch: