-
Notifications
You must be signed in to change notification settings - Fork 2.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ci] Scrape and verify metrics at the end of e2e tests (#6330)
## Which problem is this PR solving? - Part of #6278 ## Description of the changes - scrape script and usage in cit workflow - diff calculating script - cache save and restore from main workflow runs - A sample diff (txt) ``` --- +++ @@ -303,2 +303,2 @@ -rpc_server_requests_per_rpc{le="+Inf",rpc_grpc_status_code="0",rpc_method="Export",rpc_service="opentelemetry.proto.collector.trace.v1.TestService",rpc_system="grpc",service_name="jaeger",service_version=""} -rpc_server_requests_per_rpc{le="0",rpc_grpc_status_code="0",rpc_method="Export",rpc_service="opentelemetry.proto.collector.trace.v1.TraceService",rpc_system="grpc",service_name="jaeger",service_version=""} +rpc_server_requests_per_rpc{le="+Inf",rpc_grpc_status_code="0",rpc_method="Export",rpc_service="opentelemetry.proto.collector.trace.v1.TraceService",rpc_system="grpc",service_name="jaeger",service_version=""} +rpc_server_requests_per_rpc{le="0",rpc_grpc_status_code="1",rpc_method="Export",rpc_service="opentelemetry.proto.collector.trace.v1.TraceService",rpc_system="grpc",service_name="jaeger",service_version=""} @@ -321 +321 @@ -rpc_server_response_size{le="+Inf",rpc_method="Export",rpc_service="opentelemetry.proto.collector.trace.v1.TraceService",rpc_system="grpc",service_name="jaeger",service_version=""} +rpc_server_response_size{le="+Inf",rpc_service="opentelemetry.proto.collector.trace.v1.TraceService",rpc_system="grpc",service_name="jaeger",service_version="",test_change="Export"} @@ -338 +338 @@ -rpc_server_response_size{rpc_method="Export",rpc_service="opentelemetry.proto.collector.trace.v1.TraceService",rpc_system="grpc",service_name="jaeger",service_version=""} +rpc_server_response_size{rpc_method="Export",rpc_service="opentelemetry.proto.collector.trace.v1.TraceService",rpc_system="grpc",service_name="test-jaeger",service_version=""} ``` ## How was this change tested? - ## Checklist - [x] I have read https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md - [x] I have signed all commits - [ ] I have added unit tests for the new functionality - [x] I have run lint and test steps successfully - for `jaeger`: `make lint test` - for `jaeger-ui`: `yarn lint` and `yarn test` --------- Signed-off-by: chahatsagarmain <[email protected]> Signed-off-by: chahat sagar <[email protected]> Signed-off-by: Yuri Shkuro <[email protected]> Co-authored-by: Yuri Shkuro <[email protected]>
- Loading branch information
1 parent
2ee8e4c
commit 75b78cd
Showing
12 changed files
with
215 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| # Copyright (c) 2023 The Jaeger Authors. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| name: 'Verify Metric Snapshot and Upload Metrics' | ||
| description: 'Upload or cache the metrics data after verification' | ||
| inputs: | ||
| snapshot: | ||
| description: 'Path to the metric file' | ||
| required: true | ||
| artifact_key: | ||
| description: 'Artifact key used for uploading and fetching artifacts' | ||
| required: true | ||
| runs: | ||
| using: 'composite' | ||
| steps: | ||
| - name: Upload current metrics snapshot | ||
| uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 | ||
| with: | ||
| name: ${{ inputs.artifact_key }} | ||
| path: ./.metrics/${{ inputs.snapshot }}.txt | ||
| retention-days: 7 | ||
|
|
||
| # The github cache restore successfully restores when cache saved has same key and same path. | ||
| # Hence to restore release metric with name relese_{metric_name} , the name must be changed to the same. | ||
| - name: Change file name before caching | ||
| if: github.ref_name == 'main' | ||
| shell: bash | ||
| run: | | ||
| mv ./.metrics/${{ inputs.snapshot }}.txt ./.metrics/baseline_${{ inputs.snapshot }}.txt | ||
| - name: Cache metrics snapshot on main branch for longer retention | ||
| if: github.ref_name == 'main' | ||
| uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 | ||
| with: | ||
| path: ./.metrics/baseline_${{ inputs.snapshot }}.txt | ||
| key: ${{ inputs.artifact_key }}_${{ github.run_id }} | ||
|
|
||
| # Use restore keys to match prefix and fetch the latest cache | ||
| # Here , restore keys is an ordered list of prefixes that need to be matched | ||
| - name: Download the cached tagged metrics | ||
| id: download-release-snapshot | ||
| if: github.ref_name != 'main' | ||
| uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 | ||
| with: | ||
| path: ./.metrics/baseline_${{ inputs.snapshot }}.txt | ||
| key: ${{ inputs.artifact_key }} | ||
| restore-keys: | | ||
| ${{ inputs.artifact_key }} | ||
| - name: Calculate diff between the snapshots | ||
| id: compare-snapshots | ||
| if: ${{ (github.ref_name != 'main') && (steps.download-release-snapshot.outputs.cache-matched-key != '') }} | ||
| shell: bash | ||
| run: | | ||
| python3 -m pip install prometheus-client | ||
| python3 ./scripts/e2e/compare_metrics.py --file1 ./.metrics/${{ inputs.snapshot }}.txt --file2 ./.metrics/baseline_${{ inputs.snapshot }}.txt --output ./.metrics/diff_${{ inputs.snapshot }}.txt | ||
| if [ $? -eq 1 ]; then | ||
| echo "🛑 Differences found in metrics" | ||
| exit 1 | ||
| fi | ||
| - name: Upload the diff artifact | ||
| if: ${{ (github.ref_name != 'main') && (steps.compare-snapshots.outcome == 'failure') }} | ||
| uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 | ||
| with: | ||
| name: diff_${{ inputs.artifact_key }} | ||
| path: ./.metrics/diff_${{ inputs.snapshot }}.txt | ||
| retention-days: 7 | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -36,7 +36,3 @@ jobs: | |
| opensearch: | ||
| uses: ./.github/workflows/ci-e2e-opensearch.yml | ||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -50,3 +50,4 @@ sha256sum.combined.txt | |
| resource.syso | ||
| .gocache | ||
| test-results.json | ||
| .metrics/ | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| # Copyright (c) 2024 The Jaeger Authors. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| import argparse | ||
| from difflib import unified_diff | ||
| from bisect import insort | ||
| from prometheus_client.parser import text_string_to_metric_families | ||
|
|
||
| def read_metric_file(file_path): | ||
| with open(file_path, 'r') as f: | ||
| return f.readlines() | ||
|
|
||
| def parse_metrics(content): | ||
| metrics = [] | ||
| for family in text_string_to_metric_families(content): | ||
| for sample in family.samples: | ||
| labels = dict(sample.labels) | ||
| #simply pop undesirable metric labels | ||
| labels.pop('service_instance_id',None) | ||
| label_pairs = sorted(labels.items(), key=lambda x: x[0]) | ||
| label_str = ','.join(f'{k}="{v}"' for k,v in label_pairs) | ||
| metric = f"{family.name}{{{label_str}}}" | ||
| insort(metrics , metric) | ||
|
|
||
| return metrics | ||
|
|
||
|
|
||
| def generate_diff(file1_content, file2_content): | ||
| if isinstance(file1_content, list): | ||
| file1_content = ''.join(file1_content) | ||
| if isinstance(file2_content, list): | ||
| file2_content = ''.join(file2_content) | ||
|
|
||
| metrics1 = parse_metrics(file1_content) | ||
| metrics2 = parse_metrics(file2_content) | ||
|
|
||
| diff = unified_diff(metrics1, metrics2,lineterm='',n=0) | ||
|
|
||
| return '\n'.join(diff) | ||
|
|
||
| def write_diff_file(diff_lines, output_path): | ||
|
|
||
| with open(output_path, 'w') as f: | ||
| f.write(diff_lines) | ||
| f.write('\n') # Add final newline | ||
| print(f"Diff file successfully written to: {output_path}") | ||
|
|
||
| def main(): | ||
| parser = argparse.ArgumentParser(description='Generate diff between two Jaeger metric files') | ||
| parser.add_argument('--file1', help='Path to first metric file') | ||
| parser.add_argument('--file2', help='Path to second metric file') | ||
| parser.add_argument('--output', '-o', default='metrics_diff.txt', | ||
| help='Output diff file path (default: metrics_diff.txt)') | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| # Read input files | ||
| file1_lines = read_metric_file(args.file1) | ||
| file2_lines = read_metric_file(args.file2) | ||
|
|
||
| # Generate diff | ||
| diff_lines = generate_diff(file1_lines, file2_lines) | ||
|
|
||
| # Check if there are any differences | ||
| if diff_lines: | ||
| print("differences found between the metric files.") | ||
| print("=== Metrics Comparison Results ===") | ||
| print(diff_lines) | ||
| write_diff_file(diff_lines, args.output) | ||
|
|
||
| return 1 | ||
|
|
||
| print("no difference found") | ||
| return 0 | ||
|
|
||
| if __name__ == '__main__': | ||
| main() |