Add Geo monitoring tests (#426)

* Add Geo monitoring tests * use prod env * comment from AI * fail proxy test * more time * Output region * add separate suite for synth monitoring (#429) * add separate suite for synth monitoring * updates * add more tests * clean up * readme * exact in quote synth monitoring * update branches --------- Co-authored-by: Roman <[email protected]>
osmosis-labs · Aug 3, 2024 · 6f3c44b · 6f3c44b
1 parent 6943417
commit 6f3c44b
Show file tree

Hide file tree

Showing 10 changed files with 429 additions and 249 deletions.
diff --git a/.github/workflows/geo-integration-test.yml b/.github/workflows/geo-integration-test.yml
@@ -0,0 +1,79 @@
+name: Synthetic Geo Monitoring SQS tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "*/20 * * * *"
+  push:
+    branches:
+      - "main"
+      - "v[0-9]**"
+
+jobs:
+  setup-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - id: set-matrix
+        run: |
+          echo "matrix={\"include\":[{ \"server\":\"138.68.112.16:8888\", \"env\": \"prod-fra1\"}, {\"server\":\"139.59.218.19:8888\", \"env\": \"prod-sgp1\"}]}" >> "$GITHUB_OUTPUT"
+
+  sqs-quote-tests:
+    timeout-minutes: 15
+    name: ${{ matrix.env }}-sqs-quote-tests
+    needs: setup-matrix
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.setup-matrix.outputs.matrix)}}
+    steps:
+      - name: Echo IP
+        run: curl -L "https://ipinfo.io" -s
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f tests/requirements.txt ]; then pip install -r tests/requirements.txt; fi
+      - name: Run Swap tests on ${{ matrix.env }}
+        env:
+          SQS_ENVIRONMENTS: "prod"
+          SQS_API_KEY: ${{ secrets.SQS_API_KEY }}
+          HTTP_PROXY: "http://${{secrets.TEST_PROXY_CRED}}@${{ matrix.server }}"
+          HTTPS_PROXY: "http://${{secrets.TEST_PROXY_CRED}}@${{ matrix.server }}"
+        run: |
+          pytest -s -n auto tests/test_synthetic_geo.py -v
+
+
+  delete-deployments:
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      [sqs-quote-tests]
+    steps:
+      - name: Delete Previous deployments
+        uses: actions/github-script@v7
+        with:
+          debug: true
+          script: |
+            const deployments = await github.rest.repos.listDeployments({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               sha: context.sha
+             });
+             await Promise.all(
+               deployments.data.map(async (deployment) => {
+                 await github.rest.repos.createDeploymentStatus({ 
+                   owner: context.repo.owner, 
+                   repo: context.repo.repo, 
+                   deployment_id: deployment.id, 
+                   state: 'inactive' 
+                 });
+                 return github.rest.repos.deleteDeployment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  deployment_id: deployment.id
+                });
+               })
+             );
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -39,7 +39,7 @@ jobs:
           if [ -f tests/requirements.txt ]; then pip install -r tests/requirements.txt; fi
       
       - name: Execute tests
-        run: pytest -s -n auto
+        run: pytest -s -n auto --ignore=tests/test_syntheic_geo.py
         env:
           SQS_ENVIRONMENTS: ${{ github.event.inputs.environment }}
           SQS_API_KEY: ${{ secrets.SQS_API_KEY }}
@@ -69,7 +69,7 @@ jobs:
           if [ -f tests/requirements.txt ]; then pip install -r tests/requirements.txt; fi
       
       - name: Execute tests
-        run: pytest -s -n auto
+        run: pytest -s -n auto --ignore=tests/test_syntheic_geo.py
         env:
           SQS_ENVIRONMENTS: ${{ matrix.environment }}
           SQS_API_KEY: ${{ secrets.SQS_API_KEY }}
diff --git a/Makefile b/Makefile
@@ -184,10 +184,10 @@ test-prices-mainnet:
 
 # Run E2E tests in verbose mode (-s) -n 4 concurrent workers
 e2e-run-stage:
-	SQS_ENVIRONMENTS=stage pytest -s -n 4
+	SQS_ENVIRONMENTS=stage pytest -s -n 4 --ignore=tests/test_syntheic_geo.py
 
 e2e-run-local:
-	SQS_ENVIRONMENTS=local pytest -s -n 4
+	SQS_ENVIRONMENTS=local pytest -s -n 4 --ignore=tests/test_syntheic_geo.py
 
 #### E2E Python Setup
 

diff --git a/tests/README.md b/tests/README.md
@@ -108,3 +108,12 @@ Our `pytest` parses the following environment variables in `conftest.py`
 
 - `SQS_API_KEY` -> API Key to bypass rate limit. If not provided, the tests will run without API key set.
 - `SQS_ENVIRONMENTS` -> Comma separated list of environment names per "Supported Environments" to run the tests against. If not provided, the tests will run against stage.
+
+## Geo-Distributed Synthetic Monitoring
+
+We run a subset of the tests in a geo-distributed manner using Synthetic Monitoring.
+
+For that, we define a custom suite in `tests/test_synthetic_geo.py` that runs a small subset of deterministic tests against all production endpoints
+in various regions.
+
+This is controlled by the `.github/workflows/geo-integration-test.yml` GitHub Action.
diff --git a/tests/test_candidate_routes.py b/tests/test_candidate_routes.py
@@ -24,7 +24,7 @@ def test_usdc_uosmo(self, environment_url):
         config = sqs_service.get_config()
         expected_num_routes = config['Router']['MaxRoutes']
 
-        self.run_candidate_routes_test(environment_url, constants.USDC, constants.UOSMO, expected_latency_upper_bound_ms, expected_min_routes=expected_num_routes, expected_max_routes=expected_num_routes)
+        run_candidate_routes_test(environment_url, constants.USDC, constants.UOSMO, expected_latency_upper_bound_ms, expected_min_routes=expected_num_routes, expected_max_routes=expected_num_routes)
 
     # Switch token in and out denoms compared to test_usdc_uosmo
     def test_uosmo_usdc(self, environment_url):
@@ -33,7 +33,7 @@ def test_uosmo_usdc(self, environment_url):
         config = sqs_service.get_config()
         expected_num_routes = config['Router']['MaxRoutes']
 
-        self.run_candidate_routes_test(environment_url, constants.UOSMO, constants.USDC, expected_latency_upper_bound_ms, expected_min_routes=expected_num_routes, expected_max_routes=expected_num_routes)
+        run_candidate_routes_test(environment_url, constants.UOSMO, constants.USDC, expected_latency_upper_bound_ms, expected_min_routes=expected_num_routes, expected_max_routes=expected_num_routes)
 
     # Test all valid listed tokens with appropriate liquidity with dynamic parameterization
     @pytest.mark.parametrize("denom", conftest.shared_test_state.valid_listed_tokens)
@@ -44,7 +44,7 @@ def test_all_valid_tokens(self, environment_url, denom):
         expected_num_routes = config['Router']['MaxRoutes']
 
 
-        self.run_candidate_routes_test(environment_url, denom, constants.USDC, expected_latency_upper_bound_ms, expected_min_routes=1, expected_max_routes=expected_num_routes)
+        run_candidate_routes_test(environment_url, denom, constants.USDC, expected_latency_upper_bound_ms, expected_min_routes=1, expected_max_routes=expected_num_routes)
 
     def test_transmuter_tokens(self, environment_url):
         sqs_service = SERVICE_MAP[environment_url]
@@ -58,7 +58,7 @@ def test_transmuter_tokens(self, environment_url):
         config = sqs_service.get_config()
         expected_num_routes = config['Router']['MaxRoutes']
 
-        routes = self.run_candidate_routes_test(environment_url, tansmuter_token_pair[0], tansmuter_token_pair[1], expected_latency_upper_bound_ms, expected_min_routes=1, expected_max_routes=expected_num_routes)
+        routes = run_candidate_routes_test(environment_url, tansmuter_token_pair[0], tansmuter_token_pair[1], expected_latency_upper_bound_ms, expected_min_routes=1, expected_max_routes=expected_num_routes)
 
         validate_pool_id_in_route(routes, [transmuter_pool_id])
 
@@ -77,7 +77,7 @@ def test_astroport_tokens(self, environment_url):
         config = sqs_service.get_config()
         expected_num_routes = config['Router']['MaxRoutes']
 
-        routes = self.run_candidate_routes_test(environment_url, astroport_token_pair[0], astroport_token_pair[1], expected_latency_upper_bound_ms, expected_min_routes=1, expected_max_routes=expected_num_routes)
+        routes = run_candidate_routes_test(environment_url, astroport_token_pair[0], astroport_token_pair[1], expected_latency_upper_bound_ms, expected_min_routes=1, expected_max_routes=expected_num_routes)
 
         validate_pool_id_in_route(routes, [astroport_pool_id])
 
@@ -94,41 +94,41 @@ def test_misc_token_pairs(self, environment_url, pair):
         config = sqs_service.get_config()
         expected_num_routes = config['Router']['MaxRoutes']
 
-        self.run_candidate_routes_test(environment_url, pair[0], pair[1], expected_latency_upper_bound_ms, expected_min_routes=1, expected_max_routes=expected_num_routes)
+        run_candidate_routes_test(environment_url, pair[0], pair[1], expected_latency_upper_bound_ms, expected_min_routes=1, expected_max_routes=expected_num_routes)
 
-    def run_candidate_routes_test(self, environment_url, token_in, token_out, expected_latency_upper_bound_ms, expected_min_routes, expected_max_routes):
-        """
-        Runs a test for the /router/routes endpoint with the given input parameters.
+def run_candidate_routes_test(environment_url, token_in, token_out, expected_latency_upper_bound_ms, expected_min_routes, expected_max_routes):
+    """
+    Runs a test for the /router/routes endpoint with the given input parameters.
 
-        Returns routes for additional validation if needed by client
+    Returns routes for additional validation if needed by client
 
-        Validates:
-        - The number of routes returned
-        - Following pools in each route, all tokens within these pools are present and valid
-        - The latency is under the given bound
-        """
-        
-        sqs_service = SERVICE_MAP[environment_url]
+    Validates:
+    - The number of routes returned
+    - Following pools in each route, all tokens within these pools are present and valid
+    - The latency is under the given bound
+    """
+
+    sqs_service = SERVICE_MAP[environment_url]
 
-        start_time = time.time()
-        response = sqs_service.get_candidate_routes(token_in, token_out)
-        elapsed_time_ms = (time.time() - start_time) * 1000
+    start_time = time.time()
+    response = sqs_service.get_candidate_routes(token_in, token_out)
+    elapsed_time_ms = (time.time() - start_time) * 1000
 
-        # If denoms are equal, there can be no routes between them
-        if token_in == token_out:
-            assert response.status_code == 500, f"Error: {response.text}"
-            return
+    # If denoms are equal, there can be no routes between them
+    if token_in == token_out:
+        assert response.status_code == 500, f"Error: {response.text}"
+        return
 
-        assert response.status_code == 200, f"Error: {response.text}"
-        assert expected_latency_upper_bound_ms > elapsed_time_ms, f"Error: latency {elapsed_time_ms} exceeded {expected_latency_upper_bound_ms} ms, token in {token_in} and token out {token_out}" 
+    assert response.status_code == 200, f"Error: {response.text}"
+    assert expected_latency_upper_bound_ms > elapsed_time_ms, f"Error: latency {elapsed_time_ms} exceeded {expected_latency_upper_bound_ms} ms, token in {token_in} and token out {token_out}" 
 
-        response_json = response.json()
-        routes = response_json['Routes']
+    response_json = response.json()
+    routes = response_json['Routes']
 
-        validate_candidate_routes(routes, token_in, token_out, expected_min_routes, expected_max_routes)
+    validate_candidate_routes(routes, token_in, token_out, expected_min_routes, expected_max_routes)
 
-        # Return routes in case additional validation is desired
-        return routes
+    # Return routes in case additional validation is desired
+    return routes
 
 def validate_candidate_routes(routes, token_in, token_out, expected_min_routes, expected_max_routes):
     """

diff --git a/tests/test_passthrough.py b/tests/test_passthrough.py
@@ -24,42 +24,45 @@
 class TestPassthrough:
 
     def test_poortfolio_assets(self, environment_url):
-        sqs_service = SERVICE_MAP[environment_url]
-
-        # Arbitrary addresses
-        addresses = [
-            "osmo1044qatzg4a0wm63jchrfdnn2u8nwdgxxt6e524",
-            "osmo1aaa9rpq2m6tu6t0dvknqq2ps7zudxv7th209q4",
-            "osmo18sd2ujv24ual9c9pshtxys6j8knh6xaek9z83t",
-            "osmo140p7pef5hlkewuuramngaf5j6s8dlynth5zm06",
-        ]
+        run_test_portfolio_assets(environment_url)
 
-        for address in addresses:
-            response = sqs_service.get_portfolio_assets(address)
+def run_test_portfolio_assets(environment_url):
+    sqs_service = SERVICE_MAP[environment_url]
+
+    # Arbitrary addresses
+    addresses = [
+        "osmo1044qatzg4a0wm63jchrfdnn2u8nwdgxxt6e524",
+        "osmo1aaa9rpq2m6tu6t0dvknqq2ps7zudxv7th209q4",
+        "osmo18sd2ujv24ual9c9pshtxys6j8knh6xaek9z83t",
+        "osmo140p7pef5hlkewuuramngaf5j6s8dlynth5zm06",
+    ]
+
+    for address in addresses:
+        response = sqs_service.get_portfolio_assets(address)
 
-            categories = response.get('categories')
-            assert categories is not None
+        categories = response.get('categories')
+        assert categories is not None
 
-            user_balances = categories.get(user_balances_assets_category_name)
-            validate_category(user_balances, True)
+        user_balances = categories.get(user_balances_assets_category_name)
+        validate_category(user_balances, True)
 
-            unstaking = categories.get(unstaking_assets_category_name)
-            validate_category(unstaking)
+        unstaking = categories.get(unstaking_assets_category_name)
+        validate_category(unstaking)
 
-            staked = categories.get(staked_assets_category_name)
-            validate_category(staked)
+        staked = categories.get(staked_assets_category_name)
+        validate_category(staked)
 
-            inLocks = categories.get(inLocks_assets_category_name)
-            validate_category(inLocks)
+        inLocks = categories.get(inLocks_assets_category_name)
+        validate_category(inLocks)
 
-            pooled = categories.get(pooled_assets_category_name)
-            validate_category(pooled)
+        pooled = categories.get(pooled_assets_category_name)
+        validate_category(pooled)
 
-            unclaimed_rewards = categories.get(unclaimed_rewards_assets_category_name)
-            validate_category(unclaimed_rewards)
+        unclaimed_rewards = categories.get(unclaimed_rewards_assets_category_name)
+        validate_category(unclaimed_rewards)
 
-            total_assets = categories.get(total_assets_category_name)
-            validate_category(total_assets, True)
+        total_assets = categories.get(total_assets_category_name)
+        validate_category(total_assets, True)
 
 def validate_category(category, should_have_breakdown=False):
     assert category is not None