Enhancement - Leverage requests session object for network level retry management

.github/workflows/pull-request.yml

@@ -6,9 +6,6 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-#    runs-on: self-hosted
-#    runs-on: self-hosted-amd64-small-privileged
-#    runs-on: self-hosted-amd64-large-privileged-on-demand-storage
     defaults:
       run:
         shell: bash -el {0}
@@ -40,16 +37,10 @@ jobs:
     - name: Build Conda Package
       run: |
         mkdir build
-        conda build conda-recipe --output-folder build
+        ./build-package.sh
     - name: Run Integration Tests
       run: |
         anaconda-project run test:integration:slipstream
-#    - name: Run System Tests
-#      env:
-#        AE5_HOSTNAME: dev1.ae.anacondaconnect.com
-#        CI: true
-#      run: |
-#        anaconda-project run test:system
     - name: Upload to anaconda.org (Dev Build)
       env:
         ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}

.github/workflows/python-publish.yml

@@ -8,9 +8,6 @@ on:
 jobs:
   package:
     runs-on: ubuntu-latest
-#    runs-on: self-hosted
-#    runs-on: self-hosted-amd64-small-privileged
-#    runs-on: self-hosted-amd64-large-privileged-on-demand-storage
     defaults:
       run:
         shell: bash -el {0}
@@ -33,7 +30,7 @@ jobs:
     - name: Build Conda Package
       run: |
         mkdir build
-        conda build conda-recipe --output-folder build
+        ./build-package.sh
     - name: Upload to anaconda.org
       env:
         ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}

CONTRIBUTING.md

@@ -21,7 +21,6 @@ These commands are used during development for solution management.
 | test             | Default     | Run all test suites                       |
 | test:unit        | Default     | Unit Test Suite                           |
 | test:integration | Default     | Integration Test Suite                    |
-| test:system      | Default     | System Test Suite                         |
 
 ## Contributing
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2023 Anaconda, Inc.
+Copyright 2024 Anaconda, Inc.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are

README.md

@@ -18,7 +18,6 @@ There is already a fair amount of inline help, so type `ae5 --help` to get start
 For k8s service integration refer to:
 * [k8s Service Component Documentation](docs/source/K8S_SERVER.md)
 * [Project Commands Documentation](docs/source/COMMANDS.md)
-* [System and Load Testing](docs/source/SYSTEM_TESTS.md)
 * [Contributing](CONTRIBUTING.md)
 
 ## Installation
@@ -66,9 +65,10 @@ The package has the following particular dependencies:
       - `revision`: `list`, `info`, `download`
     - `sample`: `info`, `list`
     - `secret`: `list`, `add`, `delete`
+    - `role`: `add`, `remove`
     - `run`: `delete`, `info`, `list`, `log`, `stop`
     - `session`: `branches`, `changes`, `info`, `list`, `open`, `start`, `stop`
-    - `user`: `info`, `list`
+    - `user`: `info`, `list`, `create`, `delete`
 - Simple commands: `call`, `login`, `logout`
 - Login options: `--hostname`, `--username`, `--admin-username`, `--admin-hostname`, `--impersonate`
 - Output format options: `--format`, `--filter`, `--columns`, `--sort`, `--width`, `--wide`, `--no-header`

ae5_tools/api.py

@@ -15,9 +15,13 @@
 
 import requests
 from dateutil import parser
+from requests import Session
+from requests.adapters import HTTPAdapter
 from requests.packages import urllib3
+from urllib3 import Retry
 
 from .archiver import create_tar_archive
+from .common.config.environment import demand_env_var, get_env_var
 from .config import config
 from .docker import build_image, get_condarc, get_dockerfile
 from .filter import filter_list_of_dicts, filter_vars, split_filter
@@ -222,8 +226,6 @@ def __init__(self, response, method, url, **kwargs):
             msg.append(f'  json: {kwargs["json"]}')
         super(AEUnexpectedResponseError, self).__init__("\n".join(msg))
 
-    pass
-
 
 class AESessionBase(object):
     """Base class for AE5 API interactions."""
@@ -250,15 +252,60 @@ def __init__(self, hostname, username, password, prefix, persist):
         self.password = password
         self.persist = persist
         self.prefix = prefix.lstrip("/")
-        self.session = requests.Session()
-        self.session.verify = False
-        self.session.cookies = LWPCookieJar()
+        self.session: Session = AESessionBase._build_requests_session()
+
+        # Cloudflare headers need to be present on all requests (even before auth can be start).
+        self._set_cf_headers()
+
+        # Proceed with auth flow
         if self.persist:
             self._load()
         self.connected = self._connected()
         if self.connected:
             self._set_header()
 
+    def _set_cf_headers(self):
+        """
+        If cloudflare auth is enabled, then get and set the headers
+        https://developers.cloudflare.com/cloudflare-one/identity/service-tokens/#connect-your-service-to-access
+        CF-Access-Client-Id: <Client ID>
+        CF-Access-Client-Secret: <Client Secret>
+        """
+
+        if get_env_var(name="CF_ACCESS_CLIENT_ID") and get_env_var(name="CF_ACCESS_CLIENT_SECRET"):
+            self.session.headers["CF-Access-Client-Id"] = demand_env_var(name="CF_ACCESS_CLIENT_ID")
+            self.session.headers["CF-Access-Client-Secret"] = demand_env_var(name="CF_ACCESS_CLIENT_SECRET")
+
+    @staticmethod
+    def _build_requests_session() -> Session:
+        """
+        Responsible for creating the requests session object.
+        This implementation is global right now, but future work should allow more granular
+        control of retries on a per-call basis.
+        """
+
+        session: Session = Session()
+        session.cookies = LWPCookieJar()
+
+        # TODO: This should be parameterized
+        session.verify = False
+
+        # Status Code Defaults
+        # 403, 501, 502 are seen when ae5 is behind CloudFlare
+        # 502, 503, 504 can be encountered when ae5 is under heavy load
+        # TODO: this should be definable on a per command basis, and parameterized.
+        retries: Retry = Retry(
+            total=10,
+            backoff_factor=0.1,
+            status_forcelist=[403, 502, 503, 504],
+            allowed_methods={"POST", "PUT", "PATCH", "GET", "DELETE", "OPTIONS", "HEAD"},
+        )
+
+        adapter: HTTPAdapter = HTTPAdapter(max_retries=retries)
+        session.mount(prefix="https://", adapter=adapter)
+
+        return session
+
     @staticmethod
     def _auth_message(msg, nl=True):
         print(msg, file=sys.stderr, end="\n" if nl else "")
@@ -291,6 +338,9 @@ def _is_login(self, response):
         pass
 
     def authorize(self):
+        # Cloudflare headers need to be present on all requests (even before auth can be start).
+        self._set_cf_headers()
+
         key = f"{self.username}@{self.hostname}"
         need_password = self.password is None
         last_valid = True
@@ -594,6 +644,9 @@ def _set_header(self):
                 s.headers["x-xsrftoken"] = cookie.value
                 break
 
+        # Ensure that Cloudflare headers get added [back] to session when setting the other auth headers.
+        self._set_cf_headers()
+
     def _load(self):
         s = self.session
         if os.path.exists(self._filename):
@@ -1787,11 +1840,45 @@ def _set_header(self):
         self.session.headers["Authorization"] = f'Bearer {self._sdata["access_token"]}'
 
     def _connect(self, password):
-        resp = self.session.post(
-            self._login_base + "/token",
-            data={"username": self.username, "password": password, "grant_type": "password", "client_id": "admin-cli"},
-        )
-        self._sdata = {} if resp.status_code == 401 else resp.json()
+        try:
+            # Set the initial security data to an empty dictionary.
+            self._sdata = {}
+
+            # Get our auth
+            params: dict = {"username": self.username, "password": password, "grant_type": "password", "client_id": "admin-cli"}
+            resp: requests.Response = self.session.post(
+                self._login_base + "/token",
+                data=params,
+            )
+
+            if resp.status_code not in [401]:
+                try:
+                    self._sdata = resp.json()
+                except json.decoder.JSONDecodeError as error:
+                    # The response is not json parsable.
+                    # This is most likely some type of error (serialized, or html content, etc).
+                    print(f"Received an unexpected response.\nStatus Code: {resp.status_code}\n{resp.text}")
+
+        except requests.exceptions.RetryError:
+            message: str = f"Exceeded maximum retry limit on call to {self._login_base}/token"
+            try:
+                message += f", response code seen: {resp.status_code}, last response: {resp.text}"
+            except NameError:
+                # if `resp` is not defined, then we hit the retry max before it was declared
+                # during the `session.post` operation.
+                pass
+
+            print(message)
+
+        except Exception as error:
+            message: str = f"Unknown error calling {self._login_base}/token"
+            try:
+                message += f", response code seen: {resp.status_code}, last response: {resp.text}"
+            except NameError:
+                # if `resp` is not defined, just pass.
+                pass
+            print(message)
+            print(str(error))
 
     def _disconnect(self):
         if self._sdata:

ae5_tools/common/config/environment.py

@@ -1,4 +1,5 @@
 """ Helper functions for environment variables. """
+
 from __future__ import annotations
 
 import os

anaconda-project.yml

@@ -71,16 +71,6 @@ commands:
       conda install build/noarch/ae5-tools-*.tar.bz2
       py.test --cov=ae5_tools -v tests/integration  --cov-append --cov-report=xml -vv
 
-  test:load:
-    env_spec: default
-    unix: |
-      python -m tests.load.runner
-
-  test:system:
-    env_spec: default
-    unix: |
-      python -m tests.system.runner
-
   # Documentation Commands ####################################################
 
   build:apidocs:

build-package.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Perform the build
+conda build conda-recipe --no-anaconda-upload --no-test --output-folder build
+
+# Check the exit status of the cp command
+if [ $? -eq 0 ]; then
+  echo "Build successful"
+else
+  echo "Build failed"
+fi
+
+echo "Moving on ..."
+exit 0

docs/source/COMMANDS.md

@@ -15,9 +15,6 @@ These commands are used during project development.
 | test             | Default     | Run all test suites                       |
 | test:unit        | Default     | Unit Test Suite                           |
 | test:integration | Default     | Integration Test Suite                    |
-| test:system      | Default     | System Test Suite                         |
-| test:load        | Default     | Load Test Suite                           |
-
 
 ### Runtime
 

docs/source/README.md

@@ -57,9 +57,10 @@ The package has the following particular dependencies:
       - `revision`: `list`, `info`, `download`
     - `sample`: `info`, `list`
     - `secret`: `list`, `add`, `delete`
+    - `role`: `add`, `remove`
     - `run`: `delete`, `info`, `list`, `log`, `stop`
     - `session`: `branches`, `changes`, `info`, `list`, `open`, `start`, `stop`
-    - `user`: `info`, `list`
+    - `user`: `info`, `list`, `create`, `delete`
 - Simple commands: `call`, `login`, `logout`
 - Login options: `--hostname`, `--username`, `--admin-username`, `--admin-hostname`, `--impersonate`
 - Output format options: `--format`, `--filter`, `--columns`, `--sort`, `--width`, `--wide`, `--no-header`