Merge pull request #80 from sanger/Y24-033-remove-redpanda-api-keys

Y24-033: Remove API keys from RedPanda connections
sanger · Apr 22, 2024 · 3fe09a3 · 3fe09a3
2 parents 8ec5244 + 5e34c0c
commit 3fe09a3
Show file tree

Hide file tree

Showing 10 changed files with 719 additions and 509 deletions.
diff --git a/Pipfile b/Pipfile
@@ -22,7 +22,7 @@ more-itertools = "~=9.0"
 python-dotenv = "~=0.21"
 requests = "~=2.31"
 slackclient = "~=2.9"
-lab-share-lib = {editable = false,git = 'https://github.com/sanger/lab-share-lib.git',ref = 'v0.1.6'}
+lab-share-lib = {editable = false,git = 'https://github.com/sanger/lab-share-lib.git',ref = 'v0.2.0'}
 requests-mock = "*"
 types-requests = "*"
 python-snappy = "*"

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/schemas/README.md b/schemas/README.md
@@ -1,37 +1,48 @@
-How to publish schemas
-----------------------
+# Modifying Schemas Managed By RedPanda
+
+## Current Best Method Since April 2024
+
+Each instance of RedPanda has a an admin web console which lets you take all actions on schemas in a visual editor.
+The console can be accessed by appending `/console/` to the base URL of the RedPanda instance.
+It will require authentication which can be found in the team password vault.
+Only the Schema Registry section should be used in the console as we are not using the other parts of RedPanda.
+
+## Legacy Methods
+
+In order to perform create, update and delete actions on schemas via the API the restriction on HTTP methods will need to be lifted on the RedPanda instance.
+This is a config option in nginx that restricts methods to just HEAD and GET when PUSH is needed to write schemas.
+It is recommended the config is not changed, as this can cause security issues for the schemas.
+
+### Create and Update Schemas
 
 1. Ensure `jq` is installed:
 
-    ```bash
+    ```shell
     brew install jq
     ```
 
 1. Run the command:
 
-    ```bash
-    push.sh <REDPANDA_URL> <API_KEY>
+    ```shell
+    push.sh <REDPANDA_URL>
     ```
 
     where:
 
     ```text
     <REDPANDA_URL>: URL to connect to RedPanda where the schemas will be uploaded
-    <API_KEY>: secret key with write permission for redpanda
     ```
 
-How to remove last schemas created
-----------------------------------
+### Remove Schemas
 
 Run the command:
 
-```bash
-remove_all.sh <REDPANDA_URL> <API_KEY>
+```shell
+remove_all.sh <REDPANDA_URL>
 ```
 
 where:
 
 ```text
 <REDPANDA_URL>: URL to connect to RedPanda where the schemas will be uploaded
-<API_KEY>: secret key with write permission for redpanda
 ```
diff --git a/schemas/push.sh b/schemas/push.sh
@@ -1,17 +1,18 @@
 #!/bin/bash
 if [ $# -ne 2 ]; then
   echo "Syntax:"
-  echo "  push.sh <REDPANDA_URL> <API_KEY>"
+  echo "  push.sh <REDPANDA_URL>"
   echo "where:"
   echo "  <REDPANDA_URL>: URL to connect to RedPanda where the schemas will be uploaded"
-  echo "  <API_KEY>: secret key with write permission for redpanda"
+  echo ""
+  echo "Note that the RedPanda APIs on OpenStack do not allow POST methods, so this script will"
+  echo "not work unless that restriction is temporarily removed in the nginx config."
+  echo "A better option is to use the RedPanda Web Console instead."
   exit 1
 fi
 REDPANDA_URL=$1
-API_KEY=$2
-
+
 CONTENT_TYPE="Content-Type: application/vnd.schemaregistry.v1+json"
-API_KEY_HEADER="X-API-KEY: $API_KEY"
 
 pushd "$(dirname "$0")"
 
@@ -22,13 +23,13 @@ for schema in `find . -name "*.avsc"`; do
   echo "{\"schema\":" > $schema.tmp
   jq -c 'tojson' $schema >> $schema.tmp
   echo "}" >> $schema.tmp
-  
+
   echo "Uploading schema $schema_name"
-  curl -X POST -d @$schema.tmp -H "$CONTENT_TYPE" -H "$API_KEY_HEADER" "$REDPANDA_URL/subjects/$schema_name/versions"
+  curl -X POST -d @$schema.tmp -H "$CONTENT_TYPE" "$REDPANDA_URL/subjects/$schema_name/versions"
 
   # Remove temp file
   rm $schema.tmp
-  echo 
+  echo
 done
 
 popd 1>/dev/null
diff --git a/schemas/remove_all.sh b/schemas/remove_all.sh
@@ -4,22 +4,19 @@ if [ $# -ne 2 ]; then
   echo "  remove_all.sh <REDPANDA_URL> <API_KEY>"
   echo "where:"
   echo "  <REDPANDA_URL>: URL to connect to RedPanda where the schemas will be removed"
-  echo "  <API_KEY>: secret key with write permission for redpanda"
   exit 1
 fi
 REDPANDA_URL=$1
-API_KEY=$2
-
+
 CONTENT_TYPE="Content-Type: application/vnd.schemaregistry.v1+json"
-API_KEY_HEADER="X-API-KEY: $API_KEY"
 
 pushd "$(dirname "$0")"
 
 for schema in `find . -name "*.avsc"`; do
   schema_name=`dirname $schema | sed 's/\.//g' | sed 's/\///g'`
   echo "Deleting all schemas from $schema_name"
-  curl -X DELETE -H "$CONTENT_TYPE" -H "$API_KEY_HEADER" "$REDPANDA_URL/subjects/$schema_name/versions/latest"
-  echo 
+  curl -X DELETE -H "$CONTENT_TYPE" "$REDPANDA_URL/subjects/$schema_name/versions/latest"
+  echo
 done
 
 popd 1>/dev/null
diff --git a/tol_lab_share/config/defaults.py b/tol_lab_share/config/defaults.py
@@ -28,7 +28,6 @@
 # RedPanda details
 ###
 REDPANDA_BASE_URI = f"http://{os.environ.get('LOCALHOST', '127.0.0.1')}:8081"
-REDPANDA_API_KEY = ""
 
 
 RABBITMQ_PUBLISH_RETRY_DELAY = 5

diff --git a/tol_lab_share/messages/rabbit/published/create_labware_feedback_message.py b/tol_lab_share/messages/rabbit/published/create_labware_feedback_message.py
@@ -73,7 +73,7 @@ def publish(self, publisher: BasicPublisher, schema_registry: SchemaRegistry, ex
         encoded_message = None
         try:
             encoded_message = encoder.encode([message])
-        except BaseException as e:
+        except Exception as e:
             self.trigger_error(error_codes.ERROR_22_CANNOT_ENCODE_FEEDBACK_MESSAGE, text=str(e))
             return
 

diff --git a/tol_lab_share/types.py b/tol_lab_share/types.py
@@ -22,7 +22,6 @@ class Config(ModuleType):
 
     # RedPanda
     REDPANDA_BASE_URI: str
-    REDPANDA_API_KEY: str
 
     PROCESSORS: dict[str, str]
 

diff --git a/tools/demo/README.md b/tools/demo/README.md
@@ -3,7 +3,7 @@ How to run
 
 1. Create a docker image for the project that we will use to run the code.
 This saves a lot of problems when installing dependent libraries in local.
-To build you have to reference to the root folder of the project that contains the 
+To build you have to reference to the root folder of the project that contains the
 Dockerfile file, for example, if you run the command from inside this folder it should be:
 
 ```bash
@@ -17,7 +17,6 @@ SETTINGS_MODULE=tol_lab_share.config.defaults
 LOCALHOST=host.docker.internal
 REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 REDPANDA_URL=...
-REDPANDA_API_KEY=...
 RABBITMQ_HOST=...
 RABBITMQ_USERNAME=...
 RABBITMQ_PASSWORD=...
@@ -30,7 +29,6 @@ SETTINGS_MODULE=tol_lab_share.config.defaults
 LOCALHOST=host.docker.internal
 REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 REDPANDA_URL=http://host.docker.internal:8081
-REDPANDA_API_KEY=redpanda-test
 RABBITMQ_HOST=host.docker.internal
 RABBITMQ_PORT=5672
 RABBITMQ_USERNAME=admin

diff --git a/tools/demo/publisher.py b/tools/demo/publisher.py
@@ -10,7 +10,6 @@
 from testing_data import build_create_labware_96_msg, build_create_tube_msg, build_update_labware_msg
 
 REDPANDA_URL = os.getenv("REDPANDA_URL", "http://localhost")
-REDPANDA_API_KEY = os.getenv("REDPANDA_API_KEY", "test")
 RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "localhost")
 RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", "5671")
 RABBITMQ_USERNAME = os.getenv("RABBITMQ_USERNAME", "psd")
@@ -60,7 +59,7 @@ def send_message(msg, subject, registry, publisher):
 
     args = parser.parse_args()
 
-    registry = SchemaRegistry(REDPANDA_URL, REDPANDA_API_KEY, verify=False)
+    registry = SchemaRegistry(REDPANDA_URL, verify=False)
 
     rabbitmq_details = RabbitServerDetails(
         uses_ssl=True,