Adds honeybadger support.

Dockerfile

@@ -1,4 +1,8 @@
-FROM python:3.11-slim
+FROM python:3.11
+
+# Note: We cannot use a '3.x-slim' image because we want 'gcc' installed for
+# certain packages. For instance, 'honeybadger' requires gcc via a
+# dependency on 'psutil'.
 
 WORKDIR /app
 COPY requirements.txt .

README.md

@@ -33,6 +33,11 @@ to `DEBUG`, `INFO`, `WARNING`, `ERROR`, or `CRITICAL`. The `DEBUG` setting is th
 most permissive and shows all logging text. The `CRITICAL` prevents most logging
 from happening. Most logging happens at `INFO`, which is the default setting.
 
+To enable Honeybadger reporting for events in the application, provide the
+`HONEYBADGER_API_KEY` within the configuration or as an environment variable. When
+enabled, the application will notify on Exceptions raised and any `error` or `critical`
+logs.
+
 ## Local Development
 
 - Install docker
@@ -254,6 +259,23 @@ build and run the container against the Python tests.
 
 For information about testing the service, see the [Testing documentation](TESTING.md).
 
+## Secrets
+
+Secrets for this project follow the naming convention `${EnvironmentType}/${DNSRecord}/honeybadger_api_key`. Our security model controls access to secrets based on the prefix, and will deny read access to any secret not prefixed with `development/`, so we tell our application's Cloudformation template whether we are in a development, test, or production environment type via [config files](cicd/3-app/aiproxy/config).
+
+* In production: `production/aiproxy.code.org/honeybadger_api_key`
+* In test: `test/aiproxy-test.code.org/honeybadger_api_key`
+* In an "adhoc" development environment: `development/aiproxy-dev-mybranch.code.org/honeybadger_api_key`
+* A prod-like environment, not based on `main`: `production/aiproxy-otherbranch.code.org/honeybadger_api_key`
+
+Read more about environments and environment types in [cicd/README.md](cicd/README.md).
+
+### Creating new secrets
+
+To create a new secret, define it in "[cicd/3-app/aiproxy/template.yml](cicd/3-app/aiproxy/template.yml) in the "Secrets" section. This will create an empty secret once deployed.
+
+Once created, you can set the value of this secret (via the AWS Console, as an Admin). Finally, deploy your code that uses the new secret, loading it via the `GetSecretValue` function from the AWS SDK.
+
 ## CICD
 
 See [CICD Readme](./cicd/README.md)

cicd/3-app/aiproxy/config/dev.config.json

@@ -1,7 +1,8 @@
 {
     "Parameters": {
         "BaseDomainName": "code.org",
-        "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU"
+        "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU",
+        "EnvironmentType": "development"
     },
     "Tags": {
         "EnvType": "development"

cicd/3-app/aiproxy/config/production.config.json

@@ -1,7 +1,8 @@
 {
     "Parameters": {
         "BaseDomainName": "code.org",
-        "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU"
+        "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU",
+        "EnvironmentType": "production"
     },
     "Tags": {
         "EnvType": "production"

cicd/3-app/aiproxy/config/test.config.json

@@ -1,7 +1,8 @@
 {
     "Parameters": {
         "BaseDomainName": "code.org",
-        "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU"
+        "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU",
+        "EnvironmentType": "test"
     },
     "Tags": {
         "EnvType": "test"

cicd/3-app/aiproxy/template.yml

@@ -5,6 +5,9 @@ Description: Provision an instance of the AI Proxy service.
 # Dependencies: This template has dependencies, look for !ImportValue in the Resources section.
 
 Parameters:
+  EnvironmentType:
+    Type: String
+    Description: Environment type (e.g. 'development', 'staging', 'test', 'prod').
   BaseDomainName:
     Type: String
     Description: Base domain name (e.g. 'code.org' in 'aiproxy.code.org').
@@ -18,9 +21,6 @@ Parameters:
     Type: String
     Description: URI of the Docker image in ECR.
 
-# Conditions:
-#   IsDevCondition: !Equals [!Ref BaseDomainName, "dev-code.org"]
-
 Resources:
 
   # ------------------
@@ -222,13 +222,30 @@ Resources:
           Essential: true
           PortMappings:
             - ContainerPort: 80
+          Environment:
+            # The most unique identifier for the environment is the DNS record.
+            - Name: ENVIRONMENT
+              Value: !Ref DNSRecord
+            - Name: ENVIRONMENT_TYPE
+              Value: !Ref EnvironmentType
           LogConfiguration:
             LogDriver: awslogs
             Options:
               awslogs-group: !Ref LogGroup
               awslogs-region: !Ref AWS::Region
               awslogs-stream-prefix: ecs
 
+  # ------------------
+  #      Secrets
+  # ------------------
+
+  HoneybadgerApiKeySecret:
+    Type: AWS::SecretsManager::Secret
+    Properties:
+      Name: !Sub "${EnvironmentType}/${DNSRecord}/honeybadger_api_key"
+      Description: Honeybadger API key for this service
+
+
   # ------------------
   #  Logging & Alerts
   # ------------------

config.txt.sample

@@ -1,2 +1,3 @@
 OPENAI_API_KEY=
 LOG_LEVEL=INFO
+#HONEYBADGER_API_KEY=hbp_xxx

requirements.txt

@@ -2,6 +2,7 @@ numpy==1.24.4
 openai==0.28.1
 flask==3.0.0
 waitress==2.1.2
+honeybadger==0.20.1
 pytest==7.4.3
 pytest-mock==3.12.0
 requests-mock==1.11.0

src/__init__.py

@@ -9,11 +9,15 @@
 from src.openai import openai_routes
 from src.assessment import assessment_routes
 
+# AWS
+import boto3
+
 # Flask
 from flask import Flask
 
-# OpenAI library
-import openai
+# Honeybadger support
+from honeybadger.contrib import FlaskHoneybadger
+from honeybadger.contrib.logger import HoneybadgerHandler
 
 def create_app(test_config=None):
 
@@ -42,6 +46,37 @@ def create_app(test_config=None):
     logging.basicConfig(format='%(asctime)s: %(name)s:%(message)s', level=log_level)
     logging.log(100, f"Setting up application. Logging level={log_level}")
     logging.basicConfig(format='%(asctime)s: %(levelname)s:%(name)s:%(message)s', level=log_level)
+
+
+    honeybadger_api_key = get_secret('honeybadger_api_key')
+    if honeybadger_api_key:
+        # Add Honeybadger support
+        logging.info('Setting up Honeybadger configuration')
+
+        # I need to patch Flask in order for Honeybadger to load
+        # The honeybadger library uses this deprecated attribute
+        # It was deprecated because it is now always 'True'.
+        # See: https://github.com/pallets/flask/commit/04994df59f2f642e52ba46ca656088bcdb931262
+        from flask import signals
+        setattr(signals, 'signals_available', True)
+
+        # Log exceptions to Honeybadger
+        app.config['HONEYBADGER_API_KEY'] = honeybadger_api_key
+        app.config['HONEYBADGER_PARAMS_FILTERS'] = 'password, secret, openai_api_key, api_key'
+        app.config['HONEYBADGER_ENVIRONMENT'] = os.getenv('FLASK_ENV')
+        FlaskHoneybadger(app, report_exceptions=True)
+
+        # Also log ERROR/CRITICAL logs to Honeybadger
+        class NoExceptionErrorFilter(logging.Filter):
+            def filter(self, record):
+                # But ignore Python logging exceptions on 'ERROR'
+                return not record.getMessage().startswith('Exception on ')
+
+        hb_handler = HoneybadgerHandler(api_key=honeybadger_api_key)
+        hb_handler.setLevel(logging.ERROR)
+        hb_handler.addFilter(NoExceptionErrorFilter())
+        logger = logging.getLogger()
+        logger.addHandler(hb_handler)
 
     # Index (a simple HTML response that will always succeed)
     @app.route('/')
@@ -53,3 +88,23 @@ def root():
     app.register_blueprint(assessment_routes)
 
     return app
+
+# Get a secret from AWS Secrets Manager or local ENV
+def get_secret(secret_name):
+    local_env_var_name = secret_name.upper()
+    # AWS Secrets are named like `production/aiproxy.code.org/secret_name}`
+    aws_secret_id = '/'.join([os.getenv('ENVIRONMENT'), os.getenv('ENVIRONMENT_TYPE'), secret_name])
+
+    secret = ''
+    if os.getenv(local_env_var_name):
+        secret = os.getenv(local_env_var_name)
+        logging.info(f'Retrieved secret "{secret_name}" from local ENV')
+    else:
+        try:
+            client = boto3.client('secretsmanager')
+            logging.info(f'Retrieved secret "{secret_name}" from AWS Secrets Manager')
+            secret = client.get_secret_value(SecretId=aws_secret_id)
+        except Exception as e:
+            logging.error(f'Error getting "{secret_name}" from AWS Secrets Manager: {e}')
+
+    return secret

src/test.py

@@ -12,7 +12,25 @@
 def test():
     return {}
 
-# A simple JSON response that always succeeds
+# A simple failing request
+@test_routes.route('/test/exception')
+def test_exception():
+    raise Exception("This is a test")
+    return {}
+
+# A simple post of an error message.
+@test_routes.route('/test/error')
+def test_error():
+    logging.error("This is an error log.")
+    return {}
+
+# A simple post of a critical message.
+@test_routes.route('/test/critical')
+def test_critical():
+    logging.critical("This is a critical log.")
+    return {}
+
+# A simple 429 failing request.
 @test_routes.route('/test/429')
 def test_429():
     return "Too many requests", 429