From 27940fa7ffd3a4a8a7bec55cab67b181b36cc19d Mon Sep 17 00:00:00 2001 From: wilkie Date: Thu, 19 Oct 2023 21:09:09 -0400 Subject: [PATCH 1/3] Adds honeybadger support. The Dockerfile is required to be in a non-slim image because honeybadger introduces a dependency that is natively built. We will likely run into these things, anyway. Adds the `honeybadger` Python library to `requirements.txt` New configuration option via env to provide the honeybadger project key within the `HONEYBADGER_API_KEY` variable. Sets it up to log all exceptions that are unhandled within the app. Sets it up to log all CRITICAL logs that happen in the app. Sets it up to log all ERROR logs (that aren't exceptions) in the app. --- Dockerfile | 6 +++++- README.md | 5 +++++ config.txt.sample | 1 + requirements.txt | 1 + src/__init__.py | 34 ++++++++++++++++++++++++++++++++-- src/test.py | 18 ++++++++++++++++++ 6 files changed, 62 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0dd2efe..e706b8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,8 @@ -FROM python:3.11-slim +FROM python:3.11 + +# Note: We cannot use a '3.x-slim' image because we want 'gcc' installed for +# certain packages. For instance, 'honeybadger' requires gcc via a +# dependency on 'psutil'. WORKDIR /app COPY requirements.txt . diff --git a/README.md b/README.md index 0680602..eefd9c1 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,11 @@ to `DEBUG`, `INFO`, `WARNING`, `ERROR`, or `CRITICAL`. The `DEBUG` setting is th most permissive and shows all logging text. The `CRITICAL` prevents most logging from happening. Most logging happens at `INFO`, which is the default setting. +To enable Honeybadger reporting for events in the application, provide the +`HONEYBADGER_API_KEY` within the configuration or as an environment variable. When +enabled, the application will notify on Exceptions raised and any `error` or `critical` +logs. + ## Local Development All of our server code is written using [Flask](https://flask.palletsprojects.com/en/2.3.x/). diff --git a/config.txt.sample b/config.txt.sample index def47dd..51e9d2b 100644 --- a/config.txt.sample +++ b/config.txt.sample @@ -1,2 +1,3 @@ OPENAI_API_KEY= LOG_LEVEL=INFO +#HONEYBADGER_API_KEY=hbp_xxx diff --git a/requirements.txt b/requirements.txt index 2376d25..93354dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ numpy openai flask waitress +honeybadger diff --git a/src/__init__.py b/src/__init__.py index 0dafb2a..d6dc483 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -12,8 +12,9 @@ # Flask from flask import Flask -# OpenAI library -import openai +# Honeybadger support +from honeybadger.contrib import FlaskHoneybadger +from honeybadger.contrib.logger import HoneybadgerHandler def create_app(test_config=None): @@ -43,6 +44,35 @@ def create_app(test_config=None): logging.log(100, f"Setting up application. Logging level={log_level}") logging.basicConfig(format='%(asctime)s: %(levelname)s:%(name)s:%(message)s', level=log_level) + # Add Honeybadger support + if os.getenv('HONEYBADGER_API_KEY'): + logging.info('Setting up Honeybadger configuration') + + # I need to patch Flask in order for Honeybadger to load + # The honeybadger library uses this deprecated attribute + # It was deprecated because it is now always 'True'. + # See: https://github.com/pallets/flask/commit/04994df59f2f642e52ba46ca656088bcdb931262 + from flask import signals + setattr(signals, 'signals_available', True) + + # Log exceptions to Honeybadger + app.config['HONEYBADGER_API_KEY'] = os.getenv('HONEYBADGER_API_KEY') + app.config['HONEYBADGER_PARAMS_FILTERS'] = 'password, secret, openai_api_key, api_key' + app.config['HONEYBADGER_ENVIRONMENT'] = os.getenv('FLASK_ENV') + FlaskHoneybadger(app, report_exceptions=True) + + # Also log ERROR/CRITICAL logs to Honeybadger + class NoExceptionErrorFilter(logging.Filter): + def filter(self, record): + # But ignore Python logging exceptions on 'ERROR' + return not record.getMessage().startswith('Exception on ') + + hb_handler = HoneybadgerHandler(api_key=os.getenv('HONEYBADGER_API_KEY')) + hb_handler.setLevel(logging.ERROR) + hb_handler.addFilter(NoExceptionErrorFilter()) + logger = logging.getLogger() + logger.addHandler(hb_handler) + # Index (a simple HTML response that will always succeed) @app.route('/') def root(): diff --git a/src/test.py b/src/test.py index 52debd4..c9d95e8 100644 --- a/src/test.py +++ b/src/test.py @@ -17,6 +17,24 @@ def index(): def test(): return {} +# A simple failing request +@test_routes.route('/test/exception') +def test_exception(): + raise Exception("This is a test") + return {} + +# A simple post of an error message. +@test_routes.route('/test/error') +def test_error(): + logging.error("This is an error log.") + return {} + +# A simple post of a critical message. +@test_routes.route('/test/critical') +def test_critical(): + logging.critical("This is a critical log.") + return {} + # Test numpy integration @test_routes.route('/test/numpy') def test_numpy(): From c0e27ba3bc0bb51aa2bb3c5ae337361185013963 Mon Sep 17 00:00:00 2001 From: Darin Webb Date: Fri, 20 Oct 2023 13:21:56 -0500 Subject: [PATCH 2/3] Add Honeybadger secret and pass environment name to container --- README.md | 17 +++++++++++++++ cicd/3-app/aiproxy/config/dev.config.json | 3 ++- .../aiproxy/config/production.config.json | 3 ++- cicd/3-app/aiproxy/config/test.config.json | 3 ++- cicd/3-app/aiproxy/template.yml | 21 ++++++++++++++++--- 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index eefd9c1..47d178f 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,23 @@ For information about the API, see the [API documentation](API.md). For information about testing the service, see the [Testing documentation](TESTING.md). +## Secrets + +Secrets for this project follow the naming convention `${EnvironmentType}/${DNSRecord}/honeybadger_api_key`. Our security model controls access to secrets based on the prefix, and will deny read access to any secret not prefixed with `development/`, so we tell our application's Cloudformation template whether we are in a development, test, or production environment type via [config files](cicd/3-app/aiproxy/config). + +* In production: `production/aiproxy.code.org/honeybadger_api_key` +* In test: `test/aiproxy-test.code.org/honeybadger_api_key` +* In an "adhoc" development environment: `development/aiproxy-dev-mybranch.code.org/honeybadger_api_key` +* A prod-like environment, not based on `main`: `production/aiproxy-otherbranch.code.org/honeybadger_api_key` + +Read more about environments and environment types in [cicd/README.md](cicd/README.md). + +### Creating new secrets + +To create a new secret, define it in "[cicd/3-app/aiproxy/template.yml](cicd/3-app/aiproxy/template.yml) in the "Secrets" section. This will create an empty secret once deployed. + +Once created, you can set the value of this secret (via the AWS Console, as an Admin). Finally, deploy your code that uses the new secret, loading it via the `GetSecretValue` function from the AWS SDK. + ## CICD See [CICD Readme](./cicd/README.md) diff --git a/cicd/3-app/aiproxy/config/dev.config.json b/cicd/3-app/aiproxy/config/dev.config.json index 087ae22..da61a80 100644 --- a/cicd/3-app/aiproxy/config/dev.config.json +++ b/cicd/3-app/aiproxy/config/dev.config.json @@ -1,7 +1,8 @@ { "Parameters": { "BaseDomainName": "code.org", - "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU" + "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU", + "EnvironmentType": "development" }, "Tags": { "EnvType": "development" diff --git a/cicd/3-app/aiproxy/config/production.config.json b/cicd/3-app/aiproxy/config/production.config.json index 03318ca..5b44249 100644 --- a/cicd/3-app/aiproxy/config/production.config.json +++ b/cicd/3-app/aiproxy/config/production.config.json @@ -1,7 +1,8 @@ { "Parameters": { "BaseDomainName": "code.org", - "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU" + "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU", + "EnvironmentType": "production" }, "Tags": { "EnvType": "production" diff --git a/cicd/3-app/aiproxy/config/test.config.json b/cicd/3-app/aiproxy/config/test.config.json index e988064..f4ed742 100644 --- a/cicd/3-app/aiproxy/config/test.config.json +++ b/cicd/3-app/aiproxy/config/test.config.json @@ -1,7 +1,8 @@ { "Parameters": { "BaseDomainName": "code.org", - "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU" + "BaseDomainNameHostedZonedID": "Z2LCOI49SCXUGU", + "EnvironmentType": "test" }, "Tags": { "EnvType": "test" diff --git a/cicd/3-app/aiproxy/template.yml b/cicd/3-app/aiproxy/template.yml index ab7b5b5..f6ca769 100644 --- a/cicd/3-app/aiproxy/template.yml +++ b/cicd/3-app/aiproxy/template.yml @@ -5,6 +5,9 @@ Description: Provision an instance of the AI Proxy service. # Dependencies: This template has dependencies, look for !ImportValue in the Resources section. Parameters: + EnvironmentType: + Type: String + Description: Environment type (e.g. 'development', 'staging', 'test', 'prod'). BaseDomainName: Type: String Description: Base domain name (e.g. 'code.org' in 'aiproxy.code.org'). @@ -18,9 +21,6 @@ Parameters: Type: String Description: URI of the Docker image in ECR. -# Conditions: -# IsDevCondition: !Equals [!Ref BaseDomainName, "dev-code.org"] - Resources: # ------------------ @@ -185,6 +185,10 @@ Resources: Essential: true PortMappings: - ContainerPort: 80 + Environment: + # The most unique identifier for the environment is the DNS record. + - Name: ENVIRONMENT + Value: !Ref DNSRecord LogConfiguration: LogDriver: awslogs Options: @@ -192,6 +196,17 @@ Resources: awslogs-region: !Ref AWS::Region awslogs-stream-prefix: ecs + # ------------------ + # Secrets + # ------------------ + + HoneybadgerApiKeySecret: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub "${EnvironmentType}/${DNSRecord}/honeybadger_api_key" + Description: Honeybadger API key for this service + + # ------------------ # Logging & Alerts # ------------------ From 743c17f09a2b61c1c490c8c7fe7be6a8790dd85b Mon Sep 17 00:00:00 2001 From: Darin Webb Date: Fri, 20 Oct 2023 16:27:16 -0500 Subject: [PATCH 3/3] fetch honeybadger token from Secrets Manager or ENV --- cicd/3-app/aiproxy/template.yml | 2 ++ src/__init__.py | 35 ++++++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/cicd/3-app/aiproxy/template.yml b/cicd/3-app/aiproxy/template.yml index f6ca769..a500ea6 100644 --- a/cicd/3-app/aiproxy/template.yml +++ b/cicd/3-app/aiproxy/template.yml @@ -189,6 +189,8 @@ Resources: # The most unique identifier for the environment is the DNS record. - Name: ENVIRONMENT Value: !Ref DNSRecord + - Name: ENVIRONMENT_TYPE + Value: !Ref EnvironmentType LogConfiguration: LogDriver: awslogs Options: diff --git a/src/__init__.py b/src/__init__.py index d6dc483..a0f7770 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -9,6 +9,9 @@ from src.openai import openai_routes from src.assessment import assessment_routes +# AWS +import boto3 + # Flask from flask import Flask @@ -43,9 +46,11 @@ def create_app(test_config=None): logging.basicConfig(format='%(asctime)s: %(name)s:%(message)s', level=log_level) logging.log(100, f"Setting up application. Logging level={log_level}") logging.basicConfig(format='%(asctime)s: %(levelname)s:%(name)s:%(message)s', level=log_level) - - # Add Honeybadger support - if os.getenv('HONEYBADGER_API_KEY'): + + + honeybadger_api_key = get_secret('honeybadger_api_key') + if honeybadger_api_key: + # Add Honeybadger support logging.info('Setting up Honeybadger configuration') # I need to patch Flask in order for Honeybadger to load @@ -56,7 +61,7 @@ def create_app(test_config=None): setattr(signals, 'signals_available', True) # Log exceptions to Honeybadger - app.config['HONEYBADGER_API_KEY'] = os.getenv('HONEYBADGER_API_KEY') + app.config['HONEYBADGER_API_KEY'] = honeybadger_api_key app.config['HONEYBADGER_PARAMS_FILTERS'] = 'password, secret, openai_api_key, api_key' app.config['HONEYBADGER_ENVIRONMENT'] = os.getenv('FLASK_ENV') FlaskHoneybadger(app, report_exceptions=True) @@ -67,7 +72,7 @@ def filter(self, record): # But ignore Python logging exceptions on 'ERROR' return not record.getMessage().startswith('Exception on ') - hb_handler = HoneybadgerHandler(api_key=os.getenv('HONEYBADGER_API_KEY')) + hb_handler = HoneybadgerHandler(api_key=honeybadger_api_key) hb_handler.setLevel(logging.ERROR) hb_handler.addFilter(NoExceptionErrorFilter()) logger = logging.getLogger() @@ -83,3 +88,23 @@ def root(): app.register_blueprint(assessment_routes) return app + +# Get a secret from AWS Secrets Manager or local ENV +def get_secret(secret_name): + local_env_var_name = secret_name.upper() + # AWS Secrets are named like `production/aiproxy.code.org/secret_name}` + aws_secret_id = '/'.join([os.getenv('ENVIRONMENT'), os.getenv('ENVIRONMENT_TYPE'), secret_name]) + + secret = '' + if os.getenv(local_env_var_name): + secret = os.getenv(local_env_var_name) + logging.info(f'Retrieved secret "{secret_name}" from local ENV') + else: + try: + client = boto3.client('secretsmanager') + logging.info(f'Retrieved secret "{secret_name}" from AWS Secrets Manager') + secret = client.get_secret_value(SecretId=aws_secret_id) + except Exception as e: + logging.error(f'Error getting "{secret_name}" from AWS Secrets Manager: {e}') + + return secret