From 5f7fc419fe617c809570dd28c41c183a6232ee74 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:12:04 -0600 Subject: [PATCH 01/17] wip --- gen3workflow/app.py | 2 + gen3workflow/routes/s3.py | 203 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 gen3workflow/routes/s3.py diff --git a/gen3workflow/app.py b/gen3workflow/app.py index 4a73dd9..13b0a77 100644 --- a/gen3workflow/app.py +++ b/gen3workflow/app.py @@ -9,6 +9,7 @@ from gen3workflow import logger from gen3workflow.config import config from gen3workflow.routes.ga4gh_tes import router as ga4gh_tes_router +from gen3workflow.routes.s3 import router as s3_router from gen3workflow.routes.storage import router as storage_router from gen3workflow.routes.system import router as system_router @@ -28,6 +29,7 @@ def get_app(httpx_client=None) -> FastAPI: ) app.async_client = httpx_client or httpx.AsyncClient() app.include_router(ga4gh_tes_router, tags=["GA4GH TES"]) + app.include_router(s3_router, tags=["S3"]) app.include_router(storage_router, tags=["Storage"]) app.include_router(system_router, tags=["System"]) diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py new file mode 100644 index 0000000..c1621a3 --- /dev/null +++ b/gen3workflow/routes/s3.py @@ -0,0 +1,203 @@ +from datetime import datetime +import hashlib +import json +import os +import logging +import urllib.parse +import uuid + +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials +from botocore.auth import SigV4Auth +import boto3 +import hmac + +from fastapi import APIRouter, FastAPI, Request +import httpx +# import requests # TODO replace with httpx +import uvicorn + +from starlette.responses import JSONResponse, Response, StreamingResponse +from io import BytesIO + +# TODO Generate a presigned URL if the request is a GET request, see https://cdis.slack.com/archives/D01DMJWKVB5/p1733169741227879 + + +# The base URL to forward requests to +bucket = "ga4ghtes-pauline-planx-pla-net" +FORWARD_TO_BASE_URL = f"https://{bucket}.s3.amazonaws.com" + + +# app = FastAPI() + + +logging.basicConfig( + level=logging.INFO, + format='%(message)s', + handlers=[ + logging.FileHandler('server.log'), + logging.StreamHandler() + ] +) + +router = APIRouter(prefix="/s3") + + +async def _log_request(request, path): + # Get current timestamp + timestamp = datetime.now().isoformat() + + # Reading body as bytes, then decode it as string if necessary + body_bytes = await request.body() + try: + body = body_bytes.decode() + except UnicodeDecodeError: + body = str(body_bytes) # In case of binary data + try: + body = json.loads(body) + except: + pass # Keep body as string if not JSON + + # Create log entry + log_entry = { + 'timestamp': timestamp, + 'method': request.method, + 'path': path, + 'headers': dict(request.headers), + 'body': body, + } + + # Log as JSON for easier parsing + logging.info(json.dumps(log_entry, indent=2)) + + +def get_signature_key(key, date_stamp, region_name, service_name): + """ + Create a signing key using the AWS Signature Version 4 algorithm. + """ + key_date = hmac.new(f"AWS4{key}".encode('utf-8'), date_stamp.encode('utf-8'), hashlib.sha256).digest() + key_region = hmac.new(key_date, region_name.encode('utf-8'), hashlib.sha256).digest() + key_service = hmac.new(key_region, service_name.encode('utf-8'), hashlib.sha256).digest() + key_signing = hmac.new(key_service, b"aws4_request", hashlib.sha256).digest() + return key_signing + + +@router.api_route( + "/{path:path}", + methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH", "TRACE", "HEAD"] +) +async def catch_all_v4(path: str, request: Request): + if "KEY" not in os.environ: + raise Exception("No key") + await _log_request(request, path) + query_params = dict(request.query_params) + def url_enc(s): + return urllib.parse.quote_plus(s) #, safe='/', encoding=None, errors=None) + query_params_names = sorted(list(query_params.keys())) # query params have to be sorted + canonical_r_q_params = "&".join(f"{url_enc(key)}={url_enc(query_params[key])}" for key in query_params_names) + + used_path = path.split(bucket)[1] + used_url = "/".join(used_path.split("/")[1:]) + + headers = {} + # TODO try again to include all the headers + # headers = dict(request.headers) + # headers.pop("authorization") + + url = f"{FORWARD_TO_BASE_URL}/{used_url}" + body = await request.body() + + # if "Content-Type" in request.headers: + # headers["content-type"] = request.headers["Content-Type"] + headers['host'] = f'{bucket}.s3.amazonaws.com' + + # Hash the request body + body_hash = hashlib.sha256(body).hexdigest() + headers['x-amz-content-sha256'] = body_hash + + # Ensure 'x-amz-date' is included in the headers (it's needed for signature calculation) + amz_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') + headers['x-amz-date'] = amz_date + + # AWS request object + aws_request = AWSRequest( + method=request.method, + url=url, + data=body, + headers=headers + ) + + canon_headers = "".join(f"{key}:{headers[key]}\n" for key in sorted(list(headers.keys()))) + header_names = ";".join(sorted(list(headers.keys()))) + + # Construct the canonical request (with cleaned-up path) + canonical_request = ( + f"{request.method}\n" + f"{used_path}\n" + f"{canonical_r_q_params}\n" # Query parameters + f"{canon_headers}" + f"\n" + f"{header_names}\n" # Signed headers + f"{headers['x-amz-content-sha256']}" # Final Body hash + ) + logging.info(f"- Canonical Request:\n{canonical_request}") + + # AWS Credentials for signing + credentials = Credentials( + access_key=os.environ.get('KEY'), + secret_key=os.environ.get('SECRET') + ) + + # Create the string to sign based on the canonical request + region = 'us-east-1' + service = 's3' + date_stamp = headers['x-amz-date'][:8] # The date portion (YYYYMMDD) + string_to_sign = ( + f"AWS4-HMAC-SHA256\n" + f"{headers['x-amz-date']}\n" # The timestamp in 'YYYYMMDDTHHMMSSZ' format + f"{date_stamp}/{region}/{service}/aws4_request\n" # Credential scope + f"{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}" # Hash of the Canonical Request + ) + + # Log String to Sign + logging.info(f"- String to Sign:\n{string_to_sign}") + + # Generate the signing key using our `get_signature_key` function + signing_key = get_signature_key(credentials.secret_key, date_stamp, region, service) + + # Calculate the signature by signing the string to sign with the signing key + signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), hashlib.sha256).hexdigest() + + # Log Signature + logging.info(f"- Signature: {signature}") + + # Ensure all headers that are in the request are included in the SignedHeaders + signed_headers = ';'.join(sorted([k.lower() for k in headers.keys() if k != 'authorization'])) + + # Log final headers before sending the request + aws_request.headers['Authorization'] = f"AWS4-HMAC-SHA256 Credential={credentials.access_key}/{date_stamp}/{region}/{service}/aws4_request, SignedHeaders={signed_headers}, Signature={signature}" + + logging.info(f"- Signed Headers:\n{aws_request.headers}") + + # Send the signed request to S3 + prepared_request = aws_request.prepare() + logging.info(f"- Making {prepared_request.method} request to {prepared_request.url}") + + # Perform the actual HTTP request + async with httpx.AsyncClient() as client: + # print("request:", {"method": request.method, "url": url, "body": body, "headers": prepared_request.headers, "query param": query_params}) + response = await client.request( + method=request.method, + url=url, + headers=prepared_request.headers, + params=query_params, + data=body, + ) + + # Check for errors + if response.status_code != 200: + logging.error(f"- Error from AWS: {response}") + + if "Content-Type" in response.headers: + return Response(content=response.content, status_code=response.status_code, media_type=response.headers['Content-Type']) + return Response(content=response.content, status_code=response.status_code) From 3fa21ad8222677b759b1b52faec047115ea2166b Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:45:18 -0600 Subject: [PATCH 02/17] wip --- gen3workflow/routes/s3.py | 107 +++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 60 deletions(-) diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index c1621a3..37f3a1b 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -2,52 +2,26 @@ import hashlib import json import os -import logging import urllib.parse -import uuid from botocore.awsrequest import AWSRequest +from fastapi import APIRouter, Request from botocore.credentials import Credentials -from botocore.auth import SigV4Auth -import boto3 import hmac - -from fastapi import APIRouter, FastAPI, Request import httpx -# import requests # TODO replace with httpx -import uvicorn - -from starlette.responses import JSONResponse, Response, StreamingResponse -from io import BytesIO - -# TODO Generate a presigned URL if the request is a GET request, see https://cdis.slack.com/archives/D01DMJWKVB5/p1733169741227879 +from starlette.responses import Response +from gen3workflow import logger -# The base URL to forward requests to -bucket = "ga4ghtes-pauline-planx-pla-net" -FORWARD_TO_BASE_URL = f"https://{bucket}.s3.amazonaws.com" - - -# app = FastAPI() +# TODO Generate a presigned URL if the request is a GET request, see https://cdis.slack.com/archives/D01DMJWKVB5/p1733169741227879 -logging.basicConfig( - level=logging.INFO, - format='%(message)s', - handlers=[ - logging.FileHandler('server.log'), - logging.StreamHandler() - ] -) router = APIRouter(prefix="/s3") async def _log_request(request, path): - # Get current timestamp - timestamp = datetime.now().isoformat() - - # Reading body as bytes, then decode it as string if necessary + # Read body as bytes, then decode it as string if necessary body_bytes = await request.body() try: body = body_bytes.decode() @@ -57,8 +31,8 @@ async def _log_request(request, path): body = json.loads(body) except: pass # Keep body as string if not JSON - - # Create log entry + + timestamp = datetime.now().isoformat() log_entry = { 'timestamp': timestamp, 'method': request.method, @@ -66,9 +40,7 @@ async def _log_request(request, path): 'headers': dict(request.headers), 'body': body, } - - # Log as JSON for easier parsing - logging.info(json.dumps(log_entry, indent=2)) + logger.debug(f"Incoming request: {json.dumps(log_entry, indent=2)}") def get_signature_key(key, date_stamp, region_name, service_name): @@ -89,40 +61,60 @@ def get_signature_key(key, date_stamp, region_name, service_name): async def catch_all_v4(path: str, request: Request): if "KEY" not in os.environ: raise Exception("No key") - await _log_request(request, path) - query_params = dict(request.query_params) + + # await _log_request(request, path) + + # TODO get bucket from path + bucket = "ga4ghtes-pauline-planx-pla-net" + def url_enc(s): - return urllib.parse.quote_plus(s) #, safe='/', encoding=None, errors=None) + return urllib.parse.quote_plus(s) + + query_params = dict(request.query_params) query_params_names = sorted(list(query_params.keys())) # query params have to be sorted canonical_r_q_params = "&".join(f"{url_enc(key)}={url_enc(query_params[key])}" for key in query_params_names) - used_path = path.split(bucket)[1] - used_url = "/".join(used_path.split("/")[1:]) + # Example 1: + # - path = my-bucket// + # - request_path = // + # - api_endpoint = / + # Example 2: + # - path = my-bucket/pre/fix/ + # - request_path = /pre/fix/ + # - api_endpoint = pre/fix/ + request_path = path.split(bucket)[1] + api_endpoint = "/".join(request_path.split("/")[1:]) - headers = {} - # TODO try again to include all the headers # headers = dict(request.headers) # headers.pop("authorization") - - url = f"{FORWARD_TO_BASE_URL}/{used_url}" - body = await request.body() + headers = {} + # TODO try again to include all the headers + # `x-amz-content-sha256` is sometimes set to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" in + # the original request, but i was not able to get the signing working when copying it # if "Content-Type" in request.headers: # headers["content-type"] = request.headers["Content-Type"] headers['host'] = f'{bucket}.s3.amazonaws.com' # Hash the request body + body = await request.body() body_hash = hashlib.sha256(body).hexdigest() headers['x-amz-content-sha256'] = body_hash + # headers['x-amz-content-sha256'] = request.headers['x-amz-content-sha256'] + # if 'content-length' in request.headers: + # headers['content-length'] = request.headers['content-length'] + # if 'x-amz-decoded-content-length' in request.headers: + # headers['x-amz-decoded-content-length'] = request.headers['x-amz-decoded-content-length'] # Ensure 'x-amz-date' is included in the headers (it's needed for signature calculation) amz_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') headers['x-amz-date'] = amz_date # AWS request object + aws_api_url = f"https://{bucket}.s3.amazonaws.com/{api_endpoint}" aws_request = AWSRequest( method=request.method, - url=url, + url=aws_api_url, data=body, headers=headers ) @@ -133,14 +125,14 @@ def url_enc(s): # Construct the canonical request (with cleaned-up path) canonical_request = ( f"{request.method}\n" - f"{used_path}\n" + f"{request_path}\n" f"{canonical_r_q_params}\n" # Query parameters f"{canon_headers}" f"\n" f"{header_names}\n" # Signed headers f"{headers['x-amz-content-sha256']}" # Final Body hash ) - logging.info(f"- Canonical Request:\n{canonical_request}") + logger.debug(f"- Canonical Request:\n{canonical_request}") # AWS Credentials for signing credentials = Credentials( @@ -158,37 +150,32 @@ def url_enc(s): f"{date_stamp}/{region}/{service}/aws4_request\n" # Credential scope f"{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}" # Hash of the Canonical Request ) - - # Log String to Sign - logging.info(f"- String to Sign:\n{string_to_sign}") + logger.debug(f"- String to Sign:\n{string_to_sign}") # Generate the signing key using our `get_signature_key` function signing_key = get_signature_key(credentials.secret_key, date_stamp, region, service) # Calculate the signature by signing the string to sign with the signing key signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), hashlib.sha256).hexdigest() - - # Log Signature - logging.info(f"- Signature: {signature}") + logger.debug(f"- Signature: {signature}") # Ensure all headers that are in the request are included in the SignedHeaders signed_headers = ';'.join(sorted([k.lower() for k in headers.keys() if k != 'authorization'])) # Log final headers before sending the request aws_request.headers['Authorization'] = f"AWS4-HMAC-SHA256 Credential={credentials.access_key}/{date_stamp}/{region}/{service}/aws4_request, SignedHeaders={signed_headers}, Signature={signature}" - - logging.info(f"- Signed Headers:\n{aws_request.headers}") + logger.debug(f"- Signed Headers:\n{aws_request.headers}") # Send the signed request to S3 prepared_request = aws_request.prepare() - logging.info(f"- Making {prepared_request.method} request to {prepared_request.url}") + logger.debug(f"- Making {prepared_request.method} request to {prepared_request.url}") # Perform the actual HTTP request async with httpx.AsyncClient() as client: # print("request:", {"method": request.method, "url": url, "body": body, "headers": prepared_request.headers, "query param": query_params}) response = await client.request( method=request.method, - url=url, + url=aws_api_url, headers=prepared_request.headers, params=query_params, data=body, @@ -196,7 +183,7 @@ def url_enc(s): # Check for errors if response.status_code != 200: - logging.error(f"- Error from AWS: {response}") + logger.error(f"- Error from AWS: {response}") if "Content-Type" in response.headers: return Response(content=response.content, status_code=response.status_code, media_type=response.headers['Content-Type']) From 7ea2a352d5b0cbc27ea264e2b4b63a50f546b56c Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:47:51 -0600 Subject: [PATCH 03/17] working version --- gen3workflow/routes/s3.py | 94 ++++++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index 37f3a1b..b12fc73 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -4,17 +4,19 @@ import os import urllib.parse -from botocore.awsrequest import AWSRequest from fastapi import APIRouter, Request +from fastapi.security import HTTPAuthorizationCredentials from botocore.credentials import Credentials import hmac import httpx +from starlette.datastructures import Headers from starlette.responses import Response -from gen3workflow import logger +from gen3workflow import aws_utils, logger +from gen3workflow.auth import Auth -# TODO Generate a presigned URL if the request is a GET request, see https://cdis.slack.com/archives/D01DMJWKVB5/p1733169741227879 +# TODO Generate a presigned URL if the request is a GET request, see https://cdis.slack.com/archives/D01DMJWKVB5/p1733169741227879 - is that required? router = APIRouter(prefix="/s3") @@ -43,7 +45,25 @@ async def _log_request(request, path): logger.debug(f"Incoming request: {json.dumps(log_entry, indent=2)}") -def get_signature_key(key, date_stamp, region_name, service_name): +def get_access_token(headers: Headers) -> str: + """ + Extract the user's access token, which should have been provided as the key ID, from the + Authorization header in the following expected format: + `AWS4-HMAC-SHA256 Credential=////aws4_request, SignedHeaders=<>, Signature=<>` + + Args: + headers (Headers): request headers + + Returns: + str: the user's access token or "" if not found + """ + auth_header = headers.get("authorization") + if not auth_header: + return "" + return auth_header.split("Credential=")[1].split("/")[0] + + +def get_signature_key(key: str, date_stamp: str, region_name: str, service_name: str) -> str: """ Create a signing key using the AWS Signature Version 4 algorithm. """ @@ -59,20 +79,35 @@ def get_signature_key(key, date_stamp, region_name, service_name): methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH", "TRACE", "HEAD"] ) async def catch_all_v4(path: str, request: Request): + """TODO + + Args: + path (str): _description_ + request (Request): _description_ + + Raises: + Exception: _description_ + + Returns: + _type_: _description_ + """ if "KEY" not in os.environ: raise Exception("No key") # await _log_request(request, path) - # TODO get bucket from path - bucket = "ga4ghtes-pauline-planx-pla-net" - - def url_enc(s): - return urllib.parse.quote_plus(s) + # extract the user's access token from the request headers, and use it to get the name of + # the user's bucket + auth = Auth(api_request=request) + auth.bearer_token = HTTPAuthorizationCredentials(scheme="bearer", credentials=get_access_token(request.headers)) + token_claims = await auth.get_token_claims() + user_id = token_claims.get("sub") + user_bucket = aws_utils.get_safe_name_from_user_id(user_id) + user_bucket = "ga4ghtes-pauline-planx-pla-net" # TODO remove - for testing query_params = dict(request.query_params) query_params_names = sorted(list(query_params.keys())) # query params have to be sorted - canonical_r_q_params = "&".join(f"{url_enc(key)}={url_enc(query_params[key])}" for key in query_params_names) + canonical_r_q_params = "&".join(f"{urllib.parse.quote_plus(key)}={urllib.parse.quote_plus(query_params[key])}" for key in query_params_names) # Example 1: # - path = my-bucket// @@ -82,7 +117,7 @@ def url_enc(s): # - path = my-bucket/pre/fix/ # - request_path = /pre/fix/ # - api_endpoint = pre/fix/ - request_path = path.split(bucket)[1] + request_path = path.split(user_bucket)[1] api_endpoint = "/".join(request_path.split("/")[1:]) # headers = dict(request.headers) @@ -94,7 +129,7 @@ def url_enc(s): # if "Content-Type" in request.headers: # headers["content-type"] = request.headers["Content-Type"] - headers['host'] = f'{bucket}.s3.amazonaws.com' + headers['host'] = f'{user_bucket}.s3.amazonaws.com' # Hash the request body body = await request.body() @@ -110,15 +145,6 @@ def url_enc(s): amz_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') headers['x-amz-date'] = amz_date - # AWS request object - aws_api_url = f"https://{bucket}.s3.amazonaws.com/{api_endpoint}" - aws_request = AWSRequest( - method=request.method, - url=aws_api_url, - data=body, - headers=headers - ) - canon_headers = "".join(f"{key}:{headers[key]}\n" for key in sorted(list(headers.keys()))) header_names = ";".join(sorted(list(headers.keys()))) @@ -132,9 +158,10 @@ def url_enc(s): f"{header_names}\n" # Signed headers f"{headers['x-amz-content-sha256']}" # Final Body hash ) - logger.debug(f"- Canonical Request:\n{canonical_request}") + # logger.debug(f"- Canonical Request:\n{canonical_request}") # AWS Credentials for signing + # TODO support either AWS IAM key or service account credentials = Credentials( access_key=os.environ.get('KEY'), secret_key=os.environ.get('SECRET') @@ -150,40 +177,35 @@ def url_enc(s): f"{date_stamp}/{region}/{service}/aws4_request\n" # Credential scope f"{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}" # Hash of the Canonical Request ) - logger.debug(f"- String to Sign:\n{string_to_sign}") + # logger.debug(f"- String to Sign:\n{string_to_sign}") # Generate the signing key using our `get_signature_key` function signing_key = get_signature_key(credentials.secret_key, date_stamp, region, service) # Calculate the signature by signing the string to sign with the signing key signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), hashlib.sha256).hexdigest() - logger.debug(f"- Signature: {signature}") + # logger.debug(f"- Signature: {signature}") # Ensure all headers that are in the request are included in the SignedHeaders signed_headers = ';'.join(sorted([k.lower() for k in headers.keys() if k != 'authorization'])) # Log final headers before sending the request - aws_request.headers['Authorization'] = f"AWS4-HMAC-SHA256 Credential={credentials.access_key}/{date_stamp}/{region}/{service}/aws4_request, SignedHeaders={signed_headers}, Signature={signature}" - logger.debug(f"- Signed Headers:\n{aws_request.headers}") - - # Send the signed request to S3 - prepared_request = aws_request.prepare() - logger.debug(f"- Making {prepared_request.method} request to {prepared_request.url}") + headers['authorization'] = f"AWS4-HMAC-SHA256 Credential={credentials.access_key}/{date_stamp}/{region}/{service}/aws4_request, SignedHeaders={signed_headers}, Signature={signature}" + # logger.debug(f"- Signed Headers:\n{aws_request.headers}") # Perform the actual HTTP request + s3_api_url = f"https://{user_bucket}.s3.amazonaws.com/{api_endpoint}" + # logger.debug(f"- Making {request.method} request to {s3_api_url}") async with httpx.AsyncClient() as client: - # print("request:", {"method": request.method, "url": url, "body": body, "headers": prepared_request.headers, "query param": query_params}) response = await client.request( method=request.method, - url=aws_api_url, - headers=prepared_request.headers, + url=s3_api_url, + headers=headers, params=query_params, data=body, ) - - # Check for errors if response.status_code != 200: - logger.error(f"- Error from AWS: {response}") + logger.error(f"Error from AWS: {response.status_code} {response.text}") if "Content-Type" in response.headers: return Response(content=response.content, status_code=response.status_code, media_type=response.headers['Content-Type']) From d61a1fd02debdc0b314194bf4c6b1ac14251c8be Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 5 Dec 2024 10:28:45 -0600 Subject: [PATCH 04/17] assume role --- .secrets.baseline | 4 ++-- gen3workflow/config-default.yaml | 4 ++++ gen3workflow/config.py | 1 + gen3workflow/routes/s3.py | 28 +++++++++++++++++----------- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 9b3abca..6064cdc 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -151,7 +151,7 @@ "filename": "gen3workflow/config-default.yaml", "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_verified": false, - "line_number": 27 + "line_number": 31 } ], "migrations/versions/e1886270d9d2_create_system_key_table.py": [ @@ -182,5 +182,5 @@ } ] }, - "generated_at": "2024-11-19T19:43:31Z" + "generated_at": "2024-12-05T16:27:30Z" } diff --git a/gen3workflow/config-default.yaml b/gen3workflow/config-default.yaml index 270384d..652c68b 100644 --- a/gen3workflow/config-default.yaml +++ b/gen3workflow/config-default.yaml @@ -16,6 +16,10 @@ MAX_IAM_KEYS_PER_USER: 2 # the default AWS AccessKeysPerUser quota is 2 IAM_KEYS_LIFETIME_DAYS: 30 USER_BUCKETS_REGION: us-east-1 +S3_ENDPOINTS_AWS_ROLE_ARN: +S3_ENDPOINTS_AWS_ACCESS_KEY_ID: +S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY: + ############# # DATABASE # ############# diff --git a/gen3workflow/config.py b/gen3workflow/config.py index fc747da..e722abc 100644 --- a/gen3workflow/config.py +++ b/gen3workflow/config.py @@ -48,6 +48,7 @@ def validate_top_level_configs(self): "MAX_IAM_KEYS_PER_USER": {"type": "integer", "maximum": 100}, "IAM_KEYS_LIFETIME_DAYS": {"type": "integer"}, "USER_BUCKETS_REGION": {"type": "string"}, + # TODO S3_ENDPOINTS_AWS_ROLE_ARN etc "ARBORIST_URL": {"type": ["string", "null"]}, "TASK_IMAGE_WHITELIST": {"type": "array", "items": {"type": "string"}}, "TES_SERVER_URL": {"type": "string"}, diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index b12fc73..8510cd7 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -4,6 +4,7 @@ import os import urllib.parse +import boto3 from fastapi import APIRouter, Request from fastapi.security import HTTPAuthorizationCredentials from botocore.credentials import Credentials @@ -14,6 +15,7 @@ from gen3workflow import aws_utils, logger from gen3workflow.auth import Auth +from gen3workflow.config import config # TODO Generate a presigned URL if the request is a GET request, see https://cdis.slack.com/archives/D01DMJWKVB5/p1733169741227879 - is that required? @@ -91,9 +93,6 @@ async def catch_all_v4(path: str, request: Request): Returns: _type_: _description_ """ - if "KEY" not in os.environ: - raise Exception("No key") - # await _log_request(request, path) # extract the user's access token from the request headers, and use it to get the name of @@ -140,7 +139,7 @@ async def catch_all_v4(path: str, request: Request): # headers['content-length'] = request.headers['content-length'] # if 'x-amz-decoded-content-length' in request.headers: # headers['x-amz-decoded-content-length'] = request.headers['x-amz-decoded-content-length'] - + # Ensure 'x-amz-date' is included in the headers (it's needed for signature calculation) amz_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') headers['x-amz-date'] = amz_date @@ -161,14 +160,21 @@ async def catch_all_v4(path: str, request: Request): # logger.debug(f"- Canonical Request:\n{canonical_request}") # AWS Credentials for signing - # TODO support either AWS IAM key or service account - credentials = Credentials( - access_key=os.environ.get('KEY'), - secret_key=os.environ.get('SECRET') - ) + if config["S3_ENDPOINTS_AWS_ROLE_ARN"]: + sts_client = boto3.client('sts') + response = sts_client.assume_role( + RoleArn=config["S3_ENDPOINTS_AWS_ROLE_ARN"], + RoleSessionName='SessionName' + ) + credentials = response['Credentials'] + else: + credentials = Credentials( + access_key=config["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"], + secret_key=config["S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY"], + ) # Create the string to sign based on the canonical request - region = 'us-east-1' + region = config["USER_BUCKETS_REGION"] service = 's3' date_stamp = headers['x-amz-date'][:8] # The date portion (YYYYMMDD) string_to_sign = ( @@ -195,7 +201,7 @@ async def catch_all_v4(path: str, request: Request): # Perform the actual HTTP request s3_api_url = f"https://{user_bucket}.s3.amazonaws.com/{api_endpoint}" - # logger.debug(f"- Making {request.method} request to {s3_api_url}") + logger.debug(f"Making {request.method} request to {s3_api_url}") async with httpx.AsyncClient() as client: response = await client.request( method=request.method, From 018dd4632558e16b81f3ebd90b2b78d565e3c400 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:31:22 -0600 Subject: [PATCH 05/17] fix getting credentials for assumed role --- gen3workflow/routes/s3.py | 47 ++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index 8510cd7..c441b86 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -5,13 +5,14 @@ import urllib.parse import boto3 -from fastapi import APIRouter, Request +from fastapi import APIRouter, HTTPException, Request from fastapi.security import HTTPAuthorizationCredentials from botocore.credentials import Credentials import hmac import httpx from starlette.datastructures import Headers from starlette.responses import Response +from starlette.status import HTTP_401_UNAUTHORIZED from gen3workflow import aws_utils, logger from gen3workflow.auth import Auth @@ -80,7 +81,7 @@ def get_signature_key(key: str, date_stamp: str, region_name: str, service_name: "/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH", "TRACE", "HEAD"] ) -async def catch_all_v4(path: str, request: Request): +async def todo_rename(path: str, request: Request): """TODO Args: @@ -102,11 +103,6 @@ async def catch_all_v4(path: str, request: Request): token_claims = await auth.get_token_claims() user_id = token_claims.get("sub") user_bucket = aws_utils.get_safe_name_from_user_id(user_id) - user_bucket = "ga4ghtes-pauline-planx-pla-net" # TODO remove - for testing - - query_params = dict(request.query_params) - query_params_names = sorted(list(query_params.keys())) # query params have to be sorted - canonical_r_q_params = "&".join(f"{urllib.parse.quote_plus(key)}={urllib.parse.quote_plus(query_params[key])}" for key in query_params_names) # Example 1: # - path = my-bucket// @@ -116,6 +112,8 @@ async def catch_all_v4(path: str, request: Request): # - path = my-bucket/pre/fix/ # - request_path = /pre/fix/ # - api_endpoint = pre/fix/ + if user_bucket not in path: + raise HTTPException(HTTP_401_UNAUTHORIZED, f"'{path}' not allowed. You can make calls to your personal bucket, '{user_bucket}'") request_path = path.split(user_bucket)[1] api_endpoint = "/".join(request_path.split("/")[1:]) @@ -144,9 +142,30 @@ async def catch_all_v4(path: str, request: Request): amz_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') headers['x-amz-date'] = amz_date + # AWS Credentials for signing + if config["S3_ENDPOINTS_AWS_ROLE_ARN"]: + # sts_client = boto3.client('sts') + # response = sts_client.assume_role( + # RoleArn=config["S3_ENDPOINTS_AWS_ROLE_ARN"], + # RoleSessionName='SessionName' + # ) + # credentials = response['Credentials'] + session = boto3.Session() + credentials = session.get_credentials() + headers["x-amz-security-token"] = credentials.token + else: + credentials = Credentials( + access_key=config["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"], + secret_key=config["S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY"], + ) + canon_headers = "".join(f"{key}:{headers[key]}\n" for key in sorted(list(headers.keys()))) header_names = ";".join(sorted(list(headers.keys()))) + query_params = dict(request.query_params) + query_params_names = sorted(list(query_params.keys())) # query params have to be sorted + canonical_r_q_params = "&".join(f"{urllib.parse.quote_plus(key)}={urllib.parse.quote_plus(query_params[key])}" for key in query_params_names) + # Construct the canonical request (with cleaned-up path) canonical_request = ( f"{request.method}\n" @@ -159,20 +178,6 @@ async def catch_all_v4(path: str, request: Request): ) # logger.debug(f"- Canonical Request:\n{canonical_request}") - # AWS Credentials for signing - if config["S3_ENDPOINTS_AWS_ROLE_ARN"]: - sts_client = boto3.client('sts') - response = sts_client.assume_role( - RoleArn=config["S3_ENDPOINTS_AWS_ROLE_ARN"], - RoleSessionName='SessionName' - ) - credentials = response['Credentials'] - else: - credentials = Credentials( - access_key=config["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"], - secret_key=config["S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY"], - ) - # Create the string to sign based on the canonical request region = config["USER_BUCKETS_REGION"] service = 's3' From 492276f5475981e73f1ce77522f80fa01d88d25d Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:16:54 -0600 Subject: [PATCH 06/17] logs for errors --- gen3workflow/aws_utils.py | 7 +++---- gen3workflow/routes/ga4gh_tes.py | 2 ++ gen3workflow/routes/s3.py | 10 +++------- gen3workflow/routes/storage.py | 9 ++++----- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/gen3workflow/aws_utils.py b/gen3workflow/aws_utils.py index e98e8ae..9b30549 100644 --- a/gen3workflow/aws_utils.py +++ b/gen3workflow/aws_utils.py @@ -174,7 +174,6 @@ def delete_iam_user_key(user_id, key_id): ) except ClientError as e: if e.response["Error"]["Code"] == "NoSuchEntity": - raise HTTPException( - HTTP_404_NOT_FOUND, - f"No such key: '{key_id}'", - ) + err_msg = f"No such key: '{key_id}'" + logger.error(err_msg) + raise HTTPException(HTTP_404_NOT_FOUND, err_msg) diff --git a/gen3workflow/routes/ga4gh_tes.py b/gen3workflow/routes/ga4gh_tes.py index 160c03a..da242c1 100644 --- a/gen3workflow/routes/ga4gh_tes.py +++ b/gen3workflow/routes/ga4gh_tes.py @@ -119,6 +119,7 @@ async def create_task(request: Request, auth=Depends(Auth)): username=username, user_id=user_id ) except ArboristError as e: + logger.error(e.message) raise HTTPException(e.code, e.message) return res.json() @@ -195,6 +196,7 @@ async def list_tasks(request: Request, auth=Depends(Auth)): }, ) except ArboristError as e: + logger.error(e.message) raise HTTPException(e.code, e.message) # filter out tasks the current user does not have access to diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index c441b86..19a2f75 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -113,7 +113,9 @@ async def todo_rename(path: str, request: Request): # - request_path = /pre/fix/ # - api_endpoint = pre/fix/ if user_bucket not in path: - raise HTTPException(HTTP_401_UNAUTHORIZED, f"'{path}' not allowed. You can make calls to your personal bucket, '{user_bucket}'") + err_msg = f"'{path}' not allowed. You can make calls to your personal bucket, '{user_bucket}'" + logger.error(err_msg) + raise HTTPException(HTTP_401_UNAUTHORIZED, err_msg) request_path = path.split(user_bucket)[1] api_endpoint = "/".join(request_path.split("/")[1:]) @@ -144,12 +146,6 @@ async def todo_rename(path: str, request: Request): # AWS Credentials for signing if config["S3_ENDPOINTS_AWS_ROLE_ARN"]: - # sts_client = boto3.client('sts') - # response = sts_client.assume_role( - # RoleArn=config["S3_ENDPOINTS_AWS_ROLE_ARN"], - # RoleSessionName='SessionName' - # ) - # credentials = response['Credentials'] session = boto3.Session() credentials = session.get_credentials() headers["x-amz-security-token"] = credentials.token diff --git a/gen3workflow/routes/storage.py b/gen3workflow/routes/storage.py index c1cb3d9..a7d354c 100644 --- a/gen3workflow/routes/storage.py +++ b/gen3workflow/routes/storage.py @@ -8,9 +8,9 @@ HTTP_400_BAD_REQUEST, ) +from gen3workflow import aws_utils, logger from gen3workflow.auth import Auth from gen3workflow.config import config -from gen3workflow import aws_utils router = APIRouter(prefix="/storage") @@ -35,10 +35,9 @@ async def generate_user_key(request: Request, auth=Depends(Auth)): existing_keys = aws_utils.list_iam_user_keys(user_id) if len(existing_keys) >= config["MAX_IAM_KEYS_PER_USER"]: - raise HTTPException( - HTTP_400_BAD_REQUEST, - f"Too many existing keys: only {config['MAX_IAM_KEYS_PER_USER']} are allowed per user. Delete an existing key before creating a new one", - ) + err_msg = f"Too many existing keys: only {config['MAX_IAM_KEYS_PER_USER']} are allowed per user. Delete an existing key before creating a new one" + logger.error(err_msg) + raise HTTPException(HTTP_400_BAD_REQUEST, err_msg) key_id, key_secret = aws_utils.create_iam_user_and_key(user_id) return { From 90492de6ee0e870439843ea3e07bbf877730abf6 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:46:14 -0600 Subject: [PATCH 07/17] clean up --- .secrets.baseline | 4 +- gen3workflow/config-default.yaml | 1 - gen3workflow/config.py | 7 +- gen3workflow/routes/s3.py | 187 ++++++++++++++----------------- 4 files changed, 95 insertions(+), 104 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 6064cdc..c305ddb 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -151,7 +151,7 @@ "filename": "gen3workflow/config-default.yaml", "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_verified": false, - "line_number": 31 + "line_number": 30 } ], "migrations/versions/e1886270d9d2_create_system_key_table.py": [ @@ -182,5 +182,5 @@ } ] }, - "generated_at": "2024-12-05T16:27:30Z" + "generated_at": "2024-12-05T22:45:14Z" } diff --git a/gen3workflow/config-default.yaml b/gen3workflow/config-default.yaml index 652c68b..f2c23bf 100644 --- a/gen3workflow/config-default.yaml +++ b/gen3workflow/config-default.yaml @@ -16,7 +16,6 @@ MAX_IAM_KEYS_PER_USER: 2 # the default AWS AccessKeysPerUser quota is 2 IAM_KEYS_LIFETIME_DAYS: 30 USER_BUCKETS_REGION: us-east-1 -S3_ENDPOINTS_AWS_ROLE_ARN: S3_ENDPOINTS_AWS_ACCESS_KEY_ID: S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY: diff --git a/gen3workflow/config.py b/gen3workflow/config.py index e722abc..08862b6 100644 --- a/gen3workflow/config.py +++ b/gen3workflow/config.py @@ -48,7 +48,8 @@ def validate_top_level_configs(self): "MAX_IAM_KEYS_PER_USER": {"type": "integer", "maximum": 100}, "IAM_KEYS_LIFETIME_DAYS": {"type": "integer"}, "USER_BUCKETS_REGION": {"type": "string"}, - # TODO S3_ENDPOINTS_AWS_ROLE_ARN etc + "S3_ENDPOINTS_AWS_ACCESS_KEY_ID": {"type": ["string", "null"]}, + "S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY": {"type": ["string", "null"]}, "ARBORIST_URL": {"type": ["string", "null"]}, "TASK_IMAGE_WHITELIST": {"type": "array", "items": {"type": "string"}}, "TES_SERVER_URL": {"type": "string"}, @@ -56,6 +57,10 @@ def validate_top_level_configs(self): } validate(instance=self, schema=schema) + assert bool(self["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"]) == bool( + self["S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY"] + ), "Both 'S3_ENDPOINTS_AWS_ACCESS_KEY_ID' and 'S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY' must be configured, or both must be left empty" + config = Gen3WorkflowConfig(DEFAULT_CFG_PATH) try: diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index 19a2f75..4a3fba4 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone import hashlib import json import os @@ -25,29 +25,6 @@ router = APIRouter(prefix="/s3") -async def _log_request(request, path): - # Read body as bytes, then decode it as string if necessary - body_bytes = await request.body() - try: - body = body_bytes.decode() - except UnicodeDecodeError: - body = str(body_bytes) # In case of binary data - try: - body = json.loads(body) - except: - pass # Keep body as string if not JSON - - timestamp = datetime.now().isoformat() - log_entry = { - 'timestamp': timestamp, - 'method': request.method, - 'path': path, - 'headers': dict(request.headers), - 'body': body, - } - logger.debug(f"Incoming request: {json.dumps(log_entry, indent=2)}") - - def get_access_token(headers: Headers) -> str: """ Extract the user's access token, which should have been provided as the key ID, from the @@ -63,47 +40,61 @@ def get_access_token(headers: Headers) -> str: auth_header = headers.get("authorization") if not auth_header: return "" - return auth_header.split("Credential=")[1].split("/")[0] + try: + return auth_header.split("Credential=")[1].split("/")[0] + except Exception as e: + logger.error( + f"Unexpected format; unable to extract access token from authorization header: {e}" + ) + return "" -def get_signature_key(key: str, date_stamp: str, region_name: str, service_name: str) -> str: +def get_signature_key(key: str, date: str, region_name: str, service_name: str) -> str: """ Create a signing key using the AWS Signature Version 4 algorithm. """ - key_date = hmac.new(f"AWS4{key}".encode('utf-8'), date_stamp.encode('utf-8'), hashlib.sha256).digest() - key_region = hmac.new(key_date, region_name.encode('utf-8'), hashlib.sha256).digest() - key_service = hmac.new(key_region, service_name.encode('utf-8'), hashlib.sha256).digest() + key_date = hmac.new( + f"AWS4{key}".encode("utf-8"), date.encode("utf-8"), hashlib.sha256 + ).digest() + key_region = hmac.new( + key_date, region_name.encode("utf-8"), hashlib.sha256 + ).digest() + key_service = hmac.new( + key_region, service_name.encode("utf-8"), hashlib.sha256 + ).digest() key_signing = hmac.new(key_service, b"aws4_request", hashlib.sha256).digest() return key_signing @router.api_route( "/{path:path}", - methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH", "TRACE", "HEAD"] + methods=["GET", "POST", "PUT", "DELETE", "OPTIONS", "PATCH", "TRACE", "HEAD"], ) -async def todo_rename(path: str, request: Request): - """TODO - - Args: - path (str): _description_ - request (Request): _description_ - - Raises: - Exception: _description_ - - Returns: - _type_: _description_ +async def s3_endpoint(path: str, request: Request): + """ + Receive incoming S3 requests, re-sign them with the appropriate credentials to access the + current user's AWS S3 bucket, and forward them to AWS S3. """ - # await _log_request(request, path) + logger.debug(f"Incoming S3 request: '{request.method} {path}'") # extract the user's access token from the request headers, and use it to get the name of # the user's bucket auth = Auth(api_request=request) - auth.bearer_token = HTTPAuthorizationCredentials(scheme="bearer", credentials=get_access_token(request.headers)) + auth.bearer_token = HTTPAuthorizationCredentials( + scheme="bearer", credentials=get_access_token(request.headers) + ) token_claims = await auth.get_token_claims() user_id = token_claims.get("sub") user_bucket = aws_utils.get_safe_name_from_user_id(user_id) + # TODO make sure calls to bucket1 is not allowed when user's bucket is bucket12 + if user_bucket not in path: + err_msg = f"'{path}' not allowed. You can make calls to your personal bucket, '{user_bucket}'" + logger.error(err_msg) + raise HTTPException(HTTP_401_UNAUTHORIZED, err_msg) + + # extract the request path (used in the canonical request) and the API endpoint (used to make + # the request to AWS). # Example 1: # - path = my-bucket// # - request_path = // @@ -112,97 +103,88 @@ async def todo_rename(path: str, request: Request): # - path = my-bucket/pre/fix/ # - request_path = /pre/fix/ # - api_endpoint = pre/fix/ - if user_bucket not in path: - err_msg = f"'{path}' not allowed. You can make calls to your personal bucket, '{user_bucket}'" - logger.error(err_msg) - raise HTTPException(HTTP_401_UNAUTHORIZED, err_msg) request_path = path.split(user_bucket)[1] api_endpoint = "/".join(request_path.split("/")[1:]) + # generate the request headers # headers = dict(request.headers) # headers.pop("authorization") headers = {} # TODO try again to include all the headers # `x-amz-content-sha256` is sometimes set to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" in - # the original request, but i was not able to get the signing working when copying it - + # the original request, but i was not able to get the signing working when copying it. # if "Content-Type" in request.headers: # headers["content-type"] = request.headers["Content-Type"] - headers['host'] = f'{user_bucket}.s3.amazonaws.com' - - # Hash the request body + headers["host"] = f"{user_bucket}.s3.amazonaws.com" body = await request.body() body_hash = hashlib.sha256(body).hexdigest() - headers['x-amz-content-sha256'] = body_hash + headers["x-amz-content-sha256"] = body_hash # headers['x-amz-content-sha256'] = request.headers['x-amz-content-sha256'] # if 'content-length' in request.headers: # headers['content-length'] = request.headers['content-length'] # if 'x-amz-decoded-content-length' in request.headers: # headers['x-amz-decoded-content-length'] = request.headers['x-amz-decoded-content-length'] + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + headers["x-amz-date"] = timestamp - # Ensure 'x-amz-date' is included in the headers (it's needed for signature calculation) - amz_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') - headers['x-amz-date'] = amz_date - - # AWS Credentials for signing - if config["S3_ENDPOINTS_AWS_ROLE_ARN"]: - session = boto3.Session() - credentials = session.get_credentials() - headers["x-amz-security-token"] = credentials.token - else: + # get AWS credentials from the configuration or the current assumed role session + if config["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"]: credentials = Credentials( access_key=config["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"], secret_key=config["S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY"], ) + else: # running in k8s: get credentials from the assumed role + session = boto3.Session() + credentials = session.get_credentials() + headers["x-amz-security-token"] = credentials.token - canon_headers = "".join(f"{key}:{headers[key]}\n" for key in sorted(list(headers.keys()))) - header_names = ";".join(sorted(list(headers.keys()))) - + # construct the canonical request + canonical_headers = "".join( + f"{key}:{headers[key]}\n" for key in sorted(list(headers.keys())) + ) + signed_headers = ";".join(sorted([k.lower() for k in headers.keys()])) query_params = dict(request.query_params) - query_params_names = sorted(list(query_params.keys())) # query params have to be sorted - canonical_r_q_params = "&".join(f"{urllib.parse.quote_plus(key)}={urllib.parse.quote_plus(query_params[key])}" for key in query_params_names) - - # Construct the canonical request (with cleaned-up path) + # the query params in the canonical request have to be sorted: + query_params_names = sorted(list(query_params.keys())) + canonical_query_params = "&".join( + f"{urllib.parse.quote_plus(key)}={urllib.parse.quote_plus(query_params[key])}" + for key in query_params_names + ) canonical_request = ( f"{request.method}\n" f"{request_path}\n" - f"{canonical_r_q_params}\n" # Query parameters - f"{canon_headers}" + f"{canonical_query_params}\n" + f"{canonical_headers}" f"\n" - f"{header_names}\n" # Signed headers - f"{headers['x-amz-content-sha256']}" # Final Body hash + f"{signed_headers}\n" + f"{body_hash}" ) - # logger.debug(f"- Canonical Request:\n{canonical_request}") - # Create the string to sign based on the canonical request + # construct the string to sign based on the canonical request + date = timestamp[:8] # the date portion (YYYYMMDD) of the timestamp region = config["USER_BUCKETS_REGION"] - service = 's3' - date_stamp = headers['x-amz-date'][:8] # The date portion (YYYYMMDD) + service = "s3" string_to_sign = ( f"AWS4-HMAC-SHA256\n" - f"{headers['x-amz-date']}\n" # The timestamp in 'YYYYMMDDTHHMMSSZ' format - f"{date_stamp}/{region}/{service}/aws4_request\n" # Credential scope - f"{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}" # Hash of the Canonical Request + f"{timestamp}\n" + f"{date}/{region}/{service}/aws4_request\n" # credential scope + f"{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}" # canonical request hash ) - # logger.debug(f"- String to Sign:\n{string_to_sign}") - - # Generate the signing key using our `get_signature_key` function - signing_key = get_signature_key(credentials.secret_key, date_stamp, region, service) - # Calculate the signature by signing the string to sign with the signing key - signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), hashlib.sha256).hexdigest() - # logger.debug(f"- Signature: {signature}") - - # Ensure all headers that are in the request are included in the SignedHeaders - signed_headers = ';'.join(sorted([k.lower() for k in headers.keys() if k != 'authorization'])) - - # Log final headers before sending the request - headers['authorization'] = f"AWS4-HMAC-SHA256 Credential={credentials.access_key}/{date_stamp}/{region}/{service}/aws4_request, SignedHeaders={signed_headers}, Signature={signature}" - # logger.debug(f"- Signed Headers:\n{aws_request.headers}") - - # Perform the actual HTTP request + # generate the signing key, and generate the signature by signing the string to sign with the + # signing key + signing_key = get_signature_key(credentials.secret_key, date, region, service) + signature = hmac.new( + signing_key, string_to_sign.encode("utf-8"), hashlib.sha256 + ).hexdigest() + + # construct the Authorization header from the credentials and the signature, and forward the + # call to AWS S3 with the new Authorization header + headers["authorization"] = ( + f"AWS4-HMAC-SHA256 Credential={credentials.access_key}/{date}/{region}/{service}/aws4_request, SignedHeaders={signed_headers}, Signature={signature}" + ) s3_api_url = f"https://{user_bucket}.s3.amazonaws.com/{api_endpoint}" - logger.debug(f"Making {request.method} request to {s3_api_url}") + logger.debug(f"Outgoing S3 request: '{request.method} {s3_api_url}'") async with httpx.AsyncClient() as client: response = await client.request( method=request.method, @@ -214,6 +196,11 @@ async def todo_rename(path: str, request: Request): if response.status_code != 200: logger.error(f"Error from AWS: {response.status_code} {response.text}") + # return the response from AWS S3 if "Content-Type" in response.headers: - return Response(content=response.content, status_code=response.status_code, media_type=response.headers['Content-Type']) + return Response( + content=response.content, + status_code=response.status_code, + media_type=response.headers["Content-Type"], + ) return Response(content=response.content, status_code=response.status_code) From 6f7f474c69300a719c98b00a99490e9af56625f0 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:14:50 -0600 Subject: [PATCH 08/17] unit tests --- gen3workflow/config-default.yaml | 4 +- gen3workflow/routes/ga4gh_tes.py | 7 ++- gen3workflow/routes/s3.py | 21 ++++---- tests/conftest.py | 77 +++++++++++++++++++++++++---- tests/test-gen3workflow-config.yaml | 3 ++ tests/test_s3_endpoint.py | 53 ++++++++++++++++++++ 6 files changed, 140 insertions(+), 25 deletions(-) create mode 100644 tests/test_s3_endpoint.py diff --git a/gen3workflow/config-default.yaml b/gen3workflow/config-default.yaml index f2c23bf..53330db 100644 --- a/gen3workflow/config-default.yaml +++ b/gen3workflow/config-default.yaml @@ -16,6 +16,8 @@ MAX_IAM_KEYS_PER_USER: 2 # the default AWS AccessKeysPerUser quota is 2 IAM_KEYS_LIFETIME_DAYS: 30 USER_BUCKETS_REGION: us-east-1 +# configure an AWS IAM key to use when making S3 requests on behalf of users. If left empty, it +# is assumed there is an existing STS session we can get credentials from. S3_ENDPOINTS_AWS_ACCESS_KEY_ID: S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY: @@ -30,8 +32,6 @@ DB_USER: postgres DB_PASSWORD: postgres DB_DATABASE: gen3workflow_test - - ############# # GA4GH TES # ############# diff --git a/gen3workflow/routes/ga4gh_tes.py b/gen3workflow/routes/ga4gh_tes.py index da242c1..ea1aafe 100644 --- a/gen3workflow/routes/ga4gh_tes.py +++ b/gen3workflow/routes/ga4gh_tes.py @@ -99,10 +99,9 @@ async def create_task(request: Request, auth=Depends(Auth)): invalid_images = get_non_allowed_images(images_from_request, username) if invalid_images: - raise HTTPException( - HTTP_403_FORBIDDEN, - f"The specified images are not allowed: {list(invalid_images)}", - ) + err_msg = f"The specified images are not allowed: {list(invalid_images)}" + logger.error(f"{err_msg}. Allowed images: {config['TASK_IMAGE_WHITELIST']}") + raise HTTPException(HTTP_403_FORBIDDEN, err_msg) if "tags" not in body: body["tags"] = {} diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index 4a3fba4..2c5f3a3 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -72,8 +72,9 @@ def get_signature_key(key: str, date: str, region_name: str, service_name: str) ) async def s3_endpoint(path: str, request: Request): """ - Receive incoming S3 requests, re-sign them with the appropriate credentials to access the - current user's AWS S3 bucket, and forward them to AWS S3. + Receive incoming S3 requests, re-sign them (AWS Signature Version 4 algorithm) with the + appropriate credentials to access the current user's AWS S3 bucket, and forward them to + AWS S3. """ logger.debug(f"Incoming S3 request: '{request.method} {path}'") @@ -136,6 +137,7 @@ async def s3_endpoint(path: str, request: Request): else: # running in k8s: get credentials from the assumed role session = boto3.Session() credentials = session.get_credentials() + assert credentials, "No AWS credentials found" headers["x-amz-security-token"] = credentials.token # construct the canonical request @@ -185,14 +187,13 @@ async def s3_endpoint(path: str, request: Request): ) s3_api_url = f"https://{user_bucket}.s3.amazonaws.com/{api_endpoint}" logger.debug(f"Outgoing S3 request: '{request.method} {s3_api_url}'") - async with httpx.AsyncClient() as client: - response = await client.request( - method=request.method, - url=s3_api_url, - headers=headers, - params=query_params, - data=body, - ) + response = await request.app.async_client.request( + method=request.method, + url=s3_api_url, + headers=headers, + params=query_params, + data=body, + ) if response.status_code != 200: logger.error(f"Error from AWS: {response.status_code} {response.text}") diff --git a/tests/conftest.py b/tests/conftest.py index 8a6339a..1e4a6f7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,8 @@ """ import asyncio +from datetime import datetime +from dateutil.tz import tzutc import json import os from unittest.mock import MagicMock, patch @@ -14,6 +16,8 @@ import pytest_asyncio from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from starlette.config import environ +from threading import Thread +import uvicorn # Set GEN3WORKFLOW_CONFIG_PATH *before* loading the app, which loads the configuration CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -29,6 +33,38 @@ TEST_USER_ID = "64" NEW_TEST_USER_ID = "784" # a new user that does not already exist in arborist +# a "ListBucketResult" S3 response from AWS, and the corresponding response as parsed by boto3 +MOCKED_S3_RESPONSE_XML = f"""\ngen3wf-{config['HOSTNAME']}-{TEST_USER_ID}test-folder/test-file1.txt250urlfalsetest-folder/test-file1.txt2024-12-09T22:32:20.000Z"something"211somethingsomethingSTANDARD""" +MOCKED_S3_RESPONSE_DICT = { + "ResponseMetadata": { + "HTTPStatusCode": 200, + "HTTPHeaders": { + "server": "uvicorn", + "content-length": "569", + "content-type": "application/xml", + }, + "RetryAttempts": 0, + }, + "IsTruncated": False, + "Marker": "", + "Contents": [ + { + "Key": "test-folder/test-file1.txt", + "LastModified": datetime( + 2024, 12, 9, 22, 32, 20, tzinfo=tzutc() + ), + "ETag": '"something"', + "Size": 211, + "StorageClass": "STANDARD", + "Owner": {"DisplayName": "something", "ID": "something"}, + } + ], + "Name": "gen3wf-localhost-64", + "Prefix": "test-folder/test-file1.txt", + "MaxKeys": 250, + "EncodingType": "url", +} + @pytest_asyncio.fixture(scope="function") async def engine(): @@ -71,7 +107,7 @@ async def session(engine): @pytest.fixture(scope="function") -def access_token_patcher(client, request): +def access_token_patcher(request): """ The `access_token` function will return a token linked to a test user. This fixture should be used explicitely instead of the automatic @@ -270,6 +306,12 @@ async def handle_request(request: Request): body=request.content.decode(), authorized=authorized, ) + elif url.startswith(f"https://gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}.s3.amazonaws.com"): + mocked_response = httpx.Response( + status_code=200, + text=MOCKED_S3_RESPONSE_XML, + headers={"content-type": "application/xml"}, + ) if mocked_response is not None: print(f"Mocking request '{request.method} {url}'") @@ -287,11 +329,28 @@ async def handle_request(request: Request): transport=httpx.MockTransport(handle_request) ) - # the tests use a real httpx client that forwards requests to the app - async with httpx.AsyncClient( - app=app, base_url="http://test-gen3-wf" - ) as real_httpx_client: - # for easier access to the param in the tests - real_httpx_client.tes_resp_code = tes_resp_code - real_httpx_client.authorized = authorized - yield real_httpx_client + get_url = False + if hasattr(request, "param"): + get_url = request.param.get("get_url", get_url) + + if get_url: # for tests that need to hit the app URL directly + host = "0.0.0.0" + port = 8080 + def run_uvicorn(): + uvicorn.run(app, host=host, port=port) + + # start the app in a separate thread + thread = Thread(target=run_uvicorn) + thread.daemon = True # ensures the thread ends when the test ends + thread.start() + + yield f"http://{host}:{port}" # URL to use in the tests + else: + # the tests use a real httpx client that forwards requests to the app + async with httpx.AsyncClient( + app=app, base_url="http://test-gen3-wf" + ) as real_httpx_client: + # for easier access to the param in the tests + real_httpx_client.tes_resp_code = tes_resp_code + real_httpx_client.authorized = authorized + yield real_httpx_client diff --git a/tests/test-gen3workflow-config.yaml b/tests/test-gen3workflow-config.yaml index fb37faa..b3aff15 100644 --- a/tests/test-gen3workflow-config.yaml +++ b/tests/test-gen3workflow-config.yaml @@ -1,6 +1,9 @@ DEBUG: true ARBORIST_URL: http://test-arborist-server +S3_ENDPOINTS_AWS_ACCESS_KEY_ID: test-aws-key-id +S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY: test-aws-key + TES_SERVER_URL: http://external-tes-server/tes TASK_IMAGE_WHITELIST: - public.ecr.aws/random/approved/public:* diff --git a/tests/test_s3_endpoint.py b/tests/test_s3_endpoint.py new file mode 100644 index 0000000..39e3aaf --- /dev/null +++ b/tests/test_s3_endpoint.py @@ -0,0 +1,53 @@ +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError +import pytest + +from conftest import MOCKED_S3_RESPONSE_DICT, TEST_USER_ID +from gen3workflow.config import config + + +@pytest.fixture() +def s3_client(client): + """ + Return an S3 client configured to talk to the gen3-workflow `/s3` endpoint. + """ + session = boto3.session.Session() + return session.client( + service_name="s3", + aws_access_key_id="test-key-id", + aws_secret_access_key="test-key", + endpoint_url=f"{client}/s3", + # no retries; only try the call once: + config=Config(retries={"max_attempts": 0}), + ) + + +@pytest.mark.parametrize("client", [{"get_url": True}], indirect=True) +def test_s3_endpoint(s3_client, access_token_patcher): + """ + Hitting the `/s3` endpoint should result in the request being forwarded to AWS S3. + """ + res = s3_client.list_objects(Bucket=f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}") + res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).pop("date", None) + assert res == MOCKED_S3_RESPONSE_DICT + + +@pytest.mark.parametrize("client", [{"get_url": True}], indirect=True) +def test_s3_endpoint_no_token(s3_client): + """ + Hitting the `/s3` endpoint without a Gen3 access token should result in a 401 Unauthorized + error. + """ + with pytest.raises(ClientError, match="Unauthorized"): + s3_client.list_objects(Bucket=f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}") + + +@pytest.mark.parametrize("client", [{"get_url": True}], indirect=True) +def test_s3_endpoint_wrong_bucket(s3_client, access_token_patcher): + """ + Hitting the `/s3` endpoint with a bucket that is not the bucket generated by gen3-workflow for + the current user should result in a 401 Unauthorized error. + """ + with pytest.raises(ClientError, match="Unauthorized"): + s3_client.list_objects(Bucket="not-the-user-s-bucket") From 2a40477d53ea638d1b000abb4f1e8361de2e43c4 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:24:56 -0600 Subject: [PATCH 09/17] edge case unit test --- gen3workflow/routes/s3.py | 4 ++-- tests/test_s3_endpoint.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index 2c5f3a3..2ed1277 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -88,8 +88,8 @@ async def s3_endpoint(path: str, request: Request): user_id = token_claims.get("sub") user_bucket = aws_utils.get_safe_name_from_user_id(user_id) - # TODO make sure calls to bucket1 is not allowed when user's bucket is bucket12 - if user_bucket not in path: + request_bucket = path.split("?")[0].split("/")[0] + if request_bucket != user_bucket: err_msg = f"'{path}' not allowed. You can make calls to your personal bucket, '{user_bucket}'" logger.error(err_msg) raise HTTPException(HTTP_401_UNAUTHORIZED, err_msg) diff --git a/tests/test_s3_endpoint.py b/tests/test_s3_endpoint.py index 39e3aaf..6748e9f 100644 --- a/tests/test_s3_endpoint.py +++ b/tests/test_s3_endpoint.py @@ -44,10 +44,13 @@ def test_s3_endpoint_no_token(s3_client): @pytest.mark.parametrize("client", [{"get_url": True}], indirect=True) -def test_s3_endpoint_wrong_bucket(s3_client, access_token_patcher): +@pytest.mark.parametrize("bucket_name", ["not-the-user-s-bucket", f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}-2"]) +def test_s3_endpoint_wrong_bucket(s3_client, access_token_patcher, bucket_name): """ Hitting the `/s3` endpoint with a bucket that is not the bucket generated by gen3-workflow for the current user should result in a 401 Unauthorized error. + Specific edge case: if the user's bucket is "gen3wf--", a bucket name such as + "gen3wf---2" should not be allowed. """ with pytest.raises(ClientError, match="Unauthorized"): - s3_client.list_objects(Bucket="not-the-user-s-bucket") + s3_client.list_objects(Bucket=bucket_name) From fbe97ce16e21c1517ba5069d5d0d33f34a56cc9e Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:30:12 -0600 Subject: [PATCH 10/17] black and secrets --- .secrets.baseline | 15 ++++++++++++--- tests/conftest.py | 9 +++++---- tests/test_s3_endpoint.py | 5 ++++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index c305ddb..63e8630 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -151,7 +151,7 @@ "filename": "gen3workflow/config-default.yaml", "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_verified": false, - "line_number": 30 + "line_number": 32 } ], "migrations/versions/e1886270d9d2_create_system_key_table.py": [ @@ -169,7 +169,7 @@ "filename": "tests/conftest.py", "hashed_secret": "0dd78d9147bb410f0cb0199c5037da36594f77d8", "is_verified": false, - "line_number": 188 + "line_number": 222 } ], "tests/migrations/test_migration_e1886270d9d2.py": [ @@ -180,7 +180,16 @@ "is_verified": false, "line_number": 24 } + ], + "tests/test-gen3workflow-config.yaml": [ + { + "type": "Secret Keyword", + "filename": "tests/test-gen3workflow-config.yaml", + "hashed_secret": "900a7331f7bf83bff0e1b2c77f471b4a5145da0f", + "is_verified": false, + "line_number": 5 + } ] }, - "generated_at": "2024-12-05T22:45:14Z" + "generated_at": "2024-12-09T23:30:01Z" } diff --git a/tests/conftest.py b/tests/conftest.py index 1e4a6f7..8266b68 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,9 +50,7 @@ "Contents": [ { "Key": "test-folder/test-file1.txt", - "LastModified": datetime( - 2024, 12, 9, 22, 32, 20, tzinfo=tzutc() - ), + "LastModified": datetime(2024, 12, 9, 22, 32, 20, tzinfo=tzutc()), "ETag": '"something"', "Size": 211, "StorageClass": "STANDARD", @@ -306,7 +304,9 @@ async def handle_request(request: Request): body=request.content.decode(), authorized=authorized, ) - elif url.startswith(f"https://gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}.s3.amazonaws.com"): + elif url.startswith( + f"https://gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}.s3.amazonaws.com" + ): mocked_response = httpx.Response( status_code=200, text=MOCKED_S3_RESPONSE_XML, @@ -336,6 +336,7 @@ async def handle_request(request: Request): if get_url: # for tests that need to hit the app URL directly host = "0.0.0.0" port = 8080 + def run_uvicorn(): uvicorn.run(app, host=host, port=port) diff --git a/tests/test_s3_endpoint.py b/tests/test_s3_endpoint.py index 6748e9f..bcf3177 100644 --- a/tests/test_s3_endpoint.py +++ b/tests/test_s3_endpoint.py @@ -44,7 +44,10 @@ def test_s3_endpoint_no_token(s3_client): @pytest.mark.parametrize("client", [{"get_url": True}], indirect=True) -@pytest.mark.parametrize("bucket_name", ["not-the-user-s-bucket", f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}-2"]) +@pytest.mark.parametrize( + "bucket_name", + ["not-the-user-s-bucket", f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}-2"], +) def test_s3_endpoint_wrong_bucket(s3_client, access_token_patcher, bucket_name): """ Hitting the `/s3` endpoint with a bucket that is not the bucket generated by gen3-workflow for From 7fd649ccca31a24b96136cebcec6803f2bd8996f Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:48:14 -0600 Subject: [PATCH 11/17] clean up --- gen3workflow/routes/s3.py | 6 ------ tests/test_s3_endpoint.py | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index 2ed1277..f71cd0c 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -1,7 +1,5 @@ from datetime import datetime, timezone import hashlib -import json -import os import urllib.parse import boto3 @@ -9,7 +7,6 @@ from fastapi.security import HTTPAuthorizationCredentials from botocore.credentials import Credentials import hmac -import httpx from starlette.datastructures import Headers from starlette.responses import Response from starlette.status import HTTP_401_UNAUTHORIZED @@ -19,9 +16,6 @@ from gen3workflow.config import config -# TODO Generate a presigned URL if the request is a GET request, see https://cdis.slack.com/archives/D01DMJWKVB5/p1733169741227879 - is that required? - - router = APIRouter(prefix="/s3") diff --git a/tests/test_s3_endpoint.py b/tests/test_s3_endpoint.py index bcf3177..ff815ca 100644 --- a/tests/test_s3_endpoint.py +++ b/tests/test_s3_endpoint.py @@ -15,8 +15,8 @@ def s3_client(client): session = boto3.session.Session() return session.client( service_name="s3", - aws_access_key_id="test-key-id", - aws_secret_access_key="test-key", + aws_access_key_id=config["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"], + aws_secret_access_key=config["S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY"], endpoint_url=f"{client}/s3", # no retries; only try the call once: config=Config(retries={"max_attempts": 0}), From 64a40489f7dd145cb90f37072b5ea2011b993f63 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:49:33 -0600 Subject: [PATCH 12/17] include all the original headers + add comments --- gen3workflow/routes/s3.py | 46 ++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index f71cd0c..f82f6f1 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -1,4 +1,3 @@ -from datetime import datetime, timezone import hashlib import urllib.parse @@ -82,6 +81,7 @@ async def s3_endpoint(path: str, request: Request): user_id = token_claims.get("sub") user_bucket = aws_utils.get_safe_name_from_user_id(user_id) + # ensure the user is making a call to their own bucket request_bucket = path.split("?")[0].split("/")[0] if request_bucket != user_bucket: err_msg = f"'{path}' not allowed. You can make calls to your personal bucket, '{user_bucket}'" @@ -101,26 +101,31 @@ async def s3_endpoint(path: str, request: Request): request_path = path.split(user_bucket)[1] api_endpoint = "/".join(request_path.split("/")[1:]) - # generate the request headers - # headers = dict(request.headers) - # headers.pop("authorization") - headers = {} - # TODO try again to include all the headers - # `x-amz-content-sha256` is sometimes set to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" in - # the original request, but i was not able to get the signing working when copying it. - # if "Content-Type" in request.headers: - # headers["content-type"] = request.headers["Content-Type"] - headers["host"] = f"{user_bucket}.s3.amazonaws.com" body = await request.body() body_hash = hashlib.sha256(body).hexdigest() + timestamp = request.headers["x-amz-date"] + date = timestamp[:8] # the date portion (YYYYMMDD) of the timestamp + region = config["USER_BUCKETS_REGION"] + service = "s3" + + # generate the request headers: + # - first, copy all the headers from the original request. + headers = dict(request.headers) + # - remove the `authorization` header: it contains a Gen3 token instead of an AWS IAM key. + # The new `authorization` header will be added _after_ generating the signature. + headers.pop("authorization") + # - overwrite the `x-amz-content-sha256` header value with the body hash. When this header is + # set to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" in the original request (payload sent over + # multiple chunks), we replace it with the body hash (because I couldn't get the signing to + # work for "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" - I believe it requires using the signature + # from the previous chunk). + # NOTE: This may cause issues when large files are _actually_ uploaded over multiple chunks. headers["x-amz-content-sha256"] = body_hash - # headers['x-amz-content-sha256'] = request.headers['x-amz-content-sha256'] - # if 'content-length' in request.headers: - # headers['content-length'] = request.headers['content-length'] - # if 'x-amz-decoded-content-length' in request.headers: - # headers['x-amz-decoded-content-length'] = request.headers['x-amz-decoded-content-length'] - timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - headers["x-amz-date"] = timestamp + # - remove the `content-md5` header: when the `x-amz-content-sha256` header is overwritten (see + # above), the original `content-md5` value becomes incorrect. It's not required in V4 signing. + headers.pop("content-md5", None) + # - replace the `host` header, since we are re-signing and sending to a different host. + headers["host"] = f"{user_bucket}.s3.amazonaws.com" # get AWS credentials from the configuration or the current assumed role session if config["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"]: @@ -128,7 +133,7 @@ async def s3_endpoint(path: str, request: Request): access_key=config["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"], secret_key=config["S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY"], ) - else: # running in k8s: get credentials from the assumed role + else: # assume the service is running in k8s: get credentials from the assumed role session = boto3.Session() credentials = session.get_credentials() assert credentials, "No AWS credentials found" @@ -157,9 +162,6 @@ async def s3_endpoint(path: str, request: Request): ) # construct the string to sign based on the canonical request - date = timestamp[:8] # the date portion (YYYYMMDD) of the timestamp - region = config["USER_BUCKETS_REGION"] - service = "s3" string_to_sign = ( f"AWS4-HMAC-SHA256\n" f"{timestamp}\n" From 1ea76185f4e5dc54650e23e6766867543cace73f Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:51:43 -0600 Subject: [PATCH 13/17] update docs --- docs/openapi.yaml | 249 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) diff --git a/docs/openapi.yaml b/docs/openapi.yaml index a5ef6ec..a717944 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -174,6 +174,255 @@ paths: summary: Cancel Task tags: - GA4GH TES + /s3/{path}: + delete: + description: 'Receive incoming S3 requests, re-sign them (AWS Signature Version + 4 algorithm) with the + + appropriate credentials to access the current user''s AWS S3 bucket, and forward + them to + + AWS S3.' + operationId: s3_endpoint_s3__path__patch + parameters: + - in: path + name: path + required: true + schema: + title: Path + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: S3 Endpoint + tags: + - S3 + get: + description: 'Receive incoming S3 requests, re-sign them (AWS Signature Version + 4 algorithm) with the + + appropriate credentials to access the current user''s AWS S3 bucket, and forward + them to + + AWS S3.' + operationId: s3_endpoint_s3__path__patch + parameters: + - in: path + name: path + required: true + schema: + title: Path + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: S3 Endpoint + tags: + - S3 + head: + description: 'Receive incoming S3 requests, re-sign them (AWS Signature Version + 4 algorithm) with the + + appropriate credentials to access the current user''s AWS S3 bucket, and forward + them to + + AWS S3.' + operationId: s3_endpoint_s3__path__patch + parameters: + - in: path + name: path + required: true + schema: + title: Path + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: S3 Endpoint + tags: + - S3 + options: + description: 'Receive incoming S3 requests, re-sign them (AWS Signature Version + 4 algorithm) with the + + appropriate credentials to access the current user''s AWS S3 bucket, and forward + them to + + AWS S3.' + operationId: s3_endpoint_s3__path__patch + parameters: + - in: path + name: path + required: true + schema: + title: Path + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: S3 Endpoint + tags: + - S3 + patch: + description: 'Receive incoming S3 requests, re-sign them (AWS Signature Version + 4 algorithm) with the + + appropriate credentials to access the current user''s AWS S3 bucket, and forward + them to + + AWS S3.' + operationId: s3_endpoint_s3__path__patch + parameters: + - in: path + name: path + required: true + schema: + title: Path + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: S3 Endpoint + tags: + - S3 + post: + description: 'Receive incoming S3 requests, re-sign them (AWS Signature Version + 4 algorithm) with the + + appropriate credentials to access the current user''s AWS S3 bucket, and forward + them to + + AWS S3.' + operationId: s3_endpoint_s3__path__patch + parameters: + - in: path + name: path + required: true + schema: + title: Path + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: S3 Endpoint + tags: + - S3 + put: + description: 'Receive incoming S3 requests, re-sign them (AWS Signature Version + 4 algorithm) with the + + appropriate credentials to access the current user''s AWS S3 bucket, and forward + them to + + AWS S3.' + operationId: s3_endpoint_s3__path__patch + parameters: + - in: path + name: path + required: true + schema: + title: Path + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: S3 Endpoint + tags: + - S3 + trace: + description: 'Receive incoming S3 requests, re-sign them (AWS Signature Version + 4 algorithm) with the + + appropriate credentials to access the current user''s AWS S3 bucket, and forward + them to + + AWS S3.' + operationId: s3_endpoint_s3__path__patch + parameters: + - in: path + name: path + required: true + schema: + title: Path + type: string + responses: + '200': + content: + application/json: + schema: {} + description: Successful Response + '422': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + description: Validation Error + summary: S3 Endpoint + tags: + - S3 /storage/credentials: get: operationId: get_user_keys_storage_credentials_get From 959f946e6381e7bfc3ee14ba4e0114fdd60e5077 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:59:11 -0600 Subject: [PATCH 14/17] minor updates --- tests/conftest.py | 5 ++++- tests/test_s3_endpoint.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8266b68..f6e112f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,7 +57,7 @@ "Owner": {"DisplayName": "something", "ID": "something"}, } ], - "Name": "gen3wf-localhost-64", + "Name": f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}", "Prefix": "test-folder/test-file1.txt", "MaxKeys": 250, "EncodingType": "url", @@ -288,6 +288,7 @@ async def handle_request(request: Request): parsed_url = urlparse(url) mocked_response = None if url.startswith(config["TES_SERVER_URL"]): + # mock calls to the TES server path = url[len(config["TES_SERVER_URL"]) :].split("?")[0].rstrip("/") mocked_response = mock_tes_server_request( method=request.method, @@ -297,6 +298,7 @@ async def handle_request(request: Request): status_code=tes_resp_code, ) elif url.startswith(config["ARBORIST_URL"]): + # mock calls to Arborist path = url[len(config["ARBORIST_URL"]) :].split("?")[0].rstrip("/") mocked_response = mock_arborist_request( method=request.method, @@ -307,6 +309,7 @@ async def handle_request(request: Request): elif url.startswith( f"https://gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}.s3.amazonaws.com" ): + # mock calls to AWS S3 mocked_response = httpx.Response( status_code=200, text=MOCKED_S3_RESPONSE_XML, diff --git a/tests/test_s3_endpoint.py b/tests/test_s3_endpoint.py index ff815ca..43657a2 100644 --- a/tests/test_s3_endpoint.py +++ b/tests/test_s3_endpoint.py @@ -18,7 +18,7 @@ def s3_client(client): aws_access_key_id=config["S3_ENDPOINTS_AWS_ACCESS_KEY_ID"], aws_secret_access_key=config["S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY"], endpoint_url=f"{client}/s3", - # no retries; only try the call once: + # no retries; only try each call once: config=Config(retries={"max_attempts": 0}), ) @@ -52,8 +52,8 @@ def test_s3_endpoint_wrong_bucket(s3_client, access_token_patcher, bucket_name): """ Hitting the `/s3` endpoint with a bucket that is not the bucket generated by gen3-workflow for the current user should result in a 401 Unauthorized error. - Specific edge case: if the user's bucket is "gen3wf--", a bucket name such as - "gen3wf---2" should not be allowed. + Specific edge case: if the user's bucket is "gen3wf--", a bucket name which + is a superstring of that, such as "gen3wf---2", should not be allowed. """ with pytest.raises(ClientError, match="Unauthorized"): s3_client.list_objects(Bucket=bucket_name) From 56994a8c27b404c0172631604d5e362526d0cb7e Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:21:23 -0600 Subject: [PATCH 15/17] fix infinite docs generation bug --- docs/openapi.yaml | 40 ++++++++++++++++++++-------------------- run.py | 26 ++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/docs/openapi.yaml b/docs/openapi.yaml index a717944..0e384f9 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -41,7 +41,7 @@ openapi: 3.1.0 paths: /: get: - operationId: get_status__get + operationId: get_status_2 responses: '200': content: @@ -55,7 +55,7 @@ paths: - System /_status: get: - operationId: get_status__status_get + operationId: get_status responses: '200': content: @@ -69,7 +69,7 @@ paths: - System /_version: get: - operationId: get_version__version_get + operationId: get_version responses: '200': content: @@ -83,7 +83,7 @@ paths: - System /ga4gh/tes/v1/service-info: get: - operationId: service_info_ga4gh_tes_v1_service_info_get + operationId: service_info responses: '200': content: @@ -95,7 +95,7 @@ paths: - GA4GH TES /ga4gh/tes/v1/tasks: get: - operationId: list_tasks_ga4gh_tes_v1_tasks_get + operationId: list_tasks responses: '200': content: @@ -108,7 +108,7 @@ paths: tags: - GA4GH TES post: - operationId: create_task_ga4gh_tes_v1_tasks_post + operationId: create_task responses: '200': content: @@ -122,7 +122,7 @@ paths: - GA4GH TES /ga4gh/tes/v1/tasks/{task_id}: get: - operationId: get_task_ga4gh_tes_v1_tasks__task_id__get + operationId: get_task parameters: - in: path name: task_id @@ -149,7 +149,7 @@ paths: - GA4GH TES /ga4gh/tes/v1/tasks/{task_id}:cancel: post: - operationId: cancel_task_ga4gh_tes_v1_tasks__task_id__cancel_post + operationId: cancel_task parameters: - in: path name: task_id @@ -183,7 +183,7 @@ paths: them to AWS S3.' - operationId: s3_endpoint_s3__path__patch + operationId: s3_endpoint parameters: - in: path name: path @@ -214,7 +214,7 @@ paths: them to AWS S3.' - operationId: s3_endpoint_s3__path__patch + operationId: s3_endpoint parameters: - in: path name: path @@ -245,7 +245,7 @@ paths: them to AWS S3.' - operationId: s3_endpoint_s3__path__patch + operationId: s3_endpoint parameters: - in: path name: path @@ -276,7 +276,7 @@ paths: them to AWS S3.' - operationId: s3_endpoint_s3__path__patch + operationId: s3_endpoint parameters: - in: path name: path @@ -307,7 +307,7 @@ paths: them to AWS S3.' - operationId: s3_endpoint_s3__path__patch + operationId: s3_endpoint parameters: - in: path name: path @@ -338,7 +338,7 @@ paths: them to AWS S3.' - operationId: s3_endpoint_s3__path__patch + operationId: s3_endpoint parameters: - in: path name: path @@ -369,7 +369,7 @@ paths: them to AWS S3.' - operationId: s3_endpoint_s3__path__patch + operationId: s3_endpoint parameters: - in: path name: path @@ -400,7 +400,7 @@ paths: them to AWS S3.' - operationId: s3_endpoint_s3__path__patch + operationId: s3_endpoint parameters: - in: path name: path @@ -425,7 +425,7 @@ paths: - S3 /storage/credentials: get: - operationId: get_user_keys_storage_credentials_get + operationId: get_user_keys responses: '200': content: @@ -438,7 +438,7 @@ paths: tags: - Storage post: - operationId: generate_user_key_storage_credentials_post + operationId: generate_user_key responses: '201': content: @@ -452,7 +452,7 @@ paths: - Storage /storage/credentials/{key_id}: delete: - operationId: delete_user_key_storage_credentials__key_id__delete + operationId: delete_user_key parameters: - in: path name: key_id @@ -476,7 +476,7 @@ paths: - Storage /storage/info: get: - operationId: get_storage_info_storage_info_get + operationId: get_storage_info responses: '200': content: diff --git a/run.py b/run.py index c9fa102..778e903 100644 --- a/run.py +++ b/run.py @@ -6,6 +6,8 @@ import os import sys + +from fastapi.routing import APIRoute import uvicorn import yaml @@ -15,9 +17,29 @@ CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +def overwrite_openapi_operation_ids(app) -> None: + """ + The default operation ID format is `__`. + A bug is causing the operation IDs for the `/s3` endpoint, which accepts all methods, to not + be generated properly. This ensures unique operation IDs are generated for all routes. + """ + existing_routes = set() + for route in app.routes: + if not isinstance(route, APIRoute): + continue + route.operation_id = route.name + i = 2 + while route.operation_id in existing_routes: + route.operation_id = f"{route.name}_{i}" + i += 1 + existing_routes.add(route.operation_id) + + if __name__ == "__main__": - if sys.argv[-1] == "openapi": - schema = get_app().openapi() + if sys.argv[-1] == "openapi": # generate openapi docs + app = get_app() + overwrite_openapi_operation_ids(app) + schema = app.openapi() path = os.path.join(CURRENT_DIR, "docs/openapi.yaml") yaml.Dumper.ignore_aliases = lambda *args: True with open(path, "w+") as f: From 303c570d3f797310f8bc96eef4b09681eb9fdf8a Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:43:17 -0600 Subject: [PATCH 16/17] add docs about s3 endpoint --- .secrets.baseline | 20 ++++++++++- README.md | 1 + docs/local_installation.md | 27 +++++++++++--- docs/s3.md | 72 +++++++++++++++++++++++++++++++++++++ docs/s3.png | Bin 0 -> 30098 bytes 5 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 docs/s3.md create mode 100644 docs/s3.png diff --git a/.secrets.baseline b/.secrets.baseline index 63e8630..5548e9d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -145,6 +145,24 @@ "line_number": 64 } ], + "docs/local_installation.md": [ + { + "type": "Secret Keyword", + "filename": "docs/local_installation.md", + "hashed_secret": "08d2e98e6754af941484848930ccbaddfefe13d6", + "is_verified": false, + "line_number": 94 + } + ], + "docs/s3.md": [ + { + "type": "Secret Keyword", + "filename": "docs/s3.md", + "hashed_secret": "08d2e98e6754af941484848930ccbaddfefe13d6", + "is_verified": false, + "line_number": 56 + } + ], "gen3workflow/config-default.yaml": [ { "type": "Secret Keyword", @@ -191,5 +209,5 @@ } ] }, - "generated_at": "2024-12-09T23:30:01Z" + "generated_at": "2024-12-12T23:42:54Z" } diff --git a/README.md b/README.md index 8b48b55..8eab625 100644 --- a/README.md +++ b/README.md @@ -13,3 +13,4 @@ The documentation can be browsed in the [docs](docs) folder, and key documents a * [Detailed API Documentation](http://petstore.swagger.io/?url=https://raw.githubusercontent.com/uc-cdis/gen3-workflow/master/docs/openapi.yaml) * [Local installation](docs/local_installation.md) * [Authorization](docs/authorization.md) +* [S3 interaction](docs/s3.md) diff --git a/docs/local_installation.md b/docs/local_installation.md index cbf69b7..e95b622 100644 --- a/docs/local_installation.md +++ b/docs/local_installation.md @@ -75,7 +75,8 @@ Try out the API at or `http://localhost:8080` is where Gen3Workflow runs by default when started with `python run.py`. +> The Gen3Workflow URL should be set to `http://localhost:8080` in this case; this is where the service runs by default when started with `python run.py`. + +- Run a workflow: -Run a workflow: +When setting your token manually: ``` +export GEN3_TOKEN= nextflow run hello ``` +Or, with the [Gen3 Python SDK](https://github.com/uc-cdis/gen3sdk-python) configured with an API key: +``` +gen3 run nextflow run hello +``` ## AWS access diff --git a/docs/s3.md b/docs/s3.md new file mode 100644 index 0000000..c15ba4f --- /dev/null +++ b/docs/s3.md @@ -0,0 +1,72 @@ +# S3 interaction + +Note: This discussion can apply to many use cases, but it is written with a specific use case in mind: using the Gen3Workflow service to run Nextflow workflows. + +Contents: +- [Using IAM keys](#using-iam-keys) +- [Using a custom S3 endpoint](#using-a-custom-s3-endpoint) +- [Diagram](#diagram) + +## Using IAM keys + +We initially considered generating IAM keys for users to upload their input files to S3, retrieve their output files and store Nextflow intermediary files. Users would configure Nextflow with the generated IAM key ID and secret: + +``` +plugins { + id 'nf-ga4gh' +} +process { + executor = 'tes' + container = 'quay.io/nextflow/bash' +} +tes { + endpoint = '/ga4gh/tes' + oauthToken = "${GEN3_TOKEN}" +} +aws { + accessKey = "${AWS_KEY_ID}" + secretKey = "${AWS_KEY_SECRET}" + region = 'us-east-1' +} +workDir = '' +``` + +Plain-text AWS IAM keys in users' hands causes security concerns. It creates a difficult path for auditing and traceability. The ability to easily see the secrets in plain-text is also a concern. + +## Using a custom S3 endpoint + +The `/s3` endpoint was implemented to avoid using IAM keys. This endpoint receives S3 requests, re-signs them with internal credentials, and forwards them to AWS S3. Users provide their Gen3 token as the “access key ID”, which is used to verify they have the appropriate access. This key is then overwritten with internal credentials that actually have access to AWS S3. + +Nextflow supports S3-compatible storage through the `aws.client.s3PathStyleAccess` and `aws.client.endpoint` settings, this allows users to point Nextflow to our custom S3 API: + +``` +plugins { + id 'nf-ga4gh' +} +process { + executor = 'tes' + container = 'quay.io/nextflow/bash' +} +tes { + endpoint = '/ga4gh/tes' + oauthToken = "${GEN3_TOKEN}" +} +aws { + accessKey = "${GEN3_TOKEN}" + secretKey = 'N/A' + region = 'us-east-1' + client { + s3PathStyleAccess = true + endpoint = '/s3' + } +} +workDir = '' +``` + +Notes: +- We have to set the Gen3 token as the “key ID”, not the “key secret”, in order to extract it from the request. The “key secret” is hashed and cannot be extracted. +- When an `aws.accessKey` value is provided, the Nextflow configuration requires the `aws.secretKey` value to be provided as well. Users can set it to something like "N/A". + +## Diagram + +![s3 interaction diagram](s3.png) diff --git a/docs/s3.png b/docs/s3.png new file mode 100644 index 0000000000000000000000000000000000000000..662790079832937cb99879be80eed04bfd166a0f GIT binary patch literal 30098 zcmc$_WmHt*8Ze5W!T{1KNJux*(xG&BcPKeX4=saqcT0CjH#n4%(# zpt6uvS3*Jp(IX)RfRT`{9)<#TkdWLsk&yOGk&pyak&uXinXPKVNJvkHR24Pf-{0S# zot^g$47Rj&ZEf#d-`qsUrwU5QE2$phFI9T$ZHr2y;rB<5akwCFtTz4`3F0? zfR>h_zYmV$Q*+igw#Fx?50Bv2*Vo?~TKOeaojd}Kteta*aT!mWVIN1By$SN_#{+z z_I^8ieTKr;KR9`)=~+lB>YkmQH8nMDZf}X>NT}=h|8$FQ_!$= za(}O=tE?lVq#YHPY^SQW_Mq=4<_a?J zkV2oml709A0LtpQAR&=tJpMhAa|?Pj5>!_?rT3^in3%*AL;3dMCp>;oKLUC-HIC_rMVV{5>GSR7=)PaNc@ zl2j~0a=OOsTGN$%jrK#L#`^BJqGK+1wR8=G>i*Ny1`X>qx~xPIedgc(|9`kCRx#+( zr1&2x_SR}kFw!)XLyu!_Mxh9l9m?CR`?(`+)*(WL($C)8w8=^6YmD=^XT*M|QZiy! zeclE?1@ow)ETl81X$WnEph|vn3AgJxH&dKz@5g5$as#xFNZ?n?59dcf+x?s5yiCex-xWbe;j% z$)Ed4w|{FeH|RhtdudU7#;o*ag{bCtg6bmlW+Q~suNH+nd0(uhsj{MHaQb}GR3zOU z?S7il??O-I!cRg~-Se4d|EGRekAnpGqYKmSRTr0?+24MS z`CY)a&l{87WEBPMbQW9I7`(%QeYZZUsj4rH+!=%dYz}Nw;2qf))Yx}>w!5c=!gu8xyI=-pmwvWKxcF%I zmm))O4FECf|x_Bxe9Kw0ZQ!iN8Z{TcD;voO*(B1uJ~Y%)DmyBMsGtKaTUZ3-c2w-+Cp3>WGyZl zS-AuSRgMh_P0Og}a&~$ReSM-S!`p86CjZJ}k+Rd{^UtxvdCE>DPo^P5M$#4mj+X(? z2%HUm;KUC5*z+EdPsq`PTC%QEVWv>gu98FU2c4hAV%ikS)=z{n>Fq7kq}BEP6RY3Y z%IB+Xk5`k8@0+W?v(@;pm0q}82qO!;h{DMJ!B~R5TcGv>kB>X-WbOFFEMWBjXH!vgcP?<@7i}ELAt1X zEADh5|ECB8#CPOv^5HuHN1IQy4!}p5___>%B33WryHJjc%?G@P$X#z7mrM?(f z)$2dwq9$vB=iZ7OP%rZsT4B*cEb3@@z_ZW`-PXz3Xs;)c;*Xd+-Wa=E)ZPw`9SYYe z{uEr&Gz-@g<9IPuYfNOC06&YRe-TLNpeG(?;)rL=1Ku~!FuYHP#{+DlwA)(NbiUTF zxKH^X44g+#Ci9ONsbM&O5z+HAK)5%eRb#)AgmBBrG}rDIrbMM=r#rPParlx`W2HSx z>eDm94Y8Z`SK0oXInDkRTGjom)RwnAoY2>GQ+Y(I5Uf`ap%4;kSgD^z0WVBbt(f3L zo1+$sq5>Jk5Sv2vD2?T*-PW65o<*wiH)L(Hq57u`dD zg`c-E6!+0c9I8@R_bkPP1iXI}`=3ccKQrbN;*Wwnd0ReT{fO=dUXI^hkFZ~^Kd3gp ze3Y=60v3Zh%zAklbFCxtve3u52G_wpj_gtKidEvozPj^~R<|mY&yEN19ztN!C}GbN zo8E&5h>!ozg^M=3-(q0Tv8LhwA{ARJmGCk;Hl$w+t@vViN0K`t-%_RMWKI-1+4*?s{KgM1s+|dM;ibrWp zH=*74os3!<$vo~*yjXvX+vsTb$10-}F=@b|e}r9p5zx)hdK4c-t$cegJ73#KWl7UQ zm&~jJ4877@DA?a-$@PN~k4?Q&fKw&LwH5-|KJ61zzhZ##`zQ15KEJe0YHd=^+Ce5WXY<1K)`xe{Fn8_uby#K@EBeuGTTE!qCW+g@<1qE)QL%F?-tNkE2qlr& zklNnhrEO#jB<1`#h306b>!zmjp0&8{GRmQIwys`oc0|X5*0A@yrE2i#b_iRcq{5-c zdBFh~gu;=`g%v|cbYkmnmvMB;>M-GUGR+9+N*HO`JRihir^7uQs--jh+596Z(l*6Y z8&L#Ag|KxMH~o58M{)%3ug%A6_WxYF0)XS2R+(ZbY(I@zq&D0QjANo>&;MoDafjY<+J|xDg%WAA zkvSF$|Ebd;$w9H2-ECmHbnZ%S4srN)xd66EQ{df{oB&$Qs<<*mbvD5GPL2hPL^0+v z2A9qe&O%_a@g_ektj zA1_+hPScD#!cLeFH)4q%*Oy@x$+a~%@k>ge_AdFK2|4zu6O1|~us`cJ(df-{%*Gi? zd@3Kk%CO(W6{RL!Xp~lWWV0e9>#zLKA-P2dsPoTZfr+_Cp5j$9r$se7s-#$-qVcdI zWe{*C+MABDl`U0L8EoHLkb$~LyS^reYwsxv)i&sYtrxs#e!>Ruuu3!|^GtzjuF(d^ zw!7L}j+r^l5DcNBVM}rcmL^&^zQ(&#V>8_^Kcou!L)PJHqc*LyL;3vdcxBQ#6wuVS zWq?QVj?2vlJ~f#Pq%2jTxnZH9^mP{yVD=r*iUVRL!Qkv)``wq>u=KSxD(t#cw6zet zdaTggU&d9XY`(1(TM4{ZdnbD@$S?L=_bi6}*IQ97+Iz;t1+;5&7cl+n8YzqiOA;w= z+}_a+r7molVhD|**f=SWkrQ@P7sel?HpbZAzSYwnvU-+uoN{i};Hjs>vP^lW&&B}1 zA3`=k!6H1h==iVobvTCSDwG)fJSgkx&GrU~y@55(*^hTHTf)4@lSO80ob)@45h1#+Do6D6 zOGFtCtvYkq6b`}1bw4|EU*?belYydutViIY9aKDOe9L(`=*@~5y-((uNcexJF9Z&} z2{H!Uu=_nj(%{Yu=Zt(Y`;YEx(rs;m1|?+pa@oGtkxu#YiD=q>n90%Ax@)J+jLvN5 z)vB@5w#EQn)SLULzZ=fyGg++YwJi58ROFotQaDs@4s_aewA@UKqbpC1efMJOeDNh) z&t8#-^Vxmott8qHbk10T8e4PUnx|H+h^~ZWxoJMAE`FaCqO0QMJSF^Q= z-+o9-U&y>TqXU?%p4L7|uPC?esBkSnOLqa~26Fyy50&BGv@;gop>3J7WC%AGWPde` zn9gUXRru-2d4oqYrHZEdR7UmOebSL^OIDD7k)4 z{Y>xEP?8)mZF?-(Q@`!R;fgkQ-V*SR-ic$R+|o!*usVmu1`%vAE5<>P= zH}`c{uV+)X>^RC1t@>kczD$6=(Ml6*AsTu~!c=4ZzgD*!ava|PGNo-gsX;l@6W`NT zx-8U2;QLARK8@gnas&Z}gCabt{a};0z!dFX5@N<+=ojr>wux%&(gDUUFH{bD>aN`A z*J~wR7yahmNMG>+CnM<;Uyl0?5A500(EF-XwsMLp&J1YhYlXiikH^Ko;_hiH0RM3fq&x4VG%=@T?0- zp;dfN0C|^C!X9?{Gr!yLLJ=I%zNWqne5_RV6nsx6*5i#nf~*7jy&hx@<57l-#gIXPUt|3oF@@biSitZ+;u}Jp9a5k)*!-b@EVj-*FG;DOX zRm8=yvD+jEX;g!;MGR@c|NU+};VWd>NLdehrLRPhWmnRLf6OD`{DL1Pz3BxHNV8@> z-Gm1&CKu^pi9}%ZeI&QIYpYL%Q}fZl*xx^6fCTF;=Tqg0=v}o2K&cIX3fg}T;0K4; zsD4?5qRfEbb3oKq9;5c@An1?%|t@x0>wF>*{9aQE0r4>v=&&{Y422WcN z%F0s^_NIm$t3DvWfIdsA%XtAKu?DoCHx-_=fF+IX@s*#f z;u#6lZ(9Vx5skY=Dl1;3P6~_GWVXp~zaP?OJC!*uHNYr@31FJ*^bjq4NND=9;m6mB z2RO#(-(cn4!9j+y)K6nNe(OzL=7BlNTU+*?mGpp`t?gU-k!j12dmIqp?C)q+m);o{i`xQ1Y`dfy75ULjcGin1Ta6B0Eu9fY zd8ZjOwodXNeN2u>*d=&%&f4upjvez=K*pAT^Z;@A-AJP=Ya|_x8Bn;b^<(ZlgR@5! zz)yRQbC`C5VQ=K3aq;`-l!G~u@tv`+xwCLJvj(klEZYs3%?wP}Cy=ELKY!IkPlDwZ zHSJ=nosoCl$;Im_U=h~fg^lrM!PC#@Q# zJS*Df>YK8Mc))ug`UuUfo(BbfrMmGks0xE4)-05ipo9${$5xmblLKM=0kz0?DbrUj zEfv-ppJK)r{qlJ+>1&?Zu#>aSeG?90uB}tLAT!)~b`II!*nGC~WSW7t{P>WO$kD%JY!+VC~C;ph=HvW2V^=(ndy~X?42R~)wP@qktp~XHQ z@5d2bxw5osGZc5WOu*SkDX+`iww&UC6O8FR24JLdg5~=cD>?A&Ir^ovE={{XrDC)2 zBH#*@greE;f>bm2vp*lZl_+7O1H>=1`g%Y7oM^&$p?E(k!dqETC|rpHr8VS)o$VsQ zkV>mSDYd6_tM^iZk8p?Dw9^b)*bM)d*xeMF$;Gxm0Ll{btZ z2)Bx$U@Y2Mmf7OM3jqE^IZw}Qr>dHg0QqTkscdrD*@Ulsdl+>BEn*kc$4@x(a~)T$ zU^B>`Q5(bJBuqz7YgQ4;u7dK|p1FFnQL zC=F4SYSWlFOa_+a^`xkoE6+$MS_!~gp%ZymelKV1kDf1BxO~xtOyhH9H4GR zMFb?dYspU*Vc5v)r}h`Cp5_^Mlit(KQZX{x^pQpl#g<{&`LA!EF}A$cDZ=R0x|${< zfR}M_YJlaTSCY9h(rvr3id}GgpW$4;dx)J-zDNffo#Mr2!+>>g1iz*HOaAiiq_j}m z0Qsi~1%qO$#g|&*P#Z2Try#}aPe1hKf0Xe|f{O*8F_S#L6RhGjKSNerCq>a$K-Q}| zk^HFb!-gXKDny;h!i4rFK-;(YPwzB1#B~?PL@?NoFwwVCNU>Rj&*Y|6;JVlP+VQJj zhFcvPSDC9!(Ov8sC3B5}hG{(Nj29V;aDM_a6rkG3SN&Nv#xWk`z+i=TfM?t^k|z^O z6BbsWBk>E%k!Sm210AoNw5S9s-W-KW^nnVu3d5Vxxwg98i|hmS6(xn2;NcejR%pHef^=Ut=N zRd;q@Llx#XP$5kNxMH2}0spFcOv5)G$%U}`CMw&tk+EE*rmp#%hm*JU;!)DUS#Dz+ z;~Adv{TAqlWE)b(m<%CJ{ zMs!g= z*8aU0_6ZS+DA_w_wb$xralRK6hVm9a2>**B@l4BK>W<8K*?Y-D)B0JH8a-LF-Zf=~H3D5ZX!&lwKR6)Urgh~j3{)RwsN zqYd1#rL>L9Pgz2W$r3U5^cH*yURbZ-eo!1H zxQ@L91-vd`PFno3AIZBTkp0FlzmTm5H=L})iihWgf{{u3g13AMI*LsM&=Put=cvDW z$~gHh#vNeUH1*%RsnWF!!2A~(@UQ|dj=bHmQUk;E1ZzG&Hxof9e%fjr;9iWipX+HK5& z%8kxf9?Ws72A89iP-(DJ?<$6Y{zfxHB_iIfWq-w>aWUj2pDkfxm|>U=Jq6B< zcfLiuMQHzGe8z1T@P$V%%!oF3cYTh##fj(9kSO~8z5Kz}!9%j97Xf+H!r|ep(M(8!ePR!N=tI+&8;VN&=ej(uiAZ@ZWfD}^t^UHIbR{344NY`GR zI%(3iLkf(AkN-C58S;@< z^A~lyG{ya~WLcseq;z_@@JWwMh|Zi}wSCi(YX&#$h0+gOug^9y#uHnDzg0M3j*7U? z*4SanXuLoj{?4e2M=F2vp{n{0YRQl|*Ife=3F0)%NCems%3PBZqb;b=nVwB_)Rsb` zx;w6=$v8I+DbH6{A`{e|(!P&4EHj)N3-?B~=5=GSC`|)gRN(hN4k5;`Y5>vw1J*skV;u*08L55BImD zKe%!IeDxYsZx|TwHnr5q)}bIkG)fnSs69R9~iG|Dmpzm1awNKKczcR znXn&wpXE5!m~i;r?Wm?wP}!3<;#~;x9JO#@lSm@q%|;aFpIB=0kniM(6ZTlC8(PJTYi z#Guk30`cv(nRM z*TLsiUzg$td6<{axfvXu=Mke2)%^8VXc%7#Te}e?10+D00%ykVvnJFL8>EWHAE@y8 zI?MWtbuU{jB$-+caY&G1?bCs#=PTmVlBlMq4tgs=#$)$wgZ`^#qB>OZcj_%kWdA`M zqmgBG#wZxty%oeptEALbU7zNpZ<9Yfrs9$mM=s#spWGK4e zXx4eQ>r^WhB*Bbzz684>TIsD*X*FwJ6(5AEMLEcmk(e4)ALXH@a9TsEom{xSW{uoE zIts%=dkFlKo%55~9yd9mbzCFN(h{=n#K>pzyA2vYf+vHSq63g>2@lxaJWoaqUL$|i z*o8sRtNe&lZ(#I5pYd0Uu>C?b1&g|!j*>~EjUBKza9PN{lHI}OOB4gj7sY>G#*@(& zys`pnR?~L%<%|*hfzf2JR=yP_G4dgBSH6_)bSF&u*;k_!GjkQyQ zIU)B?ds_xrHwBlyLcC@+ZdAkY!-`Y8YX`x{6nH9{j&CU!>}>IG2JggoSXnRh0F1RS z)G0SHJO`nf&Q%&G+30}O^$E1Ea`VN3|>%}-5=Wrd0OJ( zD$hA+*!ey6-leIT|4S6Zf^YylPtStNu?BqtG%jKIRn;PicKP&m+Rl+3(MQZ7YXx0a z7azc?`oP}9iSTrnF zX~@ccD>EJ_MqD;ONUMxCG~Q&mnz9s-KH1*iBzPL^x=f4H*^?qe|1m5f9T<4P%hdQ= z4O9z6!8KEh?$~6~JclO*5-Yj0?>RfA-Tn`OP`MA{EO{IvJuxgiUY;JysaVcD@_}%U zoo4y#vRkTB+gtV`yde;hLQ^*8KrFOUhJO~^c4f;H6`ZT#{GE38DOQByw#U)?o?91{ zRG@)<<74`Ia0n0Es?sv-{ZWcS>caYhCg#>8pjeC*0SMIXiFW5Lp=5rrwrrM=iCz6! zH)L~3e=$RwrFj1%3Qvw6Zj$QnHfo>eo+5^W!>p-^U^Rk0Vwth`K`1Ef{|7@UUpbBQ z$zcV1oeYZSHtO?-cc%ZGm^w8a-1DEHyf`Y>Ks(+=KGqpR~4Ipeh<$j4|B%+4-_?fioXSI zYskRI|7EXXaLnZYQe`TDviN@~h!bB_`F~mQ|7%H<8@675^#6*;haPOoxnW(HgBc>)@!{&oO90vi1{_D z@|2s64S4@MA?+P<>u9(PyBdKC@7FK6bUFQL_}^sq0hYpI#!zSOf8oRJJp|;A9KIv| z2l!!$jmL-JuB848)`1U?8M|)TY;0{c3p!2ma5LrbAZO1;q2JM8`+gh#k9eE59>Xt- zg?~wY0GYZEbN?9mSVN#{S{-lxd59Iz>yp0AlNjz*!VR`4{RIWQv0FoBug7N*} zQ}70eaTFJ19~G>9b_hy?qnKpsSm31a^h$1~csu6DIxIyFOrU^mHX2fmw>_mJWn~)jnMYt5;PmK237a<(NuQ;c{$7ezdWhLxy`Zba(nC z?Ro262}n{bu38bmgD`QA3)yl>7Iby3yXx^r*>`3E{A2tcvGSgtD1DydMVu|1--%yjS?|HF@S|>? z4Sf<|R7U)?$ayyiUSwWm6;bq5wfz$cEFqD}{k7RDqvcA)kMi7#`mR6~F3O1jmc$Q9 zsm@8EU9ohQ*=Sa&y?|;!+;eWV7lM?I8{8baAnQf zVd^L}_k_1q^yF1!o2Ed;fdN-J&?YC$BH^!Q9%Ptg_L>~s^(ze{CMSB_Ie9S?d$2__ zixt|%HJxde0=Y5i5Ze5x)v|5!55__r>1Z9CKzuX}dJ8kkUkdN?6~R4pzRQiBQq8@& z*Gz9wB^0wi^P%3iu99}M&_=<%3yr>{4ljv`*|9#{JSI6LQ&2AfUW|1I9}gQbKMz!9 zv`V^@%K*E_>p_OO$!S&4Dy+wB7-taYBZF!;f6A!6eKmZ8YDz zu~$FB@}i;tm*$Aw?9xZHVjZa!speyJWGKEQsY(}pdr{h*)|Kk`41qM;KRb8>af>1g zJGG*L)WuqV9^`{T0+L|SI9SXDITEFrgVvIX_W3SJ&)tJ+q4rJSS!Ia{DbJ>F%8t0o zRRQZeI?-OGnL?lY`5rFH2Kf|X@RWmD*5b_YwaF}LL-$tx7(oi(%Y z+yqcjl;I299Gsg*cs1Ey4{AGLM0Y9uau)WPj3r0fy`weiHdxyj?os z30HL|y&|FM4lG!e`_-}o$Fd#yRV;wA#eUsdI2nsdn9*&T>?_feCr9bj91ms4Jmi*1 zgCff`!N6aJ$lDi2^h1qH(j*u$XdLiCo|XPLj=uWtLvWG=wyrZBP%0c5POfKT0`g;W5zWM-tCICmlQuQO%ZBb6GR zxhWJvaImZzzSH#Qeh51oKaKzcIg;etz(S5_!pTKHqW7M+MIr2jWtpb}=_ltfHJ25d z6A0fJ>iFXq2nevLzJj30vN3Zp_B$urMfxKsD2EVpm^D`am2mM^Vy-jcb8_NSM~d18 zzs+V;e4(+`xG+GIR->L^c;;&Y!5ayItN8OzFZ*2DzW$YC^y}iM-y+}Z^F3kGUH5b( zPWUvGh@m+GsIS{L4-bTd{C>_Cd>E z$@w;B{$T_)Zd;S_uYWt8|1_AtGoPLD7ofscVhF%R0QsL`Vs8!wifQ|)I=~w6XGWH4|9muE|cR0fdbRg3CbGk zs4>yQY2{Hkk#JRWB1Uq^~F85Vrn zI0TQtp)eKm#resFc;9@^$nkTcWa+Zi#wgeV+&1%K+w5=5#TsRj*vD!UDcpJvi=ts) z0Bj*HGr%?-!~#Y2z2#n6eTw{B)gBrn9s>(*01rE%Hq>UJ#ZJv+UYds@=0oH{-6Cj; zjmxGy+-kOFWV@ERVD9iUU61Y&rBmQ0s-kWep#)%?kGzU48&(H5lN`^e>YCLbXxhlf zMCfGY*1NX0x%diy*mrwpbaTavA9{9$b#|KY#1E}_UrWQ6N41GPk&E)~dF^(KNaVj@ zGQ^Y?3I6IPomDWL@EYRQBLu$gMz1OH|B5*|fe9=J9A#nyt-cv|&?dew+b#f*CS@;# zlNLtuVZf-W_h&4@mw1ygp`NXN2N_blX4g#xlFYd}I-v#fA9+8$UP8(q6kz*6loc~X zK-{l}*8MKS(CwdOYck_4$^~6$ni#RtP%doTa)DHz(^vLO}|J^zuW z+v}!4W>m)g%^?DA$$QSc(mBI~PU4fRqV6B?w3Wr7=5+o|wn9xkyT3E!kF3_uTd zqS9bPO2bL5to@T~+Y*xd!Tm#T#2;?Zyr}^Q=i+-m!iWQ-JsX$f+o>V>Z*+#jDo2Td z{fy`0_6oE4xhr z2~48N&z*ulFvo%3MQdbxTURhkCDY^LUjJpL#zYZC_+$^lXeeNQn2Mm_qtMAa;$ne( znmJLEG2vrGG!*sz)1S{AcujJ&buQ|I zC1@&|w8~;b{4eOMK5R5DUVsf%2w#h&f^Fm9N+hII??vX!zumocIewe5->c1GU2alm zd&pYW;=&fiTwL=qYOf`jOKe?4FMnRyEZ(o}Ac30*hBWkB9M1+RU*^+bR%}{lB3Ru?fE}8DWoN*_ zt`1^Xz)sc<6l%H;5c}O{oy;pC7AgJCv&a&>j#2xX9xABSsftc!j`v>b%fo@htAbG= zb#M}S;B^T^M7p_Hh!17lhcHx9t$~eg?U&1g%HC7A=O%UcwO;85c;6l)zN~g|Xj5|B zk0SFEW8BWCjS`vURKsB|PcneNfA2TY-4yD>#E$BcPFlsCOtm_u!p3YkI5belJTF1s zS*~a%Y}R$toBQj~QKk@sJVA?r#f^iC`DlfF8ewt&^yGSi+C;A2#^Km-6GoV?-j~8D zDT;63HDTXixDcb3Y5epgJob20Wyzm?Y_POQ2rzAT|%JE)_Ex@cDw=8l6=e)G}HiqnNH9&}v$mqaI^5=nde9 zB!Jz9fDDvqtJyr3O}S=&Gz9Wy)K+e(xgUz!>;As7DJ5bUOf0 zMfCE6ySmuQ%RqW*Q@`+kU44c47+c(Eb?}2VDsRzWrfLA^tVQk@*epwBJlDRE8b3Gb zU$BAf&X=|fAePwE?_a4klyZx;#VsBno4pv{LE{)9Szp48XH;Mu2&(<+nnVY6Mq#`oKKs%Dr%LOuH^+bAt1Bf~T~Dy}P`6M=~7eu3+MR1(g@DFvUD_hBN%o zxg#wea4~B8=~{D2L@{M6W47k>hTEC)&r~B8TPyz@lszZcZlC-n``UJ&rukoK3?o;tgB!aJ z;oReiU2U1@@67SnL(~!4{O{*?->r>i_;y zz*}>yYlYmSs@UEgZ6JP8xD&JsF$347_|;0w2#n(YDZv=BjEns-pKE;NdRqFe(ij^B%v@?i{(m-I>Z}ynMzWoBA?ZFyoA8I$7bh0 zc^zXNKP0u^8UC;KKDW`4cm1mJ;T{5!z(&>p!lKqC4-{$!E0nsbioRaTm*zbXEdCZ5 z-W|l19~}oc1%)h3F#Xn%;7-wZWLmM2{+Xl-)lpk7v?%v}ODjtvXCdTYK2!H`GFqCQ zw=o0;Y~VM&@NK7RnPaPwX>C1&jJcmy?2i| z6l6~kYy9_MkmvTtv=)OY`ZB=4ttN1vL7Nh)aB+vWU6K2e5Z3S+)}|a$k}I?Mv?A=Z%F!>Q6_{ZtV2d3Y;dBvomCabBBX?7W662g5bKK zCn+<~DMo&?!uQ?z_x1}DW*Ly;mger}vWOc|2vhUbp}ACr#zW)%mybYY z=ipi8H9jlgY71x!AT43LW!{z*xNU6-*}6GWED)~W%y{U^?{Q7b z5O~OmUv{;@*U-Z~tj?TX)hzXp3rB()?y-jtJbGM}mEZcR>eIimTnjtczUxOKv7K=_ z9Zqt}KcrNA9phfAaDKD30p|?&VPJ>O6l7fa+|P+!%zi9d`KH2+XJt#QX~dCa=f=X+JH%<<>6-Xz0Z-VQGw`p-Ne0_2-MXa;4u< z45L5Pre}#&yNvtq*&h2*??>;kLWEdO!yE}lOxJS`4tmC0-9rTIO?MiPLJrRdmOgjN z9n?(IkNdRQgucIZkZR_%V?Icoe%t(3{QbA31r-_rkFg`X6{W6u5D7#<&-%OE30R{m zo2#?3=Ds79ceyp?S6P{8)-3n7TXZ7)3ujm2v(160G>Dx)Woe>VB$x7E5C${L#+lv1 zbv4F(KVqdK63ReU;&bj9h@bm&83@FrE>W{l%HAS2AqTWdg<>}aq%RGr7)R1 z>+al{PC&N~1rGb9Y2WsDwmPcHspbC;*;BEp<2%1$(0dLf7Vo0iFC-H~* zzt0e&&za2ZC&);Ssc-<*gY@*&q`l;d+^dezbJO*(8g3oiLIJ1 z%Su&ceT`I_ht9{G?4K|$2t$MI(obP!uczMX{1862jWJ`PnAt%aQx9z7KUpojIUuqAhfx2+Lo)K7TqVEcBt4Mgiz zu&s-)s~Uc%nLh)p9&0bsX;3$9kdtBy; zM`XYB9n&XW$$5Tkh36 zS!TmeELgUv@ZO9QW#>^b=Q261Tht4it#)>1k++uGBZ^4GHfkN3N!*u)+S7mPNPmdt z*KG@KOxi}_X06_MA;N@=U4e~y@9{OQNjwOZ4;2ktO4y|Ys^GgH#b0;aot^EqueQAm zX$H)KsYO@Gg)7@cTPk|$u2c_8zh2fzH5>m5i0KT%c>_p(ZKwZ*@|;k!hb6dY zkt^Uz$?-=ae+AMFvOW}@=4smN#&-iR4D4S*5JaZTOHVz}4D;FPijL3Brub71b)=eC zAcU2tSgn)A?020d##xuGT(}FaK)y^OEa)x?2R;0a`28ii8=@L8f;yY79;uy)j&7n< zwrT%I8y7acNPw66kY28JFe#oeboR{7;3_*WU}~V#Y)&AXA8d41x^)?r8ckZiZjOa( zZK&5Qj$Uny3=t)SW8+jQ8d72Q5JG4;z)K(4SQg~A*~}{ z=<>Ekh=;iFbNkzl6{ykd2-MibYyEDVV~&{8)e_t$Cw-FA3LdCOcED_U&v!22z5mQ- zg__(1LY-NR5@))KmY$zkxWWEK1l(pF;v(;pqJKh9B_(x{DcNMk6QW1vup5lb)3I8P z6wPyBa|DtDhlSyC=ljYEWS@P6i0=GS#z3celj~Kg(}I8wF{!SPr@dl;X0!sKSBqQh z`}RgEPf=%$t>ZiCbQXpsk8rXh=o=^?4J-=yT1m`=J>DooQsDL%Rx#(4yVJj2mtaWK%o{D483%J(!=~2N*_BII{BU27d^ISqC8w9oS zut?KEL$xKxs@}6u`qB?|QiB83tlC1sS1$ci`mfJ z?x>&>cy`x1d&IC5l)zo~rCy{(a%R!^`uclPx7KIBIY2)CzVgDgud_r zurb%jz!_i0pJu3GESoIXce*FSkhfhwga8fi+X>QvoLU^5D;<7fFue{@kc^dg&H@y> zFiFz^DX;%o$+-vGQ6{vi%khx%u1q$0ymKdzh9;8>9J1o6(0ry1q~1;j8nGp;8eJB( zvej2lOL-QN9saUmKBNn3Y`Z+(6u#1=ah9*NrukaHAIKQXuMZ_cH8EvZZY~Z z1cATT_hY1q{d-FrmHz5(VK<`0$wfPTZwjZbD5a&tk6XIkPE6~(G2Y|72A8Y9m>Yy| zU~f|B`X`$F!m?~lf}g%hpGxqY&PC)y4Y^+r*NjI?<8?@LO`NBQoSbvz$E{cI-+NV5O|hqYb?@$;y;raA`&MImCYs7+cagSI zS^1H~*MWl^WbDKkUts%RSO?pXe&dD>?Yd#opKS>Rn=H}zYF22Upw(3hoE~lb@-o=v z^i|VzsEpir3>Aa@nRmA0f^oFV22Cte%oI;P)uOyZ93LKt)!(~DM>(F_n(a?wu6}+} zY4S6!=G^Ab1vd!+;4_T6HC=`uec%7cU4h-?hxbw3V;MVY5%Fa^K z;z{hj%EZyPBuZOBI<&DxU zN%tb$RR2Xtwut%Ow5h9^{*wO(3{iuG#;X5|K?@XC{nXR?7ch*!JDdz!sj{d28&QU3 z5{fxyzi9XyOAU3Fk2`L3)&Cb|5N`NJF%KTwBKX_n>Jq2DY=hQctCmB|vH6xh_W!R? zo&P^o0;H+?u5?^fdLeb&4-f+&MLj`w;e3%w_wN8iZY1gS{x9$;a)Vg+>BSw}=Iz+%L=$`nJIn?3J-xlIC6AEP^2|xz(&F#I`N~nQk ztMK4|08wLEe8Q{CI0Ylg_$^bfeQCAc`-CmRzw`MHC|9WHOO0{yl{z#} z-fVZ(JD6swi*NMMEMQ=yIpyz?c;+0DK&81z;4K>Oah7L}^tn&5oFF2&E4KSxq!l&){uwqj;Fhh-p@VCGBm4zl6&Pxx3I#?)?=0-PZl@ zHa4p>MS+d@Q^nCe<=AdHkIeg@&#!Y<7UN>FnTv%JvUi&hXv4!sjSZACXJmDG_dQ0< z4K$P@n!tID)mYD(pXwqF9k+}9!w&ZwK8sEudV)KnyB?p#@$u&_^6qBnzSNEFEjQC? zEr)%VOCvKz<5KI$yfBn~XIuSa_q)p$u5KBb;;d*KBT6@r&-hy{dpQFsB&sX^O^cZQ z;8^JZlBTQji+Yp5ZG%&zd%g?af^u>~Nc*BGmFJ6?tYqU)iPIOd17j@i($m+)MA;a# z)!L&{mkysY?iObnz7|vstac!3FY5ElKMBweua7BW;o9WOU)@jLri z1*PObrPB59J>SF8YN-cw1_(brx`fJ7V=A)91v6G94f3;Zg%)C?^i8jW4E$`_G%uoT zb>pD8X6#eJPK0r@Mj@QyDdr%_516m^)q;gq7+P)7M`gj^HpNdbGyyx3}oEAzbUmPq`^oZyi!x-u#S%9udsDs7dhbHWr_ z`IBe!Y*p_(2&jyVO;k8idYxhpH&k8~)AR%n@G+(oLqD$9V|;j9oZrrcmnGicREX{P zJ-}YHU`cY2ja_Q5@L7#(_1gm*s5-(QV;s(>4x?|AxvhXN6fx4>jv~$-^tl!P#NGK^ z&F+-}Gvn055~A4iV;Cs$X}iR0zpgcQYX@3KC~HgCrLriff6F-G!XW}8pTCc#>&>mO z$AZ( z_9+td%8WK*P#5;6SVgt`>}pG$pPIz5hfHg8oor>3x zf1<+RUJGI9{c?8L5Oc)IξTd)k0rTu^_#AqY0*{KPXl!>ZAHny3zf= zB=a00UzQGc!#?`#8UX_}yWP2&YYAkTruBSB8Dk2wRVtHO-vtDO!!i;?3u0Kz0LcV( zDjsu@@*YB$lkF+I9f7mb5oCo#jsfHDU+Tccrzr#VmgRil;*by4GGTB*sl|m^nk76u z_tq6{(^pGp-94ubcQYFk&5n3-n?Vax_vb77APm=oPl-X!5wUh#P4>LS>E%C$lRjuR z7vf~(N!VV!iL>*@dp+f@6s=Q)g(VPi=&O8lB;M6SXKct7EH2bj~JV^k#ZtGS5<=?qI6JuRzbjTR95F|Fz`=U62;n*(S8YH@LuUMT$sY3fP8 zSEDV5+@Vsp2yO_o8(stm$z8hI-I|GPZ!krdEt;I4YpYJD6jOJfl)`{UO>lg+skzI8d}?CR2yi^iR*1XyXu&4pEeckAFOnZ=E+h($SAMz z9??U?;AVe^dAEb~;VbfyHJFFY`QFX`E?TD6oy4dShDD&~^w-Re?_k?yyi47X2A zpbnn@P9v>?sEbq{*%%#x54|mpZP_o;Bu57@ zr|F4wx}_cp0`&uH%d!MVpj6PIgmTyl=|dON*CG5T-A|ykN*W&M9%rC{D9e5L_TUYM z@_N|fcn5&tmiOF_!lQ7$fE0h<0Z1@Uvs78&WtMSLeuJ*9ZJ*&w$N_7puC&|>jFbti z3Ust|nf`15Xp5e7PjZ}1L|J33jS0i(i1xdQX)z;aBu6jix3H(IG){GF>o}~(IB@I7 zvbVZvf9NpZYGZ|Ll$XEtYiV-yi?6J$&-Dj57C78oTIMJXWu-pFs=wIyT;$_`Pc?{c z=qfb81zRC|xC8qmA=_7(Gk>DpK2BR4q0D3WM%@h6M^`{{3NmVlFa!S`j|*)LXO$B0%b>e~FqMJULzW9CJw zDO6ehS5YQ(H+64x4;zZN=PqPd`)OsKdxZMxjfM;39}2D*benl+yH;9`~a6QvwEXVvwGVY6zT@bUntB!l{-( zN)U=e(O&PDnqe`H3&W(s66W+Uy)U{(C%YD(Rb#IwnaU1#42Lg<$nsE-+-Sjo>P+!7lNV&V29`aWTd!uGONi4H^_vOEdUf_K zLjNcbJd0yDj|)RWMe_MEufXHpL`y}C>#&!nikEVw)UEjnlrMF%i!{j9F4?&ggJU+p z&{sh53qN#0rq0rn7ovD?BAgOzTRO(pb=0|V+QLuKfuFH=bZqJuL|yGd`yX)QA2wL&H^yF9^weE}i zk2GH|Ig$*il{$wyUh9ZS@=cL!)XU)vZ7~ohT?umfXUG_e_I8Jtrh{@OH^YIM{HD~J z=Rp)J^chi>2$bSx@NB!M!lm@J`!Fj43rKUSgC|x8!H2z1SmH zE}M&jgL`Th6g=O>p)SDoSOXc_zI(}T!mGSwTHJz%+!hKx1fVQgq+ys+zagsmvu{nO zV|(}YEzB-j*hvwMc0sZG3G#;=jE%yG6@J*7>X>7AqjM2?l>GDIXP3wF6vHI>15K&o zV(qL{-479{r*2J5)G{Gte$vl&&)-be=h!t9$VMM3EcZk<7UW_ZidPawk1gUIN~R+l z>S-UCWme~RxG*66OS(Z7bimftWl>%czfSr>5s>ktwofj=5n3(55VD)HKQSla8R8V4 zC;t2-ELB}y%vFjn;`UVm#lg_TiPR!B@U!JQ^Pk^Pyj5++@O`$f*h^2sl8I}p5^CAn z97D|CjV+IIqF~B_s&i6BRk6d|OR9-JteOek&E`N-(X$YfoR8^AItkBuMX3QK7BOPq zE(^b_`I>$Qo9ee!W)Lr&+KrnM?Z3l~A0&H*5jsf4PPO%=3!Ia7-O0kB#Mlrt4a!~G zA*A}i)_8bEAh(C-u?zDON^AV39QFk_fdMyjoh)H@TrQf_y{{?l-B)VCeHrCpjFMv| zr7gKd>X?=WIP6Aastvh9SCLsBr|XV*{S;0=f)76GGz9zX(-}kYm=wqzcoa^9wnL(Z zt9picoYs`s=N0kv$G4i~;CsBC7H|k>!+HMBc$3xU_wZ?QIhVRz${MYVi8{dHWT%}e z1jCNnm3HZwiiD$55}enM<@6A&Z?d9J=uZ+%k$v=f{&)#kJ<5Q5dn6}hz!|Dz24c3{ z)BSkEF*+*v?H5aH)|;{{a+Ht(EBF7LlUO7gEmJSXmtlJqzTtfto#?2X^4`*ksGt2;9k9Q%=la`JCmA3w*A@jc)1~q$m9~A7{THgb z9bDH*HoDBGK`fM^Y|M&t&&YV?bD%8pFe%Rbu?X(8jJ0o9N`SCbOm-b|Bdu06_gW%0l zlD`yFaz;wJ<2w;%Cv0nmA#SFRVo5M0TT!E(dVOktnbUXl)@~+6TCUN0bDP(tvwl{P zBREfA-YZW{_@F}A5V_63tkt=!82FU)+j$kt*Mjb$#D0yBfk#Os8c$12AL1A)UQq8e z!wp4hnLnCv9IY`}+C+hiIm-ys_-^vVkwPcV zMDG{L+Seug*u$=ighAX@v+qY<=UO_})=uoqt#G6!v! zrkth}PeW1;M~IaeZf&FE533BOOvGR!{MW=h`pU~n1q|tYX7sDuv9d(Fp?VOdVZ2Fv zHKwwc=cH??)JKG0`3>6q!@X)CKMjvxZf#FxCNvnj^2NN!%EOq=P^FX^<|9e^1h!n! z2|WJ7Ee;I=c1;=r4CI=)>}ivn)`~eylp?b)1zU-T893GazpaqJ%?_J3790P7^K^qFv27t(kOy9dUK zj^=ayRmk z<=xA&gu!G%S{vzSUc(7Y;qDw8p9(_(3Z5x5n``7lrQ|qJV+q4Qv=rq+5Hh=pYoa0i z4vnG;;XEFhDz;y2OrGhnb~{w{8A#u9h~G42bOIgY0K&XT-)cCU+siKNR5*^Cyy}y| zWB+M0?OF{7hAOIgzy)s6+YEO0gSJ^5eZwDeoTRC$ZXa!^bb; zng6_w{J7T$l4OX{D>JQkGn_I>bc{tDxtzTNUu03)P8pv)aSL8#JCbsF<2JY1>`@%Q z(zJAjA+tr%_sLu^_aiO|_@`Wi(|PE6TK!a%99DWq%)!n_+kVJml+}p{SKM%a@bFKo zpuDEiKGTJfi6EjFb7!(4#crL75QjDfmg!<~)o_@n`te92!_O5Z47`#TSqr09kM&Qo zUzqA&b6gdN)R|`$_PrLyKRZ=mTsC}`aJ%yMVz?A7DQ$}Ix5t-eW|&Uhqgl=1qLDAe zZ{CDLWJGwOY)Et%9D{GX*tk z(b0&n3ZCM}(6RpBMaYW%2w~huvyv&T-mD`5Lvgf&aPG6{9INE%vs`4lkbQ&wglG@7z6z zQ=d!=`DMulYvQ)kdtc%a3Jg8`UfCImVL8vXn4_P=mIW&)Whc?nc104(g0)JEM9ViK ztFVXH$l6upW5b*>?s(?PVwIf>PqFbufWJB$qRVps3;Jc^=C%nxk1N@ks0(EQj!q?_fEwK zv5llvP7Hm2CZEpnN;6I&Jkbq-PR=_Sy*ccSDIk@I-l`^zM$clZ7rlfea3;!$oa%hx z!h!KK>J(B2H35#S^f#`!-z_9wCM)p3AbeH$0i<=5D>~G6LVkuM2q{m?UPA?itpYm z!AG&cOUy=?$Nr-r8~?d<*$@6HSx-_a;-z5}$gX*x4OFC&`0&l@bY2HK?7E$FU>mN!#7lpdflL>^p#$@7c!(pV! z&4nCNm2uFsD7-15Q))bavHLvUtt}6kEQja&6?sS7*a|OCwD^oE*LQ`0LaGlB5dsd_ zS5!;zY#xYRf#`)Q`BQy$Vf~f{S5_K(@Jc?eE3<=*fH>r&*asW`iS; z<1=|CAYaC#jJ)`EfuK8U7gv!bTMM^?KqpQ;=+41g7(d!FYAw4?%iL!wYY2)FSUcUO z?;+B{u*6i6k=0s{t!+^QX_0DF@0)Rk=S7q%4(2n#sMSNBYA`jVVZPu&q$4>YJU(yn z)8)hl`LGrgQ(<$g9TJ}N^c0ATZmUH&9?`xyTl8+bhk#WMe=>wR4hTd|+x^w<-%?Jdf zzVY4^o%%Njbj>>C{(Nq3at5=JI;dP%IwykcxMlQ8HNT7IMl4f|#%>FZca+8OjPA+T z45S>)^?^Nz!w406WzCw|C>4Nxk-w#1><{*PEcupyNPPZCUM;yXN=>FRyhh4XL@z(q zQ5JxXS|7}a+;PY2n-(MKXUbx>=%FvU{KJ2kDC^{IJ(sz@-j}{MfMVIYI->l!u(@_$ zyu0rHX_Nm}l>Ze=Xa&%tZ_AzJ9~aaV$`?%8i)mfd+{PBmUWH@t*6K>>=x5{0DOHO% zz@zL`18_@sAc*NoEWs-;DmA#pTkpQiF+-g*UbE zE6GS3VVC1p#*PQZg;KQWHmfmFT?b=o{6zfalVA88w!!l^q7934mWgSH4(M-|>ryj9 zymv#gm-85n8m-bCVm}tVU_Emc>o9!TOQux7O`kYs)R2v%HYzZMqItZ;eFl@Qb+yC| z%OIQT-gBRn05cBYtvLFRabsJLi0u9gUI=A_u=&Hm+0!(qhq)w{il#`u&M~eR~YGE8$bB+ z9R*Ha+a*VAtvQ3J^6*eu!#q8Hm+@ z@D1F)wJz(xvZs z1XwwIgFCxY%Hy#y-1@vUyl_iqh4gxAZTHAmmUIe8alnupj{EF2-^*$%X43;qie^LG z&nSlKNLs475X|rt8E7q8{a!NSPcXo*{8q;ZH$k|4xWB4XU?Xa5`x|_~9s0|M^#eC( zXJ8{4U=qc+d$rOleAt$s#MwUX{XWP=-uny_1&%roY$TE<#k2dlR%=fJiDNSV>g=@q zN_K^;DPmQMU^&cw>M)b`w_AR4)kypq_?c7YvTuCNC;9wK-$`HPlC-s_CE{V-V)ea2 z@SAY_V72dWY6zF3{a`Qw`uOo_eKZ{ja}gNuk*?rKup3m3yEU*JXD;NWezbk;8beqZ z=8?H$D&O*IW`aQnJckE#PK0~x;>L{FbTDKsy@l?rz39EPU3qoicRz1`GrT}~F^OTf z1Hx&LEqHbN`Q_-X;b=`~4pMgt6d{MLU)kNyQnxgqA(^~=%nVCv=4uPmKl%%_{`m*t z=e6OqS!}HXJVIPT;YPLXjL0Gg6y|$ao88>l>U=@`v}sZ{QaG9QgwfzG$cMZ-FVCDG zPq$71dFi^N`sQYt(u~~uUx6lX%F6luqty=M>ABJWFnqczI}9A-8}$ds5eZgrtlFL< z#e{12DzR#Hr5%6X0XsPwVBrJR_DDcn9?_BRn4wpQx0y6mBaNa($FL+_Xi+q<(t(6Z zDMy&mWOV9~xOen*L5O3<^$_kBF*4H{@>0i5%Br{$s(r|QJKUT9d^IroK+smZbnzr-r z4*s*WuS0vGZ@K+X`ALxm=hrADUB}C$wmUn{gF)Ln+4_PpjwZo&Ql0_8@B_PK^J72d zFqJw?w80a=;MJ@WG`9eL2;^)8z{*eZMB!_&lXZ#%0ZRbOmPfQVkkCl5Ve3 zQ`5dGmNN(6zU)}XK9cJ1puyS-)Rjb*8YYYGczZP4&^49!H~@RrWxCgG_zf%*IdP6~ z5*-Ys07(B@e|zeX2lPN8INX`k)14n!zk>v@Au6AByp6?H#^AV=F zTJApQ4%teBG~@vv#>jLK*&fEmd;s%XN=ZLJ_pw-;h3?1$**yRvYi?}bwl*lA#Cmd& z`r`|k`0%awd*`%6y_5OV%lEa@al~2@2`@8ne`DTXEi67!9b|9Pu*Tp=QAzHIC{*;U z(B%>HSE+lR05y?SGo8!>9og>L*%L^SF}1f0w(3aF24-SZy1j?4Y$ckK>GPXTc7h1N z=WeS;O7=7g2bjP}thm&xLZf7)B`iXJP-M>-UB!4jh^CiA>+t3{IjZuU*QHIZH`ULF z9hSSJf5uBMT1*Y!#kn(hOZoa7=dN8tef8S3=`QU#)X1o;k|br~zyXd-u077oT10Mm zo$tH}-(b|}FQ?CJ`(bd6coe`g_VWv%oz$V$=l(KJFok8JxFiZ?Ww2wHq=q)32My2( z78+?SBXyX5S=C)JYJT=vhpxnCt_37CeT~)OKOOb9Y-6igZ{o2-Ex;G2m5(`&8y0zH zYv2bJnD$0!!HjWWwbmjT%HYW zk5d5CgH8-a3YNj-(`1gK&tzGISEt#p_575oe2oMSBFXrD+A{ea`k4#gO@O;bFxu(f zbl+yxlQxpwuiaB$ww)ZagH6qsS{jmPiKjZ&$=kZBm@69<)v1W~`O!J!Fm<#@nwK1= zZ-^bUwfnE4+OugZu+;Bp6AylOuJ@9`qHF=Y7)GiSvW?3E^RyJp=+u3BzgZ{ZKr~dI z4DxY9#~~Hr@AV9OjXppL-rq**BsT>{gH`2Sby$Rp&{-=mYXF){aLHU zakeU$gBz5fjNdo7E2FPSVTX`MTmEjJ^E{?=K`@yOQ}+xs+UBcm5QW+a!iEzT10$;b z`jq$80m@;v@8e;WX94ra5HWJ7whH>_mQF|jK{r63%?_A)n3{elhg2&eJWHvHseqVk zp%g{gDe;X#9-k4;D{Wbd{Ll;BfK&~od80PgpP_WNICqgJI32lw;@S=J>Q-;v zu3l{YAuV{1V)3JEaJ2IEVN>o!7yKGWVQ0Yl@rudIM6_E!znd~cc0@xT?QQ(R6j4dw z%HmU#+dtR*ME-7?GFJ$l<;C8|g3Q{N8xm3e14(Srm~)%%*({x4BZEGSd+(NpXcSjH z2)g)}mBqD#H~kL0Q1igRquT4fFNfuDepta+X%m#(CrID>hU&2ExU+!YFOwW;x}tYt z-A{*c3zLt^B`8A#*IP!8S=nCeAMK1Z1Yhl!4P~Ii;M-?ybO*l~_#PxTTfZY%@1*%g zDU*|C7AT1s&qp2}b%FrxsOsk)cCKUk?-4X&ox%@AEjJ^MNYoarwL46<@qYCO4Z7*O-D zWsU@2-qD);Q{W)7p8v%#cpS%%Q9G>Py-#~6a$n@BUt=yOM@dgKh67I)lGk(_aP~Qe z8#aljz7N!P08o3=&h2irDO}#}on(y|?r;|glIkap`4-&+w;ZNTc!MP@%CFFil6bDQ zAXTkL@>qU2@Mv4K3s+f=5pYA9rT^&aeQHll8L2w0@3ERD+&bJva)uuZhcT&rO{1ov zGTrgA5u|1Ge8w(s($h#|sK=U%2D7mhdS!FK4^UDDkw>h0nU_mgTjN>Qo&hAYUbnsJdv{msb$6~>H4?oeeb(ew%`}4AxZ_Qsn z;69%o@IruC+XLYqL~{f9cV%7%8i|z9qjl7=6=YJIi?hWNExr~>i0mE(3+dbnTD!K4 z&H6umn3_bRZ(;x)__>Id@4E1!K(;c#m98h3`Q9ruw-t#hp^jS!qBo~-pZniLuJIt_ z_uWYby28?KVZtw{cUMiJ`c@QTVTZw2QT)5I(K=Q|Jc^r|lp0QYM55C_Bs2|jnig2Q zRyyg&KpouR^;x)OStxmy(D0S@ODV4V_mor*!tb)%<-hKSZ;?y6G6qU>-2<|P@?5bX5>!Vi@vOY8Fof5^I4gPfMRw^#-i zE@X=;UI};l7=QcGVTzTZ zwcv^@unZ2Xo#i<_kff!gj+HwKBcBHs5lBMetvw)L$^>QTkwH6%(8(AVz}vDb}o`8NnHDyQx7kF zc#bsdK@KH!iyCZPYwi1orO>jkwf2G@$i*Zs!TOTp(V1ro-}>zugiCo1Q<10mq(RbI zo>V3sonamKZILHQnP1oqJm>)znxFtP4bZfoKW6*;PYGE$|6ILpqLnq(-JG>X@gZ$j zZDLKgdw5Fm4UpL5E;oCi_4}O(`wC6jX=!^QPex+@^+wwa-OGR6ozE4Qsu9tutgEZt zi#&*l{TI9TtKa7Fpc2^da6t=pc~j-*x$Z2}2f5)QDBeq6p1$4SQ0>>UPaK9>@xW=+ zKlkMYK~KnkJvh3>)p>r^mu}?zSq;;$>!Gi2o1jXOI)jb;C?~IQ3b7sdbw)(5=9YUO zzO!$lJ(NNZ_9_K$;n|rhA79QNNt-(*J-D9D4iI!-fOb6i7(^m|dAE$|z3HRzXG24CD}h&;;UG?M>Xxr$d?K=0?94A;YSr{RxU5`kBYX zn13klvE^Q34$DkQvXSX8ng1`G&=^~BPu<@^AU#PO&VxLulHOHH$JNZ()m+fj*&O-t zh?AX@n~j4L`KQ6jCCJ4t$jQmV&MwH#u1b8T{(n@kb1<{A@cO@3cqJ!&fULmuKU;9M gvNLyaHMVp7zt`~qSt1Woee_693MdH?|M2;L03x Date: Thu, 12 Dec 2024 18:12:49 -0600 Subject: [PATCH 17/17] add authz check --- gen3workflow/routes/s3.py | 9 +++++---- tests/test_s3_endpoint.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/gen3workflow/routes/s3.py b/gen3workflow/routes/s3.py index f82f6f1..1d47d58 100644 --- a/gen3workflow/routes/s3.py +++ b/gen3workflow/routes/s3.py @@ -71,17 +71,18 @@ async def s3_endpoint(path: str, request: Request): """ logger.debug(f"Incoming S3 request: '{request.method} {path}'") - # extract the user's access token from the request headers, and use it to get the name of - # the user's bucket + # extract the user's access token from the request headers, and ensure the user has access + # to run workflows auth = Auth(api_request=request) auth.bearer_token = HTTPAuthorizationCredentials( scheme="bearer", credentials=get_access_token(request.headers) ) + await auth.authorize("create", ["/services/workflow/gen3-workflow/tasks"]) + + # get the name of the user's bucket and ensure the user is making a call to their own bucket token_claims = await auth.get_token_claims() user_id = token_claims.get("sub") user_bucket = aws_utils.get_safe_name_from_user_id(user_id) - - # ensure the user is making a call to their own bucket request_bucket = path.split("?")[0].split("/")[0] if request_bucket != user_bucket: err_msg = f"'{path}' not allowed. You can make calls to your personal bucket, '{user_bucket}'" diff --git a/tests/test_s3_endpoint.py b/tests/test_s3_endpoint.py index 43657a2..d2f5f13 100644 --- a/tests/test_s3_endpoint.py +++ b/tests/test_s3_endpoint.py @@ -43,6 +43,23 @@ def test_s3_endpoint_no_token(s3_client): s3_client.list_objects(Bucket=f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}") +""" +This test currently doesn't work because the client generated when `get_url` is True is not stopped +properly, so generating a different client (with `authorized=False` param) triggers an error: +> OSError: [Errno 48] error while attempting to bind on address ('0.0.0.0', 8080): address already + in use +TODO fix that +""" +# @pytest.mark.parametrize("client", [{"get_url": True, "authorized": False}], indirect=True) +# def test_s3_endpoint_unauthorized(s3_client, access_token_patcher): +# """ +# Hitting the `/s3` endpoint with a Gen3 access token that does not have the appropriate access +# should result in a 403 Forbidden error. +# """ +# with pytest.raises(ClientError, match="403"): +# s3_client.list_objects(Bucket=f"gen3wf-{config['HOSTNAME']}-{TEST_USER_ID}") + + @pytest.mark.parametrize("client", [{"get_url": True}], indirect=True) @pytest.mark.parametrize( "bucket_name",