Remove storage endpoints (#32)

.secrets.baseline

@@ -169,7 +169,7 @@
         "filename": "gen3workflow/config-default.yaml",
         "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
         "is_verified": false,
-        "line_number": 32
+        "line_number": 30
       }
     ],
     "migrations/versions/e1886270d9d2_create_system_key_table.py": [
@@ -187,7 +187,7 @@
         "filename": "tests/conftest.py",
         "hashed_secret": "0dd78d9147bb410f0cb0199c5037da36594f77d8",
         "is_verified": false,
-        "line_number": 222
+        "line_number": 226
       }
     ],
     "tests/migrations/test_migration_e1886270d9d2.py": [
@@ -209,5 +209,5 @@
       }
     ]
   },
-  "generated_at": "2024-12-12T23:42:54Z"
+  "generated_at": "2024-12-16T22:39:24Z"
 }

docs/authorization.md

@@ -2,6 +2,10 @@
 
 The Gen3 Workflow endpoints are protected by Arborist policies.
 
+Contents:
+- [GA4GH TES](#ga4gh-tes)
+  - [Authorization configuration example](#authorization-configuration-example)
+
 ## GA4GH TES
 
 - To create a task, users need `create` access to resource `/services/workflow/gen3-workflow/tasks` on service `gen3-workflow`.

docs/local_installation.md

@@ -100,6 +100,9 @@ aws {
 }
 workDir = '<your working directory>'
 ```
+
+A Gen3 access token is expected by most endpoints to verify the user's access (see [Authorization](authorization.md) documentation).
+
 > The Gen3Workflow URL should be set to `http://localhost:8080` in this case; this is where the service runs by default when started with `python run.py`.
 
 - Run a workflow:

docs/openapi.yaml

@@ -88,7 +88,9 @@ paths:
         '200':
           content:
             application/json:
-              schema: {}
+              schema:
+                title: Response Service Info Ga4Gh Tes V1 Service Info Get
+                type: object
           description: Successful Response
       summary: Service Info
       tags:
@@ -100,7 +102,9 @@ paths:
         '200':
           content:
             application/json:
-              schema: {}
+              schema:
+                title: Response List Tasks Ga4Gh Tes V1 Tasks Get
+                type: object
           description: Successful Response
       security:
       - HTTPBearer: []
@@ -113,7 +117,9 @@ paths:
         '200':
           content:
             application/json:
-              schema: {}
+              schema:
+                title: Response Create Task Ga4Gh Tes V1 Tasks Post
+                type: object
           description: Successful Response
       security:
       - HTTPBearer: []
@@ -134,7 +140,9 @@ paths:
         '200':
           content:
             application/json:
-              schema: {}
+              schema:
+                title: Response Get Task Ga4Gh Tes V1 Tasks  Task Id  Get
+                type: object
           description: Successful Response
         '422':
           content:
@@ -161,7 +169,9 @@ paths:
         '200':
           content:
             application/json:
-              schema: {}
+              schema:
+                title: Response Cancel Task Ga4Gh Tes V1 Tasks  Task Id  Cancel Post
+                type: object
           description: Successful Response
         '422':
           content:
@@ -423,65 +433,16 @@ paths:
       summary: S3 Endpoint
       tags:
       - S3
-  /storage/credentials:
-    get:
-      operationId: get_user_keys
-      responses:
-        '200':
-          content:
-            application/json:
-              schema: {}
-          description: Successful Response
-      security:
-      - HTTPBearer: []
-      summary: Get User Keys
-      tags:
-      - Storage
-    post:
-      operationId: generate_user_key
-      responses:
-        '201':
-          content:
-            application/json:
-              schema: {}
-          description: Successful Response
-      security:
-      - HTTPBearer: []
-      summary: Generate User Key
-      tags:
-      - Storage
-  /storage/credentials/{key_id}:
-    delete:
-      operationId: delete_user_key
-      parameters:
-      - in: path
-        name: key_id
-        required: true
-        schema:
-          title: Key Id
-          type: string
-      responses:
-        '204':
-          description: Successful Response
-        '422':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/HTTPValidationError'
-          description: Validation Error
-      security:
-      - HTTPBearer: []
-      summary: Delete User Key
-      tags:
-      - Storage
   /storage/info:
     get:
       operationId: get_storage_info
       responses:
         '200':
           content:
             application/json:
-              schema: {}
+              schema:
+                title: Response Get Storage Info Storage Info Get
+                type: object
           description: Successful Response
       security:
       - HTTPBearer: []

gen3workflow/auth.py

@@ -25,11 +25,11 @@ def __init__(
         self,
         api_request: Request,
         bearer_token: HTTPAuthorizationCredentials = Security(bearer),
-    ):
+    ) -> None:
         self.arborist_client = api_request.app.arborist_client
         self.bearer_token = bearer_token
 
-    def get_access_token(self):
+    def get_access_token(self) -> str:
         if config["MOCK_AUTH"]:
             return "123"
 
@@ -95,7 +95,14 @@ async def authorize(
 
         return authorized
 
-    async def grant_user_access_to_their_own_tasks(self, username, user_id):
+    async def grant_user_access_to_their_own_tasks(self, username, user_id) -> None:
+        """
+        Ensure the specified user exists in Arborist and has a policy granting them access to their
+        own Gen3Workflow tasks ("read" and "delete" access to resource "/users/<user ID>/gen3-workflow/tasks" for service "gen3-workflow").
+        Args:
+            username (str): The user's Gen3 username
+            user_id (str): The user's unique Gen3 ID
+        """
         logger.info(
             f"Granting user '{username}' access to their own tasks if they don't already have it"
         )

gen3workflow/aws_utils.py

@@ -1,19 +1,15 @@
-import json
+from typing import Tuple
 
 import boto3
-from botocore.exceptions import ClientError
-from fastapi import HTTPException
-from starlette.status import HTTP_404_NOT_FOUND
 
-from gen3workflow import logger
 from gen3workflow.config import config
 
 
 iam_client = boto3.client("iam")
 iam_resp_err = "Unexpected response from AWS IAM"
 
 
-def get_safe_name_from_user_id(user_id):
+def get_safe_name_from_user_id(user_id: str) -> str:
     """
     Generate a valid IAM user name or S3 bucket name for the specified user.
     - IAM user names can contain up to 64 characters. They can only contain alphanumeric characters
@@ -36,7 +32,7 @@ def get_safe_name_from_user_id(user_id):
     return safe_name
 
 
-def create_user_bucket(user_id):
+def create_user_bucket(user_id: str) -> Tuple[str, str, str]:
     """
     Create an S3 bucket for the specified user and return information about the bucket.
 
@@ -46,134 +42,8 @@ def create_user_bucket(user_id):
     Returns:
         tuple: (bucket name, prefix where the user stores objects in the bucket, bucket region)
     """
+    # TODO lifetime policy and encryption
     user_bucket_name = get_safe_name_from_user_id(user_id)
     s3_client = boto3.client("s3")
     s3_client.create_bucket(Bucket=user_bucket_name)
     return user_bucket_name, "ga4gh-tes", config["USER_BUCKETS_REGION"]
-
-
-def create_or_update_policy(policy_name, policy_document, path_prefix, tags):
-    # attempt to create the policy
-    try:
-        response = iam_client.create_policy(
-            PolicyName=policy_name,
-            Path=path_prefix,
-            PolicyDocument=json.dumps(policy_document),
-            Tags=tags,
-        )
-        assert "Arn" in response.get("Policy", {}), f"{iam_resp_err}: {response}"
-        return response["Policy"]["Arn"]
-    except ClientError as e:
-        if e.response["Error"]["Code"] != "EntityAlreadyExists":
-            raise
-
-        # policy already exists: update it.
-        # find the right policy
-        response = iam_client.list_policies(PathPrefix=path_prefix)
-        assert "Policies" in response, f"{iam_resp_err}: {response}"
-        for policy in response["Policies"]:
-            assert "PolicyName" in policy, f"{iam_resp_err}: {policy}"
-            assert "Arn" in policy, f"{iam_resp_err}: {policy}"
-            if policy["PolicyName"] == policy_name:
-                break
-
-        # there can only be up to 5 versions, so delete old versions of this policy
-        response = iam_client.list_policy_versions(PolicyArn=policy["Arn"])
-        assert "Versions" in response, f"{iam_resp_err}: {response}"
-        for version in response["Versions"]:
-            assert "VersionId" in version, f"{iam_resp_err}: {version}"
-            assert "IsDefaultVersion" in version, f"{iam_resp_err}: {version}"
-            if version["IsDefaultVersion"]:
-                continue  # do not delete the latest versions
-            iam_client.delete_policy_version(
-                PolicyArn=policy["Arn"], VersionId=version["VersionId"]
-            )
-
-        # update the policy by creating a new version
-        iam_client.create_policy_version(
-            PolicyArn=policy["Arn"],
-            PolicyDocument=json.dumps(policy_document),
-            SetAsDefault=True,
-        )
-        return policy["Arn"]
-
-
-def create_iam_user_and_key(user_id):
-    iam_user_name = get_safe_name_from_user_id(user_id)
-    escaped_hostname = config["HOSTNAME"].replace(".", "-")
-    iam_tags = [
-        {
-            "Key": "name",
-            "Value": f"gen3wf-{escaped_hostname}",
-        },
-    ]
-
-    try:
-        iam_client.create_user(UserName=iam_user_name, Tags=iam_tags)
-    except ClientError as e:
-        # if the user already exists, ignore the error and proceed
-        if e.response["Error"]["Code"] != "EntityAlreadyExists":
-            raise
-
-    # grant the IAM user access to the user's s3 bucket
-    bucket_name, bucket_prefix, _ = create_user_bucket(user_id)
-    policy_document = {
-        "Version": "2012-10-17",
-        "Statement": [
-            {
-                "Sid": "AllowListingBucketFolder",
-                "Effect": "Allow",
-                "Action": ["s3:ListBucket"],
-                "Resource": [f"arn:aws:s3:::{bucket_name}"],
-                "Condition": {"StringLike": {"s3:prefix": [f"{bucket_prefix}/*"]}},
-            },
-            {
-                "Sid": "AllowManagingBucketFolder",
-                "Effect": "Allow",
-                "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
-                "Resource": [f"arn:aws:s3:::{bucket_name}/{bucket_prefix}/*"],
-            },
-        ],
-    }
-    path_prefix = f"/{iam_user_name}/"  # used later to get existing policies' ARN
-    policy_arn = create_or_update_policy(
-        f"{iam_user_name}-policy", policy_document, path_prefix, iam_tags
-    )
-    iam_client.attach_user_policy(PolicyArn=policy_arn, UserName=iam_user_name)
-
-    # create a key for this user
-    key = iam_client.create_access_key(UserName=iam_user_name)
-    assert "AccessKeyId" in key.get("AccessKey", {}), f"{iam_resp_err}: {key}"
-    assert "SecretAccessKey" in key.get("AccessKey", {}), f"{iam_resp_err}: {key}"
-
-    return key["AccessKey"]["AccessKeyId"], key["AccessKey"]["SecretAccessKey"]
-
-
-def list_iam_user_keys(user_id):
-    iam_user_name = get_safe_name_from_user_id(user_id)
-    try:
-        response = iam_client.list_access_keys(UserName=iam_user_name)
-    except ClientError as e:
-        if e.response["Error"]["Code"] == "NoSuchEntity":
-            return []  # user does not exist in IAM, so they have no IAM keys
-        else:
-            raise
-    assert "AccessKeyMetadata" in response, f"{iam_resp_err}: {response}"
-    for key in response["AccessKeyMetadata"]:
-        assert "AccessKeyId" in key, f"{iam_resp_err}: {key}"
-        assert "CreateDate" in key, f"{iam_resp_err}: {key}"
-        assert "Status" in key, f"{iam_resp_err}: {key}"
-    return response["AccessKeyMetadata"]
-
-
-def delete_iam_user_key(user_id, key_id):
-    try:
-        iam_client.delete_access_key(
-            UserName=get_safe_name_from_user_id(user_id),
-            AccessKeyId=key_id,
-        )
-    except ClientError as e:
-        if e.response["Error"]["Code"] == "NoSuchEntity":
-            err_msg = f"No such key: '{key_id}'"
-            logger.error(err_msg)
-            raise HTTPException(HTTP_404_NOT_FOUND, err_msg)

gen3workflow/config-default.yaml

@@ -10,10 +10,8 @@ DOCS_URL_PREFIX: /gen3workflow
 ARBORIST_URL:
 
 # /!\ only use for development! Allows running gen3workflow locally without Arborist interaction
-MOCK_AUTH: false # TODO add to config validation. Also add "no unexpected props" to validation.
+MOCK_AUTH: false
 
-MAX_IAM_KEYS_PER_USER: 2  # the default AWS AccessKeysPerUser quota is 2
-IAM_KEYS_LIFETIME_DAYS: 30
 USER_BUCKETS_REGION: us-east-1
 
 # configure an AWS IAM key to use when making S3 requests on behalf of users. If left empty, it