From 6c59e9dc41d1074cf9c541733a6ad43844779770 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Thu, 23 May 2024 14:29:45 +0100 Subject: [PATCH] Allow DA admins to view share logs (#1274) ### Feature or Bugfix - Feature ### Detail - The latest logstream (last 24 hours) is loaded into modal window on Share Object page - User can see "Logs" button, if they are admins, stewards or owner of the dataset ### Relates - #1215 ### Security Please answer the questions below briefly where applicable, or write `N/A`. Based on [OWASP 10](https://owasp.org/Top10/en/). - Does this PR introduce or modify any input fields or queries - this includes fetching data from storage outside the application (e.g. a database, an S3 bucket)? - Is the input sanitized? - What precautions are you taking before deserializing the data you consume? - Is injection prevented by parametrizing queries? - Have you ensured no `eval` or similar functions are used? - Does this PR introduce any functionality or component that requires authorization? - How have you ensured it respects the existing AuthN/AuthZ mechanisms? - Are you logging failed auth attempts? - Are you using or adding any cryptographic features? - Do you use a standard proven implementations? - Are the used keys controlled by the customer? Where are they stored? - Are you introducing any new policies/roles/users? - Have you used the least-privilege principle? How? By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --------- Co-authored-by: Sofia Sazonova --- .../modules/s3_datasets_shares/api/queries.py | 9 ++ .../s3_datasets_shares/api/resolvers.py | 9 ++ .../modules/s3_datasets_shares/api/types.py | 16 +++ .../services/share_object_service.py | 65 ++++++++++ .../modules/Shares/components/ShareLogs.js | 119 ++++++++++++++++++ .../modules/Shares/services/getShareObject.js | 1 + .../src/modules/Shares/views/ShareView.js | 29 +++++ .../graphql/ShareObject/getShareLogs.js | 15 +++ .../src/services/graphql/ShareObject/index.js | 1 + 9 files changed, 264 insertions(+) create mode 100644 frontend/src/modules/Shares/components/ShareLogs.js create mode 100644 frontend/src/services/graphql/ShareObject/getShareLogs.js diff --git a/backend/dataall/modules/s3_datasets_shares/api/queries.py b/backend/dataall/modules/s3_datasets_shares/api/queries.py index 39620f498..bad081806 100644 --- a/backend/dataall/modules/s3_datasets_shares/api/queries.py +++ b/backend/dataall/modules/s3_datasets_shares/api/queries.py @@ -2,6 +2,7 @@ from dataall.modules.s3_datasets_shares.api.resolvers import ( get_dataset_shared_assume_role_url, get_share_object, + get_share_logs, list_shared_with_environment_data_items, list_shares_in_my_inbox, list_shares_in_my_outbox, @@ -70,3 +71,11 @@ resolver=get_dataset_shared_assume_role_url, test_scope='Dataset', ) + + +getShareLogs = gql.QueryField( + name='getShareLogs', + args=[gql.Argument(name='shareUri', type=gql.NonNullableType(gql.String))], + type=gql.ArrayType(gql.Ref('ShareLog')), + resolver=get_share_logs, +) diff --git a/backend/dataall/modules/s3_datasets_shares/api/resolvers.py b/backend/dataall/modules/s3_datasets_shares/api/resolvers.py index 73102e2c0..d506de7c7 100644 --- a/backend/dataall/modules/s3_datasets_shares/api/resolvers.py +++ b/backend/dataall/modules/s3_datasets_shares/api/resolvers.py @@ -15,6 +15,7 @@ from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, S3Dataset + log = logging.getLogger(__name__) @@ -132,6 +133,10 @@ def get_share_object(context, source, shareUri: str = None): return ShareObjectService.get_share_object(uri=shareUri) +def get_share_logs(context, source, shareUri: str): + return ShareObjectService.get_share_logs(shareUri) + + def resolve_user_role(context: Context, source: ShareObject, **kwargs): if not source: return None @@ -164,6 +169,10 @@ def resolve_user_role(context: Context, source: ShareObject, **kwargs): ) +def resolve_can_view_logs(context: Context, source: ShareObject): + return ShareObjectService.check_view_log_permissions(context.username, context.groups, source.shareUri) + + def resolve_dataset(context: Context, source: ShareObject, **kwargs): if not source: return None diff --git a/backend/dataall/modules/s3_datasets_shares/api/types.py b/backend/dataall/modules/s3_datasets_shares/api/types.py index 158aed212..937bed630 100644 --- a/backend/dataall/modules/s3_datasets_shares/api/types.py +++ b/backend/dataall/modules/s3_datasets_shares/api/types.py @@ -16,6 +16,7 @@ list_shareable_objects, resolve_user_role, resolve_shared_database_name, + resolve_can_view_logs, ) from dataall.core.environment.api.resolvers import resolve_environment @@ -166,6 +167,11 @@ type=gql.Ref('SharedItemSearchResult'), resolver=list_shareable_objects, ), + gql.Field( + name='canViewLogs', + resolver=resolve_can_view_logs, + type=gql.Boolean, + ), gql.Field( name='userRoleForShareObject', type=gql.Ref('ShareObjectPermission'), @@ -261,3 +267,13 @@ gql.Field(name='hasPrevious', type=gql.Boolean), ], ) + +ShareLog = gql.ObjectType( + name='ShareLog', + fields=[ + gql.Field(name='logStream', type=gql.String), + gql.Field(name='logGroup', type=gql.String), + gql.Field(name='timestamp', type=gql.String), + gql.Field(name='message', type=gql.String), + ], +) diff --git a/backend/dataall/modules/s3_datasets_shares/services/share_object_service.py b/backend/dataall/modules/s3_datasets_shares/services/share_object_service.py index 2df5ee7f5..ac898221e 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/share_object_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/share_object_service.py @@ -1,3 +1,4 @@ +import os from datetime import datetime from warnings import warn @@ -45,12 +46,24 @@ from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_TABLE_READ, DATASET_FOLDER_READ from dataall.base.aws.iam import IAM +from dataall.base.utils import Parameter +from dataall.base.db import exceptions +from dataall.core.stacks.aws.cloudwatch import CloudWatch + + import logging log = logging.getLogger(__name__) class ShareObjectService: + @staticmethod + def check_view_log_permissions(username, groups, shareUri): + with get_context().db_engine.scoped_session() as session: + share = ShareObjectRepository.get_share_by_uri(session, shareUri) + ds = DatasetRepository.get_dataset_by_uri(session, share.datasetUri) + return ds.stewards in groups or ds.SamlAdminGroupName in groups or username == ds.owner + @staticmethod def verify_principal_role(session, share: ShareObject) -> bool: role_name = share.principalIAMRoleName @@ -590,3 +603,55 @@ def attach_dataset_folder_read_permission(session, share): log.info( f'Resource permission policy {DATASET_FOLDER_READ} to table {location.itemUri} for group {share.groupUri} already exists. Skip... ' ) + + @staticmethod + def get_share_logs_name_query(shareUri): + log.info(f'Get share Logs stream name for share {shareUri}') + + query = f"""fields @logStream + |filter @message like '{shareUri}' + | sort @timestamp desc + | limit 1 + """ + return query + + @staticmethod + def get_share_logs_query(log_stream_name): + query = f"""fields @timestamp, @message, @logStream, @log as @logGroup + | sort @timestamp asc + | filter @logStream like "{log_stream_name}" + """ + return query + + @staticmethod + def get_share_logs(shareUri): + context = get_context() + if not ShareObjectService.check_view_log_permissions(context.username, context.groups, shareUri): + raise exceptions.ResourceUnauthorized( + username=context.username, + action='View Share Logs', + resource_uri=shareUri, + ) + + envname = os.getenv('envname', 'local') + log_group_name = f"/{Parameter().get_parameter(env=envname, path='resourcePrefix')}/{envname}/ecs/share-manager" + + query_for_name = ShareObjectService.get_share_logs_name_query(shareUri=shareUri) + name_query_result = CloudWatch.run_query( + query=query_for_name, + log_group_name=log_group_name, + days=1, + ) + if len(name_query_result) == 0: + return [] + + name = name_query_result[0]['logStream'] + + query = ShareObjectService.get_share_logs_query(log_stream_name=name) + results = CloudWatch.run_query( + query=query, + log_group_name=log_group_name, + days=1, + ) + log.info(f'Running Logs query {query} for log_group_name={log_group_name}') + return results diff --git a/frontend/src/modules/Shares/components/ShareLogs.js b/frontend/src/modules/Shares/components/ShareLogs.js new file mode 100644 index 000000000..2b4b01500 --- /dev/null +++ b/frontend/src/modules/Shares/components/ShareLogs.js @@ -0,0 +1,119 @@ +import Editor from '@monaco-editor/react'; +import { RefreshRounded } from '@mui/icons-material'; +import { + Box, + Button, + CircularProgress, + Dialog, + Grid, + Typography +} from '@mui/material'; +import PropTypes from 'prop-types'; +import React, { useCallback, useEffect, useState } from 'react'; +import { THEMES, useSettings } from 'design'; +import { SET_ERROR, useDispatch } from 'globalErrors'; +import { getShareLogs, useClient } from 'services'; + +export const ShareLogs = (props) => { + const { shareUri, onClose, open } = props; + const { settings } = useSettings(); + const client = useClient(); + const dispatch = useDispatch(); + const [logs, setLogs] = useState(null); + const [loading, setLoading] = useState(true); + + const getLogs = useCallback(async () => { + setLoading(true); + try { + const response = await client.query(getShareLogs(shareUri)); + if (response && !response.errors) { + setLogs(response.data.getShareLogs.map((l) => l.message)); + } else { + dispatch({ type: SET_ERROR, error: response.errors[0].message }); + } + } catch (e) { + dispatch({ type: SET_ERROR, error: e.message }); + } + setLoading(false); + }, [client, dispatch, shareUri]); + + useEffect(() => { + if (client && open) { + getLogs().catch((e) => dispatch({ type: SET_ERROR, error: e.message })); + } + }, [client, dispatch, getLogs, open]); + + return ( + + + + + + Logs for share {shareUri} + + + + + + + + + {loading ? ( + + ) : ( + + {logs && ( + +
+ 0 + ? logs.join('\n') + : 'No logs available for the last 24 hours. Logs may take few minutes after the share is processed...' + } + options={{ minimap: { enabled: false } }} + theme="vs-dark" + inDiffEditor={false} + height="35rem" + language="text" + showPrintMargin + showGutter + highlightActiveLine + editorProps={{ + $blockScrolling: Infinity + }} + setOptions={{ + enableBasicAutocompletion: true, + enableLiveAutocompletion: true, + enableSnippets: true, + showLineNumbers: true, + tabSize: 2 + }} + /> +
+ + )} + + )} + + + + ); +}; + +ShareLogs.propTypes = { + shareUri: PropTypes.string.isRequired +}; diff --git a/frontend/src/modules/Shares/services/getShareObject.js b/frontend/src/modules/Shares/services/getShareObject.js index 728ae4b8b..579aeaa37 100644 --- a/frontend/src/modules/Shares/services/getShareObject.js +++ b/frontend/src/modules/Shares/services/getShareObject.js @@ -15,6 +15,7 @@ export const getShareObject = ({ shareUri, filter }) => ({ requestPurpose rejectPurpose userRoleForShareObject + canViewLogs consumptionData { s3AccessPointName sharedGlueDatabase diff --git a/frontend/src/modules/Shares/views/ShareView.js b/frontend/src/modules/Shares/views/ShareView.js index 4e89820de..99a707d10 100644 --- a/frontend/src/modules/Shares/views/ShareView.js +++ b/frontend/src/modules/Shares/views/ShareView.js @@ -1,4 +1,5 @@ import { + Article, BlockOutlined, CheckCircleOutlined, CopyAllOutlined, @@ -73,6 +74,7 @@ import { UpdateRequestReason } from '../components'; import { generateShareItemLabel } from 'utils'; +import { ShareLogs } from '../components/ShareLogs'; function ShareViewHeader(props) { const { @@ -89,6 +91,8 @@ function ShareViewHeader(props) { const [rejecting, setRejecting] = useState(false); const [submitting, setSubmitting] = useState(false); const [isRejectShareModalOpen, setIsRejectShareModalOpen] = useState(false); + const [openLogsModal, setOpenLogsModal] = useState(null); + const submit = async () => { setSubmitting(true); const response = await client.mutate( @@ -133,6 +137,13 @@ function ShareViewHeader(props) { } }; + const handleOpenLogsModal = () => { + setOpenLogsModal(true); + }; + const handleCloseOpenLogs = () => { + setOpenLogsModal(false); + }; + const handleRejectShareModalOpen = () => { setIsRejectShareModalOpen(true); }; @@ -246,6 +257,17 @@ function ShareViewHeader(props) { > Refresh + {share.canViewLogs && ( + + )} {(share.userRoleForShareObject === 'Approvers' || share.userRoleForShareObject === 'ApproversAndRequesters') && ( <> @@ -319,6 +341,13 @@ function ShareViewHeader(props) { rejectFunction={reject} /> )} + {share.canViewLogs && ( + + )} ); } diff --git a/frontend/src/services/graphql/ShareObject/getShareLogs.js b/frontend/src/services/graphql/ShareObject/getShareLogs.js new file mode 100644 index 000000000..8940f69dd --- /dev/null +++ b/frontend/src/services/graphql/ShareObject/getShareLogs.js @@ -0,0 +1,15 @@ +import { gql } from 'apollo-boost'; + +export const getShareLogs = (shareUri) => ({ + variables: { + shareUri + }, + query: gql` + query getShareLogs($shareUri: String!) { + getShareLogs(shareUri: $shareUri) { + message + timestamp + } + } + ` +}); diff --git a/frontend/src/services/graphql/ShareObject/index.js b/frontend/src/services/graphql/ShareObject/index.js index 033e2b72f..62e364ada 100644 --- a/frontend/src/services/graphql/ShareObject/index.js +++ b/frontend/src/services/graphql/ShareObject/index.js @@ -1,2 +1,3 @@ export * from './createShareObject'; export * from './getShareRequestsToMe'; +export * from './getShareLogs';