Merge branch 'main' into data-dot-allgh-904-central-catalog-support

# Conflicts:
#	backend/dataall/modules/dataset_sharing/services/share_processors/lakeformation_process_share.py

backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py

@@ -203,6 +203,21 @@ def grant_pivot_role_all_database_permissions_to_shared_database(self) -> True:
         )
         return True
 
+    def grant_pivot_role_drop_permissions_to_resource_link_table(self, table: DatasetTable) -> True:
+        """
+        Grants 'DROP' Lake Formation permissions to pivot role to the resource link table in target account
+        :param table: DatasetTable
+        :return: True if it is successful
+        """
+        self.lf_client_in_target.grant_permissions_to_table(
+            principals=[SessionHelper.get_delegation_role_arn(self.target_environment.AwsAccountId)],
+            database_name=self.shared_db_name,
+            table_name=table.GlueTableName,
+            catalog_id=self.target_environment.AwsAccountId,
+            permissions=['DROP']
+        )
+        return True
+
     def grant_principals_database_permissions_to_shared_database(self) -> True:
         """
         Grants 'DESCRIBE' Lake Formation permissions to share principals to the shared database in target account

backend/dataall/modules/dataset_sharing/services/share_processors/lakeformation_process_share.py

@@ -164,6 +164,7 @@ def process_revoked_shares(self) -> bool:
         success = True
 
         try:
+            self.grant_pivot_role_all_database_permissions_to_shared_database()
             if not self.check_catalog_account_exists_and_update_processor():
                 success = False
                 return success
@@ -201,6 +202,7 @@ def process_revoked_shares(self) -> bool:
 
                     if (self.is_new_share and not other_table_shares_in_env) or not self.is_new_share:
                         warn('self.is_new_share will be deprecated in v2.6.0', DeprecationWarning, stacklevel=2)
+                        self.grant_pivot_role_drop_permissions_to_resource_link_table(table)
                         self.delete_resource_link_table_in_shared_database(table)
 
                 if not other_table_shares_in_env:

backend/dataall/modules/datasets/api/dataset/input_types.py

@@ -19,7 +19,7 @@
         gql.Argument(
             name='businessOwnerDelegationEmails', type=gql.ArrayType(gql.String)
         ),
-        gql.Argument('confidentiality', gql.Ref('ConfidentialityClassification')),
+        gql.Argument('confidentiality', gql.String),
         gql.Argument(name='stewards', type=gql.String),
         gql.Argument(name='autoApprovalEnabled', type=gql.Boolean)
     ],
@@ -36,7 +36,7 @@
         gql.Argument('businessOwnerDelegationEmails', gql.ArrayType(gql.String)),
         gql.Argument('businessOwnerEmail', gql.String),
         gql.Argument('language', gql.Ref('Language')),
-        gql.Argument('confidentiality', gql.Ref('ConfidentialityClassification')),
+        gql.Argument('confidentiality', gql.String),
         gql.Argument(name='stewards', type=gql.String),
         gql.Argument('KmsAlias', gql.NonNullableType(gql.String)),
         gql.Argument(name='autoApprovalEnabled', type=gql.Boolean)
@@ -103,7 +103,7 @@
         gql.Argument(
             name='businessOwnerDelegationEmails', type=gql.ArrayType(gql.String)
         ),
-        gql.Argument('confidentiality', gql.Ref('ConfidentialityClassification')),
+        gql.Argument('confidentiality', gql.String),
         gql.Argument(name='stewards', type=gql.String),
         gql.Argument(name='autoApprovalEnabled', type=gql.Boolean)
 

backend/dataall/modules/datasets/api/dataset/resolvers.py

@@ -9,7 +9,7 @@
 from dataall.base.db.exceptions import RequiredParameter, InvalidInput
 from dataall.modules.dataset_sharing.db.share_object_models import ShareObject
 from dataall.modules.datasets_base.db.dataset_models import Dataset
-from dataall.modules.datasets_base.services.datasets_base_enums import DatasetRole
+from dataall.modules.datasets_base.services.datasets_base_enums import DatasetRole, ConfidentialityClassification
 from dataall.modules.datasets.services.dataset_service import DatasetService
 
 log = logging.getLogger(__name__)
@@ -201,6 +201,7 @@ def validate_creation_request(data):
             raise RequiredParameter('group')
         if not data.get('label'):
             raise RequiredParameter('label')
+        ConfidentialityClassification.validate_confidentiality_level(data.get('confidentiality', ''))
         if len(data['label']) > 52:
             raise InvalidInput(
                 'Dataset name', data['label'], 'less than 52 characters'

backend/dataall/modules/datasets/api/dataset/types.py

@@ -129,7 +129,7 @@
         ),
         gql.Field(name='topics', type=gql.ArrayType(gql.Ref('Topic'))),
         gql.Field(
-            name='confidentiality', type=gql.Ref('ConfidentialityClassification')
+            name='confidentiality', type=gql.String
         ),
         gql.Field(name='language', type=gql.Ref('Language')),
         gql.Field(

backend/dataall/modules/datasets/cdk/dataset_stack.py

@@ -25,6 +25,8 @@
 from dataall.modules.datasets.aws.lf_dataset_client import LakeFormationDatasetClient
 from dataall.modules.datasets_base.db.dataset_models import Dataset
 from dataall.base.utils.cdk_nag_utils import CDKNagUtil
+from dataall.base.config import config
+
 
 logger = logging.getLogger(__name__)
 
@@ -535,7 +537,8 @@ def __init__(self, scope, id, target_uri: str = None, **kwargs):
             )
             trigger.node.add_dependency(job)
 
-        Tags.of(self).add('Classification', dataset.confidentiality)
+        if config.get_property('modules.datasets.features.confidentiality_dropdown', False):
+            Tags.of(self).add('Classification', dataset.confidentiality)
 
         TagsUtil.add_tags(stack=self, model=Dataset, target_type="dataset")
 

backend/dataall/modules/datasets/indexers/dataset_indexer.py

@@ -1,4 +1,6 @@
 """Indexes Datasets in OpenSearch"""
+import re
+
 from dataall.core.environment.services.environment_service import EnvironmentService
 from dataall.core.organizations.db.organization_repositories import OrganizationRepository
 from dataall.modules.vote.db.vote_repositories import VoteRepository
@@ -34,7 +36,7 @@ def upsert(cls, session, dataset_uri: str):
                     'source': dataset.S3BucketName,
                     'resourceKind': 'dataset',
                     'description': dataset.description,
-                    'classification': dataset.confidentiality,
+                    'classification': re.sub('[^A-Za-z0-9]+', '', dataset.confidentiality),
                     'tags': [t.replace('-', '') for t in dataset.tags or []],
                     'topics': dataset.topics,
                     'region': dataset.region.replace('-', ''),

backend/dataall/modules/datasets/services/dataset_column_service.py

@@ -33,7 +33,7 @@ def paginate_active_columns_for_table(uri: str, filter=None):
             table: DatasetTable = DatasetTableRepository.get_dataset_table_by_uri(session, uri)
             dataset = DatasetRepository.get_dataset_by_uri(session, table.datasetUri)
             if (
-                    dataset.confidentiality != ConfidentialityClassification.Unclassified.value
+                    ConfidentialityClassification.get_confidentiality_level(dataset.confidentiality) != ConfidentialityClassification.Unclassified.value
             ):
                 ResourcePolicy.check_user_resource_permission(
                     session=session,

backend/dataall/modules/datasets/services/dataset_profiling_service.py

@@ -110,7 +110,7 @@ def _check_preview_permissions_if_needed(session, table_uri):
             session, table_uri
         )
         dataset = DatasetRepository.get_dataset_by_uri(session, table.datasetUri)
-        if dataset.confidentiality != ConfidentialityClassification.Unclassified.value:
+        if ConfidentialityClassification.get_confidentiality_level(dataset.confidentiality) != ConfidentialityClassification.Unclassified.value:
             ResourcePolicy.check_user_resource_permission(
                 session=session,
                 username=context.username,

backend/dataall/modules/datasets/services/dataset_table_service.py

@@ -85,7 +85,7 @@ def preview(table_uri: str):
             )
             dataset = DatasetRepository.get_dataset_by_uri(session, table.datasetUri)
             if (
-                    dataset.confidentiality != ConfidentialityClassification.Unclassified.value
+                ConfidentialityClassification.get_confidentiality_level(dataset.confidentiality) != ConfidentialityClassification.Unclassified.value
             ):
                 ResourcePolicy.check_user_resource_permission(
                     session=session,

backend/dataall/modules/datasets_base/services/datasets_base_enums.py

@@ -1,4 +1,8 @@
 from dataall.base.api.constants import GraphQLEnumMapper
+from dataall.base.config import config
+from dataall.base.db.exceptions import InvalidInput
+
+custom_confidentiality_mapping = config.get_property('modules.datasets.features.custom_confidentiality_mapping', {})
 
 
 class DatasetRole(GraphQLEnumMapper):
@@ -22,6 +26,20 @@ class ConfidentialityClassification(GraphQLEnumMapper):
     Official = 'Official'
     Secret = 'Secret'
 
+    @staticmethod
+    def get_confidentiality_level(confidentiality):
+        return confidentiality if not custom_confidentiality_mapping else custom_confidentiality_mapping.get(
+            confidentiality, None)
+
+    @staticmethod
+    def validate_confidentiality_level(confidentiality):
+        if config.get_property('modules.datasets.features.confidentiality_dropdown', False):
+            confidentiality = ConfidentialityClassification.get_confidentiality_level(confidentiality)
+            if confidentiality not in [item.value for item in list(ConfidentialityClassification)]:
+                raise InvalidInput('Confidentiality Name', confidentiality,
+                                   'does not conform to the confidentiality classification. Hint: Check your confidentiality value OR check your mapping if you are using custom confidentiality values')
+        return True
+
 
 class Language(GraphQLEnumMapper):
     English = 'English'

config.json

@@ -24,7 +24,9 @@
                     }
                 },
                 "preview_data": true,
-                "glue_crawler": true
+                "glue_crawler": true,
+                "confidentiality_dropdown" : true,
+                "topics_dropdown" : true
             }
         },
         "worksheets": {

frontend/src/modules/Catalog/views/Catalog.js

@@ -32,6 +32,7 @@ import {
   useSettings
 } from 'design';
 import { GlossarySearchWrapper, GlossarySearchResultItem } from '../components';
+import config from '../../../generated/config.json';
 
 const useStyles = makeStyles((theme) => ({
   mainSearch: {
@@ -171,7 +172,14 @@ const Catalog = () => {
   const classes = useStyles();
   const anchorRef = useRef(null);
   const [openMenu, setOpenMenu] = useState(false);
-  const [filterItems] = useState([
+  const dataFieldList = ['label', 'name', 'description', 'region', 'tags'];
+
+  if (config.modules.datasets.features.topics_dropdown === true)
+    dataFieldList.push('topics');
+  if (config.modules.datasets.features.confidentiality_dropdown === true)
+    dataFieldList.push('classification');
+
+  const filterItemsInit = [
     {
       title: 'Type',
       dataField: 'resourceKind',
@@ -184,25 +192,30 @@ const Catalog = () => {
       componentId: 'TagSensor',
       filterLabel: 'Tags'
     },
-    {
-      title: 'Topics',
-      dataField: 'topics',
-      componentId: 'TopicSensor',
-      filterLabel: 'Topics'
-    },
     {
       title: 'Region',
       dataField: 'region',
       componentId: 'RegionSensor',
       filterLabel: 'Region'
-    },
-    {
+    }
+  ];
+
+  if (config.modules.datasets.features.topics_dropdown === true)
+    filterItemsInit.push({
+      title: 'Topics',
+      dataField: 'topics',
+      componentId: 'TopicSensor',
+      filterLabel: 'Topics'
+    });
+  if (config.modules.datasets.features.confidentiality_dropdown === true)
+    filterItemsInit.push({
       title: 'Classification',
       dataField: 'classification',
       componentId: 'ClassificationSensor',
       filterLabel: 'Classification'
-    }
-  ]);
+    });
+
+  const [filterItems] = useState(filterItemsInit);
   const [listClass, setListClass] = useState(
     settings.theme === THEMES.LIGHT
       ? classes.lightListSearch
@@ -337,14 +350,7 @@ const Catalog = () => {
                     fuzziness="AUTO"
                     componentId="SearchSensor"
                     filterLabel="text"
-                    dataField={[
-                      'label',
-                      'name',
-                      'description',
-                      'region',
-                      'topics',
-                      'tags'
-                    ]}
+                    dataField={dataFieldList}
                     placeholder="Search"
                   />
                 </Box>

frontend/src/modules/Datasets/components/DatasetGovernance.js

@@ -9,6 +9,7 @@ import {
 } from '@mui/material';
 import PropTypes from 'prop-types';
 import { Label } from 'design';
+import { isFeatureEnabled } from 'utils';
 
 export const DatasetGovernance = (props) => {
   const { dataset } = props;
@@ -48,31 +49,36 @@ export const DatasetGovernance = (props) => {
           </Label>
         </Box>
       </CardContent>
-      <CardContent>
-        <Typography color="textSecondary" variant="subtitle2">
-          Classification
-        </Typography>
-        <Box sx={{ mt: 1 }}>
-          <Label color="primary">{dataset.confidentiality}</Label>
-        </Box>
-      </CardContent>
-      <CardContent>
-        <Typography color="textSecondary" variant="subtitle2">
-          Topics
-        </Typography>
-        <Box sx={{ mt: 1 }}>
-          {dataset.topics &&
-            dataset.topics.length > 0 &&
-            dataset.topics.map((t) => (
-              <Chip
-                sx={{ mr: 0.5, mb: 0.5 }}
-                key={t}
-                label={t}
-                variant="outlined"
-              />
-            ))}
-        </Box>
-      </CardContent>
+      {isFeatureEnabled('datasets', 'confidentiality_dropdown') && (
+        <CardContent>
+          <Typography color="textSecondary" variant="subtitle2">
+            Classification
+          </Typography>
+          <Box sx={{ mt: 1 }}>
+            <Label color="primary">{dataset.confidentiality}</Label>
+          </Box>
+        </CardContent>
+      )}
+      {isFeatureEnabled('datasets', 'topics_dropdown') && (
+        <CardContent>
+          <Typography color="textSecondary" variant="subtitle2">
+            Topics
+          </Typography>
+          <Box sx={{ mt: 1 }}>
+            {dataset.topics &&
+              dataset.topics.length > 0 &&
+              dataset.topics.map((t) => (
+                <Chip
+                  sx={{ mr: 0.5, mb: 0.5 }}
+                  key={t}
+                  label={t}
+                  variant="outlined"
+                />
+              ))}
+          </Box>
+        </CardContent>
+      )}
+
       <CardContent>
         <Typography color="textSecondary" variant="subtitle2">
           Tags