Merge branch 'data-dot-all:main' into main

.gitignore

@@ -71,3 +71,4 @@ yarn-debug.log*
 yarn-error.log*
 .idea
 /.ruff_cache/
+/testdata.json

backend/dataall/base/api/constants.py

@@ -21,6 +21,14 @@ def to_label(cls, value):
                 return c.name
         return None
 
+    @classmethod
+    def has_value(cls, value):
+        return value in cls._value2member_map_
+
+    @classmethod
+    def has_key(cls, key):
+        return key in cls._member_map_
+
 
 class SortDirection(GraphQLEnumMapper):
     asc = 'asc'

backend/dataall/base/api/gql/graphql_type_modifiers.py

@@ -36,7 +36,7 @@ def gql(self):
             elif isinstance(self.of_type, Thunk):
                 return template(self.of_type.target.name)
             else:
-                raise Exception('Cant gql ')
+                raise Exception(f'Cant gql {self.of_type}')
 
     return Modifier
 

backend/dataall/modules/metadata_forms/__init__.py

@@ -0,0 +1,13 @@
+from dataall.base.loader import ModuleInterface, ImportMode
+
+
+class MetadataFormsApiModuleInterface(ModuleInterface):
+    """Implements ModuleInterface for Metadata Forms GraphQl lambda"""
+
+    @classmethod
+    def is_supported(cls, modes):
+        return ImportMode.API in modes
+
+    def __init__(self):
+        import dataall.modules.metadata_forms.api
+        import dataall.modules.metadata_forms.db.enums

backend/dataall/modules/metadata_forms/api/__init__.py

@@ -0,0 +1,10 @@
+from . import (
+    input_types,
+    types,
+    resolvers,
+    enums,
+    mutations,
+    queries,
+)
+
+__all__ = ['enums', 'resolvers', 'types', 'input_types', 'queries', 'mutations']

backend/dataall/modules/metadata_forms/api/enums.py

@@ -0,0 +1,5 @@
+from dataall.base.api import GraphQLEnumMapper
+
+
+class EnvironmentSortField(GraphQLEnumMapper):
+    name = 'name'

backend/dataall/modules/metadata_forms/api/input_types.py

@@ -0,0 +1,22 @@
+from dataall.base.api import gql
+
+NewMetadataFormInput = gql.InputType(
+    name='NewMetadataFormInput',
+    arguments=[
+        gql.Field(name='name', type=gql.NonNullableType(gql.String)),
+        gql.Field(name='description', type=gql.String),
+        gql.Field(name='SamlGroupName', type=gql.NonNullableType(gql.String)),
+        gql.Field(name='visibility', type=gql.NonNullableType(gql.String)),
+        gql.Field(name='homeEntity', type=gql.String),
+    ],
+)
+
+
+MetadataFormFilter = gql.InputType(
+    name='MetadataFormFilter',
+    arguments=[
+        gql.Argument('page', gql.Integer),
+        gql.Argument('search_input', gql.String),
+        gql.Argument('pageSize', gql.Integer),
+    ],
+)

backend/dataall/modules/metadata_forms/api/mutations.py

@@ -0,0 +1,20 @@
+from dataall.base.api import gql
+from dataall.modules.metadata_forms.api.resolvers import create_metadata_form, delete_metadata_form
+
+createMetadataForm = gql.MutationField(
+    name='createMetadataForm',
+    args=[gql.Argument(name='input', type=gql.NonNullableType(gql.Ref('NewMetadataFormInput')))],
+    type=gql.Ref('MetadataForm'),
+    resolver=create_metadata_form,
+    test_scope='MetadataForm',
+)
+
+deleteMetadataForm = gql.MutationField(
+    name='deleteMetadataForm',
+    args=[
+        gql.Argument(name='formUri', type=gql.NonNullableType(gql.String)),
+    ],
+    type=gql.Boolean,
+    resolver=delete_metadata_form,
+    test_scope='MetadataForm',
+)

backend/dataall/modules/metadata_forms/api/queries.py

@@ -0,0 +1,10 @@
+from dataall.base.api import gql
+from dataall.modules.metadata_forms.api.resolvers import list_metadata_forms
+
+listMetadataForms = gql.QueryField(
+    name='listMetadataForms',
+    args=[gql.Argument('filter', gql.Ref('MetadataFormFilter'))],
+    type=gql.Ref('MetadataFormSearchResult'),
+    resolver=list_metadata_forms,
+    test_scope='MetadataForm',
+)

backend/dataall/modules/metadata_forms/api/resolvers.py

@@ -0,0 +1,19 @@
+from dataall.base.api.context import Context
+from dataall.modules.metadata_forms.db.metadata_form_models import MetadataForm
+from dataall.modules.metadata_forms.services.metadata_form_service import MetadataFormService
+
+
+def create_metadata_form(context: Context, source, input):
+    return MetadataFormService.create_metadata_form(input)
+
+
+def delete_metadata_form(context: Context, source, formUri):
+    return MetadataFormService.delete_metadata_form_by_uri(formUri)
+
+
+def list_metadata_forms(context: Context, source, filter=None):
+    return MetadataFormService.paginated_metadata_form_list(filter)
+
+
+def get_home_entity_name(context: Context, source: MetadataForm):
+    return MetadataFormService.get_home_entity_name(source)

backend/dataall/modules/metadata_forms/api/types.py

@@ -0,0 +1,30 @@
+from dataall.base.api import gql
+from dataall.modules.metadata_forms.api.resolvers import get_home_entity_name
+
+MetadataForm = gql.ObjectType(
+    name='MetadataForm',
+    fields=[
+        gql.Field(name='uri', type=gql.ID),
+        gql.Field(name='name', type=gql.String),
+        gql.Field(name='description', type=gql.String),
+        gql.Field(name='SamlGroupName', type=gql.String),
+        gql.Field(name='visibility', type=gql.String),
+        gql.Field(name='homeEntity', type=gql.String),
+        gql.Field(name='homeEntityName', type=gql.String, resolver=get_home_entity_name),
+    ],
+)
+
+MetadataFormSearchResult = gql.ObjectType(
+    name='MetadataFormSearchResult',
+    fields=[
+        gql.Field(name='count', type=gql.Integer),
+        gql.Field(name='nodes', type=gql.ArrayType(gql.Ref('MetadataForm'))),
+        gql.Field(name='pageSize', type=gql.Integer),
+        gql.Field(name='nextPage', type=gql.Integer),
+        gql.Field(name='pages', type=gql.Integer),
+        gql.Field(name='page', type=gql.Integer),
+        gql.Field(name='previousPage', type=gql.Integer),
+        gql.Field(name='hasNext', type=gql.Boolean),
+        gql.Field(name='hasPrevious', type=gql.Boolean),
+    ],
+)

backend/dataall/modules/metadata_forms/db/metadata_form_repository.py

@@ -0,0 +1,36 @@
+from sqlalchemy import or_
+
+from dataall.modules.metadata_forms.db.metadata_form_models import MetadataForm
+from dataall.modules.metadata_forms.db.enums import MetadataFormVisibility
+
+
+class MetadataFormRepository:
+    @staticmethod
+    def create_metadata_form(session, data=None):
+        mf: MetadataForm = MetadataForm(
+            name=data.get('name'),
+            description=data.get('description'),
+            SamlGroupName=data.get('SamlGroupName'),
+            visibility=data.get('visibility'),
+            homeEntity=data.get('homeEntity'),
+        )
+        session.add(mf)
+        session.commit()
+        return mf
+
+    @staticmethod
+    def get_metadata_form(session, uri):
+        return session.query(MetadataForm).get(uri)
+
+    @staticmethod
+    def list_metadata_forms(session, filter=None):
+        query = session.query(MetadataForm)
+        if filter and filter.get('search_input'):
+            query = query.filter(
+                or_(
+                    MetadataForm.name.ilike('%' + filter.get('search_input') + '%'),
+                    MetadataForm.description.ilike('%' + filter.get('search_input') + '%'),
+                )
+            )
+
+        return query.order_by(MetadataForm.name)

backend/dataall/modules/metadata_forms/services/metadata_form_service.py

@@ -0,0 +1,71 @@
+from dataall.base.context import get_context
+from dataall.base.db import exceptions, paginate
+from dataall.core.organizations.db.organization_repositories import OrganizationRepository
+from dataall.core.environment.db.environment_repositories import EnvironmentRepository
+
+from dataall.modules.metadata_forms.db.enums import MetadataFormVisibility
+from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
+
+
+class MetadataFormParamValidationService:
+    @staticmethod
+    def validate_create_form_params(data):
+        visibility = data.get('visibility', MetadataFormVisibility.Team.value)
+        if not MetadataFormVisibility.has_value(visibility):
+            data['visibility'] = MetadataFormVisibility.Global.value
+
+        if not data.get('SamlGroupName'):
+            raise exceptions.RequiredParameter('SamlGroupName')
+
+        if (not data.get('homeEntity')) and (visibility != MetadataFormVisibility.Global.value):
+            raise exceptions.RequiredParameter('homeEntity')
+
+        if not data.get('name'):
+            raise exceptions.RequiredParameter('name')
+
+
+class MetadataFormService:
+    @staticmethod
+    def create_metadata_form(data):
+        MetadataFormParamValidationService.validate_create_form_params(data)
+        with get_context().db_engine.scoped_session() as session:
+            form = MetadataFormRepository.create_metadata_form(session, data)
+            return form
+
+    # toDo: add permission check
+    @staticmethod
+    def get_metadata_form_by_uri(uri):
+        with get_context().db_engine.scoped_session() as session:
+            return MetadataFormRepository.get_metadata_form(session, uri)
+
+    # toDo: add permission check
+    # toDo: deletion logic
+    @staticmethod
+    def delete_metadata_form_by_uri(uri):
+        mf = MetadataFormService.get_metadata_form_by_uri(uri)
+        with get_context().db_engine.scoped_session() as session:
+            return session.delete(mf)
+
+    @staticmethod
+    def paginated_metadata_form_list(data=None) -> dict:
+        context = get_context()
+        data = data if data is not None else {}
+        with context.db_engine.scoped_session() as session:
+            return paginate(
+                query=MetadataFormRepository.list_metadata_forms(session, data),
+                page=data.get('page', 1),
+                page_size=data.get('pageSize', 5),
+            ).to_dict()
+
+    @staticmethod
+    def get_home_entity_name(metadata_form):
+        if metadata_form.visibility == MetadataFormVisibility.Team.value:
+            return metadata_form.homeEntity
+        elif metadata_form.visibility == MetadataFormVisibility.Organization.value:
+            with get_context().db_engine.scoped_session() as session:
+                return OrganizationRepository.get_organization_by_uri(session, metadata_form.homeEntity).name
+        elif metadata_form.visibility == MetadataFormVisibility.Environment.value:
+            with get_context().db_engine.scoped_session() as session:
+                return EnvironmentRepository.get_environment_by_uri(session, metadata_form.homeEntity).name
+        else:
+            return ''

backend/dataall/modules/redshift_datasets/api/connections/input_types.py

@@ -5,6 +5,7 @@
     name='CreateRedshiftConnectionInput',
     arguments=[
         gql.Argument('connectionName', gql.NonNullableType(gql.String)),
+        gql.Argument('connectionType', gql.NonNullableType(gql.String)),
         gql.Argument('environmentUri', gql.NonNullableType(gql.String)),
         gql.Argument('SamlGroupName', gql.NonNullableType(gql.String)),
         gql.Argument('redshiftType', gql.NonNullableType(gql.String)),
@@ -26,5 +27,6 @@
         gql.Argument('pageSize', gql.Integer),
         gql.Argument('environmentUri', gql.String),
         gql.Argument('groupUri', gql.String),
+        gql.Argument('connectionType', gql.String),
     ],
 )

backend/dataall/modules/redshift_datasets/api/connections/types.py

@@ -15,6 +15,7 @@
         gql.Field('database', gql.String),
         gql.Field('redshiftUser', gql.String),
         gql.Field('secretArn', gql.String),
+        gql.Field('encryptionType', gql.String),
     ],
 )
 

backend/dataall/modules/redshift_datasets/aws/kms_redshift.py

@@ -0,0 +1,37 @@
+import logging
+
+from dataall.base.aws.sts import SessionHelper
+from botocore.exceptions import ClientError
+
+
+log = logging.getLogger(__name__)
+
+
+class KmsClient:
+    def __init__(self, account_id: str, region: str):
+        session = SessionHelper.remote_session(accountid=account_id, region=region)
+        self._client = session.client('kms', region_name=region)
+        self._account_id = account_id
+        self.region = region
+
+    def describe_kms_key(self, key_id: str):
+        # The same client function is defined in the data_sharing module. Duplication is allowed to avoid coupling.
+        try:
+            response = self._client.describe_key(
+                KeyId=key_id,
+            )
+            log.info(f'KMS key used to encrypt cluster {response=}')
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have kms:DescribeKey Permission for key {key_id}: {e}'
+                )
+            log.error(f'Failed to describe key {key_id}: {e}')
+            return None
+        else:
+            return response['KeyMetadata']
+
+
+def kms_redshift_client(account_id: str, region: str) -> KmsClient:
+    "Factory of Client"
+    return KmsClient(account_id=account_id, region=region)

backend/dataall/modules/redshift_datasets/aws/redshift.py

@@ -13,9 +13,16 @@ def __init__(self, account_id: str, region: str) -> None:
         self.client = session.client(service_name='redshift', region_name=region)
 
     def describe_cluster(self, clusterId: str):
-        try:
-            log.info(f'Describing cluster {clusterId=}')
-            return self.client.describe_clusters(ClusterIdentifier=clusterId)['Clusters'][0]
-        except ClientError as e:
-            log.error(e)
-            raise e
+        log.info(f'Describing cluster {clusterId=}')
+        return self.client.describe_clusters(ClusterIdentifier=clusterId)['Clusters'][0]
+
+    def get_cluster_namespaceId(self, clusterId: str):
+        log.info(f'Describing cluster {clusterId=}')
+        return self.client.describe_clusters(ClusterIdentifier=clusterId)['Clusters'][0]['ClusterNamespaceArn'].split(
+            ':'
+        )[-1]
+
+
+def redshift_client(account_id: str, region: str) -> RedshiftClient:
+    "Factory of Client"
+    return RedshiftClient(account_id=account_id, region=region)

backend/dataall/modules/redshift_datasets/aws/redshift_data.py

@@ -143,3 +143,8 @@ def list_redshift_table_columns(self, schema: str, table: str):
         except ClientError as e:
             log.error(e)
             raise e
+
+
+def redshift_data_client(account_id: str, region: str, connection: RedshiftConnection) -> RedshiftDataClient:
+    "Factory of Client"
+    return RedshiftDataClient(account_id=account_id, region=region, connection=connection)

backend/dataall/modules/redshift_datasets/aws/redshift_serverless.py

@@ -14,12 +14,14 @@ def __init__(self, account_id: str, region: str, role=None) -> None:
         self.client = session.client(service_name='redshift-serverless', region_name=region)
 
     def get_namespace_by_id(self, namespace_id: str):
+        log.info(f'Get namespace with {namespace_id=}')
         response = self.client.list_namespaces()
         namespaces = response.get('namespaces', [])
         namespaces_filtered = [namespace for namespace in namespaces if namespace['namespaceId'] == namespace_id]
         return namespaces_filtered[0] if namespaces_filtered else None
 
     def list_workgroups_in_namespace(self, namespace_name: str) -> List[dict]:
+        log.info(f'Listing workgroups in {namespace_name=}')
         response = self.client.list_workgroups()
         workgroups = response.get('workgroups', [])
         return [wg for wg in workgroups if wg['namespaceName'] == namespace_name]
@@ -32,3 +34,8 @@ def get_workgroup_arn(self, workgroup_name: str) -> str:
         except ClientError as e:
             log.error(e)
             raise e
+
+
+def redshift_serverless_client(account_id: str, region: str, role: str = None) -> RedshiftServerlessClient:
+    "Factory of Client"
+    return RedshiftServerlessClient(account_id=account_id, region=region, role=role)