Merge branch 'os-main' into feat/worksheet-nlq-beta

backend/dataall/core/environment/services/environment_service.py

@@ -729,6 +729,11 @@ def paginated_user_environment_groups(uri, data=None) -> dict:
     def get_all_environment_groups(session, uri, filter) -> Query:
         return EnvironmentRepository.query_all_environment_groups(session, uri, filter)
 
+    @staticmethod
+    def list_all_environment_groups(uri, data=None) -> [str]:
+        with get_context().db_engine.scoped_session() as session:
+            return [g.groupUri for g in EnvironmentRepository.query_all_environment_groups(session, uri, data).all()]
+
     @staticmethod
     @ResourcePolicyService.has_resource_permission(environment_permissions.LIST_ENVIRONMENT_GROUPS)
     def paginated_all_environment_groups(uri, data=None) -> dict:

backend/dataall/modules/metadata_forms/db/metadata_form_models.py

@@ -17,24 +17,42 @@ class MetadataForm(Base):
     homeEntity = Column(String, nullable=True)
 
 
+class MetadataFormVersion(Base):
+    __tablename__ = 'metadata_form_version'
+    metadataFormUri = Column(String, ForeignKey('metadata_form.uri'), nullable=False)
+    version = Column(Integer, nullable=False)
+
+    __table_args__ = (
+        PrimaryKeyConstraint('metadataFormUri', 'version'),
+        ForeignKeyConstraint(
+            ('metadataFormUri',), ('metadata_form.uri',), name='f_key_version_metadata', ondelete='CASCADE'
+        ),
+    )
+
+
 class MetadataFormEnforcementRule(Base):
     __tablename__ = 'metadata_form_enforcement_rule'
     uri = Column(String, primary_key=True, default=utils.uuid('rule'))
     metadataFormUri = Column(String, ForeignKey('metadata_form.uri'), nullable=False)
+    version = Column(Integer, nullable=False)
     level = Column(String, nullable=False)  # enum MetadataFormEnforcementScope
     entityTypes = Column(ARRAY(String), nullable=False)  # enum MetadataFormEntityTypes
     severity = Column(String, nullable=False)  # enum MetadataFormEnforcementSeverity
 
     __table_args__ = (
         ForeignKeyConstraint(
-            ('metadataFormUri',), ('metadata_form.uri',), name='f_key_enforcement_metadata', ondelete='CASCADE'
+            ['metadataFormUri', 'version'],
+            ['metadata_form_version.metadataFormUri', 'metadata_form_version.version'],
+            name='f_key_enforcement_version_metadata',
+            ondelete='CASCADE',
         ),
     )
 
 
 class MetadataFormField(Base):
     __tablename__ = 'metadata_form_field'
     metadataFormUri = Column(String)
+    version = Column(Integer, nullable=False)
     uri = Column(String, primary_key=True, default=utils.uuid('field'))
     displayNumber = Column(Integer, nullable=False)
     description = Column(String, nullable=True)
@@ -46,21 +64,28 @@ class MetadataFormField(Base):
 
     __table_args__ = (
         ForeignKeyConstraint(
-            ('metadataFormUri',), ('metadata_form.uri',), name='fk_mf_filed_form_uri', ondelete='CASCADE'
+            ['metadataFormUri', 'version'],
+            ['metadata_form_version.metadataFormUri', 'metadata_form_version.version'],
+            name='fk_version',
+            ondelete='CASCADE',
         ),
     )
 
 
 class AttachedMetadataForm(Base):
     __tablename__ = 'attached_metadata_form'
     metadataFormUri = Column(String, nullable=False)
+    version = Column(Integer, nullable=False)
     uri = Column(String, primary_key=True, default=utils.uuid('attached_form'))
     entityUri = Column(String, nullable=False)
     entityType = Column(String, nullable=False)
 
     __table_args__ = (
         ForeignKeyConstraint(
-            ('metadataFormUri',), ('metadata_form.uri',), name='fk_attached_mf_uri', ondelete='CASCADE'
+            ['metadataFormUri', 'version'],
+            ['metadata_form_version.metadataFormUri', 'metadata_form_version.version'],
+            name='fk_attached_mf_version_uri',
+            ondelete='CASCADE',
         ),
     )
 

backend/dataall/modules/metadata_forms/db/metadata_form_repository.py

@@ -1,5 +1,6 @@
 from sqlalchemy import or_, and_
 from sqlalchemy.orm import with_polymorphic
+from sqlalchemy import func
 
 from dataall.modules.metadata_forms.db.enums import MetadataFormVisibility, MetadataFormFieldType
 from dataall.modules.metadata_forms.db.metadata_form_models import (
@@ -11,6 +12,7 @@
     BooleanAttachedMetadataFormField,
     IntegerAttachedMetadataFormField,
     GlossaryTermAttachedMetadataFormField,
+    MetadataFormVersion,
 )
 
 import json
@@ -40,10 +42,39 @@ def create_metadata_form(session, data=None):
         session.commit()
         return mf
 
+    @staticmethod
+    def create_metadata_form_version(session, metadataFormUri, version_num):
+        version = MetadataFormVersion(metadataFormUri=metadataFormUri, version=version_num)
+        session.add(version)
+        session.commit()
+        return version
+
+    @staticmethod
+    def create_metadata_form_version_next(session, metadataFormUri):
+        version_num = MetadataFormRepository.get_metadata_form_version_number_latest(session, metadataFormUri)
+        version = MetadataFormVersion(metadataFormUri=metadataFormUri, version=version_num + 1)
+        session.add(version)
+        session.commit()
+        return version
+
+    @staticmethod
+    def get_metadata_form_version_number_latest(session, metadataFormUri):
+        return (
+            session.query(func.max(MetadataFormVersion.version))
+            .filter(MetadataFormVersion.metadataFormUri == metadataFormUri)
+            .scalar()
+        )
+
+    @staticmethod
+    def get_metadata_form_version_latest(session, metadataFormUri):
+        version_num = MetadataFormRepository.get_metadata_form_version_number_latest(session, metadataFormUri)
+        return session.query(MetadataFormVersion).get((metadataFormUri, version_num))
+
     @staticmethod
     def create_attached_metadata_form(session, uri, data=None):
+        version_num = MetadataFormRepository.get_metadata_form_version_number_latest(session, uri)
         amf: AttachedMetadataForm = AttachedMetadataForm(
-            metadataFormUri=uri, entityUri=data.get('entityUri'), entityType=data.get('entityType')
+            metadataFormUri=uri, version=version_num, entityUri=data.get('entityUri'), entityType=data.get('entityType')
         )
         session.add(amf)
         session.commit()
@@ -175,8 +206,10 @@ def get_metadata_form_fields(session, form_uri):
 
     @staticmethod
     def create_metadata_form_field(session, uri, data):
+        version_num = MetadataFormRepository.get_metadata_form_version_number_latest(session, uri)
         field: MetadataFormField = MetadataFormField(
             metadataFormUri=uri,
+            version=version_num,
             name=data.get('name'),
             description=data.get('description'),
             type=data.get('type'),

backend/dataall/modules/metadata_forms/services/metadata_form_service.py

@@ -115,7 +115,12 @@ def create_metadata_form(data):
                 )
 
             form = MetadataFormRepository.create_metadata_form(session, data)
-            return form
+            try:
+                MetadataFormRepository.create_metadata_form_version(session, form.uri, 1)
+                return form
+            except Exception as e:
+                session.delete(form)
+                raise e
 
     # toDo: add permission check
     @staticmethod

backend/dataall/modules/redshift_datasets/services/redshift_connection_permissions.py

@@ -27,8 +27,8 @@
 REDSHIFT_CONNECTION_READ = [GET_REDSHIFT_CONNECTION]
 
 DELETE_REDSHIFT_CONNECTION = 'DELETE_REDSHIFT_CONNECTION'
-MANAGE_REDSHIFT_CONNECTION_PERMISSIONS = 'MANAGE_REDSHIFT_CONNECTION_PERMISSIONS'
-REDSHIFT_CONNECTION_WRITE = [DELETE_REDSHIFT_CONNECTION, MANAGE_REDSHIFT_CONNECTION_PERMISSIONS]
+EDIT_REDSHIFT_CONNECTION_PERMISSIONS = 'EDIT_REDSHIFT_CONNECTION_PERMISSIONS'
+REDSHIFT_CONNECTION_WRITE = [DELETE_REDSHIFT_CONNECTION, EDIT_REDSHIFT_CONNECTION_PERMISSIONS]
 
 CREATE_SHARE_REQUEST_WITH_CONNECTION = 'CREATE_SHARE_REQUEST_WITH_CONNECTION'
 REDSHIFT_GRANTABLE_PERMISSIONS = [CREATE_SHARE_REQUEST_WITH_CONNECTION]

backend/dataall/modules/redshift_datasets/services/redshift_connection_service.py

@@ -14,7 +14,7 @@
     REDSHIFT_GRANTABLE_PERMISSIONS,
     DELETE_REDSHIFT_CONNECTION,
     GET_REDSHIFT_CONNECTION,
-    MANAGE_REDSHIFT_CONNECTION_PERMISSIONS,
+    EDIT_REDSHIFT_CONNECTION_PERMISSIONS,
     CREATE_REDSHIFT_CONNECTION,
     LIST_ENVIRONMENT_REDSHIFT_CONNECTIONS,
 )
@@ -150,7 +150,7 @@ def list_schema_tables(uri, schema):
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_REDSHIFT_CONNECTIONS)
-    @ResourcePolicyService.has_resource_permission(MANAGE_REDSHIFT_CONNECTION_PERMISSIONS)
+    @ResourcePolicyService.has_resource_permission(EDIT_REDSHIFT_CONNECTION_PERMISSIONS)
     def add_group_permissions(uri, group, permissions) -> bool:
         context = get_context()
         connection = RedshiftConnectionService.get_redshift_connection_by_uri(uri=uri)
@@ -161,8 +161,12 @@ def add_group_permissions(uri, group, permissions) -> bool:
                 param_value=permissions,
                 constraint=f'one of the possible grantable permissions {REDSHIFT_GRANTABLE_PERMISSIONS}',
             )
-
+        env_groups = EnvironmentService.list_all_environment_groups(uri=connection.environmentUri)
         with context.db_engine.scoped_session() as session:
+            if group not in env_groups:
+                raise exceptions.InvalidInput(
+                    param_name='Team', param_value=group, constraint='a team invited to the Environment.'
+                )
             ResourcePolicyService.attach_resource_policy(
                 session=session,
                 group=group,
@@ -174,7 +178,7 @@ def add_group_permissions(uri, group, permissions) -> bool:
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_REDSHIFT_CONNECTIONS)
-    @ResourcePolicyService.has_resource_permission(MANAGE_REDSHIFT_CONNECTION_PERMISSIONS)
+    @ResourcePolicyService.has_resource_permission(EDIT_REDSHIFT_CONNECTION_PERMISSIONS)
     def delete_group_permissions(uri, group) -> bool:
         context = get_context()
         connection = RedshiftConnectionService.get_redshift_connection_by_uri(uri=uri)
@@ -194,7 +198,7 @@ def delete_group_permissions(uri, group) -> bool:
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_REDSHIFT_CONNECTIONS)
-    @ResourcePolicyService.has_resource_permission(MANAGE_REDSHIFT_CONNECTION_PERMISSIONS)
+    @ResourcePolicyService.has_resource_permission(EDIT_REDSHIFT_CONNECTION_PERMISSIONS)
     def list_connection_group_permissions(uri, filter):
         context = get_context()
         permissions = REDSHIFT_GRANTABLE_PERMISSIONS
@@ -205,7 +209,7 @@ def list_connection_group_permissions(uri, filter):
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_REDSHIFT_CONNECTIONS)
-    @ResourcePolicyService.has_resource_permission(MANAGE_REDSHIFT_CONNECTION_PERMISSIONS)
+    @ResourcePolicyService.has_resource_permission(EDIT_REDSHIFT_CONNECTION_PERMISSIONS)
     def list_connection_group_no_permissions(uri, filter):
         context = get_context()
         with context.db_engine.scoped_session() as session:

backend/migrations/versions/5a798acc6282_create_version_mf.py

@@ -0,0 +1,125 @@
+"""create_version_mf
+
+Revision ID: 5a798acc6282
+Revises: 075d344ae2cc
+Create Date: 2024-10-10 17:25:09.687099
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from dataall.modules.metadata_forms.db.metadata_form_models import (
+    MetadataForm,
+    MetadataFormVersion,
+    AttachedMetadataForm,
+    MetadataFormField,
+)
+
+# revision identifiers, used by Alembic.
+revision = '5a798acc6282'
+down_revision = '075d344ae2cc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    bind = op.get_bind()
+    session = orm.Session(bind=bind)
+
+    op.create_table(
+        'metadata_form_version',
+        sa.Column('metadataFormUri', sa.String(), nullable=False),
+        sa.Column('version', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['metadataFormUri'], ['metadata_form.uri'], name='f_key_version_metadata', ondelete='CASCADE'
+        ),
+        sa.PrimaryKeyConstraint('metadataFormUri', 'version'),
+    )
+    for mf in session.query(MetadataForm).all():
+        version = MetadataFormVersion(metadataFormUri=mf.uri, version=1)
+        session.add(version)
+        session.commit()
+
+    op.add_column('attached_metadata_form', sa.Column('version', sa.Integer()))
+    for amf in session.query(AttachedMetadataForm).all():
+        amf.version = 1
+        session.commit()
+    op.alter_column('attached_metadata_form', 'version', nullable=False)
+    op.drop_constraint('fk_attached_mf_uri', 'attached_metadata_form', type_='foreignkey')
+    op.create_foreign_key(
+        'fk_attached_mf_version_uri',
+        'attached_metadata_form',
+        'metadata_form_version',
+        ['metadataFormUri', 'version'],
+        ['metadataFormUri', 'version'],
+        ondelete='CASCADE',
+    )
+
+    op.add_column('metadata_form_enforcement_rule', sa.Column('version', sa.Integer(), nullable=False))
+    op.create_foreign_key(
+        'f_key_enforcement_version_metadata',
+        'metadata_form_enforcement_rule',
+        'metadata_form_version',
+        ['metadataFormUri', 'version'],
+        ['metadataFormUri', 'version'],
+        ondelete='CASCADE',
+    )
+    op.drop_constraint('f_key_enforcement_metadata', 'metadata_form_enforcement_rule', type_='foreignkey')
+
+    op.add_column('metadata_form_field', sa.Column('version', sa.Integer()))
+    for field in session.query(MetadataFormField).all():
+        field.version = 1
+        session.commit()
+    op.alter_column('metadata_form_field', 'version', nullable=False)
+    op.drop_constraint('fk_mf_filed_form_uri', 'metadata_form_field', type_='foreignkey')
+    op.create_foreign_key(
+        'fk_version',
+        'metadata_form_field',
+        'metadata_form_version',
+        ['metadataFormUri', 'version'],
+        ['metadataFormUri', 'version'],
+        ondelete='CASCADE',
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('fk_version', 'metadata_form_field', type_='foreignkey')
+    op.drop_column('metadata_form_field', 'version')
+    op.drop_constraint('f_key_enforcement_version_metadata', 'metadata_form_enforcement_rule', type_='foreignkey')
+    op.drop_column('metadata_form_enforcement_rule', 'version')
+    op.drop_constraint('fk_attached_mf_version_uri', 'attached_metadata_form', type_='foreignkey')
+    op.drop_column('attached_metadata_form', 'version')
+    op.drop_table('metadata_form_version')
+
+    op.create_foreign_key(
+        'fk_mf_filed_form_uri',
+        'metadata_form_field',
+        'metadata_form',
+        ['metadataFormUri'],
+        ['uri'],
+        ondelete='CASCADE',
+    )
+
+    op.create_foreign_key(
+        'f_key_enforcement_metadata',
+        'metadata_form_enforcement_rule',
+        'metadata_form',
+        ['metadataFormUri'],
+        ['uri'],
+        ondelete='CASCADE',
+    )
+
+    op.create_foreign_key(
+        'fk_attached_mf_uri',
+        'attached_metadata_form',
+        'metadata_form',
+        ['metadataFormUri'],
+        ['uri'],
+        ondelete='CASCADE',
+    )
+    # ### end Alembic commands ###

tests/modules/conftest.py

@@ -5,6 +5,7 @@
 from dataall.core.environment.db.environment_models import Environment, EnvironmentGroup, EnvironmentParameter
 from dataall.core.organizations.db.organization_models import Organization
 from dataall.core.permissions.services.environment_permissions import ENVIRONMENT_ALL
+from dataall.core.permissions.services.organization_permissions import ORGANIZATION_ALL
 from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
 from dataall.core.stacks.db.stack_repositories import StackRepository
 from dataall.core.stacks.db.stack_models import KeyValueTag
@@ -128,6 +129,14 @@ def factory(name, group, user):
             )
             session.add(org)
             session.commit()
+            ResourcePolicyService.attach_resource_policy(
+                session=session,
+                resource_uri=org.organizationUri,
+                group=group.name,
+                permissions=ORGANIZATION_ALL,
+                resource_type=Organization.__name__,
+            )
+            session.commit()
             return org
 
     yield factory