Skip to content

Commit

Permalink
Resolve incorrect diffs os main + nlq branch
Browse files Browse the repository at this point in the history
  • Loading branch information
@noah-paige
noah-paige committed Oct 14, 2024
1 parent 1b7151d commit 242498a
Show file tree
Hide file tree
Showing 15 changed files with 137 additions and 358 deletions.
6 changes: 5 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,11 @@ check-security: upgrade-pip install-backend install-cdkproxy
pip install bandit
pip install safety
bandit -lll -r backend
safety check --ignore=51668,70612,70624,59234
safety check --ignore=51668,70612,70624

checkov-synth: upgrade-pip install-backend install-cdkproxy install-tests
export PYTHONPATH=./backend:/./tests && \
python -m pytest -v -ra -k test_checkov tests

test:
export PYTHONPATH=./backend:/./tests && \
Expand Down
Binary file modified UserGuide.pdf
View file
Binary file not shown.
4 changes: 2 additions & 2 deletions backend/dataall/core/resource_threshold/db/resource_threshold.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from dataall.base.db import Base
from dataall.base.db import Base, utils
from sqlalchemy import String, Integer, Column, Date, func
import uuid


class ResourceTreshold(Base):
__tablename__ = 'resource_threshold'
actionId = Column(String(64), primary_key=True, default=lambda: str(uuid.uuid4()))
actionId = Column(String(64), primary_key=True, default=lambda: utils.uuid('resource_threshold'))
username = Column(String(64), nullable=False)
actionType = Column(String(64), nullable=False)
date = Column(Date, default=func.current_date(), nullable=False)
Expand Down
183 changes: 0 additions & 183 deletions backend/dataall/modules/worksheets/aws/glue_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,66 +15,6 @@ def __init__(self, account_id, region, database):
self._account_id = account_id
self._region = region

def create_database(self, location):
try:
log.info(f'Creating database {self._database} ' f'in account {self._account_id}...')
existing_database = self.get_glue_database()
if existing_database:
glue_database_created = True
else:
self._create_glue_database(location)
glue_database_created = True
log.info(f'Successfully created database {self._database}' f'in account {self._account_id}')
return glue_database_created
except ClientError as e:
log.error(f'Failed to create database {self._database} ' f'in account {self._account_id} ' f'due to {e}')
raise e

def _create_glue_database(self, location):
database = self._database
try:
db_input = {
'Name': database,
'Description': 'dataall database {} '.format(database),
'CreateTableDefaultPermissions': [],
}
if location:
db_input['LocationUri'] = location
response = self._client.create_database(CatalogId=self._account_id, DatabaseInput=db_input)
return response
except ClientError as e:
raise e

def get_glue_database(self):
try:
log.info(f'Getting database {self._database} ' f'in account {self._account_id}...')
database = self._client.get_database(CatalogId=self._account_id, Name=self._database)
return database
except ClientError:
log.info(f'Database {self._database} not found in account {self._account_id}')
return False

def database_exists(self, database_name):
try:
log.info(f'Check database exists {self._database} ' f'in account {self._account_id}...')
self._client.get_database(CatalogId=self._account_id, Name=database_name)
return True
except ClientError:
log.info(f'Database {database_name} not found in account {self._account_id}')
return False

def table_exists(self, table_name):
try:
log.info(
f'Check table exists {table_name} ' f'in database {self._database} ' f'in account {self._account_id}...'
)
table = self._client.get_table(CatalogId=self._account_id, DatabaseName=self._database, Name=table_name)
log.info(f'Glue table {table_name} found in account {self._account_id} in database {self._database}')
return table
except ClientError:
log.info(f'Glue table not found: {table_name}')
return None

def get_metadata(self, table_name):
table_metadata = self._client.get_table(DatabaseName=self._database, Name=table_name)
table_name = table_metadata['Table']['Name']
Expand All @@ -87,126 +27,3 @@ def get_metadata(self, table_name):
Partition Metadata: {partition_metadata}
"""
return meta_data

def delete_table(self, table_name):
database = self._database
try:
log.info(
f'Deleting table {table_name} ' f'in database {self._database} ' f'in catalog {self._account_id}...'
)
response = self._client.delete_table(CatalogId=self._account_id, DatabaseName=database, Name=table_name)
log.info(
f'Successfully deleted table {table_name} '
f'in database {database} '
f'in catalog {self._account_id} '
f'response: {response}'
)
return response
except ClientError as e:
log.error(
f'Could not delete table {table_name} '
f'in database {database} '
f'in catalog {self._account_id} '
f'due to: {e}'
)
raise e

def create_resource_link(self, resource_link_name, table, catalog_id, database):
account_id = self._account_id
shared_database = self._database
resource_link_input = {
'Name': table.GlueTableName,
'TargetTable': {
'CatalogId': catalog_id,
'DatabaseName': database,
'Name': table.GlueTableName,
},
}

try:
log.info(f'Creating ResourceLink {resource_link_name} ' f'in database {shared_database}...')
resource_link = self.table_exists(resource_link_name)
if resource_link:
log.info(
f'ResourceLink {resource_link_name} '
f'in database {account_id}://{shared_database}'
f'already exists: {resource_link}'
)
else:
resource_link = self._client.create_table(
CatalogId=account_id,
DatabaseName=shared_database,
TableInput=resource_link_input,
)
log.info(
f'Successfully created ResourceLink {resource_link_name} '
f'in database {account_id}://{shared_database} '
f'response: {resource_link}'
)
return resource_link
except ClientError as e:
log.error(
f'Could not create ResourceLink {resource_link_name} '
f'in database {account_id}://{shared_database} '
f'due to: {e}'
)
raise e

def delete_database(self):
account_id = self._account_id
database = self._database
try:
log.info(f'Deleting database {self._database} ' f'in account {self._account_id}...')
existing_database = self.get_glue_database()
if existing_database:
self._client.delete_database(CatalogId=account_id, Name=database)
log.info(f'Successfully deleted database {database} ' f'in account {account_id}')
return True
except ClientError as e:
log.error(f'Could not delete database {database} ' f'in account {account_id} ' f'due to: {e}')
raise e

def get_source_catalog(self):
"""Get the source catalog account details"""
try:
log.info(f'Fetching source catalog details for database {self._database}...')
response = self._client.get_database(CatalogId=self._account_id, Name=self._database)
linked_database = response.get('Database', {}).get('TargetDatabase', {})
log.info(f'Fetched source catalog details for database {self._database} are: {linked_database}...')
if linked_database:
return {
'data_base': self._database,
'account_id': linked_database.get('CatalogId'),
'database_name': linked_database.get('DatabaseName'),
'region': linked_database.get('Region', self._region),
}

except self._client.exceptions.EntityNotFoundException as enoFnd:
log.exception(f'Could not fetch source catalog details for database {self._database} due to {enoFnd}')
raise enoFnd
except Exception as e:
log.exception(f'Error fetching source catalog details for database {self._database} due to {e}')
raise e
return None

def get_database_tags(self):
# Get tags from the glue database
account_id = self._account_id
database = self._database
region = self._region

try:
log.info(f'Getting tags for database {database}')
resource_arn = f'arn:aws:glue:{region}:{account_id}:database/{database}'
response = self._client.get_tags(ResourceArn=resource_arn)
tags = response['Tags']

log.info(f'Successfully retrieved tags: {tags}')

return tags
except self._client.exceptions.EntityNotFoundException as entNotFound:
log.exception(f'Could not get tags for database {database} due to {entNotFound}')
raise entNotFound
except Exception as e:
log.exception(f'Error fetching tags for {database} due to {e}')
raise e
4 changes: 2 additions & 2 deletions backend/migrations/versions/2258cd8d6e9f_resource_treshholds_added.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""describe_changes_shortly
Revision ID: 2258cd8d6e9f
Revises: 9efe5f7c69a1
Revises: 5a798acc6282
Create Date: 2024-08-22 12:31:38.465650
"""
Expand All @@ -12,7 +12,7 @@

# revision identifiers, used by Alembic.
revision = '2258cd8d6e9f'
down_revision = '9efe5f7c69a1'
down_revision = '5a798acc6282'
branch_labels = None
depends_on = None

Expand Down
57 changes: 1 addition & 56 deletions cdk.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,61 +4,6 @@
"@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": false,
"@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true,
"@aws-cdk/aws-rds:lowercaseDbIdentifier": false,
"@aws-cdk/core:stackRelativeExports": false,
"tooling_region": "string_TOOLING_REGION|DEFAULT=eu-west-1",
"tooling_vpc_id": "string_IMPORT_AN_EXISTING_VPC_FROM_TOOLING|DEFAULT=None",
"tooling_vpc_restricted_nacl": "boolean_CREATE_CUSTOM_NACL|DEFAULT=false",
"git_branch": "string_GIT_BRANCH_NAME|DEFAULT=dataall",
"git_release": "boolean_MANAGE_GIT_RELEASE|DEFAULT=false",
"quality_gate": "boolean_MANAGE_QUALITY_GATE_STAGE|DEFAULT=true",
"resource_prefix": "string_PREFIX_FOR_ALL_RESOURCES_CREATED_BY_THIS_APP|DEFAULT=dataall",
"repository_source": "string_VERSION_CONTROL_SERVICE|(codecommit, codestar_connection) DEFAULT=codecommit",
"repo_string": "string_REPOSITORY_IN_GITHUB_OWNER/REPOSITORY|DEFAULT=awslabs/aws-dataall, REQUIRED if repository_source=codestar_connection",
"repo_connection_arn": "string_CODESTAR_SOURCE_CONNECTION_ARN_FOR_GITHUB_arn:aws:codestar-connections:region:account-id:connection/connection-id|DEFAULT=None, REQUIRED if repository_source=codestar_connection",
"DeploymentEnvironments": [
{
"envname": "string_ENVIRONMENT_NAME|REQUIRED",
"account": "string_DEPLOYMENT_ACCOUNT|REQUIRED",
"region": "string_DEPLOYMENT_REGION|REQUIRED",
"with_approval": "boolean_ADD_CODEPIPELINE_APPROVAL_STEP|DEFAULT=false",
"vpc_id": "string_DEPLOY_WITHIN_AN_EXISTING_VPC|DEFAULT=None",
"vpc_endpoints_sg": "string_DEPLOY_WITHIN_EXISTING_VPC_SG|DEFAULT=None",
"vpc_restricted_nacl": "boolean_CREATE_CUSTOM_NACL|DEFAULT=false",
"internet_facing": "boolean_CLOUDFRONT_IF_TRUE_ELSE_ECS_BEHIND_INTERNAL_ALB|DEFAULT=true",
"custom_domain": {
"hosted_zone_name": "string_ROUTE_53_EXISTING_DOMAIN_NAME|DEFAULT=None, REQUIRED if internet_facing=false",
"hosted_zone_id": "string_ROUTE_53_EXISTING_HOSTED_ZONE_ID|DEFAULT=None",
"certificate_arn": "string_AWS_CERTIFICATE_MANAGER_EXISTING_CERTIFICATE_ARN|DEFAULT=None, REQUIRED if internet_facing=false",
"email_notification_sender_email_id":"string_EMAIL_NOTIFICATION_SENDER_EMAIL_ID|DEFAULT=noreply"
},
"ip_ranges": "list_of_strings_IP_RANGES_TO_ALLOW_IF_NOT_INTERNET_FACING|DEFAULT=None",
"apig_vpce": "string_USE_AN_EXISTING_VPCE_FOR_APIG_IF_NOT_INTERNET_FACING|DEFAULT=None",
"prod_sizing": "boolean_SET_INFRA_SIZING_TO_PROD_VALUES_IF_TRUE|DEFAULT=true",
"enable_cw_rum": "boolean_SET_CLOUDWATCH_RUM_APP_MONITOR|DEFAULT=false",
"enable_cw_canaries": "boolean_SET_CLOUDWATCH_CANARIES_FOR_FRONTEND_TESTING|DEFAULT=false",
"shared_dashboards_sessions": "string_TYPE_SESSION_SHARED_DASHBOARDS|(reader, anonymous) DEFAULT=anonymous",
"enable_pivot_role_auto_create": "boolean_ENABLE_PIVOT_ROLE_AUTO_CREATE_IN_ENVIRONMENT|DEFAULT=false",
"enable_update_dataall_stacks_in_cicd_pipeline": "boolean_ENABLE_UPDATE_DATAALL_STACKS_IN_CICD_PIPELINE|DEFAULT=false",
"enable_opensearch_serverless": "boolean_USE_OPENSEARCH_SERVERLESS|DEFAULT=false",
"cognito_user_session_timeout_inmins": "integer_COGNITO_USER_SESSION_TIMEOUT_INMINS|DEFAULT=43200",
"reauth_config": {
"reauth_apis": "list_of_strings_OPERATION_NAMES_TO_REQUIRE_REAUTH_ON|DEFAULT=None",
"ttl": "int_TIME_IN_MINUTES_TO_ALLOW_USER_TO_PERFORM_SENSITIVE_APIS_BEFORE_FORCING_REAUTH|DEFAULT=5"
},
"custom_auth": {
"provider": "string_EXTERNAL_IDP_PROVIDER_NAME|DEFAULT=None",
"url" : "string_ISSUER_URL_OF_THE_EXTERNAL_IDP|DEFAULT=None",
"redirect_url" : "string_REDIRECT_URL_OF_THE_EXTERNAL_IDP|DEFAULT=None",
"client_id": "string_EXTERNAL_IDP_CLIENT_ID|DEFAULT=None",
"response_types": "string_EXTERNAL_RESPONSE_TYPES_USED_IN_OIDC_FLOW|DEFAULT=None",
"scopes": "string_EXTERNAL_IDP_SCOPES_SPACE_SEPARATED|DEFAULT=None",
"jwks_url" : "string_EXTERNAL_IDP_JWKS_URL|DEFAULT=None",
"claims_mapping": {
"user_id": "string_USER_ID_CLAIM_NAME_MAPPING_FOR_EXTERNAL_IDP|DEFAULT=None",
"email": "string_EMAIL_ID_CLAIM_NAME_MAPPING_FOR_EXTERNAL_IDP|DEFAULT=None"
}
}
}
]
"@aws-cdk/core:stackRelativeExports": false
}
}
Loading

0 comments on commit 242498a

Please sign in to comment.