diff --git a/cartography/data/indexes.cypher b/cartography/data/indexes.cypher index 2756f9d9c..84984e1e7 100644 --- a/cartography/data/indexes.cypher +++ b/cartography/data/indexes.cypher @@ -8,6 +8,7 @@ CREATE INDEX ON :AWSDNSRecord(id); CREATE INDEX ON :AWSDNSZone(name); CREATE INDEX ON :AWSDNSZone(zoneid); CREATE INDEX ON :AWSGroup(arn); +CREATE INDEX ON :AWSInternetGateway(id); CREATE INDEX ON :AWSIpv4CidrBlock(id); CREATE INDEX ON :AWSIpv6CidrBlock(id); CREATE INDEX ON :AWSLambda(id); diff --git a/cartography/data/jobs/cleanup/aws_import_internet_Gateways_cleanup.json b/cartography/data/jobs/cleanup/aws_import_internet_Gateways_cleanup.json new file mode 100644 index 000000000..a318a47b6 --- /dev/null +++ b/cartography/data/jobs/cleanup/aws_import_internet_Gateways_cleanup.json @@ -0,0 +1,10 @@ +{ + "statements": [ + { + "query": "MATCH (:AWSAccount{id:{AWS_ID}})-[:RESOURCE]->(n:AWSInternetGateway) WHERE n.lastupdated <> {UPDATE_TAG} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + } + ], + "name": "cleanup AWSInternetGateway" + } diff --git a/cartography/intel/aws/ec2/internet_gateways.py b/cartography/intel/aws/ec2/internet_gateways.py new file mode 100644 index 000000000..338b07320 --- /dev/null +++ b/cartography/intel/aws/ec2/internet_gateways.py @@ -0,0 +1,80 @@ +import logging +from typing import Dict +from typing import List + +import boto3 +import neo4j + +from .util import get_botocore_config +from cartography.util import aws_handle_regions +from cartography.util import run_cleanup_job +from cartography.util import timeit + +logger = logging.getLogger(__name__) + + +@timeit +@aws_handle_regions +def get_internet_gateways(boto3_session: boto3.session.Session, region: str) -> List[Dict]: + client = boto3_session.client('ec2', region_name=region, config=get_botocore_config()) + return client.describe_internet_gateways()['InternetGateways'] + + +@timeit +def load_internet_gateways( + neo4j_session: neo4j.Session, internet_gateways: List[Dict], region: str, + current_aws_account_id: str, update_tag: int, +) -> None: + logger.info("Loading %d Internet Gateways in %s.", len(internet_gateways), region) + # TODO: Right now this won't work in non-AWS commercial (GovCloud, China) as partition is hardcoded + query = """ + UNWIND {internet_gateways} as igw + MERGE (ig:AWSInternetGateway{id: igw.InternetGatewayId}) + ON CREATE SET + ig.firstseen = timestamp(), + ig.region = {region} + SET + ig.ownerid = igw.OwnerId, + ig.lastupdated = {aws_update_tag}, + ig.arn = "arn:aws:ec2:"+{region}+":"+igw.OwnerId+":internet-gateway/"+igw.InternetGatewayId + WITH igw, ig + + MATCH (awsAccount:AWSAccount {id: {aws_account_id}}) + MERGE (awsAccount)-[r:RESOURCE]->(ig) + ON CREATE SET r.firstseen = timestamp() + SET r.lastupdated = {aws_update_tag} + WITH igw, ig + + UNWIND igw.Attachments as attachment + MATCH (vpc:AWSVpc{id: attachment.VpcId}) + MERGE (ig)-[r:ATTACHED_TO]->(vpc) + ON CREATE SET r.firstseen = timestamp() + SET r.lastupdated = {aws_update_tag} + """ + + neo4j_session.run( + query, + internet_gateways=internet_gateways, + region=region, + aws_account_id=current_aws_account_id, + aws_update_tag=update_tag, + ).consume() + + +@timeit +def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: + logger.debug("Running Internet Gateway cleanup job.") + run_cleanup_job('aws_import_internet_gateways_cleanup.json', neo4j_session, common_job_parameters) + + +@timeit +def sync_internet_gateways( + neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str, + update_tag: int, common_job_parameters: Dict, +) -> None: + for region in regions: + logger.info("Syncing Internet Gateways for region '%s' in account '%s'.", region, current_aws_account_id) + internet_gateways = get_internet_gateways(boto3_session, region) + load_internet_gateways(neo4j_session, internet_gateways, region, current_aws_account_id, update_tag) + + cleanup(neo4j_session, common_job_parameters) diff --git a/cartography/intel/aws/resources.py b/cartography/intel/aws/resources.py index c2f53f541..dc80223c3 100644 --- a/cartography/intel/aws/resources.py +++ b/cartography/intel/aws/resources.py @@ -18,6 +18,7 @@ from . import s3 from .ec2.auto_scaling_groups import sync_ec2_auto_scaling_groups from .ec2.instances import sync_ec2_instances +from .ec2.internet_gateways import sync_internet_gateways from .ec2.key_pairs import sync_ec2_key_pairs from .ec2.load_balancer_v2s import sync_load_balancer_v2s from .ec2.load_balancers import sync_load_balancers @@ -43,6 +44,7 @@ 'ec2:tgw': sync_transit_gateways, 'ec2:vpc': sync_vpc, 'ec2:vpc_peering': sync_vpc_peering, + 'ec2:internet_gateway': sync_internet_gateways, 'ecr': ecr.sync, 'eks': eks.sync, 'elasticache': elasticache.sync, diff --git a/cartography/intel/azure/subscription.py b/cartography/intel/azure/subscription.py index c8ad7ae10..9c7304914 100644 --- a/cartography/intel/azure/subscription.py +++ b/cartography/intel/azure/subscription.py @@ -25,7 +25,7 @@ def get_all_azure_subscriptions(credentials: Credentials) -> List[Dict]: logger.error( f'failed to fetch subscriptions for the credentials \ The provided credentials do not have access to any subscriptions - \ - {e}' + {e}', ) return [] @@ -54,7 +54,7 @@ def get_current_azure_subscription(credentials: Credentials, subscription_id: st logger.error( f'failed to fetch subscription for the credentials \ The provided credentials do not have access to this subscription: {subscription_id} - \ - {e}' + {e}', ) return [] diff --git a/docs/schema/aws.md b/docs/schema/aws.md index e39ddbfa2..bfd0f6c95 100644 --- a/docs/schema/aws.md +++ b/docs/schema/aws.md @@ -58,64 +58,66 @@ - [Relationships](#relationships-24) - [EC2Subnet](#ec2subnet) - [Relationships](#relationships-25) -- [ECRRepository](#ecrrepository) +- [AWSInternetGateway](#awsinternetgateway) - [Relationships](#relationships-26) -- [ECRRepositoryImage](#ecrrepositoryimage) +- [ECRRepository](#ecrrepository) - [Relationships](#relationships-27) -- [ECRImage](#ecrimage) +- [ECRRepositoryImage](#ecrrepositoryimage) - [Relationships](#relationships-28) -- [Package](#package) +- [ECRImage](#ecrimage) - [Relationships](#relationships-29) -- [ECRScanFinding (:Risk:CVE)](#ecrscanfinding-riskcve) +- [Package](#package) - [Relationships](#relationships-30) -- [EKSCluster](#ekscluster) +- [ECRScanFinding (:Risk:CVE)](#ecrscanfinding-riskcve) - [Relationships](#relationships-31) -- [EMRCluster](#emrcluster) +- [EKSCluster](#ekscluster) - [Relationships](#relationships-32) -- [ESDomain](#esdomain) +- [EMRCluster](#emrcluster) - [Relationships](#relationships-33) -- [Endpoint](#endpoint) +- [ESDomain](#esdomain) - [Relationships](#relationships-34) -- [Endpoint::ELBListener](#endpointelblistener) +- [Endpoint](#endpoint) - [Relationships](#relationships-35) -- [Endpoint::ELBV2Listener](#endpointelbv2listener) +- [Endpoint::ELBListener](#endpointelblistener) - [Relationships](#relationships-36) -- [Ip](#ip) +- [Endpoint::ELBV2Listener](#endpointelbv2listener) - [Relationships](#relationships-37) -- [IpRule](#iprule) +- [Ip](#ip) - [Relationships](#relationships-38) -- [IpRule::IpPermissionInbound](#ipruleippermissioninbound) +- [IpRule](#iprule) - [Relationships](#relationships-39) -- [LoadBalancer](#loadbalancer) +- [IpRule::IpPermissionInbound](#ipruleippermissioninbound) - [Relationships](#relationships-40) -- [LoadBalancerV2](#loadbalancerv2) +- [LoadBalancer](#loadbalancer) - [Relationships](#relationships-41) -- [Nameserver](#nameserver) +- [LoadBalancerV2](#loadbalancerv2) - [Relationships](#relationships-42) -- [NetworkInterface](#networkinterface) +- [Nameserver](#nameserver) - [Relationships](#relationships-43) -- [RedshiftCluster](#redshiftcluster) +- [NetworkInterface](#networkinterface) - [Relationships](#relationships-44) -- [RDSInstance](#rdsinstance) +- [RedshiftCluster](#redshiftcluster) - [Relationships](#relationships-45) -- [S3Acl](#s3acl) +- [RDSInstance](#rdsinstance) - [Relationships](#relationships-46) -- [S3Bucket](#s3bucket) +- [S3Acl](#s3acl) - [Relationships](#relationships-47) -- [KMSKey](#kmskey) +- [S3Bucket](#s3bucket) - [Relationships](#relationships-48) -- [KMSAlias](#kmsalias) +- [KMSKey](#kmskey) - [Relationships](#relationships-49) -- [KMSGrant](#kmsgrant) +- [KMSAlias](#kmsalias) - [Relationships](#relationships-50) -- [APIGatewayRestAPI](#apigatewayrestapi) +- [KMSGrant](#kmsgrant) - [Relationships](#relationships-51) -- [APIGatewayStage](#apigatewaystage) +- [APIGatewayRestAPI](#apigatewayrestapi) - [Relationships](#relationships-52) -- [APIGatewayClientCertificate](#apigatewayclientcertificate) +- [APIGatewayStage](#apigatewaystage) - [Relationships](#relationships-53) -- [APIGatewayResource](#apigatewayresource) +- [APIGatewayClientCertificate](#apigatewayclientcertificate) - [Relationships](#relationships-54) +- [APIGatewayResource](#apigatewayresource) + - [Relationships](#relationships-55) @@ -1059,6 +1061,30 @@ Representation of an AWS EC2 [Subnet](https://docs.aws.amazon.com/AWSEC2/latest/ ``` +## AWSInternetGateway + + Representation of an AWS [Interent Gateway](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_InternetGateway.html). + + | Field | Description | + |--------|-----------| + | **id** | Internet gateway ID | + | arn | Amazon Resource Name | + | region | The region of the gateway | + + + ### Relationships + + - Internet Gateways are attached to a VPC. + + ``` + (AWSInternetGateway)-[ATTACHED_TO]->(AWSVpc) + ``` + + - Internet Gateways belong to AWS Accounts + + ``` + (AWSAccount)-[RESOURCE]->(AWSInternetGateway) + ``` ## ECRRepository diff --git a/tests/data/aws/ec2/internet_gateway.py b/tests/data/aws/ec2/internet_gateway.py new file mode 100644 index 000000000..ccc939f82 --- /dev/null +++ b/tests/data/aws/ec2/internet_gateway.py @@ -0,0 +1,45 @@ +DESCRIBE_GATEWAYS = [ + { + "Attachments": [ + { + "State": "available", + "VpcId": "vpc-XXXXXXX", + }, + ], + "InternetGatewayId": "igw-1234XXX", + "OwnerId": "012345678912", + "Tags": [ + { + "Key": "Name", + "Value": "InternetGateway", + }, + ], + }, + { + "Attachments": [ + { + "State": "available", + "VpcId": "vpc-XXXXXXX", + }, + ], + "InternetGatewayId": "igw-7e3a7c18", + "OwnerId": "012345678912", + "Tags": [ + { + "Key": "AWSServiceAccount", + "Value": "697148468905", + }, + ], + }, + { + "Attachments": [ + { + "State": "available", + "VpcId": "vpc-XXXXXXX", + }, + ], + "InternetGatewayId": "igw-f1c81494", + "OwnerId": "012345678912", + "Tags": [], + }, +] diff --git a/tests/integration/cartography/intel/aws/common.py b/tests/integration/cartography/intel/aws/common.py new file mode 100644 index 000000000..50d78e82c --- /dev/null +++ b/tests/integration/cartography/intel/aws/common.py @@ -0,0 +1,11 @@ +def create_test_account(neo4j_session, test_account_id, test_update_tag): + # Create Test AWSAccount + neo4j_session.run( + """ + MERGE (aws:AWSAccount{id: {aws_account_id}}) + ON CREATE SET aws.firstseen = timestamp() + SET aws.lastupdated = {aws_update_tag} + """, + aws_account_id=test_account_id, + aws_update_tag=test_update_tag, + ) diff --git a/tests/integration/cartography/intel/aws/ec2/test_internet_gateway.py b/tests/integration/cartography/intel/aws/ec2/test_internet_gateway.py new file mode 100644 index 000000000..ff1f93e94 --- /dev/null +++ b/tests/integration/cartography/intel/aws/ec2/test_internet_gateway.py @@ -0,0 +1,64 @@ +import cartography.intel.aws.ec2 +import tests.data.aws.ec2.internet_gateway +import tests.integration.cartography.intel.aws.common + +TEST_ACCOUNT_ID = '012345678912' +TEST_REGION = 'us-east-1' +TEST_UPDATE_TAG = 123456789 + + +def test_load_internet_gateways(neo4j_session): + data = tests.data.aws.ec2.internet_gateway.DESCRIBE_GATEWAYS + cartography.intel.aws.ec2.internet_gateways.load_internet_gateways( + neo4j_session, + data, + TEST_REGION, + TEST_ACCOUNT_ID, + TEST_UPDATE_TAG, + ) + + expected_nodes = { + "igw-1234XXX", + "igw-7e3a7c18", + "igw-f1c81494", + } + + nodes = neo4j_session.run( + """ + MATCH (n:AWSInternetGateway) RETURN n.id; + """, + ) + actual_nodes = {n['n.id'] for n in nodes} + + assert actual_nodes == expected_nodes + + +def test_load_internet_gateway_relationships(neo4j_session): + tests.integration.cartography.intel.aws.common.create_test_account(neo4j_session, TEST_ACCOUNT_ID, TEST_UPDATE_TAG) + + data = tests.data.aws.ec2.internet_gateway.DESCRIBE_GATEWAYS + cartography.intel.aws.ec2.internet_gateways.load_internet_gateways( + neo4j_session, + data, + TEST_REGION, + TEST_ACCOUNT_ID, + TEST_UPDATE_TAG, + ) + + expected = { + (TEST_ACCOUNT_ID, 'igw-1234XXX'), + (TEST_ACCOUNT_ID, 'igw-7e3a7c18'), + (TEST_ACCOUNT_ID, 'igw-f1c81494'), + } + + # Fetch relationships + result = neo4j_session.run( + """ + MATCH (n1:AWSInternetGateway)<-[:RESOURCE]-(n2:AWSAccount) RETURN n1.id, n2.id; + """, + ) + actual = { + (n['n2.id'], n['n1.id']) for n in result + } + + assert actual == expected