-
Notifications
You must be signed in to change notification settings - Fork 343
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add in AWS internet gateway support (#556)
* Add in internet gateway support * Update tests * Cleanup docs * Handle unattached IGWS * Address log level PR comment * Break out account relationship test * Update to use more efficient unwind
- Loading branch information
Showing
9 changed files
with
270 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
10 changes: 10 additions & 0 deletions
10
cartography/data/jobs/cleanup/aws_import_internet_Gateways_cleanup.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| { | ||
| "statements": [ | ||
| { | ||
| "query": "MATCH (:AWSAccount{id:{AWS_ID}})-[:RESOURCE]->(n:AWSInternetGateway) WHERE n.lastupdated <> {UPDATE_TAG} DETACH DELETE (n) return COUNT(*) as TotalCompleted", | ||
| "iterative": true, | ||
| "iterationsize": 100 | ||
| } | ||
| ], | ||
| "name": "cleanup AWSInternetGateway" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| import logging | ||
| from typing import Dict | ||
| from typing import List | ||
|
|
||
| import boto3 | ||
| import neo4j | ||
|
|
||
| from .util import get_botocore_config | ||
| from cartography.util import aws_handle_regions | ||
| from cartography.util import run_cleanup_job | ||
| from cartography.util import timeit | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| @timeit | ||
| @aws_handle_regions | ||
| def get_internet_gateways(boto3_session: boto3.session.Session, region: str) -> List[Dict]: | ||
| client = boto3_session.client('ec2', region_name=region, config=get_botocore_config()) | ||
| return client.describe_internet_gateways()['InternetGateways'] | ||
|
|
||
|
|
||
| @timeit | ||
| def load_internet_gateways( | ||
| neo4j_session: neo4j.Session, internet_gateways: List[Dict], region: str, | ||
| current_aws_account_id: str, update_tag: int, | ||
| ) -> None: | ||
| logger.info("Loading %d Internet Gateways in %s.", len(internet_gateways), region) | ||
| # TODO: Right now this won't work in non-AWS commercial (GovCloud, China) as partition is hardcoded | ||
| query = """ | ||
| UNWIND {internet_gateways} as igw | ||
| MERGE (ig:AWSInternetGateway{id: igw.InternetGatewayId}) | ||
| ON CREATE SET | ||
| ig.firstseen = timestamp(), | ||
| ig.region = {region} | ||
| SET | ||
| ig.ownerid = igw.OwnerId, | ||
| ig.lastupdated = {aws_update_tag}, | ||
| ig.arn = "arn:aws:ec2:"+{region}+":"+igw.OwnerId+":internet-gateway/"+igw.InternetGatewayId | ||
| WITH igw, ig | ||
| MATCH (awsAccount:AWSAccount {id: {aws_account_id}}) | ||
| MERGE (awsAccount)-[r:RESOURCE]->(ig) | ||
| ON CREATE SET r.firstseen = timestamp() | ||
| SET r.lastupdated = {aws_update_tag} | ||
| WITH igw, ig | ||
| UNWIND igw.Attachments as attachment | ||
| MATCH (vpc:AWSVpc{id: attachment.VpcId}) | ||
| MERGE (ig)-[r:ATTACHED_TO]->(vpc) | ||
| ON CREATE SET r.firstseen = timestamp() | ||
| SET r.lastupdated = {aws_update_tag} | ||
| """ | ||
|
|
||
| neo4j_session.run( | ||
| query, | ||
| internet_gateways=internet_gateways, | ||
| region=region, | ||
| aws_account_id=current_aws_account_id, | ||
| aws_update_tag=update_tag, | ||
| ).consume() | ||
|
|
||
|
|
||
| @timeit | ||
| def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: | ||
| logger.debug("Running Internet Gateway cleanup job.") | ||
| run_cleanup_job('aws_import_internet_gateways_cleanup.json', neo4j_session, common_job_parameters) | ||
|
|
||
|
|
||
| @timeit | ||
| def sync_internet_gateways( | ||
| neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str, | ||
| update_tag: int, common_job_parameters: Dict, | ||
| ) -> None: | ||
| for region in regions: | ||
| logger.info("Syncing Internet Gateways for region '%s' in account '%s'.", region, current_aws_account_id) | ||
| internet_gateways = get_internet_gateways(boto3_session, region) | ||
| load_internet_gateways(neo4j_session, internet_gateways, region, current_aws_account_id, update_tag) | ||
|
|
||
| cleanup(neo4j_session, common_job_parameters) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| DESCRIBE_GATEWAYS = [ | ||
| { | ||
| "Attachments": [ | ||
| { | ||
| "State": "available", | ||
| "VpcId": "vpc-XXXXXXX", | ||
| }, | ||
| ], | ||
| "InternetGatewayId": "igw-1234XXX", | ||
| "OwnerId": "012345678912", | ||
| "Tags": [ | ||
| { | ||
| "Key": "Name", | ||
| "Value": "InternetGateway", | ||
| }, | ||
| ], | ||
| }, | ||
| { | ||
| "Attachments": [ | ||
| { | ||
| "State": "available", | ||
| "VpcId": "vpc-XXXXXXX", | ||
| }, | ||
| ], | ||
| "InternetGatewayId": "igw-7e3a7c18", | ||
| "OwnerId": "012345678912", | ||
| "Tags": [ | ||
| { | ||
| "Key": "AWSServiceAccount", | ||
| "Value": "697148468905", | ||
| }, | ||
| ], | ||
| }, | ||
| { | ||
| "Attachments": [ | ||
| { | ||
| "State": "available", | ||
| "VpcId": "vpc-XXXXXXX", | ||
| }, | ||
| ], | ||
| "InternetGatewayId": "igw-f1c81494", | ||
| "OwnerId": "012345678912", | ||
| "Tags": [], | ||
| }, | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| def create_test_account(neo4j_session, test_account_id, test_update_tag): | ||
| # Create Test AWSAccount | ||
| neo4j_session.run( | ||
| """ | ||
| MERGE (aws:AWSAccount{id: {aws_account_id}}) | ||
| ON CREATE SET aws.firstseen = timestamp() | ||
| SET aws.lastupdated = {aws_update_tag} | ||
| """, | ||
| aws_account_id=test_account_id, | ||
| aws_update_tag=test_update_tag, | ||
| ) |
64 changes: 64 additions & 0 deletions
64
tests/integration/cartography/intel/aws/ec2/test_internet_gateway.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| import cartography.intel.aws.ec2 | ||
| import tests.data.aws.ec2.internet_gateway | ||
| import tests.integration.cartography.intel.aws.common | ||
|
|
||
| TEST_ACCOUNT_ID = '012345678912' | ||
| TEST_REGION = 'us-east-1' | ||
| TEST_UPDATE_TAG = 123456789 | ||
|
|
||
|
|
||
| def test_load_internet_gateways(neo4j_session): | ||
| data = tests.data.aws.ec2.internet_gateway.DESCRIBE_GATEWAYS | ||
| cartography.intel.aws.ec2.internet_gateways.load_internet_gateways( | ||
| neo4j_session, | ||
| data, | ||
| TEST_REGION, | ||
| TEST_ACCOUNT_ID, | ||
| TEST_UPDATE_TAG, | ||
| ) | ||
|
|
||
| expected_nodes = { | ||
| "igw-1234XXX", | ||
| "igw-7e3a7c18", | ||
| "igw-f1c81494", | ||
| } | ||
|
|
||
| nodes = neo4j_session.run( | ||
| """ | ||
| MATCH (n:AWSInternetGateway) RETURN n.id; | ||
| """, | ||
| ) | ||
| actual_nodes = {n['n.id'] for n in nodes} | ||
|
|
||
| assert actual_nodes == expected_nodes | ||
|
|
||
|
|
||
| def test_load_internet_gateway_relationships(neo4j_session): | ||
| tests.integration.cartography.intel.aws.common.create_test_account(neo4j_session, TEST_ACCOUNT_ID, TEST_UPDATE_TAG) | ||
|
|
||
| data = tests.data.aws.ec2.internet_gateway.DESCRIBE_GATEWAYS | ||
| cartography.intel.aws.ec2.internet_gateways.load_internet_gateways( | ||
| neo4j_session, | ||
| data, | ||
| TEST_REGION, | ||
| TEST_ACCOUNT_ID, | ||
| TEST_UPDATE_TAG, | ||
| ) | ||
|
|
||
| expected = { | ||
| (TEST_ACCOUNT_ID, 'igw-1234XXX'), | ||
| (TEST_ACCOUNT_ID, 'igw-7e3a7c18'), | ||
| (TEST_ACCOUNT_ID, 'igw-f1c81494'), | ||
| } | ||
|
|
||
| # Fetch relationships | ||
| result = neo4j_session.run( | ||
| """ | ||
| MATCH (n1:AWSInternetGateway)<-[:RESOURCE]-(n2:AWSAccount) RETURN n1.id, n2.id; | ||
| """, | ||
| ) | ||
| actual = { | ||
| (n['n2.id'], n['n1.id']) for n in result | ||
| } | ||
|
|
||
| assert actual == expected |