From d515f6ce89b0feae1759bbe19565df1a19d49679 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Wed, 8 Apr 2020 14:56:18 -0700 Subject: [PATCH] #253 - Ingest AWS tags for multiple resource types using resourcegroupstaggingapi (#276) * Initial commit * Adjust indices * Use `id` as primary identifier in S3Buckets and EC2Instances for consistency * Add EC2 instance integration tests * Update schema * Add S3 bucket integration tests, update schema * Fix schema * Linter happy * Incremental commit * Implement feature with resource type mappings * Add EC2 NICs, SGs, subnets, VPCs. Update tests. * Schema docs and update Readme with supported datatypes * Typo * Add missing cleanup jobs, test data --- README.md | 4 +- cartography/data/indexes.cypher | 1 + .../jobs/cleanup/aws_import_tags_cleanup.json | 95 ++++++++++++++ cartography/intel/aws/__init__.py | 4 + .../intel/aws/resourcegroupstaggingapi.py | 119 +++++++++++++++++ docs/schema/aws.md | 123 ++++++++++++++---- tests/data/aws/resourcegroupstaggingapi.py | 31 +++++ .../aws/test_resourcegroupstaggingapi.py | 45 +++++++ .../aws/test_resourcegroupstaggingapi.py | 33 +++++ 9 files changed, 426 insertions(+), 29 deletions(-) create mode 100644 cartography/data/jobs/cleanup/aws_import_tags_cleanup.json create mode 100644 cartography/intel/aws/resourcegroupstaggingapi.py create mode 100644 tests/data/aws/resourcegroupstaggingapi.py create mode 100644 tests/integration/cartography/intel/aws/test_resourcegroupstaggingapi.py create mode 100644 tests/unit/cartography/intel/aws/test_resourcegroupstaggingapi.py diff --git a/README.md b/README.md index b043e7520..17125e250 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ You can learn more about the story behind Cartography in our [presentation at BS Start [here](docs/setup/install.md). ## Supported platforms -- [Amazon Web Services](docs/setup/config/aws.md) - EC2, Elasticsearch, DynamoDB, IAM, RDS, Route53, S3, STS -- [Google Cloud Platform](docs/setup/config/gcp.md) - Cloud Resource Manager, Compute, Storage +- [Amazon Web Services](docs/setup/config/aws.md) - EC2, Elasticsearch, Elastic Kubernetes Service, DynamoDB, IAM, RDS, Route53, S3, STS, Tags +- [Google Cloud Platform](docs/setup/config/gcp.md) - Cloud Resource Manager, Compute, Storage, Google Kubernetes Engine - [Google GSuite](docs/setup/config/gsuite.md) - users, groups - [Duo CRXcavator](docs/setup/config/crxcavator.md) - Chrome extensions, GSuite users - [Okta](docs/setup/config/okta.md) - users, groups, organizations, roles, applications, factors, trusted origins, reply URIs diff --git a/cartography/data/indexes.cypher b/cartography/data/indexes.cypher index c33c9af86..b1ca8250a 100644 --- a/cartography/data/indexes.cypher +++ b/cartography/data/indexes.cypher @@ -9,6 +9,7 @@ CREATE INDEX ON :AWSIpv6CidrBlock(id); CREATE INDEX ON :AWSPolicy(arn); CREATE INDEX ON :AWSPrincipal(arn); CREATE INDEX ON :AWSRole(arn); +CREATE INDEX ON :AWSTag(id); CREATE INDEX ON :AWSUser(arn); CREATE INDEX ON :AWSUser(name); CREATE INDEX ON :AWSVpc(id); diff --git a/cartography/data/jobs/cleanup/aws_import_tags_cleanup.json b/cartography/data/jobs/cleanup/aws_import_tags_cleanup.json new file mode 100644 index 000000000..fcd19d89b --- /dev/null +++ b/cartography/data/jobs/cleanup/aws_import_tags_cleanup.json @@ -0,0 +1,95 @@ +{ + "statements": [ + { + "query": "MATCH (n:AWSTag)<-[:TAGGED]-(:EC2Instance) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSTag)<-[r:TAGGED]-(:EC2Instance) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (n:AWSTag)<-[:TAGGED]-(:NetworkInterface) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSTag)<-[r:TAGGED]-(:NetworkInterface) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (n:AWSTag)<-[:TAGGED]-(:EC2SecurityGroup) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSTag)<-[r:TAGGED]-(:EC2SecurityGroup) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (n:AWSTag)<-[:TAGGED]-(:EC2Subnet) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSTag)<-[r:TAGGED]-(:EC2Subnet) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (n:AWSTag)<-[:TAGGED]-(:AWSVpc) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSTag)<-[r:TAGGED]-(:AWSVpc) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (n:AWSTag)<-[:TAGGED]-(:ESDomain) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSTag)<-[r:TAGGED]-(:ESDomain) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (n:AWSTag)<-[:TAGGED]-(:RDSInstance) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSTag)<-[r:TAGGED]-(:RDSInstance) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (n:AWSTag)<-[:TAGGED]-(:DBSubnetGroup) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSTag)<-[r:TAGGED]-(:DBSubnetGroup) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (n:AWSTag)<-[:TAGGED]-(:S3Bucket) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + }, + { + "query": "MATCH (:AWSTag)<-[r:TAGGED]-(:S3Bucket) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", + "iterative": true, + "iterationsize": 100 + } + ], + "name": "cleanup AWS Tags" +} diff --git a/cartography/intel/aws/__init__.py b/cartography/intel/aws/__init__.py index 6c7587a60..54f18629a 100644 --- a/cartography/intel/aws/__init__.py +++ b/cartography/intel/aws/__init__.py @@ -10,6 +10,7 @@ from . import iam from . import organizations from . import rds +from . import resourcegroupstaggingapi from . import route53 from . import s3 from cartography.util import run_analysis_job @@ -47,6 +48,9 @@ def _sync_one_account(neo4j_session, boto3_session, account_id, sync_tag, common # NOTE clean up all DNS records, regardless of which job created them run_cleanup_job('aws_account_dns_cleanup.json', neo4j_session, common_job_parameters) + # AWS Tags - Must always be last. + resourcegroupstaggingapi.sync(neo4j_session, boto3_session, regions, sync_tag, common_job_parameters) + def _sync_multiple_accounts(neo4j_session, accounts, sync_tag, common_job_parameters): logger.debug("Syncing AWS accounts: %s", ', '.join(accounts.values())) diff --git a/cartography/intel/aws/resourcegroupstaggingapi.py b/cartography/intel/aws/resourcegroupstaggingapi.py new file mode 100644 index 000000000..a9063bcc6 --- /dev/null +++ b/cartography/intel/aws/resourcegroupstaggingapi.py @@ -0,0 +1,119 @@ +import logging +from string import Template + +from cartography.util import run_cleanup_job + +logger = logging.getLogger(__name__) + + +def get_short_id_from_ec2_arn(arn): + """ + Return the short-form resource ID from an EC2 ARN. + For example, for "arn:aws:ec2:us-east-1:test_account:instance/i-1337", return 'i-1337'. + :param arn: The ARN + :return: The resource ID + """ + return arn.split('/')[-1] + + +def get_bucket_name_from_arn(bucket_arn): + """ + Return the bucket name from an S3 bucket ARN. + For example, for "arn:aws:s3:::bucket_name", return 'bucket_name'. + :param arn: The S3 bucket's full ARN + :return: The S3 bucket's name + """ + return bucket_arn.split(':')[-1] + + +# We maintain a mapping from AWS resource types to their associated labels and unique identifiers. +# label: the node label used in cartography for this resource type +# property: the field of this node that uniquely identified this resource type +# id_func: [optional] - EC2 instances and S3 buckets in cartography currently use non-ARNs as their primary identifiers +# so we need to supply a function pointer to translate the ARN returned by the resourcegroupstaggingapi to the form that +# cartography uses. +# TODO - we should make EC2 and S3 assets query-able by their full ARN so that we don't need this workaround. +TAG_RESOURCE_TYPE_MAPPINGS = { + 'ec2:instance': {'label': 'EC2Instance', 'property': 'id', 'id_func': get_short_id_from_ec2_arn}, + 'ec2:network-interface': {'label': 'NetworkInterface', 'property': 'id', 'id_func': get_short_id_from_ec2_arn}, + 'ec2:security-group': {'label': 'EC2SecurityGroup', 'property': 'id', 'id_func': get_short_id_from_ec2_arn}, + 'ec2:subnet': {'label': 'EC2Subnet', 'property': 'subnetid', 'id_func': get_short_id_from_ec2_arn}, + 'ec2:vpc': {'label': 'AWSVpc', 'property': 'id', 'id_func': get_short_id_from_ec2_arn}, + 'es:domain': {'label': 'ESDomain', 'property': 'id'}, + 'rds:db': {'label': 'RDSInstance', 'property': 'id'}, + 'rds:subgrp': {'label': 'DBSubnetGroup', 'property': 'id'}, + # Buckets are the only objects in the S3 service: https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html + 's3': {'label': 'S3Bucket', 'property': 'id', 'id_func': get_bucket_name_from_arn}, +} + + +def get_tags(boto3_session, resource_types, region): + """ + Create boto3 client and retrieve tag data. + """ + client = boto3_session.client('resourcegroupstaggingapi', region_name=region) + paginator = client.get_paginator('get_resources') + resources = [] + for page in paginator.paginate( + # Only ingest tags for resources that Cartography supports. + # This is just a starting list; there may be others supported by this API. + ResourceTypeFilters=resource_types, + ): + resources.extend(page['ResourceTagMappingList']) + return resources + + +def load_tags(neo4j_session, tag_data, resource_type, region, aws_update_tag): + INGEST_TAG_TEMPLATE = Template(""" + MATCH (resource:$resource_label{$property:{ResourceId}}) + MERGE(aws_tag:AWSTag:Tag{id:{TagId}}) + ON CREATE SET aws_tag.firstseen = timestamp() + SET aws_tag.lastupdated = {UpdateTag}, + aws_tag.key = {TagKey}, + aws_tag.value = {TagValue}, + aws_tag.region = {Region} + MERGE (resource)-[r:TAGGED]->(aws_tag) + SET r.lastupdated = {UpdateTag}, + r.firstseen = timestamp() + """) + for tag_mapping in tag_data: + for tag in tag_mapping['Tags']: + neo4j_session.run( + INGEST_TAG_TEMPLATE.safe_substitute( + resource_label=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['label'], + property=TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['property'], + ), + ResourceId=tag_mapping['resource_id'], + TagId=f'{tag["Key"]}:{tag["Value"]}', + UpdateTag=aws_update_tag, + TagKey=tag['Key'], + TagValue=tag['Value'], + Region=region, + ) + + +def transform_tags(tag_data, resource_type): + for tag_mapping in tag_data: + tag_mapping['resource_id'] = compute_resource_id(tag_mapping, resource_type) + + +def compute_resource_id(tag_mapping, resource_type): + resource_id = tag_mapping['ResourceARN'] + if 'id_func' in TAG_RESOURCE_TYPE_MAPPINGS[resource_type]: + parse_resource_id_from_arn = TAG_RESOURCE_TYPE_MAPPINGS[resource_type]['id_func'] + resource_id = parse_resource_id_from_arn(tag_mapping['ResourceARN']) + return resource_id + + +def cleanup(neo4j_session, common_job_parameters): + run_cleanup_job('aws_import_tags_cleanup.json', neo4j_session, common_job_parameters) + + +def sync(neo4j_session, boto3_session, regions, aws_update_tag, common_job_parameters): + for region in regions: + logger.info("Syncing AWS tags for region '%s'.", region) + for resource_type in TAG_RESOURCE_TYPE_MAPPINGS.keys(): + tag_data = get_tags(boto3_session, [resource_type], region) + transform_tags(tag_data, resource_type) + load_tags(neo4j_session, tag_data, resource_type, region, aws_update_tag) + cleanup(neo4j_session, common_job_parameters) diff --git a/docs/schema/aws.md b/docs/schema/aws.md index 80a38c1ba..0cd384a21 100644 --- a/docs/schema/aws.md +++ b/docs/schema/aws.md @@ -22,60 +22,62 @@ - [Relationships](#relationships-6) - [AWSVpc](#awsvpc) - [Relationships](#relationships-7) -- [AccountAccessKey](#accountaccesskey) +- [Tag::AWSTag](#tagawstag) - [Relationships](#relationships-8) -- [DBSubnetGroup](#dbsubnetgroup) +- [AccountAccessKey](#accountaccesskey) - [Relationships](#relationships-9) -- [DNSRecord](#dnsrecord) +- [DBSubnetGroup](#dbsubnetgroup) - [Relationships](#relationships-10) -- [DNSRecord::AWSDNSRecord](#dnsrecordawsdnsrecord) +- [DNSRecord](#dnsrecord) - [Relationships](#relationships-11) -- [DNSZone](#dnszone) +- [DNSRecord::AWSDNSRecord](#dnsrecordawsdnsrecord) - [Relationships](#relationships-12) -- [DNSZone::AWSDNSZone](#dnszoneawsdnszone) +- [DNSZone](#dnszone) - [Relationships](#relationships-13) -- [DynamoDBTable](#dynamodbtable) +- [DNSZone::AWSDNSZone](#dnszoneawsdnszone) - [Relationships](#relationships-14) -- [EC2Instance](#ec2instance) +- [DynamoDBTable](#dynamodbtable) - [Relationships](#relationships-15) -- [EC2KeyPair](#ec2keypair) +- [EC2Instance](#ec2instance) - [Relationships](#relationships-16) -- [EC2Reservation](#ec2reservation) +- [EC2KeyPair](#ec2keypair) - [Relationships](#relationships-17) -- [EC2SecurityGroup](#ec2securitygroup) +- [EC2Reservation](#ec2reservation) - [Relationships](#relationships-18) -- [EC2Subnet](#ec2subnet) +- [EC2SecurityGroup](#ec2securitygroup) - [Relationships](#relationships-19) -- [EKSCluster](#ekscluster) +- [EC2Subnet](#ec2subnet) - [Relationships](#relationships-20) -- [ESDomain](#esdomain) +- [EKSCluster](#ekscluster) - [Relationships](#relationships-21) -- [Endpoint](#endpoint) +- [ESDomain](#esdomain) - [Relationships](#relationships-22) -- [Endpoint::ELBListener](#endpointelblistener) +- [Endpoint](#endpoint) - [Relationships](#relationships-23) -- [Endpoint::ELBV2Listener](#endpointelbv2listener) +- [Endpoint::ELBListener](#endpointelblistener) - [Relationships](#relationships-24) -- [Ip](#ip) +- [Endpoint::ELBV2Listener](#endpointelbv2listener) - [Relationships](#relationships-25) -- [IpRule](#iprule) +- [Ip](#ip) - [Relationships](#relationships-26) -- [IpRule::IpPermissionInbound](#ipruleippermissioninbound) +- [IpRule](#iprule) - [Relationships](#relationships-27) -- [LoadBalancer](#loadbalancer) +- [IpRule::IpPermissionInbound](#ipruleippermissioninbound) - [Relationships](#relationships-28) -- [LoadBalancerV2](#loadbalancerv2) +- [LoadBalancer](#loadbalancer) - [Relationships](#relationships-29) -- [Nameserver](#nameserver) +- [LoadBalancerV2](#loadbalancerv2) - [Relationships](#relationships-30) -- [NetworkInterface](#networkinterface) +- [Nameserver](#nameserver) - [Relationships](#relationships-31) -- [RDSInstance](#rdsinstance) +- [NetworkInterface](#networkinterface) - [Relationships](#relationships-32) -- [S3Acl](#s3acl) +- [RDSInstance](#rdsinstance) - [Relationships](#relationships-33) -- [S3Bucket](#s3bucket) +- [S3Acl](#s3acl) - [Relationships](#relationships-34) +- [S3Bucket](#s3bucket) + - [Relationships](#relationships-35) @@ -370,6 +372,30 @@ More information on https://docs.aws.amazon.com/cli/latest/reference/ec2/describ ``` (AWSVpc)<-[MEMBER_OF_EC2_SECURITY_GROUP]-(EC2SecurityGroup) ``` +- AWS VPCs can be tagged with AWSTags. + ``` + (AWSVpc)-[TAGGED]->(AWSTag) + ``` + + +## Tag::AWSTag + +Representation of an AWS [Tag](https://docs.aws.amazon.com/resourcegroupstagging/latest/APIReference/API_Tag.html). AWS Tags can be applied to many objects. + +| Field | Description | +|-------|-------------| +| firstseen| Timestamp of when a sync job first discovered this node | +| lastupdated | Timestamp of the last time the node was updated | +| **id** | This tag's unique identifier of the format `{TagKey}:{TagValue}`. We fabricated this ID. | +| key | One part of a key-value pair that makes up a tag.| +| value | One part of a key-value pair that makes up a tag. | +| region | The region where this tag was discovered.| + +### Relationships +- AWS VPCs, DB Subnet Groups, EC2 Instances, EC2 SecurityGroups, EC2 Subnets, EC2 Network Interfaces, RDS Instances, and S3 Buckets can be tagged with AWSTags. + ``` + (AWSVpc, DBSubnetGroup, EC2Instance, EC2SecurityGroup, EC2Subnet, NetworkInterface, RDSInstance, S3Bucket)-[TAGGED]->(AWSTag) + ``` ## AccountAccessKey @@ -418,6 +444,12 @@ Representation of an RDS [DB Subnet Group](https://docs.aws.amazon.com/AmazonRDS (DBSubnetGroup)-[:RESOURCE]->(EC2Subnet) ``` +- DB Subnet Groups can be tagged with AWSTags. + + ``` + (DBSubnetGroup)-[TAGGED]->(AWSTag) + ``` + ## DNSRecord @@ -639,6 +671,12 @@ Our representation of an AWS [EC2 Instance](https://docs.aws.amazon.com/AWSEC2/l (AWSAccount)-[RESOURCE]->(EC2Instance) ``` +- EC2 Instances can be tagged with AWSTags. + + ``` + (EC2Instance)-[TAGGED]->(AWSTag) + ``` + ## EC2KeyPair @@ -745,6 +783,12 @@ Representation of an AWS EC2 [Security Group](https://docs.aws.amazon.com/AWSEC2 (AWSAccount)-[RESOURCE]->(EC2SecurityGroup) ``` +- EC2 SecurityGroups can be tagged with AWSTags. + + ``` + (EC2SecurityGroup)-[TAGGED]->(AWSTag) + ``` + ## EC2Subnet @@ -786,6 +830,13 @@ Representation of an AWS EC2 [Subnet](https://docs.aws.amazon.com/AWSEC2/latest/ ``` +- EC2 Subnets can be tagged with AWSTags. + + ``` + (EC2Subnet)-[TAGGED]->(AWSTag) + ``` + + ## EKSCluster Representation of an AWS [EKS Cluster](https://docs.aws.amazon.com/eks/latest/APIReference/API_Cluster.html). @@ -1154,6 +1205,12 @@ Representation of a generic Network Interface. Currently however, we only creat (EC2Instance)-[NETWORK_INTERFACE]->(NetworkInterface) ``` +- EC2 Network Interfaces can be tagged with AWSTags. + + ``` + (NetworkInterface)-[TAGGED]->(AWSTag) + ``` + ## RDSInstance @@ -1220,6 +1277,12 @@ Representation of an AWS Relational Database Service [DBInstance](https://docs.a (RDSInstance)-[:MEMBER_OF_DB_SUBNET_GROUP]->(DBSubnetGroup) ``` +- RDS Instances can be tagged with AWSTags. + + ``` + (RDSInstance)-[TAGGED]->(AWSTag) + ``` + ## S3Acl Representation of an AWS S3 [Access Control List](https://docs.aws.amazon.com/AmazonS3/latest/API/API_control_S3AccessControlList.html). @@ -1272,3 +1335,9 @@ Representation of an AWS S3 [Bucket](https://docs.aws.amazon.com/AmazonS3/latest ``` (S3Acl)-[APPLIES_TO]->(S3Bucket) ``` + +- S3 Buckets can be tagged with AWSTags. + + ``` + (S3Bucket)-[TAGGED]->(AWSTag) + ``` diff --git a/tests/data/aws/resourcegroupstaggingapi.py b/tests/data/aws/resourcegroupstaggingapi.py new file mode 100644 index 000000000..b35a72c4e --- /dev/null +++ b/tests/data/aws/resourcegroupstaggingapi.py @@ -0,0 +1,31 @@ +GET_RESOURCES_RESPONSE = [ + { + 'ResourceARN': 'arn:aws:ec2:us-east-1:1234:instance/i-01', + 'Tags': [{ + 'Key': 'TestKey', + 'Value': 'TestValue', + }], + }, { + 'ResourceARN': 'arn:aws:s3:::bucket-1', + 'Tags': [ + { + 'Key': 'Department', + 'Value': 'Engineering', + }, { + 'Key': 'Owner', + 'Value': 'cartography', + }, + ], + }, { + 'ResourceARN': 'arn:aws:rds:us-east-1:1234:db:rds-db-1', + 'Tags': [ + { + 'Key': 'Department', + 'Value': 'Engineering', + }, { + 'Key': 'LastReviewed', + 'Value': 'January', + }, + ], + }, +] diff --git a/tests/integration/cartography/intel/aws/test_resourcegroupstaggingapi.py b/tests/integration/cartography/intel/aws/test_resourcegroupstaggingapi.py new file mode 100644 index 000000000..33af9923f --- /dev/null +++ b/tests/integration/cartography/intel/aws/test_resourcegroupstaggingapi.py @@ -0,0 +1,45 @@ +import cartography.intel.aws.ec2 +import cartography.intel.aws.resourcegroupstaggingapi as rgta +import tests.data.aws.ec2.instances +import tests.data.aws.resourcegroupstaggingapi + + +TEST_ACCOUNT_ID = '1234' +TEST_REGION = 'us-east-1' +TEST_UPDATE_TAG = 123456789 + + +def _ensure_local_neo4j_has_test_ec2_instance_data(neo4j_session): + data = tests.data.aws.ec2.instances.DESCRIBE_INSTANCES + cartography.intel.aws.ec2.load_ec2_instances(neo4j_session, data, TEST_REGION, TEST_ACCOUNT_ID, TEST_UPDATE_TAG) + + +def test_transform_and_load_ec2_tags(neo4j_session): + """ + Verify that (:EC2Instance)-[:TAGGED]->(:AWSTag) relationships work as expected. + """ + _ensure_local_neo4j_has_test_ec2_instance_data(neo4j_session) + resource_type = 'ec2:instance' + rgta.transform_tags(tests.data.aws.resourcegroupstaggingapi.GET_RESOURCES_RESPONSE, resource_type) + rgta.load_tags( + neo4j_session, + tests.data.aws.resourcegroupstaggingapi.GET_RESOURCES_RESPONSE, + resource_type, + TEST_REGION, + TEST_UPDATE_TAG, + ) + expected = { + ('i-01', 'TestKey:TestValue'), + } + + # Fetch relationships + result = neo4j_session.run( + """ + MATCH (n1:EC2Instance)-[:TAGGED]->(n2:AWSTag) RETURN n1.id, n2.id; + """ + ) + actual = { + (r['n1.id'], r['n2.id']) for r in result + } + + assert actual == expected diff --git a/tests/unit/cartography/intel/aws/test_resourcegroupstaggingapi.py b/tests/unit/cartography/intel/aws/test_resourcegroupstaggingapi.py new file mode 100644 index 000000000..c52407227 --- /dev/null +++ b/tests/unit/cartography/intel/aws/test_resourcegroupstaggingapi.py @@ -0,0 +1,33 @@ +import cartography.intel.aws.resourcegroupstaggingapi as rgta +import tests.data.aws.resourcegroupstaggingapi as test_data + + +def test_compute_resource_id(): + """ + Test that the id_func function pointer behaves as expected and returns the instanceid from an EC2Instance's ARN. + """ + tag_mapping = { + 'ResourceARN': 'arn:aws:ec2:us-east-1:1234:instance/i-abcd', + 'Tags': [{ + 'Key': 'my_key', + 'Value': 'my_value', + }], + } + ec2_short_id = 'i-abcd' + assert ec2_short_id == rgta.compute_resource_id(tag_mapping, 'ec2:instance') + + +def test_get_bucket_name_from_arn(): + arn = 'arn:aws:s3:::bucket_name' + assert 'bucket_name' == rgta.get_bucket_name_from_arn(arn) + + +def test_get_short_id_from_ec2_arn(): + arn = 'arn:aws:ec2:us-east-1:test_account:instance/i-1337' + assert 'i-1337' == rgta.get_short_id_from_ec2_arn(arn) + + +def test_transform_tags(): + assert 'resource_id' not in test_data.GET_RESOURCES_RESPONSE[0] + rgta.transform_tags(test_data.GET_RESOURCES_RESPONSE, 'ec2:instance') + assert 'resource_id' in test_data.GET_RESOURCES_RESPONSE[0]