Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove duplicate EC2 cleanup jobs #1151

Merged
merged 2 commits into from
Apr 3, 2023
Merged

Remove duplicate EC2 cleanup jobs #1151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
73bab50
Remove duplicate EC2 cleanup jobs
achantavy Apr 1, 2023
a24252d
lint
achantavy Apr 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 0 additions & 20 deletions cartography/data/jobs/cleanup/aws_import_ec2_key_pairs_cleanup.json
View file
Open in desktop

This file was deleted.

5 changes: 0 additions & 5 deletions cartography/data/jobs/cleanup/aws_import_ec2_security_groupinfo_cleanup.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,5 @@
{
"statements": [
{
"query": "MATCH (n:EC2SecurityGroup)<-[:RESOURCE]-(:AWSAccount{id: $AWS_ID}) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
Copy link
Contributor Author

@achantavy achantavy Apr 3, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is covered by autocleanups in models.aws.ec2.securitygroups. Additionally the autocleanups also handle a previously unhandled case where we need to cleanup stale rels between SGs and instances.

"iterative": true,
"iterationsize": 100
},
{
"query": "MATCH (n:IpRule)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(:EC2SecurityGroup)<-[:RESOURCE]-(:AWSAccount{id: $AWS_ID}) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
"iterative": true,
Expand Down
20 changes: 0 additions & 20 deletions cartography/data/jobs/cleanup/aws_import_volumes_cleanup.json
View file
Open in desktop

This file was deleted.

15 changes: 0 additions & 15 deletions cartography/data/jobs/cleanup/aws_ingest_network_interfaces_cleanup.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,10 @@
"iterationsize": 100,
"iterative": true
},
{
"query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:EC2Instance)-[r:PART_OF_SUBNET]->(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is covered by autocleanups in models.aws.ec2.subnets.

"iterationsize": 100,
"iterative": true
},
{
"query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:LoadBalancer)-[r:PART_OF_SUBNET]->(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
"iterationsize": 100,
"iterative": true
},
{
"query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(n:NetworkInterface) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)",
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is covered by autocleanups in models.aws.ec2.networkinterfaces.

"iterative": true,
"iterationsize": 100
},
{
"query": "MATCH (:AWSAccount{id: $AWS_ID})-[r:RESOURCE]->(:NetworkInterface) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)",
"iterative": true,
"iterationsize": 100
}
],
"name": "cleanup NetworkInterface"
Expand Down
5 changes: 0 additions & 5 deletions cartography/intel/aws/ec2/instances.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,11 +308,6 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any])
logger.debug("Running EC2 instance cleanup")
GraphJob.from_node_schema(EC2ReservationSchema(), common_job_parameters).run(neo4j_session)
GraphJob.from_node_schema(EC2InstanceSchema(), common_job_parameters).run(neo4j_session)
GraphJob.from_node_schema(EC2SubnetSchema(), common_job_parameters).run(neo4j_session)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are all moved from here to their respective modules. Specifically,

  • EC2SubnetSchema -> intel.aws.ec2.subnets.cleanup_subnets
  • EC2SecurityGroupSchema -> intel.aws.ec2.security_groups.cleanup_ec2_security_groupinfo
  • EC2KeyPairSchema -> intel.aws.ec2.key_pairs.cleanup_ec2_key_pairs
  • EC2NetworkInterfaceSchema -> intel.aws.ec2.network_interfaces.cleanup_network_interfaces
  • EBSVolumeSchema -> intel.aws.ec2.cleanup_volumes

GraphJob.from_node_schema(EC2SecurityGroupSchema(), common_job_parameters).run(neo4j_session)
GraphJob.from_node_schema(EC2KeyPairSchema(), common_job_parameters).run(neo4j_session)
GraphJob.from_node_schema(EC2NetworkInterfaceSchema(), common_job_parameters).run(neo4j_session)
GraphJob.from_node_schema(EBSVolumeSchema(), common_job_parameters).run(neo4j_session)


@timeit
Expand Down
5 changes: 3 additions & 2 deletions cartography/intel/aws/ec2/key_pairs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@
import neo4j

from .util import get_botocore_config
from cartography.graph.job import GraphJob
from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema
from cartography.util import aws_handle_regions
from cartography.util import run_cleanup_job
from cartography.util import timeit

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -55,7 +56,7 @@ def load_ec2_key_pairs(

@timeit
def cleanup_ec2_key_pairs(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
run_cleanup_job('aws_import_ec2_key_pairs_cleanup.json', neo4j_session, common_job_parameters)
GraphJob.from_node_schema(EC2KeyPairSchema(), common_job_parameters).run(neo4j_session)


@timeit
Expand Down
3 changes: 3 additions & 0 deletions cartography/intel/aws/ec2/network_interfaces.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import neo4j

from .util import get_botocore_config
from cartography.graph.job import GraphJob
from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema
from cartography.util import aws_handle_regions
from cartography.util import run_cleanup_job
from cartography.util import timeit
Expand Down Expand Up @@ -265,6 +267,7 @@ def load(neo4j_session: neo4j.Session, data: List[Dict], region: str, aws_accoun
@timeit
def cleanup_network_interfaces(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
run_cleanup_job('aws_ingest_network_interfaces_cleanup.json', neo4j_session, common_job_parameters)
GraphJob.from_node_schema(EC2NetworkInterfaceSchema(), common_job_parameters).run(neo4j_session)


@timeit
Expand Down
3 changes: 3 additions & 0 deletions cartography/intel/aws/ec2/security_groups.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import neo4j

from .util import get_botocore_config
from cartography.graph.job import GraphJob
from cartography.models.aws.ec2.securitygroups import EC2SecurityGroupSchema
from cartography.util import aws_handle_regions
from cartography.util import run_cleanup_job
from cartography.util import timeit
Expand Down Expand Up @@ -146,6 +148,7 @@ def cleanup_ec2_security_groupinfo(neo4j_session: neo4j.Session, common_job_para
neo4j_session,
common_job_parameters,
)
GraphJob.from_node_schema(EC2SecurityGroupSchema(), common_job_parameters).run(neo4j_session)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We still need to keep both, for now, right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes this is intentional because the other resources cleaned up in aws_import_ec2_security_groupinfo_cleanup.json have not been refactored to the new model yet.



@timeit
Expand Down
3 changes: 3 additions & 0 deletions cartography/intel/aws/ec2/subnets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import neo4j

from .util import get_botocore_config
from cartography.graph.job import GraphJob
from cartography.models.aws.ec2.subnets import EC2SubnetSchema
from cartography.util import aws_handle_regions
from cartography.util import run_cleanup_job
from cartography.util import timeit
Expand Down Expand Up @@ -76,6 +78,7 @@ def load_subnets(
@timeit
def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
run_cleanup_job('aws_ingest_subnets_cleanup.json', neo4j_session, common_job_parameters)
GraphJob.from_node_schema(EC2SubnetSchema(), common_job_parameters).run(neo4j_session)


@timeit
Expand Down
9 changes: 3 additions & 6 deletions cartography/intel/aws/ec2/volumes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@
import boto3
import neo4j

from cartography.graph.job import GraphJob
from cartography.models.aws.ec2.volumes import EBSVolumeSchema
from cartography.util import aws_handle_regions
from cartography.util import run_cleanup_job
from cartography.util import timeit

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -97,11 +98,7 @@ def load_volume_relationships(

@timeit
def cleanup_volumes(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
run_cleanup_job(
'aws_import_volumes_cleanup.json',
neo4j_session,
common_job_parameters,
)
GraphJob.from_node_schema(EBSVolumeSchema(), common_job_parameters).run(neo4j_session)


@timeit
Expand Down