diff --git a/providers/aws/connection/connection.go b/providers/aws/connection/connection.go index c4caef50f..d38149d9c 100644 --- a/providers/aws/connection/connection.go +++ b/providers/aws/connection/connection.go @@ -6,6 +6,7 @@ package connection import ( "context" "errors" + "slices" "strconv" "strings" @@ -37,33 +38,43 @@ type AwsConnection struct { PlatformOverride string connectionOptions map[string]string Filters DiscoveryFilters - RegionLimits []string scope string } type DiscoveryFilters struct { - Ec2DiscoveryFilters Ec2DiscoveryFilters - EcrDiscoveryFilters EcrDiscoveryFilters - EcsDiscoveryFilters EcsDiscoveryFilters - GeneralDiscoveryFilters GeneralResourceDiscoveryFilters + Ec2DiscoveryFilters Ec2DiscoveryFilters + EcrDiscoveryFilters EcrDiscoveryFilters + EcsDiscoveryFilters EcsDiscoveryFilters + DiscoveryFilters GeneralDiscoveryFilters +} + +// ensure all underlying reference types aren't `nil` +func EmptyDiscoveryFilters() DiscoveryFilters { + return DiscoveryFilters{ + DiscoveryFilters: GeneralDiscoveryFilters{Regions: []string{}, ExcludeRegions: []string{}}, + Ec2DiscoveryFilters: Ec2DiscoveryFilters{InstanceIds: []string{}, ExcludeInstanceIds: []string{}, Tags: map[string]string{}, ExcludeTags: map[string]string{}}, + EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}, ExcludeTags: []string{}}, + EcsDiscoveryFilters: EcsDiscoveryFilters{}, + } } -type GeneralResourceDiscoveryFilters struct { - Tags map[string]string - Regions []string +type GeneralDiscoveryFilters struct { + Regions []string + ExcludeRegions []string } type Ec2DiscoveryFilters struct { - Regions []string - Tags map[string]string InstanceIds []string - ExcludeRegions []string - ExcludeTags map[string]string ExcludeInstanceIds []string + Tags map[string]string + ExcludeTags map[string]string } + type EcrDiscoveryFilters struct { - Tags []string + Tags []string + ExcludeTags []string } + type EcsDiscoveryFilters struct { OnlyRunningContainers bool DiscoverImages bool @@ -82,6 +93,7 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config) // check flags for connection options c := &AwsConnection{ awsConfigOptions: []func(*config.LoadOptions) error{}, + Filters: EmptyDiscoveryFilters(), } opts := parseFlagsForConnectionOptions(asset.Options, conf.Options, conf.GetCredentials()) for _, opt := range opts { @@ -125,38 +137,30 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config) c.connectionOptions = asset.Options if conf.Discover != nil { c.Filters = parseOptsToFilters(conf.Discover.Filter) - c.RegionLimits = c.Filters.GeneralDiscoveryFilters.Regions } return c, nil } func parseOptsToFilters(opts map[string]string) DiscoveryFilters { - d := DiscoveryFilters{ - Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}}, - EcsDiscoveryFilters: EcsDiscoveryFilters{}, - EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}}, - GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}}, - } + d := EmptyDiscoveryFilters() for k, v := range opts { switch { - case strings.HasPrefix(k, "ec2:tag:"): - d.Ec2DiscoveryFilters.Tags[strings.TrimPrefix(k, "ec2:tag:")] = v - case strings.HasPrefix(k, "exclude:ec2:tag:"): - d.Ec2DiscoveryFilters.ExcludeTags[strings.TrimPrefix(k, "exclude:ec2:tag:")] = v - case k == "ec2:regions": - d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, strings.Split(v, ",")...) - case k == "exclude:ec2:regions": - d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, strings.Split(v, ",")...) - case k == "all:regions", k == "regions": - d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, strings.Split(v, ",")...) + case k == "regions": + d.DiscoveryFilters.Regions = append(d.DiscoveryFilters.Regions, strings.Split(v, ",")...) + case k == "exclude:regions": + d.DiscoveryFilters.ExcludeRegions = append(d.DiscoveryFilters.ExcludeRegions, strings.Split(v, ",")...) case k == "ec2:instance-ids": d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, strings.Split(v, ",")...) - case k == "exclude:ec2:instance-ids": + case k == "ec2:exclude:instance-ids": d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, strings.Split(v, ",")...) - case strings.HasPrefix(k, "all:tag:"): - d.GeneralDiscoveryFilters.Tags[strings.TrimPrefix(k, "all:tag:")] = v + case strings.HasPrefix(k, "ec2:tag:"): + d.Ec2DiscoveryFilters.Tags[strings.TrimPrefix(k, "ec2:tag:")] = v + case strings.HasPrefix(k, "ec2:exclude:tag:"): + d.Ec2DiscoveryFilters.ExcludeTags[strings.TrimPrefix(k, "ec2:exclude:tag:")] = v case k == "ecr:tags": d.EcrDiscoveryFilters.Tags = append(d.EcrDiscoveryFilters.Tags, strings.Split(v, ",")...) + case k == "ecr:exclude:tags": + d.EcrDiscoveryFilters.ExcludeTags = append(d.EcrDiscoveryFilters.ExcludeTags, strings.Split(v, ",")...) case k == "ecs:only-running-containers": parsed, err := strconv.ParseBool(v) if err == nil { @@ -360,15 +364,17 @@ func (h *AwsConnection) Regions() ([]string, error) { log.Debug().Msg("use regions from cache") return c.Data.([]string), nil } - log.Debug().Msg("no region cache found. fetching regions") - if len(h.RegionLimits) > 0 { - log.Debug().Interface("regions", h.RegionLimits).Msg("using region limits") + // include filters have precedense over exclude filters. in any normal situation they should be mutually exclusive. + regionLimits := h.Filters.DiscoveryFilters.Regions + if len(regionLimits) > 0 { + log.Debug().Interface("regions", regionLimits).Msg("using region limits") // cache the regions as part of the provider instance - h.clientcache.Store("_regions", &CacheEntry{Data: h.RegionLimits}) - return h.RegionLimits, nil + h.clientcache.Store("_regions", &CacheEntry{Data: regionLimits}) + return regionLimits, nil } // if no cache, get regions using ec2 client (using the ssm list global regions does not give the same list) + log.Debug().Msg("no region cache or region limits found. fetching regions") regions := []string{} svc := h.Ec2("us-east-1") ctx := context.Background() @@ -383,7 +389,10 @@ func (h *AwsConnection) Regions() ([]string, error) { } } for _, region := range res.Regions { - regions = append(regions, *region.RegionName) + // ensure excluded regions are discarded + if !slices.Contains(h.Filters.DiscoveryFilters.ExcludeRegions, *region.RegionName) { + regions = append(regions, *region.RegionName) + } } // cache the regions as part of the provider instance h.clientcache.Store("_regions", &CacheEntry{Data: regions}) diff --git a/providers/aws/connection/connection_test.go b/providers/aws/connection/connection_test.go index dc9561eb0..f39c755dd 100644 --- a/providers/aws/connection/connection_test.go +++ b/providers/aws/connection/connection_test.go @@ -9,76 +9,47 @@ import ( "github.com/stretchr/testify/require" ) -// testParseOptsToFilters accepts a map which doesn't guarantee a deterministic iteration order. this means that slices -// in the parsed filters need to be compared individually ensuring their elements match regardless of their order. -func compareFilters(t *testing.T, expected, actual DiscoveryFilters) { - require.ElementsMatch(t, expected.Ec2DiscoveryFilters.Regions, actual.Ec2DiscoveryFilters.Regions) - require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeRegions, actual.Ec2DiscoveryFilters.ExcludeRegions) - - require.ElementsMatch(t, expected.Ec2DiscoveryFilters.InstanceIds, actual.Ec2DiscoveryFilters.InstanceIds) - require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeInstanceIds, actual.Ec2DiscoveryFilters.ExcludeInstanceIds) - - require.Equal(t, expected.Ec2DiscoveryFilters.Tags, actual.Ec2DiscoveryFilters.Tags) - require.Equal(t, expected.Ec2DiscoveryFilters.ExcludeTags, actual.Ec2DiscoveryFilters.ExcludeTags) - - require.Equal(t, expected.EcsDiscoveryFilters, actual.EcsDiscoveryFilters) - - require.ElementsMatch(t, expected.EcrDiscoveryFilters.Tags, actual.EcrDiscoveryFilters.Tags) - - require.ElementsMatch(t, expected.GeneralDiscoveryFilters.Regions, actual.GeneralDiscoveryFilters.Regions) - require.Equal(t, expected.GeneralDiscoveryFilters.Tags, actual.GeneralDiscoveryFilters.Tags) -} - func TestParseOptsToFilters(t *testing.T) { t.Run("all opts are mapped to discovery filters correctly", func(t *testing.T) { opts := map[string]string{ + // DiscoveryFilters.Regions + "regions": "us-east-1,us-west-1,eu-west-1", + // DiscoveryFilters.ExcludeRegions + "exclude:regions": "us-east-2,us-west-2,eu-west-2", + // Ec2DiscoveryFilters.InstanceIds + "ec2:instance-ids": "iid-1,iid-2", + // Ec2DiscoveryFilters.ExcludeInstanceIds + "ec2:exclude:instance-ids": "iid-1,iid-2", // Ec2DiscoveryFilters.Tags "ec2:tag:key1": "val1", "ec2:tag:key2": "val2", // Ec2DiscoveryFilters.ExcludeTags - "exclude:ec2:tag:key1": "val1", - "exclude:ec2:tag:key2": "val2", - // Ec2DiscoveryFilters.Regions - "ec2:regions": "us-east-1,us-west-1", - // Ec2DiscoveryFilters.ExcludeRegions - "exclude:ec2:regions": "us-east-1,us-west-1", - // Ec2DiscoveryFilters.InstanceIds - "ec2:instance-ids": "iid-1,iid-2", - // Ec2DiscoveryFilters.ExcludeInstanceIds - "exclude:ec2:instance-ids": "iid-1,iid-2", - // GeneralDiscoveryFilters.Regions - "all:regions": "us-east-1,us-west-1,eu-west-1", - // GeneralDiscoveryFilters.Tags - "all:tag:key1": "val1", - "all:tag:key2": "val2", + "ec2:exclude:tag:key1": "val1,val2", + "ec2:exclude:tag:key2": "val3", // EcrDiscoveryFilters.Tags "ecr:tags": "tag1,tag2", + // EcrDiscoveryFilters.ExcludeTags + "ecr:exclude:tags": "tag1,tag2", // EcsDiscoveryFilters "ecs:only-running-containers": "true", "ecs:discover-images": "T", "ecs:discover-instances": "false", } expected := DiscoveryFilters{ + DiscoveryFilters: GeneralDiscoveryFilters{ + Regions: []string{"us-east-1", "us-west-1", "eu-west-1"}, + ExcludeRegions: []string{"us-east-2", "us-west-2", "eu-west-2"}, + }, Ec2DiscoveryFilters: Ec2DiscoveryFilters{ - Regions: []string{ - "us-east-1", "us-west-1", - }, - ExcludeRegions: []string{ - "us-east-1", "us-west-1", - }, - InstanceIds: []string{ - "iid-1", "iid-2", - }, - ExcludeInstanceIds: []string{ - "iid-1", "iid-2", - }, + InstanceIds: []string{"iid-1", "iid-2"}, + ExcludeInstanceIds: []string{"iid-1", "iid-2"}, Tags: map[string]string{ "key1": "val1", "key2": "val2", }, ExcludeTags: map[string]string{ - "key1": "val1", - "key2": "val2", + "key1": "val1,val2", + "key2": "val3", }, }, EcsDiscoveryFilters: EcsDiscoveryFilters{ @@ -86,33 +57,19 @@ func TestParseOptsToFilters(t *testing.T) { DiscoverImages: true, DiscoverInstances: false, }, - EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{ - "tag1", "tag2", - }}, - GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{ - Regions: []string{ - "us-east-1", "us-west-1", "eu-west-1", - }, - Tags: map[string]string{ - "key1": "val1", - "key2": "val2", - }, + EcrDiscoveryFilters: EcrDiscoveryFilters{ + Tags: []string{"tag1", "tag2"}, + ExcludeTags: []string{"tag1", "tag2"}, }, } actual := parseOptsToFilters(opts) - compareFilters(t, expected, actual) + require.Equal(t, expected, actual) }) t.Run("empty opts are mapped to discovery filters correctly", func(t *testing.T) { - expected := DiscoveryFilters{ - Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}}, - EcsDiscoveryFilters: EcsDiscoveryFilters{}, - EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}}, - GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}}, - } - + expected := EmptyDiscoveryFilters() actual := parseOptsToFilters(map[string]string{}) - compareFilters(t, expected, actual) + require.Equal(t, expected, actual) }) } diff --git a/providers/aws/provider/provider.go b/providers/aws/provider/provider.go index 497b0c5a5..88e294760 100644 --- a/providers/aws/provider/provider.go +++ b/providers/aws/provider/provider.go @@ -84,16 +84,18 @@ func parseFlagsToFiltersOpts(m map[string]*llx.Primitive) map[string]string { if x, ok := m["filters"]; ok && len(x.Map) != 0 { knownTagPrefixes := []string{ - "ec2:tag:", - "exclude:ec2:tag:", - "ec2:regions", - "exclude:ec2:regions", - "all:regions", + // general filters "regions", + "exclude:regions", + // ec2 filters "ec2:instance-ids", - "exclude:ec2:instance-ids", - "all:tag:", + "ec2:exclude:instance-ids", + "ec2:tag:", + "ec2:exclude:tag:", + // ecr filters "ecr:tags", + "ecr:exclude:tags", + // ecs filters "ecs:only-running-containers", "ecs:discover-instances", "ecs:discover-images", diff --git a/providers/aws/resources/aws_ec2.go b/providers/aws/resources/aws_ec2.go index e5a209a17..b13d8b3d3 100644 --- a/providers/aws/resources/aws_ec2.go +++ b/providers/aws/resources/aws_ec2.go @@ -26,7 +26,6 @@ import ( "go.mondoo.com/cnquery/v11/providers-sdk/v1/util/jobpool" "go.mondoo.com/cnquery/v11/providers/aws/connection" "go.mondoo.com/cnquery/v11/types" - "go.mondoo.com/cnquery/v11/utils/stringx" ) func (e *mqlAwsEc2) id() (string, error) { @@ -765,20 +764,20 @@ func (a *mqlAwsEc2) instances() ([]interface{}, error) { return res, nil } -func (a *mqlAwsEc2) getEc2Instances(ctx context.Context, svc *ec2.Client, filters connection.Ec2DiscoveryFilters) ([]ec2types.Reservation, error) { +func (a *mqlAwsEc2) getEc2Instances(ctx context.Context, svc *ec2.Client, filters connection.DiscoveryFilters) ([]ec2types.Reservation, error) { res := []ec2types.Reservation{} nextToken := aws.String("no_token_to_start_with") params := &ec2.DescribeInstancesInput{ Filters: []ec2types.Filter{}, } - for k, v := range filters.Tags { + for k, v := range filters.Ec2DiscoveryFilters.Tags { params.Filters = append(params.Filters, ec2types.Filter{ Name: aws.String(fmt.Sprintf("tag:%s", k)), Values: strings.Split(v, ","), }) } - if len(filters.InstanceIds) > 0 { - params.InstanceIds = filters.InstanceIds + if len(filters.Ec2DiscoveryFilters.InstanceIds) > 0 { + params.InstanceIds = filters.Ec2DiscoveryFilters.InstanceIds } for nextToken != nil { @@ -801,7 +800,6 @@ func (a *mqlAwsEc2) getInstances(conn *connection.AwsConnection) []*jobpool.Job if err != nil { return []*jobpool.Job{{Err: err}} } - regions = determineApplicableRegions(regions, conn.Filters.Ec2DiscoveryFilters.Regions, conn.Filters.Ec2DiscoveryFilters.ExcludeRegions) for _, region := range regions { regionVal := region f := func() (jobpool.JobResult, error) { @@ -811,7 +809,7 @@ func (a *mqlAwsEc2) getInstances(conn *connection.AwsConnection) []*jobpool.Job ctx := context.Background() var res []interface{} - instances, err := a.getEc2Instances(ctx, svc, conn.Filters.Ec2DiscoveryFilters) + instances, err := a.getEc2Instances(ctx, svc, conn.Filters) if err != nil { if Is400AccessDeniedError(err) { log.Warn().Str("region", regionVal).Msg("error accessing region for AWS API") @@ -1780,13 +1778,13 @@ func (a *mqlAwsEc2Vgwtelemetry) id() (string, error) { } // true if the instance should be excluded from results. filtering for excluded regions should happen before we retrieve the EC2 instance. -func shouldExcludeInstance(instance ec2types.Instance, filters connection.Ec2DiscoveryFilters) bool { - for _, id := range filters.ExcludeInstanceIds { +func shouldExcludeInstance(instance ec2types.Instance, ec2Filters connection.Ec2DiscoveryFilters) bool { + for _, id := range ec2Filters.ExcludeInstanceIds { if instance.InstanceId != nil && *instance.InstanceId == id { return true } } - for k, v := range filters.ExcludeTags { + for k, v := range ec2Filters.ExcludeTags { for _, tagValue := range strings.Split(v, ",") { for _, iTag := range instance.Tags { if iTag.Key != nil && *iTag.Key == k && @@ -1798,19 +1796,3 @@ func shouldExcludeInstance(instance ec2types.Instance, filters connection.Ec2Dis } return false } - -// given an initial set of regions, applies the allowed regions filter and the excluded regions filter on it -// returning a resulting slice of only applicable region to which queries should be targeted -func determineApplicableRegions(regions, regionsToInclude, regionsToExclude []string) []string { - if len(regionsToInclude) > 0 { - return regionsToInclude - } - res := []string{} - for _, r := range regions { - if !stringx.Contains(regionsToExclude, r) { - res = append(res, r) - } - } - - return res -} diff --git a/providers/aws/resources/aws_ec2_test.go b/providers/aws/resources/aws_ec2_test.go index 4404b6a64..4ca3adf2d 100644 --- a/providers/aws/resources/aws_ec2_test.go +++ b/providers/aws/resources/aws_ec2_test.go @@ -78,6 +78,7 @@ func TestShouldExcludeInstance(t *testing.T) { t.Run("should exclude instances with matching values for the same tag", func(t *testing.T) { filters := connection.Ec2DiscoveryFilters{ + ExcludeInstanceIds: []string{}, ExcludeTags: map[string]string{ "key-1": "val-1,val-2,val-3", }, @@ -87,6 +88,7 @@ func TestShouldExcludeInstance(t *testing.T) { t.Run("should not exclude instances when no tag values match", func(t *testing.T) { filters := connection.Ec2DiscoveryFilters{ + ExcludeInstanceIds: []string{}, ExcludeTags: map[string]string{ "key-1": "val-2,val-3", "key-2": "val-1,val-3", @@ -96,35 +98,3 @@ func TestShouldExcludeInstance(t *testing.T) { require.False(t, shouldExcludeInstance(instance, filters)) }) } - -func TestDetermineApplicableRegions(t *testing.T) { - t.Run("allow regions override initial region list", func(t *testing.T) { - initialRegions := []string{"a", "b"} - allowedRegions := []string{"b", "c"} - excludedRegions := []string{} - - expected := []string{"b", "c"} - actual := determineApplicableRegions(initialRegions, allowedRegions, excludedRegions) - require.ElementsMatch(t, expected, actual) - }) - - t.Run("excluded regions work correctly", func(t *testing.T) { - initialRegions := []string{"a", "b"} - allowedRegions := []string{} - excludedRegions := []string{"b"} - - expected := []string{"a"} - actual := determineApplicableRegions(initialRegions, allowedRegions, excludedRegions) - require.ElementsMatch(t, expected, actual) - }) - - t.Run("excluded regions not present in the initial slice are ignored", func(t *testing.T) { - initialRegions := []string{"a", "b"} - allowedRegions := []string{} - excludedRegions := []string{"b", "c", "d", "e"} - - expected := []string{"a"} - actual := determineApplicableRegions(initialRegions, allowedRegions, excludedRegions) - require.ElementsMatch(t, expected, actual) - }) -} diff --git a/providers/aws/resources/aws_ecr.go b/providers/aws/resources/aws_ecr.go index c5a1b015d..48b3721fa 100644 --- a/providers/aws/resources/aws_ecr.go +++ b/providers/aws/resources/aws_ecr.go @@ -159,11 +159,10 @@ func (a *mqlAwsEcrRepository) images() ([]interface{}, error) { return nil, err } - for i := range res.ImageDetails { - image := res.ImageDetails[i] + for _, image := range res.ImageDetails { tags := []interface{}{} - for i := range image.ImageTags { - tags = append(tags, image.ImageTags[i]) + for _, imageTag := range image.ImageTags { + tags = append(tags, imageTag) } mqlImage, err := CreateResource(a.MqlRuntime, "aws.ecr.image", map[string]*llx.RawData{ diff --git a/providers/aws/resources/aws_ecs.go b/providers/aws/resources/aws_ecs.go index 845cd8c20..63a518735 100644 --- a/providers/aws/resources/aws_ecs.go +++ b/providers/aws/resources/aws_ecs.go @@ -136,10 +136,10 @@ func (a *mqlAwsEcs) getECSClusters(conn *connection.AwsConnection) []*jobpool.Jo if resp.NextToken != nil { params.NextToken = nextToken } - for _, cluster := range resp.ClusterArns { + for _, clusterArn := range resp.ClusterArns { mqlCluster, err := NewResource(a.MqlRuntime, "aws.ecs.cluster", map[string]*llx.RawData{ - "arn": llx.StringData(cluster), + "arn": llx.StringData(clusterArn), }) if err != nil { return nil, err @@ -179,7 +179,7 @@ func initAwsEcsCluster(runtime *plugin.Runtime, args map[string]*llx.RawData) (m } svc := conn.Ecs(region) ctx := context.Background() - clusterDetails, err := svc.DescribeClusters(ctx, &ecs.DescribeClustersInput{Clusters: []string{a}}) + clusterDetails, err := svc.DescribeClusters(ctx, &ecs.DescribeClustersInput{Clusters: []string{a}, Include: []ecstypes.ClusterField{ecstypes.ClusterFieldTags}}) if err != nil { return nil, nil, err } @@ -192,7 +192,7 @@ func initAwsEcsCluster(runtime *plugin.Runtime, args map[string]*llx.RawData) (m return nil, nil, err } args["name"] = llx.StringDataPtr(c.ClusterName) - args["tags"] = llx.MapData(ecsTags(c.Tags), types.String) + args["tags"] = llx.MapData(ecsTagsToMap(c.Tags), types.String) args["runningTasksCount"] = llx.IntData(int64(c.RunningTasksCount)) args["pendingTasksCount"] = llx.IntData(int64(c.PendingTasksCount)) args["registeredContainerInstancesCount"] = llx.IntData(int64(c.RegisteredContainerInstancesCount)) @@ -202,17 +202,6 @@ func initAwsEcsCluster(runtime *plugin.Runtime, args map[string]*llx.RawData) (m return args, nil, nil } -func ecsTags(t []ecstypes.Tag) map[string]interface{} { - res := map[string]interface{}{} - for i := range t { - tag := t[i] - if tag.Key != nil && tag.Value != nil { - res[*tag.Key] = *tag.Value - } - } - return res -} - func (a *mqlAwsEcsCluster) containerInstances() ([]interface{}, error) { conn := a.MqlRuntime.Connection.(*connection.AwsConnection) clustera := a.Arn.Data @@ -302,10 +291,10 @@ func (a *mqlAwsEcsCluster) tasks() ([]interface{}, error) { if resp.NextToken != nil { params.NextToken = nextToken } - for _, task := range resp.TaskArns { + for _, taskArn := range resp.TaskArns { mqlTask, err := NewResource(a.MqlRuntime, "aws.ecs.task", map[string]*llx.RawData{ - "arn": llx.StringData(task), + "arn": llx.StringData(taskArn), "clusterName": llx.StringData(name), }) if err != nil { @@ -351,7 +340,7 @@ func initAwsEcsTask(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[ } svc := conn.Ecs(region) ctx := context.Background() - params := &ecs.DescribeTasksInput{Tasks: []string{a}, Cluster: &clusterName} + params := &ecs.DescribeTasksInput{Tasks: []string{a}, Cluster: &clusterName, Include: []ecstypes.TaskField{ecstypes.TaskFieldTags}} params.Cluster = &clusterName taskDetails, err := svc.DescribeTasks(ctx, params) if err != nil { @@ -367,7 +356,7 @@ func initAwsEcsTask(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[ args["lastStatus"] = llx.StringData(convert.ToString(t.LastStatus)) args["platformFamily"] = llx.StringData(convert.ToString(t.PlatformFamily)) args["platformVersion"] = llx.StringData(convert.ToString(t.PlatformVersion)) - args["tags"] = llx.MapData(ecsTags(t.Tags), types.String) + args["tags"] = llx.MapData(ecsTagsToMap(t.Tags), types.String) res, err := CreateResource(runtime, "aws.ecs.task", args) if err != nil { return args, nil, err @@ -487,3 +476,13 @@ func getPublicIpForContainer(ctx context.Context, conn *connection.AwsConnection func (s *mqlAwsEcsContainer) id() (string, error) { return s.Arn.Data, nil } + +func ecsTagsToMap(tags []ecstypes.Tag) map[string]interface{} { + res := map[string]interface{}{} + for _, tag := range tags { + if tag.Key != nil && tag.Value != nil { + res[convert.ToString(tag.Key)] = convert.ToString(tag.Value) + } + } + return res +} diff --git a/providers/aws/resources/discovery.go b/providers/aws/resources/discovery.go index d799d3668..10abb493e 100644 --- a/providers/aws/resources/discovery.go +++ b/providers/aws/resources/discovery.go @@ -105,46 +105,22 @@ func containsInterfaceSlice(sl []interface{}, s string) bool { return false } -func imageMatchesFilters(image *mqlAwsEcrImage, filters connection.DiscoveryFilters) bool { - f := filters.EcrDiscoveryFilters - if len(f.Tags) > 0 { - for i := range f.Tags { - t := f.Tags[i] - if !containsInterfaceSlice(image.Tags.Data, t) { - return false - } +func imageMatchesFilters(image *mqlAwsEcrImage, filters connection.EcrDiscoveryFilters) bool { + for _, t := range filters.Tags { + if !containsInterfaceSlice(image.Tags.Data, t) { + return false } } - return true -} - -func containerMatchesFilters(container *mqlAwsEcsContainer, filters connection.DiscoveryFilters) bool { - f := filters.EcsDiscoveryFilters - if f.OnlyRunningContainers { - if container.Status.Data != "RUNNING" { + for _, t := range filters.ExcludeTags { + if containsInterfaceSlice(image.Tags.Data, t) { return false } } return true } -func shouldScanEcsContainerInstances(filters connection.DiscoveryFilters) bool { - return filters.EcsDiscoveryFilters.DiscoverInstances -} - -func shouldScanEcsContainerImages(filters connection.DiscoveryFilters) bool { - return filters.EcsDiscoveryFilters.DiscoverImages -} - -func discoveredAssetMatchesGeneralFilters(asset *inventory.Asset, filters connection.GeneralResourceDiscoveryFilters) bool { - if len(filters.Tags) > 0 { - for k, v := range filters.Tags { - if asset.Labels[k] != v { - return false - } - } - } - return true +func containerMatchesFilters(container *mqlAwsEcsContainer, ecsFilters connection.EcsDiscoveryFilters) bool { + return container.Status.Data == "RUNNING" || !ecsFilters.OnlyRunningContainers } func Discover(runtime *plugin.Runtime) (*inventory.Inventory, error) { @@ -168,15 +144,6 @@ func Discover(runtime *plugin.Runtime) (*inventory.Inventory, error) { log.Error().Err(err).Msg("error during discovery") continue } - if len(conn.Filters.GeneralDiscoveryFilters.Tags) > 0 { - newList := []*inventory.Asset{} - for i := range list { - if discoveredAssetMatchesGeneralFilters(list[i], conn.Filters.GeneralDiscoveryFilters) { - newList = append(newList, list[i]) - } - } - list = newList - } in.Spec.Assets = append(in.Spec.Assets, list...) } return in, nil @@ -277,7 +244,7 @@ func discover(runtime *plugin.Runtime, awsAccount *mqlAwsAccount, target string, for i := range images.Data { a := images.Data[i].(*mqlAwsEcrImage) - if !imageMatchesFilters(a, filters) { + if !imageMatchesFilters(a, filters.EcrDiscoveryFilters) { continue } ecrAsset := addConnectionInfoToEcrAsset(a, conn) @@ -302,12 +269,12 @@ func discover(runtime *plugin.Runtime, awsAccount *mqlAwsAccount, target string, for i := range containers.Data { c := containers.Data[i].(*mqlAwsEcsContainer) - if !containerMatchesFilters(c, filters) { + if !containerMatchesFilters(c, filters.EcsDiscoveryFilters) { continue } assetList = append(assetList, addConnectionInfoToECSContainerAsset(c, accountId, conn)) } - if shouldScanEcsContainerInstances(filters) { + if filters.EcsDiscoveryFilters.DiscoverInstances { containerInst := ecs.GetContainerInstances() if containerInst == nil { return assetList, nil @@ -336,7 +303,7 @@ func discover(runtime *plugin.Runtime, awsAccount *mqlAwsAccount, target string, for i := range containers.Data { c := containers.Data[i].(*mqlAwsEcsContainer) - if !containerMatchesFilters(c, filters) { + if !containerMatchesFilters(c, filters.EcsDiscoveryFilters) { continue } assetList = append(assetList, MqlObjectToAsset(accountId, @@ -364,7 +331,7 @@ func discover(runtime *plugin.Runtime, awsAccount *mqlAwsAccount, target string, for i := range images.Data { a := images.Data[i].(*mqlAwsEcrImage) - if !imageMatchesFilters(a, filters) { + if !imageMatchesFilters(a, filters.EcrDiscoveryFilters) { continue } l := make(map[string]string) diff --git a/providers/aws/resources/discovery_test.go b/providers/aws/resources/discovery_test.go index 9d0989c76..f96288af3 100644 --- a/providers/aws/resources/discovery_test.go +++ b/providers/aws/resources/discovery_test.go @@ -15,70 +15,31 @@ import ( ) func TestFilters(t *testing.T) { + // image filters require.True(t, imageMatchesFilters(&mqlAwsEcrImage{ Tags: plugin.TValue[[]interface{}]{Data: []interface{}{"latest"}}, - }, connection.DiscoveryFilters{})) + }, connection.EcrDiscoveryFilters{})) + require.True(t, imageMatchesFilters(&mqlAwsEcrImage{ Tags: plugin.TValue[[]interface{}]{Data: []interface{}{"latest"}}, - }, connection.DiscoveryFilters{ - EcrDiscoveryFilters: connection.EcrDiscoveryFilters{ - Tags: []string{"latest"}, - }, - })) + }, connection.EcrDiscoveryFilters{Tags: []string{"latest"}})) + require.False(t, imageMatchesFilters(&mqlAwsEcrImage{ Tags: plugin.TValue[[]interface{}]{Data: []interface{}{"ubu", "test"}}, - }, connection.DiscoveryFilters{ - EcrDiscoveryFilters: connection.EcrDiscoveryFilters{ - Tags: []string{"latest"}, - }, - })) + }, connection.EcrDiscoveryFilters{Tags: []string{"latest"}})) + // container filters require.True(t, containerMatchesFilters(&mqlAwsEcsContainer{ Status: plugin.TValue[string]{Data: "RUNNING"}, - }, connection.DiscoveryFilters{})) + }, connection.EcsDiscoveryFilters{})) require.True(t, containerMatchesFilters(&mqlAwsEcsContainer{ Status: plugin.TValue[string]{Data: "RUNNING"}, - }, connection.DiscoveryFilters{EcsDiscoveryFilters: connection.EcsDiscoveryFilters{ - OnlyRunningContainers: true, - }})) + }, connection.EcsDiscoveryFilters{OnlyRunningContainers: true})) require.False(t, containerMatchesFilters(&mqlAwsEcsContainer{ Status: plugin.TValue[string]{Data: "STOPPED"}, - }, connection.DiscoveryFilters{EcsDiscoveryFilters: connection.EcsDiscoveryFilters{ - OnlyRunningContainers: true, - }})) - - require.True(t, discoveredAssetMatchesGeneralFilters(&inventory.Asset{ - Labels: map[string]string{"test": "val", "another": "value"}, - }, connection.GeneralResourceDiscoveryFilters{})) - require.True(t, discoveredAssetMatchesGeneralFilters(&inventory.Asset{ - Labels: nil, - }, connection.GeneralResourceDiscoveryFilters{})) - require.True(t, discoveredAssetMatchesGeneralFilters(&inventory.Asset{ - Labels: map[string]string{"test": "val", "another": "value"}, - }, connection.GeneralResourceDiscoveryFilters{Tags: map[string]string{"another": "value"}})) - - require.False(t, discoveredAssetMatchesGeneralFilters(&inventory.Asset{ - Labels: map[string]string{"test": "val", "another": "value"}, - }, connection.GeneralResourceDiscoveryFilters{Tags: map[string]string{"something": "else"}})) - require.False(t, discoveredAssetMatchesGeneralFilters(&inventory.Asset{ - Labels: nil, - }, connection.GeneralResourceDiscoveryFilters{Tags: map[string]string{"something": "else"}})) - - require.True(t, shouldScanEcsContainerImages(connection.DiscoveryFilters{ - EcsDiscoveryFilters: connection.EcsDiscoveryFilters{ - DiscoverImages: true, - }, - })) - require.False(t, shouldScanEcsContainerImages(connection.DiscoveryFilters{})) - require.True(t, shouldScanEcsContainerInstances(connection.DiscoveryFilters{ - EcsDiscoveryFilters: connection.EcsDiscoveryFilters{ - DiscoverImages: false, - DiscoverInstances: true, - }, - })) - require.False(t, shouldScanEcsContainerInstances(connection.DiscoveryFilters{})) + }, connection.EcsDiscoveryFilters{OnlyRunningContainers: true})) } func TestAddConnInfoToEc2Instances(t *testing.T) {