[Asset Inventory][Azure] Add Acitive Directory fetcher to fetch Service Principals

go.mod

@@ -62,6 +62,9 @@ require (
 	github.com/huandu/xstrings v1.5.0
 	github.com/magefile/mage v1.15.0
 	github.com/masahiro331/go-xfs-filesystem v0.0.0-20231205045356-1b22259a6c44
+	github.com/microsoft/kiota-abstractions-go v1.7.0
+	github.com/microsoftgraph/msgraph-sdk-go v1.51.0
+	github.com/microsoftgraph/msgraph-sdk-go-core v1.2.1
 	github.com/mikefarah/yq/v4 v4.44.3
 	github.com/mitchellh/gox v1.0.1
 	github.com/mitchellh/mapstructure v1.5.0
@@ -102,6 +105,7 @@ require (
 	github.com/bitnami/go-version v0.0.0-20231130084017-bb00604d650c // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cjlapao/common-go v0.0.39 // indirect
 	github.com/containerd/cgroups/v3 v3.0.2 // indirect
 	github.com/containerd/containerd/api v1.7.19 // indirect
 	github.com/containerd/errdefs v0.3.0 // indirect
@@ -127,6 +131,12 @@ require (
 	github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 // indirect
 	github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491 // indirect
 	github.com/mattn/go-shellwords v1.0.12 // indirect
+	github.com/microsoft/kiota-authentication-azure-go v1.1.0 // indirect
+	github.com/microsoft/kiota-http-go v1.4.4 // indirect
+	github.com/microsoft/kiota-serialization-form-go v1.0.0 // indirect
+	github.com/microsoft/kiota-serialization-json-go v1.0.8 // indirect
+	github.com/microsoft/kiota-serialization-multipart-go v1.0.0 // indirect
+	github.com/microsoft/kiota-serialization-text-go v1.0.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/sys/user v0.3.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
@@ -145,6 +155,7 @@ require (
 	github.com/sigstore/sigstore v1.8.3 // indirect
 	github.com/sigstore/timestamp-authority v1.2.2 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/std-uritemplate/std-uritemplate/go v0.0.57 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
 	github.com/theupdateframework/go-tuf v0.7.0 // indirect
 	github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
@@ -339,7 +350,7 @@ require (
 	github.com/google/licenseclassifier/v2 v2.0.0 // indirect
 	github.com/google/s2a-go v0.1.8 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.6.0 // indirect
+	github.com/google/uuid v1.6.0
 	github.com/google/wire v0.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect

go.sum

@@ -578,6 +578,8 @@ github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb2
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cjlapao/common-go v0.0.39 h1:bAAUrj2B9v0kMzbAOhzjSmiyDy+rd56r2sy7oEiQLlA=
+github.com/cjlapao/common-go v0.0.39/go.mod h1:M3dzazLjTjEtZJbbxoA5ZDiGCiHmpwqW9l4UWaddwOA=
 github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=
 github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -1266,6 +1268,24 @@ github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxU
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/microsoft/go-rustaudit v0.0.0-20220808201409-204dfee52032 h1:TLygBUBxikNJJfLwgm+Qwdgq1FtfV8Uh7bcxRyTzK8s=
 github.com/microsoft/go-rustaudit v0.0.0-20220808201409-204dfee52032/go.mod h1:vYT9HE7WCvL64iVeZylKmCsWKfE+JZ8105iuh2Trk8g=
+github.com/microsoft/kiota-abstractions-go v1.7.0 h1:/0OKSSEe94Z1qgpcGE7ZFI9P+4iAnsDQo9v9UOk+R8E=
+github.com/microsoft/kiota-abstractions-go v1.7.0/go.mod h1:FI1I2OHg0E7bK5t8DPnw+9C/CHVyLP6XeqDBT+95pTE=
+github.com/microsoft/kiota-authentication-azure-go v1.1.0 h1:HudH57Enel9zFQ4TEaJw6lMiyZ5RbBdrRHwdU0NP2RY=
+github.com/microsoft/kiota-authentication-azure-go v1.1.0/go.mod h1:zfPFOiLdEqM77Hua5B/2vpcXrVaGqSWjHSRzlvAWEgc=
+github.com/microsoft/kiota-http-go v1.4.4 h1:HM0KT/Q7o+JsGatFkkbTIqJL24Jzo5eMI5NNe9N4TQ4=
+github.com/microsoft/kiota-http-go v1.4.4/go.mod h1:Kup5nMDD3a9sjdgRKHCqZWqtrv3FbprjcPaGjLR6FzM=
+github.com/microsoft/kiota-serialization-form-go v1.0.0 h1:UNdrkMnLFqUCccQZerKjblsyVgifS11b3WCx+eFEsAI=
+github.com/microsoft/kiota-serialization-form-go v1.0.0/go.mod h1:h4mQOO6KVTNciMF6azi1J9QB19ujSw3ULKcSNyXXOMA=
+github.com/microsoft/kiota-serialization-json-go v1.0.8 h1:+aViv9k6wqaw1Fx6P49fl5GIB1hN3b6CG0McNTcUYBc=
+github.com/microsoft/kiota-serialization-json-go v1.0.8/go.mod h1:O8+v11U0EUwHlCz7hrW38KxDmdhKAHfv4Q89uvsBalY=
+github.com/microsoft/kiota-serialization-multipart-go v1.0.0 h1:3O5sb5Zj+moLBiJympbXNaeV07K0d46IfuEd5v9+pBs=
+github.com/microsoft/kiota-serialization-multipart-go v1.0.0/go.mod h1:yauLeBTpANk4L03XD985akNysG24SnRJGaveZf+p4so=
+github.com/microsoft/kiota-serialization-text-go v1.0.0 h1:XOaRhAXy+g8ZVpcq7x7a0jlETWnWrEum0RhmbYrTFnA=
+github.com/microsoft/kiota-serialization-text-go v1.0.0/go.mod h1:sM1/C6ecnQ7IquQOGUrUldaO5wj+9+v7G2W3sQ3fy6M=
+github.com/microsoftgraph/msgraph-sdk-go v1.51.0 h1:IfRY0uVHToT8X9k6Ri19tKdt8hwPomji2yx5YsKoaw4=
+github.com/microsoftgraph/msgraph-sdk-go v1.51.0/go.mod h1:MVTeFCCih3qXy9D0q+f4NdOyumFnMZ+Ppcpurgd30TY=
+github.com/microsoftgraph/msgraph-sdk-go-core v1.2.1 h1:P1wpmn3xxfPMFJHg+PJPcusErfRkl63h6OdAnpDbkS8=
+github.com/microsoftgraph/msgraph-sdk-go-core v1.2.1/go.mod h1:vFmWQGWyLlhxCESNLv61vlE4qesBU+eWmEVH7DJSESA=
 github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs=
 github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ=
 github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
@@ -1545,6 +1565,8 @@ github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
 github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
 github.com/spiffe/go-spiffe/v2 v2.2.0 h1:9Vf06UsvsDbLYK/zJ4sYsIsHmMFknUD+feA7IYoWMQY=
 github.com/spiffe/go-spiffe/v2 v2.2.0/go.mod h1:Urzb779b3+IwDJD2ZbN8fVl3Aa8G4N/PiUe6iXC0XxU=
+github.com/std-uritemplate/std-uritemplate/go v0.0.57 h1:GHGjptrsmazP4IVDlUprssiEf9ESVkbjx15xQXXzvq4=
+github.com/std-uritemplate/std-uritemplate/go v0.0.57/go.mod h1:rG/bqh/ThY4xE5de7Rap3vaDkYUT76B0GPJ0loYeTTc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=

internal/flavors/assetinventory/strategy.go

@@ -34,6 +34,7 @@ import (
 	azure_auth "github.com/elastic/cloudbeat/internal/resources/providers/azurelib/auth"
 	gcp_auth "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/auth"
 	gcp_inventory "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
+	"github.com/elastic/cloudbeat/internal/resources/providers/msgraph"
 )
 
 type Strategy interface {
@@ -84,10 +85,15 @@ func (s *strategy) initAzureFetchers(_ context.Context) ([]inventory.AssetFetche
 	initializer := &azurelib.ProviderInitializer{}
 	provider, err := initializer.Init(s.logger, *azureConfig)
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialize azure config: %w", err)
+		return nil, fmt.Errorf("failed to initialize azure provider: %w", err)
+	}
+
+	msgraphProvider, err := msgraph.NewProvider(s.logger, *azureConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize azure msgraph provider: %w", err)
 	}
 
-	return azurefetcher.New(s.logger, provider, azureConfig), nil
+	return azurefetcher.New(s.logger, provider, msgraphProvider), nil
 }
 
 func (s *strategy) initGcpFetchers(ctx context.Context) ([]inventory.AssetFetcher, error) {

internal/flavors/assetinventory/strategy_test.go

@@ -0,0 +1,131 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package assetinventory
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
+	"github.com/elastic/elastic-agent-libs/logp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/cloudbeat/internal/config"
+	"github.com/elastic/cloudbeat/internal/inventory"
+)
+
+func TestStrategyPicks(t *testing.T) {
+	testCases := []struct {
+		name        string
+		cfg         *config.Config
+		expectedErr string
+	}{
+		{
+			"expected error: asset_inventory_provider not set",
+			&config.Config{},
+			"missing config.v1.asset_inventory_provider",
+		},
+		{
+			"expected error: unsupported provider",
+			&config.Config{
+				AssetInventoryProvider: "NOPE",
+			},
+			"unsupported Asset Inventory provider \"NOPE\"",
+		},
+		{
+			"expected success: Azure",
+			&config.Config{
+				AssetInventoryProvider: config.ProviderAzure,
+			},
+			"",
+		},
+		{
+			"expected error: GCP missing account type",
+			&config.Config{
+				AssetInventoryProvider: config.ProviderGCP,
+			},
+			"invalid gcp account type",
+		},
+		{
+			"expected success: GCP",
+			&config.Config{
+				AssetInventoryProvider: config.ProviderGCP,
+				CloudConfig: config.CloudConfig{
+					Gcp: config.GcpConfig{
+						AccountType:    config.SingleAccount,
+						ProjectId:      "nonexistent",
+						OrganizationId: "nonexistent",
+						GcpClientOpt: config.GcpClientOpt{
+							CredentialsJSON: "{\"type\": \"service_account\"}",
+						},
+					},
+				},
+			},
+			"could not parse key",
+		},
+		{
+			"expected error: AWS unsupported account type",
+			&config.Config{
+				AssetInventoryProvider: config.ProviderAWS,
+				CloudConfig: config.CloudConfig{
+					Aws: config.AwsConfig{
+						AccountType: "NOPE",
+					},
+				},
+			},
+			"unsupported account_type: \"NOPE\"",
+		},
+		{
+			"expected success: AWS",
+			&config.Config{
+				AssetInventoryProvider: config.ProviderAWS,
+				CloudConfig: config.CloudConfig{
+					Aws: config.AwsConfig{
+						AccountType: config.SingleAccount,
+						Cred: aws.ConfigAWS{
+							AccessKeyID:     "key",
+							SecretAccessKey: "key",
+						},
+					},
+				},
+			},
+			"STS: GetCallerIdentity",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			s := strategy{
+				logger: logp.NewLogger("strategy_test"),
+				cfg:    tc.cfg,
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+			defer cancel()
+			obj, err := s.NewAssetInventory(ctx, nil)
+			if tc.expectedErr != "" {
+				assert.Equal(t, inventory.AssetInventory{}, obj)
+				require.Error(t, err)
+				require.ErrorContains(t, err, tc.expectedErr)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}

internal/inventory/ASSETS.md

@@ -98,8 +98,8 @@ Infrastructure: 24% (18/73)
 
 ## AZURE Resources
 
-**Progress: 29% (15/51)**
-Identity: 0% (0/8)
+**Progress: 31% (16/51)**
+Identity: 12% (1/8)
 Infrastructure: 34% (15/43)
 
 <details> <summary>Full table</summary>
@@ -110,7 +110,7 @@ Infrastructure: 34% (15/43)
 | Identity | Access Management | Role | Azure Role | No ❌ |
 | Identity | Application | Application | Azure AD Application | No ❌ |
 | Identity | Digital Identity | Administrator | Azure Server AD Administrator | No ❌ |
-| Identity | Digital Identity | Principal | Azure Principal | No ❌ |
+| Identity | Digital Identity | Principal | Azure Principal | Yes ✅ |
 | Identity | Directory | Group | Azure AD Group | No ❌ |
 | Identity | Directory | User | Azure AD User | No ❌ |
 | Identity | Service Identity | Service Principal | Azure AD Service Principal | No ❌ |

internal/inventory/asset.go

@@ -64,6 +64,7 @@ const (
 	TypeObjectStorage       AssetType = "object-storage"
 	TypePeering             AssetType = "peering"
 	TypePolicy              AssetType = "policy"
+	TypePrincipal           AssetType = "principal"
 	TypeRegistry            AssetType = "registry"
 	TypeRelationalDatabase  AssetType = "relational"
 	TypeResourceGroup       AssetType = "resource-group"
@@ -96,6 +97,7 @@ const (
 	SubTypeAzureCosmosDBSQLDatabase AssetSubType = "azure-cosmos-db-sql-database"
 	SubTypeAzureDisk                AssetSubType = "azure-disk"
 	SubTypeAzureElasticPool         AssetSubType = "azure-elastic-pool"
+	SubTypeAzurePrincipal           AssetSubType = "azure-principal"
 	SubTypeAzureResourceGroup       AssetSubType = "azure-resource-group"
 	SubTypeAzureSQLDatabase         AssetSubType = "azure-sql-database"
 	SubTypeAzureSQLServer           AssetSubType = "azure-sql-server"
@@ -196,6 +198,7 @@ var (
 	AssetClassificationAzureResourceGroup       = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeResourceGroup, SubType: SubTypeAzureResourceGroup}
 	AssetClassificationAzureSQLDatabase         = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryDatabase, Type: TypeRelationalDatabase, SubType: SubTypeAzureSQLDatabase}
 	AssetClassificationAzureSQLServer           = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryDatabase, Type: TypeRelationalDatabase, SubType: SubTypeAzureSQLServer}
+	AssetClassificationAzureServicePrincipal    = AssetClassification{Category: CategoryIdentity, SubCategory: SubCategoryDigitalIdentity, Type: TypePrincipal, SubType: SubTypeAzurePrincipal}
 	AssetClassificationAzureSnapshot            = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryStorage, Type: TypeSnapshot, SubType: SubTypeAzureSnapshot}
 	AssetClassificationAzureStorageAccount      = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryStorage, Type: TypeStorage, SubType: SubTypeAzureStorageAccount}
 	AssetClassificationAzureStorageBlobService  = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryStorage, Type: TypeObjectStorage, SubType: SubTypeAzureStorageBlobService}

internal/inventory/azurefetcher/azurefetchers.go

@@ -22,12 +22,13 @@ import (
 
 	"github.com/elastic/cloudbeat/internal/inventory"
 	"github.com/elastic/cloudbeat/internal/resources/providers/azurelib"
-	azure_auth "github.com/elastic/cloudbeat/internal/resources/providers/azurelib/auth"
+	"github.com/elastic/cloudbeat/internal/resources/providers/msgraph"
 )
 
-func New(logger *logp.Logger, provider azurelib.ProviderAPI, _ *azure_auth.AzureFactoryConfig) []inventory.AssetFetcher {
+func New(logger *logp.Logger, provider azurelib.ProviderAPI, msgraphProvider msgraph.ProviderAPI) []inventory.AssetFetcher {
 	return []inventory.AssetFetcher{
 		newAccountFetcher(logger, provider),
+		newActiveDirectoryFetcher(logger, msgraphProvider),
 		newResourceGraphFetcher(logger, provider),
 		newStorageFetcher(logger, provider),
 	}

internal/inventory/azurefetcher/fetcher_activedirectory.go

@@ -0,0 +1,84 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package azurefetcher
+
+import (
+	"context"
+
+	"github.com/elastic/elastic-agent-libs/logp"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+
+	"github.com/elastic/cloudbeat/internal/inventory"
+	"github.com/elastic/cloudbeat/internal/resources/utils/pointers"
+)
+
+type activedirectoryFetcher struct {
+	logger   *logp.Logger
+	provider activedirectoryProvider
+}
+
+type (
+	activedirectoryProvider interface {
+		ListServicePrincipals(ctx context.Context) ([]*models.ServicePrincipal, error)
+	}
+)
+
+func newActiveDirectoryFetcher(logger *logp.Logger, provider activedirectoryProvider) inventory.AssetFetcher {
+	return &activedirectoryFetcher{
+		logger:   logger,
+		provider: provider,
+	}
+}
+
+func (f *activedirectoryFetcher) Fetch(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
+	f.fetchServicePrincipals(ctx, assetChan)
+}
+
+func (f *activedirectoryFetcher) fetchServicePrincipals(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
+	f.logger.Info("Fetching Service Principals")
+	defer f.logger.Info("Fetching Service Principals - Finished")
+
+	items, err := f.provider.ListServicePrincipals(ctx)
+	if err != nil {
+		f.logger.Errorf("Could not fetch Service Principals: %v", err)
+	}
+
+	for _, item := range items {
+		var tenantId string
+		if uuid := item.GetAppOwnerOrganizationId(); uuid != nil {
+			tenantId = uuid.String()
+		}
+		assetChan <- inventory.NewAssetEvent(
+			inventory.AssetClassificationAzureServicePrincipal,
+			[]string{pointers.Deref(item.GetId())},
+			pointers.Deref(item.GetDisplayName()),
+			inventory.WithRawAsset(
+				item.GetBackingStore().Enumerate(),
+			),
+			inventory.WithCloud(inventory.AssetCloud{
+				Provider: inventory.AzureCloudProvider,
+				Account: inventory.AssetCloudAccount{
+					Id: tenantId,
+				},
+				Service: &inventory.AssetCloudService{
+					Name: "Azure",
+				},
+			}),
+		)
+	}
+}