[Asset Inventory][Azure] Add missing resources and ECS fields required for GA

.github/workflows/ci-pull_request.yml

@@ -94,7 +94,7 @@ jobs:
         run: |
           go install gotest.tools/gotestsum
           GOOS=linux TEST_DIRECTORY=./... gotestsum --format pkgname -- -race -coverpkg=./... -coverprofile=cover.out.tmp
-          cat cover.out.tmp | grep -v "mock_.*.go" | grep -v "elastic/cloudbeat/deploy" > cover.out # remove mock files and deploy dir
+          cat cover.out.tmp | grep -v "mock_.*.go" | grep -v "elastic/cloudbeat/deploy" | grep -v "internal/inventory/asset.go" > cover.out # remove mock files and deploy dir
 
       - name: Upload coverage artifact
         uses: actions/upload-artifact@v4

internal/flavors/assetinventory/strategy.go

@@ -93,7 +93,7 @@ func (s *strategy) initAzureFetchers(_ context.Context) ([]inventory.AssetFetche
 		return nil, fmt.Errorf("failed to initialize azure msgraph provider: %w", err)
 	}
 
-	return azurefetcher.New(s.logger, provider, msgraphProvider), nil
+	return azurefetcher.New(s.logger, s.cfg.CloudConfig.Azure.Credentials.TenantID, provider, msgraphProvider), nil
 }
 
 func (s *strategy) initGcpFetchers(ctx context.Context) ([]inventory.AssetFetcher, error) {

internal/inventory/ASSETS.md

@@ -115,10 +115,11 @@ Container Registry: 100% (1/1)
 Database: 100% (4/4)
 File System Service: 100% (2/2)
 Host: 100% (1/1)
-Identity: 9% (1/11)
+Identity: 0% (0/10)
 Infrastructure: 8% (2/23)
 Messaging Service: 100% (2/2)
 Private Endpoint: 100% (1/1)
+Service Account: 100% (1/1)
 Service Usage Technology: 100% (2/2)
 Snapshot: 100% (1/1)
 Storage Bucket: 100% (1/1)
@@ -146,7 +147,6 @@ Web Service: 100% (1/1)
 | Identity | Azure AD Group |  | No ❌ |
 | Identity | Azure AD Service Principal |  | No ❌ |
 | Identity | Azure AD User |  | No ❌ |
-| Identity | Azure Principal | Azure Principal | Yes ✅ |
 | Identity | Azure Role |  | No ❌ |
 | Identity | Azure Role Assignment |  | No ❌ |
 | Identity | Azure Server AD Administrator |  | No ❌ |
@@ -177,6 +177,7 @@ Web Service: 100% (1/1)
 | Messaging Service | Azure Storage Queue | Azure Storage Queue | Yes ✅ |
 | Messaging Service | Azure Storage Queue Service | Azure Storage Queue Service | Yes ✅ |
 | Private Endpoint | Azure Storage Account | Azure Storage Account | Yes ✅ |
+| Service Account | Azure Principal | Azure Principal | Yes ✅ |
 | Service Usage Technology | Azure Storage Blob Service | Azure Storage Blob Service | Yes ✅ |
 | Service Usage Technology | Azure Storage Table Service |  | Yes ✅ |
 | Snapshot | Azure Snapshot | Azure Snapshot | Yes ✅ |

internal/inventory/asset.go

@@ -32,6 +32,7 @@ const (
 	CategoryFileSystemService      AssetCategory = "File System Service"
 	CategoryFirewall               AssetCategory = "Firewall"
 	CategoryGateway                AssetCategory = "Gateway"
+	CategoryGroup                  AssetCategory = "Group"
 	CategoryHost                   AssetCategory = "Host"
 	CategoryIdentity               AssetCategory = "Identity"
 	CategoryInfrastructure         AssetCategory = "Infrastructure"
@@ -99,10 +100,13 @@ var (
 	AssetClassificationAzureCosmosDBSQLDatabase  = AssetClassification{CategoryInfrastructure, "Azure Cosmos DB SQL Database"}
 	AssetClassificationAzureDisk                 = AssetClassification{CategoryVolume, "Azure Disk"}
 	AssetClassificationAzureElasticPool          = AssetClassification{CategoryDatabase, "Azure Elastic Pool"}
+	AssetClassificationAzureEntraGroup           = AssetClassification{CategoryGroup, "Azure Microsoft Entra ID Group"}
+	AssetClassificationAzureEntraUser            = AssetClassification{CategoryIdentity, "Azure Microsoft Entra ID User"}
 	AssetClassificationAzureResourceGroup        = AssetClassification{CategoryAccessManagement, "Azure Resource Group"}
+	AssetClassificationAzureRoleDefinition       = AssetClassification{CategoryAccessManagement, "Azure RoleDefinition"}
 	AssetClassificationAzureSQLDatabase          = AssetClassification{CategoryDatabase, "Azure SQL Database"}
 	AssetClassificationAzureSQLServer            = AssetClassification{CategoryDatabase, "Azure SQL Server"}
-	AssetClassificationAzureServicePrincipal     = AssetClassification{CategoryIdentity, "Azure Principal"}
+	AssetClassificationAzureServicePrincipal     = AssetClassification{CategoryServiceAccount, "Azure Principal"}
 	AssetClassificationAzureSnapshot             = AssetClassification{CategorySnapshot, "Azure Snapshot"}
 	AssetClassificationAzureStorageAccount       = AssetClassification{CategoryPrivateEndpoint, "Azure Storage Account"}
 	AssetClassificationAzureStorageBlobContainer = AssetClassification{CategoryStorageBucket, "Azure Storage Blob Container"}
@@ -140,9 +144,12 @@ type AssetEvent struct {
 	Event         Event
 	Network       *Network
 	Cloud         *Cloud
+	Group         *Group
 	Host          *Host
+	Organization  *Organization
 	User          *User
 	Labels        map[string]string
+	Tags          []string
 	RawAttributes *any
 }
 
@@ -178,6 +185,12 @@ type Cloud struct {
 	ProjectName      string `json:"project.name,omitempty"`
 }
 
+type Group struct {
+	ID     string `json:"id,omitempty"`
+	Name   string `json:"name,omitempty"`
+	Domain string `json:"domain,omitempty"`
+}
+
 type Host struct {
 	ID           string   `json:"id,omitempty"`
 	Name         string   `json:"name,omitempty"`
@@ -187,6 +200,11 @@ type Host struct {
 	MacAddress   []string `json:"mac,omitempty"`
 }
 
+type Organization struct {
+	ID   string `json:"id,omitempty"`
+	Name string `json:"name,omitempty"`
+}
+
 type User struct {
 	ID   string `json:"id,omitempty"`
 	Name string `json:"name,omitempty"`
@@ -245,6 +263,25 @@ func WithLabels(labels map[string]string) AssetEnricher {
 	}
 }
 
+func WithLabelsFromAny(labels map[string]any) AssetEnricher {
+	return func(a *AssetEvent) {
+		if len(labels) == 0 {
+			return
+		}
+		a.Labels = map[string]string{}
+		for k, v := range labels {
+			sv, ok := v.(string)
+			if ok {
+				a.Labels[k] = sv
+			}
+			spv, ok := v.(*string)
+			if ok {
+				a.Labels[k] = *spv
+			}
+		}
+	}
+}
+
 func WithNetwork(network Network) AssetEnricher {
 	return func(a *AssetEvent) {
 		a.Network = &network
@@ -257,12 +294,33 @@ func WithCloud(cloud Cloud) AssetEnricher {
 	}
 }
 
+func WithGroup(group Group) AssetEnricher {
+	return func(a *AssetEvent) {
+		a.Group = &group
+	}
+}
+
 func WithHost(host Host) AssetEnricher {
 	return func(a *AssetEvent) {
 		a.Host = &host
 	}
 }
 
+func WithOrganization(org Organization) AssetEnricher {
+	return func(a *AssetEvent) {
+		a.Organization = &org
+	}
+}
+
+func WithTags(tags []string) AssetEnricher {
+	return func(a *AssetEvent) {
+		if len(tags) == 0 {
+			return
+		}
+		a.Tags = tags
+	}
+}
+
 func WithUser(user User) AssetEnricher {
 	return func(a *AssetEvent) {
 		a.User = &user

internal/inventory/azurefetcher/azurefetchers.go

@@ -24,11 +24,11 @@ import (
 	"github.com/elastic/cloudbeat/internal/resources/providers/msgraph"
 )
 
-func New(logger *clog.Logger, provider azurelib.ProviderAPI, msgraphProvider msgraph.ProviderAPI) []inventory.AssetFetcher {
+func New(logger *clog.Logger, tenantID string, provider azurelib.ProviderAPI, msgraphProvider msgraph.ProviderAPI) []inventory.AssetFetcher {
 	return []inventory.AssetFetcher{
-		newAccountFetcher(logger, provider),
-		newActiveDirectoryFetcher(logger, msgraphProvider),
-		newResourceGraphFetcher(logger, provider),
-		newStorageFetcher(logger, provider),
+		newAccountFetcher(logger, tenantID, provider),
+		newActiveDirectoryFetcher(logger, tenantID, msgraphProvider),
+		newResourceGraphFetcher(logger, tenantID, provider),
+		newStorageFetcher(logger, tenantID, provider),
 	}
 }

internal/inventory/azurefetcher/fetcher_account.go

@@ -27,6 +27,7 @@ import (
 
 type accountFetcher struct {
 	logger   *clog.Logger
+	tenantID string //nolint:unused
 	provider accountProvider
 }
 
@@ -38,9 +39,10 @@ type (
 	}
 )
 
-func newAccountFetcher(logger *clog.Logger, provider accountProvider) inventory.AssetFetcher {
+func newAccountFetcher(logger *clog.Logger, tenantID string, provider accountProvider) inventory.AssetFetcher {
 	return &accountFetcher{
 		logger:   logger,
+		tenantID: tenantID,
 		provider: provider,
 	}
 }
@@ -80,6 +82,10 @@ func (f *accountFetcher) fetch(ctx context.Context, resourceName string, functio
 				AccountID:   item.TenantId,
 				ServiceName: "Azure",
 			}),
+			inventory.WithLabelsFromAny(item.Tags),
+			inventory.WithOrganization(inventory.Organization{
+				ID: item.TenantId,
+			}),
 		)
 	}
 }

internal/inventory/azurefetcher/fetcher_account_test.go

@@ -48,6 +48,9 @@ func TestAccountFetcher_Fetch_Tenants(t *testing.T) {
 				AccountID:   "<tenant UUID>",
 				ServiceName: "Azure",
 			}),
+			inventory.WithOrganization(inventory.Organization{
+				ID: "<tenant UUID>",
+			}),
 		),
 	}
 
@@ -56,7 +59,7 @@ func TestAccountFetcher_Fetch_Tenants(t *testing.T) {
 	provider := newMockAccountProvider(t)
 	provider.EXPECT().ListTenants(mock.Anything).Return(azureAssets, nil)
 	provider.EXPECT().ListSubscriptions(mock.Anything).Return(nil, nil)
-	fetcher := newAccountFetcher(logger, provider)
+	fetcher := newAccountFetcher(logger, "<tenant UUID>", provider)
 	// test & compare
 	testutil.CollectResourcesAndMatch(t, fetcher, expected)
 }
@@ -81,6 +84,9 @@ func TestAccountFetcher_Fetch_Subscriptions(t *testing.T) {
 				AccountID:   "<sub UUID>",
 				ServiceName: "Azure",
 			}),
+			inventory.WithOrganization(inventory.Organization{
+				ID: "<sub UUID>",
+			}),
 		),
 	}
 
@@ -89,7 +95,7 @@ func TestAccountFetcher_Fetch_Subscriptions(t *testing.T) {
 	provider := newMockAccountProvider(t)
 	provider.EXPECT().ListTenants(mock.Anything).Return(nil, nil)
 	provider.EXPECT().ListSubscriptions(mock.Anything).Return(azureAssets, nil)
-	fetcher := newAccountFetcher(logger, provider)
+	fetcher := newAccountFetcher(logger, "<tenant UUID>", provider)
 	// test & compare
 	testutil.CollectResourcesAndMatch(t, fetcher, expected)
 }

internal/inventory/azurefetcher/fetcher_activedirectory.go

@@ -29,24 +29,32 @@ import (
 
 type activedirectoryFetcher struct {
 	logger   *clog.Logger
+	tenantID string
 	provider activedirectoryProvider
 }
 
 type (
 	activedirectoryProvider interface {
 		ListServicePrincipals(ctx context.Context) ([]*models.ServicePrincipal, error)
+		ListDirectoryRoles(context.Context) ([]*models.DirectoryRole, error)
+		ListGroups(context.Context) ([]*models.Group, error)
+		ListUsers(context.Context) ([]*models.User, error)
 	}
 )
 
-func newActiveDirectoryFetcher(logger *clog.Logger, provider activedirectoryProvider) inventory.AssetFetcher {
+func newActiveDirectoryFetcher(logger *clog.Logger, tenantID string, provider activedirectoryProvider) inventory.AssetFetcher {
 	return &activedirectoryFetcher{
 		logger:   logger,
+		tenantID: tenantID,
 		provider: provider,
 	}
 }
 
 func (f *activedirectoryFetcher) Fetch(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
 	f.fetchServicePrincipals(ctx, assetChan)
+	f.fetchDirectoryRoles(ctx, assetChan)
+	f.fetchGroups(ctx, assetChan)
+	f.fetchUsers(ctx, assetChan)
 }
 
 func (f *activedirectoryFetcher) fetchServicePrincipals(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
@@ -75,6 +83,103 @@ func (f *activedirectoryFetcher) fetchServicePrincipals(ctx context.Context, ass
 				AccountID:   tenantId,
 				ServiceName: "Azure",
 			}),
+			inventory.WithTags(item.GetTags()),
+		)
+	}
+}
+
+func (f *activedirectoryFetcher) fetchDirectoryRoles(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
+	f.logger.Info("Fetching Directory Roles")
+	defer f.logger.Info("Fetching Directory Roles - Finished")
+
+	items, err := f.provider.ListDirectoryRoles(ctx)
+	if err != nil {
+		f.logger.Errorf("Could not fetch Directory Roles: %v", err)
+	}
+
+	for _, item := range items {
+		assetChan <- inventory.NewAssetEvent(
+			inventory.AssetClassificationAzureRoleDefinition,
+			pointers.Deref(item.GetId()),
+			pointers.Deref(item.GetDisplayName()),
+			inventory.WithRawAsset(
+				item.GetBackingStore().Enumerate(),
+			),
+			inventory.WithCloud(inventory.Cloud{
+				Provider:    inventory.AzureCloudProvider,
+				AccountID:   f.tenantID,
+				ServiceName: "Azure",
+			}),
+			inventory.WithUser(inventory.User{
+				ID:   pointers.Deref(item.GetId()),
+				Name: pointers.Deref(item.GetDisplayName()),
+			}),
+		)
+	}
+}
+
+func (f *activedirectoryFetcher) fetchGroups(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
+	f.logger.Info("Fetching Groups")
+	defer f.logger.Info("Fetching Groups - Finished")
+
+	items, err := f.provider.ListGroups(ctx)
+	if err != nil {
+		f.logger.Errorf("Could not fetch Groups: %v", err)
+	}
+
+	for _, item := range items {
+		// TODO(kuba): How to test this without being able to test Groups?
+		// var labels map[string]string
+		// for _, l := range item.GetAssignedLabels() {
+		// 	fmt.Println(l)
+		// }
+		assetChan <- inventory.NewAssetEvent(
+			inventory.AssetClassificationAzureEntraGroup,
+			pointers.Deref(item.GetId()),
+			pointers.Deref(item.GetDisplayName()),
+			inventory.WithRawAsset(
+				item.GetBackingStore().Enumerate(),
+			),
+			inventory.WithCloud(inventory.Cloud{
+				Provider:    inventory.AzureCloudProvider,
+				AccountID:   f.tenantID,
+				ServiceName: "Azure",
+			}),
+			inventory.WithGroup(inventory.Group{
+				ID:   pointers.Deref(item.GetId()),
+				Name: pointers.Deref(item.GetDisplayName()),
+			}),
+			// inventory.WithLabels(labels),
+		)
+	}
+}
+
+func (f *activedirectoryFetcher) fetchUsers(ctx context.Context, assetChan chan<- inventory.AssetEvent) {
+	f.logger.Info("Fetching Users")
+	defer f.logger.Info("Fetching Users - Finished")
+
+	items, err := f.provider.ListUsers(ctx)
+	if err != nil {
+		f.logger.Errorf("Could not fetch Users: %v", err)
+	}
+
+	for _, item := range items {
+		assetChan <- inventory.NewAssetEvent(
+			inventory.AssetClassificationAzureEntraUser,
+			pointers.Deref(item.GetId()),
+			pointers.Deref(item.GetDisplayName()),
+			inventory.WithRawAsset(
+				item.GetBackingStore().Enumerate(),
+			),
+			inventory.WithCloud(inventory.Cloud{
+				Provider:    inventory.AzureCloudProvider,
+				AccountID:   f.tenantID,
+				ServiceName: "Azure",
+			}),
+			inventory.WithUser(inventory.User{
+				ID:   pointers.Deref(item.GetId()),
+				Name: pointers.Deref(item.GetDisplayName()),
+			}),
 		)
 	}
 }