Add ability to split integration tests into different groups

docs/test-framework-dev-guide.md

@@ -67,6 +67,16 @@ between, and it can be very specific or not very specific.
 > **_NOTE:_**  This only filters down the tests based on the platform. It will not execute a tests on a platform unless
 > the test defines as supporting it.
 
+#### Selecting specific group
+
+By default, the runner will run all test groups. Each group runs on a dedicated machine instance. When working on groups of tests it's better to limit to a specific
+group of tests instead of running all tests. This can be done by using the `TEST_GROUPS="default upgrade-standalone"`
+environment variable. This variable can take multiple groups with a space between.
+
+- `TEST_GROUPS="default" mage integration:test` to execute only tests in the "default" group.
+- `TEST_GROUPS="default upgrade-standalone" mage integration:test` to execute only tests in the "default" or
+"upgrade-standalone" group.
+
 #### Passing additional go test flags
 
 When running the tests we can pass additional go test flag using the env variable `GOTEST_FLAGS`.
@@ -168,6 +178,17 @@ the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test
 framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools`
 package for helper utilities.
 
+### Test group
+
+Every `define.Require` must define a `Group` that it belongs too. Each group is executed on a separate instance with all tests with in the same group executed
+on the same instance. Placing similar tests in the same group allows those tests to run on its own instance
+as well as provides a way for a developer to select a specific group of tests with `TEST_GROUP="{group-name}"`.
+
+Grouping tests is another way of spreading out the testing load across multiple instances. The more groups that
+are defined the more instances will be provisioned to complete all tests. A balance between a small good set of
+groups is better than a ton of groups each executing a small set of tests, as the time to set up an instance can
+out weight the benefits of creating another group.
+
 ### Test namespaces
 
 Every test has access to its own unique namespace (a string value). This namespace can

magefile.go

@@ -1778,6 +1778,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 		StateDir:          ".integration-cache",
 		DiagnosticsDir:    diagDir,
 		Platforms:         testPlatforms(),
+		Groups:            testGroups(),
 		Matrix:            matrix,
 		SingleTest:        singleTest,
 		VerboseMode:       mg.Verbose(),
@@ -1868,6 +1869,20 @@ func testPlatforms() []string {
 	return platforms
 }
 
+func testGroups() []string {
+	groupsStr := os.Getenv("TEST_GROUPS")
+	if groupsStr == "" {
+		return nil
+	}
+	var groups []string
+	for _, g := range strings.Split(groupsStr, " ") {
+		if g != "" {
+			groups = append(groups, g)
+		}
+	}
+	return groups
+}
+
 // Pre-requisite: user must have the gcloud CLI installed
 func authGCP(ctx context.Context) error {
 	// We only need the service account token to exist.

pkg/testing/define/batch.go

@@ -41,15 +41,16 @@ var defaultOS = []OS{
 
 // Batch is a grouping of tests that all have the same requirements.
 type Batch struct {
+	// Group must be set on each test to define which group the tests belongs.
+	// Tests that are in the same group are executed on the same runner.
+	Group string `json:"group"`
+
 	// OS defines the operating systems this test batch needs.
 	OS OS `json:"os"`
 
 	// Stack defines the stack required for this batch.
 	Stack *Stack `json:"stack,omitempty"`
 
-	// Isolate defines that this batch is isolated to a single test.
-	Isolate bool `json:"isolate"`
-
 	// Tests define the set of packages and tests that do not require sudo
 	// privileges to be performed.
 	Tests []BatchPackageTests `json:"tests"`
@@ -177,15 +178,12 @@ func appendTest(batches []Batch, tar testActionResult, req Requirements) []Batch
 	}
 	for _, o := range set {
 		var batch Batch
-		batchIdx := -1
-		if !req.Isolate {
-			batchIdx = findBatchIdx(batches, o, req.Stack)
-		}
+		batchIdx := findBatchIdx(batches, req.Group, o, req.Stack)
 		if batchIdx == -1 {
 			// new batch required
 			batch = Batch{
+				Group:     req.Group,
 				OS:        o,
-				Isolate:   req.Isolate,
 				Tests:     nil,
 				SudoTests: nil,
 			}
@@ -241,10 +239,10 @@ func appendPackageTest(tests []BatchPackageTests, pkg string, name string, stack
 	return tests
 }
 
-func findBatchIdx(batches []Batch, os OS, stack *Stack) int {
+func findBatchIdx(batches []Batch, group string, os OS, stack *Stack) int {
 	for i, b := range batches {
-		if b.Isolate {
-			// never add to an isolate batch
+		if b.Group != group {
+			// must be in the same group
 			continue
 		}
 		if b.OS.Type != os.Type || b.OS.Arch != os.Arch {

pkg/testing/define/batch_test.go

@@ -93,6 +93,7 @@ func TestBatch(t *testing.T) {
 	}
 	expected := []Batch{
 		{
+			Group: Default,
 			OS: OS{
 				Type: Darwin,
 				Arch: AMD64,
@@ -101,6 +102,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: darwinSudoTests,
 		},
 		{
+			Group: Default,
 			OS: OS{
 				Type: Darwin,
 				Arch: ARM64,
@@ -109,6 +111,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: darwinSudoTests,
 		},
 		{
+			Group: Default,
 			OS: OS{
 				Type: Linux,
 				Arch: AMD64,
@@ -117,6 +120,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: linuxSudoTests,
 		},
 		{
+			Group: Default,
 			OS: OS{
 				Type:    Linux,
 				Arch:    ARM64,
@@ -152,6 +156,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: linuxSudoTests,
 		},
 		{
+			Group: Default,
 			OS: OS{
 				Type: Windows,
 				Arch: AMD64,
@@ -160,170 +165,47 @@ func TestBatch(t *testing.T) {
 			SudoTests: windowsSudoTests,
 		},
 		{
+			Group: "one",
 			OS: OS{
-				Type: Darwin,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Darwin,
-				Arch: ARM64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Linux,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Linux,
-				Arch: ARM64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Windows,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Darwin,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestDarwinIsolate",
-						},
-					},
-				},
+				Type:    Linux,
+				Arch:    ARM64,
+				Version: "20.04",
+				Distro:  "ubuntu",
 			},
-		},
-		{
-			OS: OS{
-				Type: Darwin,
-				Arch: ARM64,
+			Stack: &Stack{
+				Version: "8.8.0",
 			},
-			Isolate: true,
 			Tests: []BatchPackageTests{
 				{
 					Name: pkgName,
 					Tests: []BatchPackageTest{
 						{
-							Name: "TestDarwinIsolate",
+							Name:  "TestGroup_One_One",
+							Stack: true,
 						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Linux,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
 						{
-							Name: "TestLinuxIsolate",
+							Name:  "TestGroup_One_Two",
+							Stack: true,
 						},
 					},
 				},
 			},
 		},
 		{
+			Group: "two",
 			OS: OS{
 				Type: Linux,
 				Arch: ARM64,
 			},
-			Isolate: true,
 			Tests: []BatchPackageTests{
 				{
 					Name: pkgName,
 					Tests: []BatchPackageTest{
 						{
-							Name: "TestLinuxIsolate",
+							Name: "TestGroup_Two_One",
 						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Windows,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
 						{
-							Name: "TestWindowsIsolate",
+							Name: "TestGroup_Two_Two",
 						},
 					},
 				},
@@ -344,6 +226,7 @@ var testLinuxLocalTests = []BatchPackageTest{
 
 var testLinuxLocalBatch = []Batch{
 	{
+		Group: Default,
 		OS: OS{
 			Type: "linux",
 			Arch: "amd64",
@@ -356,6 +239,7 @@ var testLinuxLocalBatch = []Batch{
 		},
 	},
 	{
+		Group: Default,
 		OS: OS{
 			Type: "linux",
 			Arch: "arm64",