[8.9](backport #3097) Fix flaky TestFQDN (#3105)

* [WIP] Fix flaky `TestFQDN` (#3097) * Pass namespace to FQDN ES query * Add documentation on namespace usage * Add warning (cherry picked from commit d2162bb) # Conflicts: # docs/test-framework-dev-guide.md * Update test-framework-dev-guide.md --------- Co-authored-by: Shaunak Kashyap <[email protected]> Co-authored-by: Pierre HILBERT <[email protected]>
elastic · Jul 20, 2023 · 6452b9b · 6452b9b
1 parent 18b2242
commit 6452b9b
Show file tree

Hide file tree

Showing 2 changed files with 79 additions and 22 deletions.
diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md
@@ -14,20 +14,6 @@ ESS (QA) API Key to create on https://console.qa.cld.elstc.co/deployment-feature
 
 Warning: if you never created a deployment on it, you won't have permission to get this key so you will need to create one first.
 
-## Writing tests
-
-Write integration and E2E tests by adding them to the `testing/integration`
-folder.
-
-// TODO: Replace with a comprehensive write up of `define.*` directives,
-// environment variables, etc. useful when writing tests. Until then...
-
-Look at existing tests under the `testing/integration` for examples of how
-to write tests using the integration and E2E testing framework. Also look at
-the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test
-framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools`
-package for helper utilities.
-
 ## Running tests
 
 Some one-time setup is required to run any integration and E2E tests. Run
@@ -49,6 +35,39 @@ Run `mage integration:single [testName]` to execute a single test under the `tes
 
 Run `mage integration:matrix` to run all tests on the complete matrix of supported operating systems and architectures of the Elastic Agent.
 
+## Writing tests
+
+Write integration and E2E tests by adding them to the `testing/integration`
+folder.
+
+// TODO: Replace with a comprehensive write up of `define.*` directives,
+// environment variables, etc. useful when writing tests. Until then...
+
+Look at existing tests under the `testing/integration` for examples of how
+to write tests using the integration and E2E testing framework. Also look at
+the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test
+framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools`
+package for helper utilities.
+
+### Test namespaces
+
+Every test has access to its own unique namespace (a string value). This namespace can
+be accessed from the `info.Namespace` field, where `info` is the struct value returned
+from the `define.Require(...)` call made at the start of the test.
+
+Namespaces should be used whenever test data is being written to or read from a persistent store that's
+shared across all tests. Most commonly, this store will be the Elasticsearch cluster that Agent
+components may index their data into. All tests share a single stack deployment and, therefore,
+a single Elasticsearch cluster as well.
+
+Some examples of where namespaces should be used:
+* When creating a policy in Fleet. The Create Policy and Update Policy APIs takes a namespace parameter.
+* When searching for documents in `logs-*` or `metrics-*` data streams. Every document in these
+  data streams has a `data_stream.namespace` field.
+
+:warning: Not using namespaces when accessing data in a shared persistent store can cause tests to
+be flaky.
+
 ## Troubleshooting Tips
 
 ### Error: GCE service token missing; run 'mage integration:auth'
@@ -75,3 +94,14 @@ that includes the `-SNAPSHOT` suffix when running `mage integration:test` or
 If you encounter any errors mentioning `ogc`, try running `mage integration:clean` and then
 re-running whatever `mage integration:*` target you were trying to run originally when you
 encountered the error.
+
+### Using a different agent version from the stack version
+
+The agent version is used as a fallback for the stack version to use in integration tests
+if no other version is specified.
+
+If we need to use a different version between agent and stack we can specify the stack version
+using a separate env variable `AGENT_STACK_VERSION` like in this example (we used a
+custom package version for the agent):
+
+```AGENT_VERSION="8.10.0-testpkgversion.1-SNAPSHOT" AGENT_STACK_VERSION="8.10.0-SNAPSHOT" mage integration:test```
diff --git a/testing/integration/fqdn_test.go b/testing/integration/fqdn_test.go
@@ -6,6 +6,7 @@
 package integration
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -98,8 +99,8 @@ func TestFQDN(t *testing.T) {
 	agent := verifyAgentName(t, shortName, info.KibanaClient)
 
 	t.Log("Verify that hostname in `logs-*` and `metrics-*` is short hostname")
-	verifyHostNameInIndices(t, "logs-*", shortName, info.ESClient)
-	verifyHostNameInIndices(t, "metrics-*", shortName, info.ESClient)
+	verifyHostNameInIndices(t, "logs-*", shortName, info.Namespace, info.ESClient)
+	verifyHostNameInIndices(t, "metrics-*", shortName, info.Namespace, info.ESClient)
 
 	t.Log("Update Agent policy to enable FQDN")
 	policy.AgentFeatures = []map[string]interface{}{
@@ -129,8 +130,8 @@ func TestFQDN(t *testing.T) {
 	verifyAgentName(t, fqdn, info.KibanaClient)
 
 	t.Log("Verify that hostname in `logs-*` and `metrics-*` is FQDN")
-	verifyHostNameInIndices(t, "logs-*", fqdn, info.ESClient)
-	verifyHostNameInIndices(t, "metrics-*", fqdn, info.ESClient)
+	verifyHostNameInIndices(t, "logs-*", fqdn, info.Namespace, info.ESClient)
+	verifyHostNameInIndices(t, "metrics-*", fqdn, info.Namespace, info.ESClient)
 
 	t.Log("Update Agent policy to disable FQDN")
 	policy.AgentFeatures = []map[string]interface{}{
@@ -185,7 +186,34 @@ func verifyAgentName(t *testing.T, hostname string, kibClient *kibana.Client) *k
 	return agent
 }
 
-func verifyHostNameInIndices(t *testing.T, indices, hostname string, esClient *elasticsearch.Client) {
+func verifyHostNameInIndices(t *testing.T, indices, hostname, namespace string, esClient *elasticsearch.Client) {
+	queryRaw := map[string]interface{}{
+		"query": map[string]interface{}{
+			"bool": map[string]interface{}{
+				"must": []map[string]interface{}{
+					{
+						"term": map[string]interface{}{
+							"host.name": map[string]interface{}{
+								"value": hostname,
+							},
+						},
+					},
+					{
+						"term": map[string]interface{}{
+							"data_stream.namespace": map[string]interface{}{
+								"value": namespace,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	var buf bytes.Buffer
+	err := json.NewEncoder(&buf).Encode(queryRaw)
+	require.NoError(t, err)
+
 	search := esClient.Search
 
 	require.Eventually(
@@ -196,6 +224,7 @@ func verifyHostNameInIndices(t *testing.T, indices, hostname string, esClient *e
 				search.WithSort("@timestamp:desc"),
 				search.WithFilterPath("hits.hits"),
 				search.WithSize(1),
+				search.WithBody(&buf),
 			)
 			require.NoError(t, err)
 			require.False(t, resp.IsError())
@@ -216,9 +245,7 @@ func verifyHostNameInIndices(t *testing.T, indices, hostname string, esClient *e
 			err = decoder.Decode(&body)
 			require.NoError(t, err)
 
-			require.Len(t, body.Hits.Hits, 1)
-			hit := body.Hits.Hits[0]
-			return hostname == hit.Source.Host.Name
+			return len(body.Hits.Hits) == 1
 		},
 		2*time.Minute,
 		5*time.Second,