diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md index fc5ac72180e..28600775e80 100644 --- a/docs/test-framework-dev-guide.md +++ b/docs/test-framework-dev-guide.md @@ -14,20 +14,6 @@ ESS (QA) API Key to create on https://console.qa.cld.elstc.co/deployment-feature Warning: if you never created a deployment on it, you won't have permission to get this key so you will need to create one first. -## Writing tests - -Write integration and E2E tests by adding them to the `testing/integration` -folder. - -// TODO: Replace with a comprehensive write up of `define.*` directives, -// environment variables, etc. useful when writing tests. Until then... - -Look at existing tests under the `testing/integration` for examples of how -to write tests using the integration and E2E testing framework. Also look at -the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test -framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools` -package for helper utilities. - ## Running tests Some one-time setup is required to run any integration and E2E tests. Run @@ -49,6 +35,39 @@ Run `mage integration:single [testName]` to execute a single test under the `tes Run `mage integration:matrix` to run all tests on the complete matrix of supported operating systems and architectures of the Elastic Agent. +## Writing tests + +Write integration and E2E tests by adding them to the `testing/integration` +folder. + +// TODO: Replace with a comprehensive write up of `define.*` directives, +// environment variables, etc. useful when writing tests. Until then... + +Look at existing tests under the `testing/integration` for examples of how +to write tests using the integration and E2E testing framework. Also look at +the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test +framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools` +package for helper utilities. + +### Test namespaces + +Every test has access to its own unique namespace (a string value). This namespace can +be accessed from the `info.Namespace` field, where `info` is the struct value returned +from the `define.Require(...)` call made at the start of the test. + +Namespaces should be used whenever test data is being written to or read from a persistent store that's +shared across all tests. Most commonly, this store will be the Elasticsearch cluster that Agent +components may index their data into. All tests share a single stack deployment and, therefore, +a single Elasticsearch cluster as well. + +Some examples of where namespaces should be used: +* When creating a policy in Fleet. The Create Policy and Update Policy APIs takes a namespace parameter. +* When searching for documents in `logs-*` or `metrics-*` data streams. Every document in these + data streams has a `data_stream.namespace` field. + +:warning: Not using namespaces when accessing data in a shared persistent store can cause tests to +be flaky. + ## Troubleshooting Tips ### Error: GCE service token missing; run 'mage integration:auth' @@ -75,3 +94,14 @@ that includes the `-SNAPSHOT` suffix when running `mage integration:test` or If you encounter any errors mentioning `ogc`, try running `mage integration:clean` and then re-running whatever `mage integration:*` target you were trying to run originally when you encountered the error. + +### Using a different agent version from the stack version + +The agent version is used as a fallback for the stack version to use in integration tests +if no other version is specified. + +If we need to use a different version between agent and stack we can specify the stack version +using a separate env variable `AGENT_STACK_VERSION` like in this example (we used a +custom package version for the agent): + +```AGENT_VERSION="8.10.0-testpkgversion.1-SNAPSHOT" AGENT_STACK_VERSION="8.10.0-SNAPSHOT" mage integration:test``` diff --git a/testing/integration/fqdn_test.go b/testing/integration/fqdn_test.go index 758bab6e521..e5c7a60e007 100644 --- a/testing/integration/fqdn_test.go +++ b/testing/integration/fqdn_test.go @@ -6,6 +6,7 @@ package integration import ( + "bytes" "context" "encoding/json" "fmt" @@ -98,8 +99,8 @@ func TestFQDN(t *testing.T) { agent := verifyAgentName(t, shortName, info.KibanaClient) t.Log("Verify that hostname in `logs-*` and `metrics-*` is short hostname") - verifyHostNameInIndices(t, "logs-*", shortName, info.ESClient) - verifyHostNameInIndices(t, "metrics-*", shortName, info.ESClient) + verifyHostNameInIndices(t, "logs-*", shortName, info.Namespace, info.ESClient) + verifyHostNameInIndices(t, "metrics-*", shortName, info.Namespace, info.ESClient) t.Log("Update Agent policy to enable FQDN") policy.AgentFeatures = []map[string]interface{}{ @@ -129,8 +130,8 @@ func TestFQDN(t *testing.T) { verifyAgentName(t, fqdn, info.KibanaClient) t.Log("Verify that hostname in `logs-*` and `metrics-*` is FQDN") - verifyHostNameInIndices(t, "logs-*", fqdn, info.ESClient) - verifyHostNameInIndices(t, "metrics-*", fqdn, info.ESClient) + verifyHostNameInIndices(t, "logs-*", fqdn, info.Namespace, info.ESClient) + verifyHostNameInIndices(t, "metrics-*", fqdn, info.Namespace, info.ESClient) t.Log("Update Agent policy to disable FQDN") policy.AgentFeatures = []map[string]interface{}{ @@ -185,7 +186,34 @@ func verifyAgentName(t *testing.T, hostname string, kibClient *kibana.Client) *k return agent } -func verifyHostNameInIndices(t *testing.T, indices, hostname string, esClient *elasticsearch.Client) { +func verifyHostNameInIndices(t *testing.T, indices, hostname, namespace string, esClient *elasticsearch.Client) { + queryRaw := map[string]interface{}{ + "query": map[string]interface{}{ + "bool": map[string]interface{}{ + "must": []map[string]interface{}{ + { + "term": map[string]interface{}{ + "host.name": map[string]interface{}{ + "value": hostname, + }, + }, + }, + { + "term": map[string]interface{}{ + "data_stream.namespace": map[string]interface{}{ + "value": namespace, + }, + }, + }, + }, + }, + }, + } + + var buf bytes.Buffer + err := json.NewEncoder(&buf).Encode(queryRaw) + require.NoError(t, err) + search := esClient.Search require.Eventually( @@ -196,6 +224,7 @@ func verifyHostNameInIndices(t *testing.T, indices, hostname string, esClient *e search.WithSort("@timestamp:desc"), search.WithFilterPath("hits.hits"), search.WithSize(1), + search.WithBody(&buf), ) require.NoError(t, err) require.False(t, resp.IsError()) @@ -216,9 +245,7 @@ func verifyHostNameInIndices(t *testing.T, indices, hostname string, esClient *e err = decoder.Decode(&body) require.NoError(t, err) - require.Len(t, body.Hits.Hits, 1) - hit := body.Hits.Hits[0] - return hostname == hit.Source.Host.Name + return len(body.Hits.Hits) == 1 }, 2*time.Minute, 5*time.Second,