From 3bbbd061b7a96ca1d31dff579123faff6aaf9960 Mon Sep 17 00:00:00 2001 From: Paolo Chila Date: Thu, 5 Oct 2023 17:30:43 +0200 Subject: [PATCH] change default ESS region to staging CFT --- magefile.go | 2 +- .../create_deployment_csp_configuration.yaml | 15 + .../ess/create_deployment_request.tmpl.json | 101 +++++++ pkg/testing/ess/deployment.go | 270 +++--------------- pkg/testing/ess/provisioner.go | 11 +- 5 files changed, 171 insertions(+), 228 deletions(-) create mode 100644 pkg/testing/ess/create_deployment_csp_configuration.yaml create mode 100644 pkg/testing/ess/create_deployment_request.tmpl.json diff --git a/magefile.go b/magefile.go index 183889b3e69..2ad7b452281 100644 --- a/magefile.go +++ b/magefile.go @@ -1736,7 +1736,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche // Valid values are gcp-us-central1 (default), azure-eastus2 essRegion := os.Getenv("TEST_INTEG_AUTH_ESS_REGION") if essRegion == "" { - essRegion = "gcp-us-central1" + essRegion = "aws-eu-central-1" } instanceProvisionerMode := os.Getenv("INSTANCE_PROVISIONER") diff --git a/pkg/testing/ess/create_deployment_csp_configuration.yaml b/pkg/testing/ess/create_deployment_csp_configuration.yaml new file mode 100644 index 00000000000..d798092c08a --- /dev/null +++ b/pkg/testing/ess/create_deployment_csp_configuration.yaml @@ -0,0 +1,15 @@ +gcp: + integrations_server_conf_id: "gcp.es.datahot.n2.68x10x45" + elasticsearch_conf_id: "gcp.es.datahot.n2.68x10x45" + elasticsearch_deployment_template_id: "gcp-storage-optimized-v5" + kibana_instance_configuration_id: "gcp.kibana.n2.68x32x45" +azure: + integrations_server_conf_id: "azure.es.datahot.edsv4" + elasticsearch_conf_id: "azure.es.datahot.edsv4" + elasticsearch_deployment_template_id: "azure-storage-optimized-v2" + kibana_instance_configuration_id: "azure.kibana.fsv2" +aws: + integrations_server_conf_id: "aws.integrationsserver.c5d.2.1" + elasticsearch_conf_id: "aws.es.datahot.i3.1.1" + elasticsearch_deployment_template_id: "aws-storage-optimized-v5" + kibana_instance_configuration_id: "aws.kibana.c5d.1.1" \ No newline at end of file diff --git a/pkg/testing/ess/create_deployment_request.tmpl.json b/pkg/testing/ess/create_deployment_request.tmpl.json new file mode 100644 index 00000000000..daa97eae262 --- /dev/null +++ b/pkg/testing/ess/create_deployment_request.tmpl.json @@ -0,0 +1,101 @@ +{ + "resources": { + "integrations_server": [ + { + "elasticsearch_cluster_ref_id": "main-elasticsearch", + "region": "{{ .request.Region }}", + "plan": { + "cluster_topology": [ + { + "instance_configuration_id": "{{ .integrations_server_conf_id }}", + "zone_count": 1, + "size": { + "resource": "memory", + "value": 1024 + } + } + ], + "integrations_server": { + "version": "{{ .request.Version }}" + } + }, + "ref_id": "main-integrations_server" + } + ], + "elasticsearch": [ + { + "region": "{{ .request.Region }}", + "settings": { + "dedicated_masters_threshold": 6 + }, + "plan": { + "cluster_topology": [ + { + "zone_count": 1, + "elasticsearch": { + "node_attributes": { + "data": "hot" + } + }, + "instance_configuration_id": "{{.elasticsearch_conf_id}}", + "node_roles": [ + "master", + "ingest", + "transform", + "data_hot", + "remote_cluster_client", + "data_content" + ], + "id": "hot_content", + "size": { + "resource": "memory", + "value": 8192 + } + } + ], + "elasticsearch": { + "version": "{{ .request.Version }}", + "enabled_built_in_plugins": [] + }, + "deployment_template": { + "id": "{{ .elasticsearch_deployment_template_id }}" + } + }, + "ref_id": "main-elasticsearch" + } + ], + "enterprise_search": [], + "kibana": [ + { + "elasticsearch_cluster_ref_id": "main-elasticsearch", + "region": "{{ .request.Region }}", + "plan": { + "cluster_topology": [ + { + "instance_configuration_id": "{{.kibana_instance_configuration_id}}", + "zone_count": 1, + "size": { + "resource": "memory", + "value": 1024 + } + } + ], + "kibana": { + "version": "{{ .request.Version }}", + "user_settings_json": { + "xpack.fleet.enableExperimental": ["agentTamperProtectionEnabled"] + } + } + }, + "ref_id": "main-kibana" + } + ] + }, + "settings": { + "autoscaling_enabled": false + }, + "name": "{{ .request.Name }}", + "metadata": { + "system_owned": false + } +} \ No newline at end of file diff --git a/pkg/testing/ess/deployment.go b/pkg/testing/ess/deployment.go index 9d9469036b0..35a315d166c 100644 --- a/pkg/testing/ess/deployment.go +++ b/pkg/testing/ess/deployment.go @@ -7,6 +7,7 @@ package ess import ( "bytes" "context" + _ "embed" "encoding/json" "fmt" "html/template" @@ -14,6 +15,8 @@ import ( "net/url" "strings" "time" + + "gopkg.in/yaml.v2" ) type CreateDeploymentRequest struct { @@ -85,21 +88,16 @@ type DeploymentStatusResponse struct { // CreateDeployment creates the deployment with the specified configuration. func (c *Client) CreateDeployment(ctx context.Context, req CreateDeploymentRequest) (*CreateDeploymentResponse, error) { - tpl, err := deploymentTemplateFactory(req) + reqBodyBytes, err := generateCreateDeploymentRequestBody(req) if err != nil { return nil, err } - var buf bytes.Buffer - if err := tpl.Execute(&buf, req); err != nil { - return nil, fmt.Errorf("unable to create deployment creation request body: %w", err) - } - createResp, err := c.doPost( ctx, "deployments", "application/json", - &buf, + bytes.NewReader(reqBodyBytes), ) if err != nil { return nil, fmt.Errorf("error calling deployment creation API: %w", err) @@ -308,233 +306,59 @@ func overallStatus(statuses ...DeploymentStatus) DeploymentStatus { return overallStatus } -func deploymentTemplateFactory(req CreateDeploymentRequest) (*template.Template, error) { +//go:embed create_deployment_request.tmpl.json +var createDeploymentRequestTemplate string + +//go:embed create_deployment_csp_configuration.yaml +var cloudProviderSpecificValues []byte + +func generateCreateDeploymentRequestBody(req CreateDeploymentRequest) ([]byte, error) { regionParts := strings.Split(req.Region, "-") if len(regionParts) < 2 { return nil, fmt.Errorf("unable to parse CSP out of region [%s]", req.Region) } csp := regionParts[0] - var tplStr string - switch csp { - case "gcp": - tplStr = createDeploymentRequestTemplateGCP - case "azure": - tplStr = createDeploymentRequestTemplateAzure - default: - return nil, fmt.Errorf("unsupported CSP [%s]", csp) + templateContext, err := createDeploymentTemplateContext(csp, req) + if err != nil { + return nil, fmt.Errorf("creating request template context: %w", err) } - tpl, err := template.New("create_deployment_request").Parse(tplStr) + tpl, err := template.New("create_deployment_request").Parse(createDeploymentRequestTemplate) if err != nil { return nil, fmt.Errorf("unable to parse deployment creation template: %w", err) } - return tpl, nil + var bBuf bytes.Buffer + err = tpl.Execute(&bBuf, templateContext) + if err != nil { + return nil, fmt.Errorf("rendering create deployment request template with context %v : %w", templateContext, err) + } + return bBuf.Bytes(), nil } -const createDeploymentRequestTemplateGCP = ` -{ - "resources": { - "integrations_server": [ - { - "elasticsearch_cluster_ref_id": "main-elasticsearch", - "region": "{{ .Region }}", - "plan": { - "cluster_topology": [ - { - "instance_configuration_id": "gcp.integrationsserver.n2.68x32x45.2", - "zone_count": 1, - "size": { - "resource": "memory", - "value": 1024 - } - } - ], - "integrations_server": { - "version": "{{ .Version }}" - } - }, - "ref_id": "main-integrations_server" - } - ], - "elasticsearch": [ - { - "region": "{{ .Region }}", - "settings": { - "dedicated_masters_threshold": 6 - }, - "plan": { - "cluster_topology": [ - { - "zone_count": 1, - "elasticsearch": { - "node_attributes": { - "data": "hot" - } - }, - "instance_configuration_id": "gcp.es.datahot.n2.68x10x45", - "node_roles": [ - "master", - "ingest", - "transform", - "data_hot", - "remote_cluster_client", - "data_content" - ], - "id": "hot_content", - "size": { - "resource": "memory", - "value": 8192 - } - } - ], - "elasticsearch": { - "version": "{{ .Version }}", - "enabled_built_in_plugins": [] - }, - "deployment_template": { - "id": "gcp-storage-optimized-v5" - } - }, - "ref_id": "main-elasticsearch" - } - ], - "enterprise_search": [], - "kibana": [ - { - "elasticsearch_cluster_ref_id": "main-elasticsearch", - "region": "{{ .Region }}", - "plan": { - "cluster_topology": [ - { - "instance_configuration_id": "gcp.kibana.n2.68x32x45", - "zone_count": 1, - "size": { - "resource": "memory", - "value": 1024 - } - } - ], - "kibana": { - "version": "{{ .Version }}", - "user_settings_json": { - "xpack.fleet.enableExperimental": ["agentTamperProtectionEnabled"] - } - } - }, - "ref_id": "main-kibana" - } - ] - }, - "settings": { - "autoscaling_enabled": false - }, - "name": "{{ .Name }}", - "metadata": { - "system_owned": false - } -}` - -const createDeploymentRequestTemplateAzure = ` -{ - "resources": { - "integrations_server": [ - { - "elasticsearch_cluster_ref_id": "main-elasticsearch", - "region": "{{ .Region }}", - "plan": { - "cluster_topology": [ - { - "instance_configuration_id": "azure.integrationsserver.fsv2.2", - "zone_count": 1, - "size": { - "resource": "memory", - "value": 1024 - } - } - ], - "integrations_server": { - "version": "{{ .Version }}" - } - }, - "ref_id": "main-integrations_server" - } - ], - "elasticsearch": [ - { - "region": "{{ .Region }}", - "settings": { - "dedicated_masters_threshold": 6 - }, - "plan": { - "cluster_topology": [ - { - "zone_count": 1, - "elasticsearch": { - "node_attributes": { - "data": "hot" - } - }, - "instance_configuration_id": "azure.es.datahot.edsv4", - "node_roles": [ - "master", - "ingest", - "transform", - "data_hot", - "remote_cluster_client", - "data_content" - ], - "id": "hot_content", - "size": { - "resource": "memory", - "value": 8192 - } - } - ], - "elasticsearch": { - "version": "{{ .Version }}", - "enabled_built_in_plugins": [] - }, - "deployment_template": { - "id": "azure-storage-optimized-v2" - } - }, - "ref_id": "main-elasticsearch" - } - ], - "enterprise_search": [], - "kibana": [ - { - "elasticsearch_cluster_ref_id": "main-elasticsearch", - "region": "{{ .Region }}", - "plan": { - "cluster_topology": [ - { - "instance_configuration_id": "azure.kibana.fsv2", - "zone_count": 1, - "size": { - "resource": "memory", - "value": 1024 - } - } - ], - "kibana": { - "version": "{{ .Version }}", - "user_settings_json": { - "xpack.fleet.enableExperimental": ["agentTamperProtectionEnabled"] - } - } - }, - "ref_id": "main-kibana" - } - ] - }, - "settings": { - "autoscaling_enabled": false - }, - "name": "{{ .Name }}", - "metadata": { - "system_owned": false - } -}` +func createDeploymentTemplateContext(csp string, req CreateDeploymentRequest) (map[string]any, error) { + cspSpecificContext, err := loadCspValues(csp) + if err != nil { + return nil, fmt.Errorf("loading csp-specific values for %q: %w", csp, err) + } + + cspSpecificContext["request"] = req + + return cspSpecificContext, nil +} + +func loadCspValues(csp string) (map[string]any, error) { + var cspValues map[string]map[string]any + + err := yaml.Unmarshal(cloudProviderSpecificValues, &cspValues) + if err != nil { + return nil, fmt.Errorf("unmarshalling error: %w", err) + } + values, supportedCSP := cspValues[csp] + if !supportedCSP { + return nil, fmt.Errorf("csp %s not supported", csp) + } + + return values, nil +} diff --git a/pkg/testing/ess/provisioner.go b/pkg/testing/ess/provisioner.go index 941cf5bcaf7..6ebade07569 100644 --- a/pkg/testing/ess/provisioner.go +++ b/pkg/testing/ess/provisioner.go @@ -67,7 +67,7 @@ func (p *provisioner) Provision(ctx context.Context, requests []runner.StackRequ for _, r := range requests { // allow up to 2 minutes for each create request createCtx, createCancel := context.WithTimeout(ctx, 2*time.Minute) - resp, err := p.createDeployment(createCtx, r) + resp, err := p.createDeployment(createCtx, r, map[string]string{"elastic-agent-integration-tests": "true"}) createCancel() if err != nil { return nil, err @@ -131,17 +131,20 @@ func (p *provisioner) Clean(ctx context.Context, stacks []runner.Stack) error { return nil } -func (p *provisioner) createDeployment(ctx context.Context, r runner.StackRequest) (*CreateDeploymentResponse, error) { +func (p *provisioner) createDeployment(ctx context.Context, r runner.StackRequest, _ map[string]string) (*CreateDeploymentResponse, error) { ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() p.logger.Logf("Creating stack %s (%s)", r.Version, r.ID) name := fmt.Sprintf("%s-%s", strings.Replace(p.cfg.Identifier, ".", "-", -1), r.ID) - resp, err := p.client.CreateDeployment(ctx, CreateDeploymentRequest{ + createDeploymentRequest := CreateDeploymentRequest{ Name: name, Region: p.cfg.Region, Version: r.Version, - }) + } + //TODO handle tags + + resp, err := p.client.CreateDeployment(ctx, createDeploymentRequest) if err != nil { p.logger.Logf("Failed to create ESS cloud %s: %s", r.Version, err) return nil, fmt.Errorf("failed to create ESS cloud for version %s: %w", r.Version, err)