Load stack configuration from profile (#1428)

Commands that require an stack to connect with will try to connect with the
stack of the current profile by default. This behaviour can be overridden
by setting the usual environment variables provided by `elastic-package shellinit`.

With this change `elastic-package shellinit` is not necessary when working
with stacks managed by elastic-package. shellinit is still useful to export
the configuration for other commands or scripts.
The environment variables can also be used to work with stacks not managed
by elastic-package.

README.md

@@ -437,7 +437,20 @@ Dump stack data for debug purposes.
 
 _Context: global_
 
-Export environment variables.
+Use this command to export to the current shell the configuration of the stack managed by elastic-package.
+
+The output of this command is intended to be evaluated by the current shell. For example in bash: 'eval $(elastic-package stack shellinit)'.
+
+Relevant environment variables are:
+
+- ELASTIC_PACKAGE_ELASTICSEARCH_HOST
+- ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME
+- ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD
+- ELASTIC_PACKAGE_KIBANA_HOST
+- ELASTIC_PACKAGE_CA_CERT
+
+You can also provide these environment variables manually. In that case elastic-package commands will use these settings.
+
 
 ### `elastic-package stack status`
 

cmd/benchmark.go

@@ -165,7 +165,12 @@ func pipelineCommandAction(cmd *cobra.Command, args []string) error {
 		return errors.New("no pipeline benchmarks found")
 	}
 
-	esClient, err := stack.NewElasticsearchClient()
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return err
+	}
+
+	esClient, err := stack.NewElasticsearchClientFromProfile(profile)
 	if err != nil {
 		return fmt.Errorf("can't create Elasticsearch client: %w", err)
 	}
@@ -269,7 +274,7 @@ func systemCommandAction(cmd *cobra.Command, args []string) error {
 
 	signal.Enable()
 
-	esClient, err := stack.NewElasticsearchClient()
+	esClient, err := stack.NewElasticsearchClientFromProfile(profile)
 	if err != nil {
 		return fmt.Errorf("can't create Elasticsearch client: %w", err)
 	}
@@ -278,7 +283,7 @@ func systemCommandAction(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	kc, err := stack.NewKibanaClient()
+	kc, err := stack.NewKibanaClientFromProfile(profile)
 	if err != nil {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}

cmd/dump.go

@@ -12,6 +12,7 @@ import (
 	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/dump"
 	"github.com/elastic/elastic-package/internal/elasticsearch"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/stack"
 )
@@ -58,6 +59,7 @@ func setupDumpCommand() *cobraext.Command {
 		Long:  dumpLongDescription,
 	}
 	cmd.PersistentFlags().StringP(cobraext.DumpOutputFlagName, "o", "package-dump", cobraext.DumpOutputFlagDescription)
+	cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))
 
 	cmd.AddCommand(dumpInstalledObjectsCmd)
 	cmd.AddCommand(dumpAgentPoliciesCmd)
@@ -82,7 +84,13 @@ func dumpInstalledObjectsCmdAction(cmd *cobra.Command, args []string) error {
 	if tlsSkipVerify {
 		clientOptions = append(clientOptions, elasticsearch.OptionWithSkipTLSVerify())
 	}
-	client, err := stack.NewElasticsearchClient(clientOptions...)
+
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return err
+	}
+
+	client, err := stack.NewElasticsearchClientFromProfile(profile, clientOptions...)
 	if err != nil {
 		return fmt.Errorf("failed to initialize Elasticsearch client: %w", err)
 	}
@@ -122,7 +130,13 @@ func dumpAgentPoliciesCmdAction(cmd *cobra.Command, args []string) error {
 	if tlsSkipVerify {
 		clientOptions = append(clientOptions, kibana.TLSSkipVerify())
 	}
-	kibanaClient, err := stack.NewKibanaClient(clientOptions...)
+
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return err
+	}
+
+	kibanaClient, err := stack.NewKibanaClientFromProfile(profile, clientOptions...)
 	if err != nil {
 		return fmt.Errorf("failed to initialize Kibana client: %w", err)
 	}

cmd/export.go

@@ -14,6 +14,7 @@ import (
 	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/common"
 	"github.com/elastic/elastic-package/internal/export"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/stack"
 )
@@ -42,6 +43,7 @@ func setupExportCommand() *cobraext.Command {
 		Long:  exportLongDescription,
 	}
 	cmd.AddCommand(exportDashboardCmd)
+	cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))
 
 	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
 }
@@ -67,7 +69,12 @@ func exportDashboardsCmd(cmd *cobra.Command, args []string) error {
 		return cobraext.FlagParsingError(err, cobraext.AllowSnapshotFlagName)
 	}
 
-	kibanaClient, err := stack.NewKibanaClient(opts...)
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return err
+	}
+
+	kibanaClient, err := stack.NewKibanaClientFromProfile(profile, opts...)
 	if err != nil {
 		return fmt.Errorf("can't create Kibana client: %w", err)
 	}

cmd/install.go

@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/packages/installer"
 	"github.com/elastic/elastic-package/internal/stack"
@@ -32,6 +33,7 @@ func setupInstallCommand() *cobraext.Command {
 	cmd.Flags().StringP(cobraext.PackageRootFlagName, cobraext.PackageRootFlagShorthand, "", cobraext.PackageRootFlagDescription)
 	cmd.Flags().StringP(cobraext.ZipPackageFilePathFlagName, cobraext.ZipPackageFilePathFlagShorthand, "", cobraext.ZipPackageFilePathFlagDescription)
 	cmd.Flags().Bool(cobraext.BuildSkipValidationFlagName, false, cobraext.BuildSkipValidationFlagDescription)
+	cmd.Flags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))
 
 	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
 }
@@ -50,7 +52,12 @@ func installCommandAction(cmd *cobra.Command, _ []string) error {
 		return cobraext.FlagParsingError(err, cobraext.BuildSkipValidationFlagName)
 	}
 
-	kibanaClient, err := stack.NewKibanaClient()
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return err
+	}
+
+	kibanaClient, err := stack.NewKibanaClientFromProfile(profile)
 	if err != nil {
 		return fmt.Errorf("could not create kibana client: %w", err)
 	}

cmd/service.go

@@ -68,7 +68,7 @@ func upCommandAction(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	kibanaClient, err := stack.NewKibanaClient()
+	kibanaClient, err := stack.NewKibanaClientFromProfile(profile)
 	if err != nil {
 		return fmt.Errorf("cannot create Kibana client: %w", err)
 	}

cmd/stack.go

@@ -48,6 +48,21 @@ For details on how to connect the service with the Elastic stack, see the [servi
 
 You can customize your stack using profile settings, see [Elastic Package profiles](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-profiles-1) section. These settings can be also overriden with the --parameter flag. Settings configured this way are not persisted.`
 
+const stackShellinitLongDescription = `Use this command to export to the current shell the configuration of the stack managed by elastic-package.
+
+The output of this command is intended to be evaluated by the current shell. For example in bash: 'eval $(elastic-package stack shellinit)'.
+
+Relevant environment variables are:
+
+- ELASTIC_PACKAGE_ELASTICSEARCH_HOST
+- ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME
+- ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD
+- ELASTIC_PACKAGE_KIBANA_HOST
+- ELASTIC_PACKAGE_CA_CERT
+
+You can also provide these environment variables manually. In that case elastic-package commands will use these settings.
+`
+
 func setupStackCommand() *cobraext.Command {
 	upCommand := &cobra.Command{
 		Use:   "up",
@@ -99,7 +114,6 @@ func setupStackCommand() *cobraext.Command {
 			profile.RuntimeOverrides(userParameters)
 
 			cmd.Printf("Using profile %s.\n", profile.ProfilePath)
-			cmd.Println(`Remember to load stack environment variables using 'eval "$(elastic-package stack shellinit)"'.`)
 			err = provider.BootUp(stack.Options{
 				DaemonMode:   daemonMode,
 				StackVersion: stackVersion,
@@ -192,6 +206,7 @@ func setupStackCommand() *cobraext.Command {
 	shellInitCommand := &cobra.Command{
 		Use:   "shellinit",
 		Short: "Export environment variables",
+		Long:  stackShellinitLongDescription,
 		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			shellName, err := cmd.Flags().GetString(cobraext.ShellInitShellFlagName)

cmd/testrunner.go

@@ -215,7 +215,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 			return err
 		}
 
-		esClient, err := stack.NewElasticsearchClient()
+		esClient, err := stack.NewElasticsearchClientFromProfile(profile)
 		if err != nil {
 			return fmt.Errorf("can't create Elasticsearch client: %w", err)
 		}
@@ -224,7 +224,7 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 			return err
 		}
 
-		kibanaClient, err := stack.NewKibanaClient()
+		kibanaClient, err := stack.NewKibanaClientFromProfile(profile)
 		if err != nil {
 			return fmt.Errorf("can't create Kibana client: %w", err)
 		}

cmd/uninstall.go

@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/packages/installer"
 	"github.com/elastic/elastic-package/internal/stack"
@@ -28,6 +29,7 @@ func setupUninstallCommand() *cobraext.Command {
 		Args:  cobra.NoArgs,
 		RunE:  uninstallCommandAction,
 	}
+	cmd.Flags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))
 
 	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
 }
@@ -41,7 +43,12 @@ func uninstallCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("locating package root failed: %w", err)
 	}
 
-	kibanaClient, err := stack.NewKibanaClient()
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return err
+	}
+
+	kibanaClient, err := stack.NewKibanaClientFromProfile(profile)
 	if err != nil {
 		return fmt.Errorf("could not create kibana client: %w", err)
 	}

docs/howto/asset_testing.md

@@ -36,12 +36,6 @@ elastic-package stack up -d
 
 For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`.
 
-Next, you must set environment variables needed for further `elastic-package` commands.
-
-```
-$(elastic-package stack shellinit)
-```
-
 Next, you must invoke the asset loading test runner. This corresponds to steps 3 through 5 as described in the [_Conceptual process_](#Conceptual-process) section.
 
 Navigate to the package's root folder (or any sub-folder under it) and run the following command.

docs/howto/install_package.md

@@ -9,7 +9,6 @@ For versions of `Kibana<8.7.0`, the packages must be exposed via the Package Reg
 In case of development, this means that the package should be built previously and then the Elastic stack must be started. Or at least, package-registry service needs to be restarted in the Elastic stack.
 
 ```shell
-eval "$(elastic-package stack shellinit)"
 elastic-package build -v
 elastic-package stack up -v -d  # elastic-package stack up -v -d --services package-registry
 elastic-package install -v
@@ -30,7 +29,6 @@ From Kibana 8.7.0 version, `elastic-package install` is able to install packages
 Example of using `--zip` parameter:
 ```shell
  $ elastic-package stack up -v -d
- $ eval "$(elastic-package stack shellinit)"
  $ elastic-package install --zip /home/user/Coding/work/integrations/build/packages/elastic_package_registry-0.0.6.zip -v
 2023/02/23 18:44:59 DEBUG Enable verbose logging
 2023/02/23 18:44:59 DEBUG Distribution built without a version tag, can't determine release chronology. Please consider using official releases at https://github.com/elastic/elastic-package/releases
@@ -50,7 +48,6 @@ Done
 Example of using `elastic-package install`
 ```shell
  $ elastic-package stack up -v -d
- $ eval "$(elastic-package stack shellinit)"
  $ elastic-package install -v
 2023/02/28 12:34:44 DEBUG Enable verbose logging
 2023/02/28 12:34:44 DEBUG Distribution built without a version tag, can't determine release chronology. Please consider using official releases at https://github.com/elastic/elastic-package/releases

docs/howto/pipeline_benchmarking.md

@@ -101,12 +101,6 @@ elastic-package stack up -d --services=elasticsearch
 
 For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`.
 
-Next, you must set environment variables needed for further `elastic-package` commands.
-
-```
-$(elastic-package stack shellinit)
-```
-
 Next, you must invoke the pipeline benchmark runner. This corresponds to steps 2 through 4 as described in the [_Conceptual process_](#Conceptual-process) section.
 
 If you want to run pipeline benchmarks for **all data streams** in a package, navigate to the package's root folder (or any sub-folder under it) and run the following command.

docs/howto/pipeline_testing.md

@@ -156,12 +156,6 @@ elastic-package stack up -d --services=elasticsearch
 
 For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`.
 
-Next, you must set environment variables needed for further `elastic-package` commands.
-
-```
-$(elastic-package stack shellinit)
-```
-
 Next, you must invoke the pipeline tests runner. This corresponds to steps 2 through 4 as described in the [_Conceptual process_](#Conceptual-process) section.
 
 If you want to run pipeline tests for **all data streams** in a package, navigate to the package's root folder (or any sub-folder under it) and run the following command.

docs/howto/system_benchmarking.md

@@ -312,12 +312,6 @@ elastic-package stack up -d
 
 For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`.
 
-Next, you must set environment variables needed for further `elastic-package` commands.
-
-```
-$(elastic-package stack shellinit)
-```
-
 Next, you must invoke the system benchmark runner.
 
 ```
@@ -430,4 +424,4 @@ Ingest pipelines metrics are only collected at the end since its own collection
 
 You can see a sample collected metric [here](./sample_metric.json)
 
-Additionally, if the `reindex-to-metricstore` flag is used, the data generated during the benchmark will be sent to the metricstore into an index called `bench-reindex-{datastream}-{testRunID}` for further analysis. The events will be enriched with metadata related to the benchmark run.
+Additionally, if the `reindex-to-metricstore` flag is used, the data generated during the benchmark will be sent to the metricstore into an index called `bench-reindex-{datastream}-{testRunID}` for further analysis. The events will be enriched with metadata related to the benchmark run.

docs/howto/system_testing.md

@@ -532,12 +532,6 @@ elastic-package stack up -d
 
 For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`.
 
-Next, you must set environment variables needed for further `elastic-package` commands.
-
-```
-$(elastic-package stack shellinit)
-```
-
 Next, you must invoke the system tests runner. This corresponds to steps 3 through 7 as described in the [_Conceptual process_](#Conceptual-process) section.
 
 If you want to run system tests for **all data streams** in a package, navigate to the package's root folder (or any sub-folder under it) and run the following command.

internal/servicedeployer/custom_agent.go

@@ -55,9 +55,9 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err
 		return nil, fmt.Errorf("can't read application configuration: %w", err)
 	}
 
-	caCertPath, ok := os.LookupEnv(stack.CACertificateEnv)
-	if !ok {
-		return nil, fmt.Errorf("can't locate CA certificate: %s environment variable not set", stack.CACertificateEnv)
+	caCertPath, err := stack.FindCACertificate(d.profile)
+	if err != nil {
+		return nil, fmt.Errorf("can't locate CA certificate: %w", err)
 	}
 
 	env := append(