Add options for API key in clients (#2317)

Add also a new environment variable for shellinit.

These changes are enough to run some commands, but more changes
are required to run system tests.

Author: jsoriano

README.md

@@ -480,6 +480,7 @@ The output of this command is intended to be evaluated by the current shell. For
 
 Relevant environment variables are:
 
+- ELASTIC_PACKAGE_ELASTICSEARCH_API_KEY
 - ELASTIC_PACKAGE_ELASTICSEARCH_HOST
 - ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME
 - ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD
@@ -690,13 +691,15 @@ There are available some environment variables that could be used to change some
 
 - To configure the Elastic stack to be used by `elastic-package`:
     - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)
-    - `ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME`: User name to connect to elasticsearch (e.g. elastic)
+    - `ELASTIC_PACKAGE_ELASTICSEARCH_API_KEY`: API key to connect to elasticsearch and kibana. When set it takes precedence over username and password.
+    - `ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME`: User name to connect to elasticsearch and kibana (e.g. elastic)
     - `ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD`: Password of that user.
     - `ELASTIC_PACKAGE_ELASTICSEARCH_KIBANA_HOST`: Kibana URL (e.g. https://127.0.0.1:5601)
     - `ELASTIC_PACKAGE_ELASTICSEARCH_CA_CERT`: Path to the CA certificate to connect to the Elastic stack services.
 
 - To configure an external metricstore while running benchmarks (more info at [system benchmarking docs](https://github.com/elastic/elastic-package/blob/main/docs/howto/system_benchmarking.md#setting-up-an-external-metricstore) or [rally benchmarking docs](https://github.com/elastic/elastic-package/blob/main/docs/howto/rally_benchmarking.md#setting-up-an-external-metricstore)):
     - `ELASTIC_PACKAGE_ESMETRICSTORE_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)
+    - `ELASTIC_PACKAGE_ESMETRICSTORE_API_KEY`: API key to connect to elasticsearch and kibana. When set it takes precedence over username and password.
     - `ELASTIC_PACKAGE_ESMETRICSTORE_USERNAME`: Username to connect to elasticsearch (e.g. elastic)
     - `ELASTIC_PACKAGE_ESMETRICSTORE_PASSWORD`: Password for the user.
     - `ELASTIC_PACKAGE_ESMETRICSTORE_CA_CERT`: Path to the CA certificate to connect to the Elastic stack services.

cmd/benchmark.go

@@ -655,16 +655,18 @@ func systemCommandAction(cmd *cobra.Command, args []string) error {
 
 func initializeESMetricsClient(ctx context.Context) (*elasticsearch.Client, error) {
 	address := os.Getenv(benchcommon.ESMetricstoreHostEnv)
+	apiKey := os.Getenv(benchcommon.ESMetricstoreAPIKeyEnv)
 	user := os.Getenv(benchcommon.ESMetricstoreUsernameEnv)
 	pass := os.Getenv(benchcommon.ESMetricstorePasswordEnv)
 	cacert := os.Getenv(benchcommon.ESMetricstoreCACertificateEnv)
-	if address == "" || user == "" || pass == "" {
+	if address == "" || ((user == "" || pass == "") && apiKey == "") {
 		logger.Debugf("can't initialize metricstore, missing environment configuration")
 		return nil, nil
 	}
 
 	esClient, err := stack.NewElasticsearchClient(
 		elasticsearch.OptionWithAddress(address),
+		elasticsearch.OptionWithAPIKey(apiKey),
 		elasticsearch.OptionWithUsername(user),
 		elasticsearch.OptionWithPassword(pass),
 		elasticsearch.OptionWithCertificateAuthority(cacert),

cmd/stack.go

@@ -55,6 +55,7 @@ The output of this command is intended to be evaluated by the current shell. For
 
 Relevant environment variables are:
 
+- ELASTIC_PACKAGE_ELASTICSEARCH_API_KEY
 - ELASTIC_PACKAGE_ELASTICSEARCH_HOST
 - ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME
 - ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD

internal/benchrunner/runners/common/env.go

@@ -7,6 +7,7 @@ package common
 import "github.com/elastic/elastic-package/internal/environment"
 
 var (
+	ESMetricstoreAPIKeyEnv        = environment.WithElasticPackagePrefix("ESMETRICSTORE_API_KEY")
 	ESMetricstoreHostEnv          = environment.WithElasticPackagePrefix("ESMETRICSTORE_HOST")
 	ESMetricstoreUsernameEnv      = environment.WithElasticPackagePrefix("ESMETRICSTORE_USERNAME")
 	ESMetricstorePasswordEnv      = environment.WithElasticPackagePrefix("ESMETRICSTORE_PASSWORD")

internal/elasticsearch/client.go

@@ -35,6 +35,7 @@ type ClusterStateRequest = esapi.ClusterStateRequest
 // clientOptions are used to configure a client.
 type clientOptions struct {
 	address  string
+	apiKey   string
 	username string
 	password string
 
@@ -47,6 +48,13 @@ type clientOptions struct {
 
 type ClientOption func(*clientOptions)
 
+// OptionWithAPIKey sets the API key to be used by the client for authentication.
+func OptionWithAPIKey(apiKey string) ClientOption {
+	return func(opts *clientOptions) {
+		opts.apiKey = apiKey
+	}
+}
+
 // OptionWithAddress sets the address to be used by the client.
 func OptionWithAddress(address string) ClientOption {
 	return func(opts *clientOptions) {
@@ -109,6 +117,7 @@ func NewConfig(customOptions ...ClientOption) (elasticsearch.Config, error) {
 
 	config := elasticsearch.Config{
 		Addresses: []string{options.address},
+		APIKey:    options.apiKey,
 		Username:  options.username,
 		Password:  options.password,
 	}

internal/kibana/client.go

@@ -27,6 +27,7 @@ var ErrUndefinedHost = errors.New("missing kibana host")
 // Client is responsible for exporting dashboards from Kibana.
 type Client struct {
 	host     string
+	apiKey   string
 	username string
 	password string
 
@@ -94,6 +95,13 @@ func Address(address string) ClientOption {
 	}
 }
 
+// APIKey option sets the API key to be used by the client for authentication.
+func APIKey(apiKey string) ClientOption {
+	return func(c *Client) {
+		c.apiKey = apiKey
+	}
+}
+
 // TLSSkipVerify option disables TLS verification.
 func TLSSkipVerify() ClientOption {
 	return func(c *Client) {
@@ -182,7 +190,11 @@ func (c *Client) newRequest(ctx context.Context, method, resourcePath string, re
 		return nil, fmt.Errorf("could not create %v request to Kibana API resource: %s: %w", method, resourcePath, err)
 	}
 
-	req.SetBasicAuth(c.username, c.password)
+	if c.apiKey != "" {
+		req.Header.Set("Authorization", "ApiKey "+c.apiKey)
+	} else {
+		req.SetBasicAuth(c.username, c.password)
+	}
 	req.Header.Add("content-type", "application/json")
 	req.Header.Add("kbn-xsrf", install.DefaultStackVersion)
 

internal/stack/clients.go

@@ -21,6 +21,7 @@ import (
 func NewElasticsearchClient(customOptions ...elasticsearch.ClientOption) (*elasticsearch.Client, error) {
 	options := []elasticsearch.ClientOption{
 		elasticsearch.OptionWithAddress(os.Getenv(ElasticsearchHostEnv)),
+		elasticsearch.OptionWithAPIKey(os.Getenv(ElasticsearchAPIKeyEnv)),
 		elasticsearch.OptionWithPassword(os.Getenv(ElasticsearchPasswordEnv)),
 		elasticsearch.OptionWithUsername(os.Getenv(ElasticsearchUsernameEnv)),
 		elasticsearch.OptionWithCertificateAuthority(os.Getenv(CACertificateEnv)),
@@ -58,6 +59,10 @@ func NewElasticsearchClientFromProfile(profile *profile.Profile, customOptions .
 		elasticsearchHost = profileConfig.ElasticsearchHostPort
 		logger.Debugf("Connecting with Elasticsearch host from current profile (profile: %s, host: %q)", profile.ProfileName, elasticsearchHost)
 	}
+	elasticsearchAPIKey, found := os.LookupEnv(ElasticsearchAPIKeyEnv)
+	if !found {
+		elasticsearchAPIKey = profileConfig.ElasticsearchAPIKey
+	}
 	elasticsearchPassword, found := os.LookupEnv(ElasticsearchPasswordEnv)
 	if !found {
 		elasticsearchPassword = profileConfig.ElasticsearchPassword
@@ -73,6 +78,7 @@ func NewElasticsearchClientFromProfile(profile *profile.Profile, customOptions .
 
 	options := []elasticsearch.ClientOption{
 		elasticsearch.OptionWithAddress(elasticsearchHost),
+		elasticsearch.OptionWithAPIKey(elasticsearchAPIKey),
 		elasticsearch.OptionWithPassword(elasticsearchPassword),
 		elasticsearch.OptionWithUsername(elasticsearchUsername),
 		elasticsearch.OptionWithCertificateAuthority(caCertificate),
@@ -86,6 +92,7 @@ func NewElasticsearchClientFromProfile(profile *profile.Profile, customOptions .
 func NewKibanaClient(customOptions ...kibana.ClientOption) (*kibana.Client, error) {
 	options := []kibana.ClientOption{
 		kibana.Address(os.Getenv(KibanaHostEnv)),
+		kibana.APIKey(os.Getenv(ElasticsearchAPIKeyEnv)),
 		kibana.Password(os.Getenv(ElasticsearchPasswordEnv)),
 		kibana.Username(os.Getenv(ElasticsearchUsernameEnv)),
 		kibana.CertificateAuthority(os.Getenv(CACertificateEnv)),
@@ -123,6 +130,10 @@ func NewKibanaClientFromProfile(profile *profile.Profile, customOptions ...kiban
 		kibanaHost = profileConfig.KibanaHostPort
 		logger.Debugf("Connecting with Kibana host from current profile (profile: %s, host: %q)", profile.ProfileName, kibanaHost)
 	}
+	elasticsearchAPIKey, found := os.LookupEnv(ElasticsearchAPIKeyEnv)
+	if !found {
+		elasticsearchAPIKey = profileConfig.ElasticsearchAPIKey
+	}
 	elasticsearchPassword, found := os.LookupEnv(ElasticsearchPasswordEnv)
 	if !found {
 		elasticsearchPassword = profileConfig.ElasticsearchPassword
@@ -138,6 +149,7 @@ func NewKibanaClientFromProfile(profile *profile.Profile, customOptions ...kiban
 
 	options := []kibana.ClientOption{
 		kibana.Address(kibanaHost),
+		kibana.APIKey(elasticsearchAPIKey),
 		kibana.Password(elasticsearchPassword),
 		kibana.Username(elasticsearchUsername),
 		kibana.CertificateAuthority(caCertificate),
@@ -158,5 +170,10 @@ func FindCACertificate(profile *profile.Profile) (string, error) {
 		caCertPath = profileConfig.CACertificatePath
 	}
 
+	// Avoid returning an empty certificate path, fallback to the default path.
+	if caCertPath == "" {
+		caCertPath = profile.Path(CACertificateFile)
+	}
+
 	return caCertPath, nil
 }

internal/stack/config.go

@@ -20,6 +20,7 @@ type Config struct {
 	Provider   string            `json:"provider,omitempty"`
 	Parameters map[string]string `json:"parameters,omitempty"`
 
+	ElasticsearchAPIKey   string `json:"elasticsearch_api_key,omitempty"`
 	ElasticsearchHost     string `json:"elasticsearch_host,omitempty"`
 	ElasticsearchUsername string `json:"elasticsearch_username,omitempty"`
 	ElasticsearchPassword string `json:"elasticsearch_password,omitempty"`

internal/stack/initconfig.go

@@ -9,6 +9,7 @@ import (
 )
 
 type InitConfig struct {
+	ElasticsearchAPIKey   string
 	ElasticsearchHostPort string
 	ElasticsearchUsername string
 	ElasticsearchPassword string
@@ -23,6 +24,7 @@ func StackInitConfig(profile *profile.Profile) (*InitConfig, error) {
 	}
 
 	return &InitConfig{
+		ElasticsearchAPIKey:   config.ElasticsearchAPIKey,
 		ElasticsearchHostPort: config.ElasticsearchHost,
 		ElasticsearchUsername: config.ElasticsearchUsername,
 		ElasticsearchPassword: config.ElasticsearchPassword,

internal/stack/shellinit.go

@@ -20,9 +20,10 @@ import (
 
 // Environment variables describing the stack.
 var (
+	ElasticsearchAPIKeyEnv   = environment.WithElasticPackagePrefix("ELASTICSEARCH_API_KEY")
 	ElasticsearchHostEnv     = environment.WithElasticPackagePrefix("ELASTICSEARCH_HOST")
-	ElasticsearchUsernameEnv = environment.WithElasticPackagePrefix("ELASTICSEARCH_USERNAME")
 	ElasticsearchPasswordEnv = environment.WithElasticPackagePrefix("ELASTICSEARCH_PASSWORD")
+	ElasticsearchUsernameEnv = environment.WithElasticPackagePrefix("ELASTICSEARCH_USERNAME")
 	KibanaHostEnv            = environment.WithElasticPackagePrefix("KIBANA_HOST")
 	CACertificateEnv         = environment.WithElasticPackagePrefix("CA_CERT")
 )
@@ -38,60 +39,70 @@ func ShellInit(elasticStackProfile *profile.Profile, shellType string) (string,
 	if err != nil {
 		return "", nil
 	}
+	return shellInitWithConfig(config, shellType)
+}
 
-	// NOTE: to add new env vars, the template need to be adjusted
-	t, err := initTemplate(shellType)
+func shellInitWithConfig(config *InitConfig, shellType string) (string, error) {
+	pattern, err := selectPattern(shellType)
 	if err != nil {
 		return "", fmt.Errorf("cannot get shell init template: %w", err)
 	}
 
-	return fmt.Sprintf(t,
-		ElasticsearchHostEnv, config.ElasticsearchHostPort,
-		ElasticsearchUsernameEnv, config.ElasticsearchUsername,
-		ElasticsearchPasswordEnv, config.ElasticsearchPassword,
-		KibanaHostEnv, config.KibanaHostPort,
-		CACertificateEnv, config.CACertificatePath,
-	), nil
+	template := genTemplate(pattern)
+	return template([]generatorEnvVar{
+		{ElasticsearchAPIKeyEnv, config.ElasticsearchAPIKey},
+		{ElasticsearchHostEnv, config.ElasticsearchHostPort},
+		{ElasticsearchUsernameEnv, config.ElasticsearchUsername},
+		{ElasticsearchPasswordEnv, config.ElasticsearchPassword},
+		{KibanaHostEnv, config.KibanaHostPort},
+		{CACertificateEnv, config.CACertificatePath},
+	}), nil
+}
+
+type generatorEnvVar struct {
+	name  string
+	value string
 }
 
 const (
 	// shell init code for POSIX compliant shells.
 	// IEEE POSIX Shell and Tools portion of the IEEE POSIX specification (IEEE Standard 1003.1)
-	posixTemplate = `export %s=%s
-export %s=%s
-export %s=%s
-export %s=%s
-export %s=%s`
+	posixPattern = `export %s=%s`
 
 	// fish shell init code.
 	// fish shell is similar but not compliant to POSIX.
-	fishTemplate = `set -x %s %s;
-set -x %s %s;
-set -x %s %s;
-set -x %s %s;
-set -x %s %s;`
+	fishPattern = `set -x %s %s;`
 
 	// PowerShell init code.
 	// Output to be evaluated with `elastic-package stack shellinit | Invoke-Expression
-	powershellTemplate = `$Env:%s="%s";
-$Env:%s="%s";
-$Env:%s="%s";
-$Env:%s="%s";
-$Env:%s="%s";`
+	powershellPattern = `$Env:%s="%s";`
 )
 
+func genTemplate(pattern string) func([]generatorEnvVar) string {
+	return func(vars []generatorEnvVar) string {
+		var builder strings.Builder
+		for i, v := range vars {
+			fmt.Fprintf(&builder, pattern, v.name, v.value)
+			if i < len(vars)-1 {
+				builder.WriteString("\n")
+			}
+		}
+		return builder.String()
+	}
+}
+
 // availableShellTypes list all available values for s in initTemplate
 var availableShellTypes = []string{"bash", "dash", "fish", "sh", "zsh", "pwsh", "powershell"}
 
-// InitTemplate returns code templates for shell initialization
-func initTemplate(s string) (string, error) {
+// SelectPattern returns the patterns to generate list of environment variables for each shell.
+func selectPattern(s string) (string, error) {
 	switch s {
 	case "bash", "dash", "sh", "zsh":
-		return posixTemplate, nil
+		return posixPattern, nil
 	case "fish":
-		return fishTemplate, nil
+		return fishPattern, nil
 	case "pwsh", "powershell":
-		return powershellTemplate, nil
+		return powershellPattern, nil
 	default:
 		return "", errors.New("shell type is unknown, should be one of " + strings.Join(availableShellTypes, ", "))
 	}