Share logic to start local services

Author: jsoriano

internal/agentdeployer/agent.go

@@ -258,10 +258,11 @@ func (d *DockerComposeAgentDeployer) installDockerCompose(agentInfo AgentInfo) (
 	fleetURL := "https://fleet-server:8220"
 	kibanaHost := "https://kibana:5601"
 	stackVersion := d.stackVersion
-	if config.Provider == stack.ProviderServerless {
-		fleetURL = config.Parameters[stack.ParamServerlessFleetURL]
-		kibanaHost = config.KibanaHost
-		stackVersion = config.Parameters[stack.ParamServerlessLocalStackVersion]
+	if url, ok := config.Parameters[stack.ParamServerlessFleetURL]; ok {
+		fleetURL = url
+	}
+	if version, ok := config.Parameters[stack.ParamServerlessLocalStackVersion]; ok {
+		stackVersion = version
 	}
 
 	agentImage, err := selectElasticAgentImage(stackVersion, agentInfo.Agent.BaseImage)

internal/stack/_static/elastic-agent.env.tmpl

@@ -3,8 +3,13 @@ FLEET_ENROLL=1
 FLEET_URL={{ fact "fleet_url" }}
 KIBANA_FLEET_HOST={{ fact "kibana_host" }}
 KIBANA_HOST={{ fact "kibana_host" }}
+{{ $api_key := fact "api_key" }}
+{{- if eq $api_key "" }}
 ELASTICSEARCH_USERNAME={{ fact "username" }}
 ELASTICSEARCH_PASSWORD={{ fact "password" }}
+{{- else }}
+ELASTICSEARCH_API_KEY={{ $api_key }}
+{{- end }}
 {{ if not (semverLessThan $version "8.0.0") }}
 FLEET_TOKEN_POLICY_NAME=Elastic-Agent (elastic-package)
 {{ end }}

internal/stack/_static/serverless-docker-compose.yml.tmpl renamed to internal/stack/_static/local-services-docker-compose.yml.tmpl

@@ -9,12 +9,17 @@ input {
 }
 
 {{ $elasticsearch_host := fact "elasticsearch_host" -}}
+{{ $api_key := fact "api_key" -}}
 filter {
   elastic_integration {
     remove_field => ['@version']
     hosts => ["{{ $elasticsearch_host }}"]
+  {{- if eq $api_key "" }}
     username => '{{ fact "username" }}'
     password => '{{ fact "password" }}'
+  {{- else }}
+    api_key => '{{ $api_key }}'
+  {{- end }}
     ssl_enabled => true
     ssl_verification_mode => "none"
   }
@@ -24,8 +29,12 @@ output {
   if [@metadata][_ingest_document][id] {
     elasticsearch {
       hosts => ["{{ $elasticsearch_host }}"]
+  {{- if eq $api_key "" -}}
       user => '{{ fact "username" }}'
       password => '{{ fact "password" }}'
+  {{- else -}}
+      api_key => '{{ $api_key }}'
+  {{- end }}
       ssl_enabled => true
   {{- if eq $elasticsearch_host "https://elasticsearch:9200" }}
       ssl_certificate_authorities => "/usr/share/logstash/config/certs/ca-cert.pem"

internal/stack/_static/logstash.conf.tmpl

@@ -32,7 +32,9 @@ var tlsServices = []tlsService{
 	{Name: "elastic-agent", IsClient: true},
 }
 
-var tlsServicesServerless = []tlsService{
+// tlsLocalServices is the list of server TLS certificates that will
+// be created for local services when the stack is not local.
+var tlsLocalServices = []tlsService{
 	{Name: "logstash"},
 	{Name: "elastic-agent", IsClient: true},
 }

internal/stack/certs.go

@@ -21,7 +21,7 @@ func TestTLSCertsInitialization(t *testing.T) {
 		services []tlsService
 	}{
 		{"tlsServices", tlsServices},
-		{"tlsServicesServerless", tlsServicesServerless},
+		{"tlsServicesServerless", tlsLocalServices},
 	}
 	profilePath := t.TempDir()
 	caCertFile := filepath.Join(profilePath, "certs", "ca-cert.pem")

internal/stack/certs_test.go

@@ -61,7 +61,7 @@ func dumpStackLogs(ctx context.Context, options DumpOptions) ([]DumpResult, erro
 		return nil, fmt.Errorf("can't remove output location: %w", err)
 	}
 
-	services, err := localServiceNames(DockerComposeProjectName(options.Profile))
+	services, err := localServiceNames(options.Profile)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get local services: %w", err)
 	}

internal/stack/dump.go

@@ -0,0 +1,128 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package stack
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/elastic/elastic-package/internal/elasticsearch"
+	"github.com/elastic/elastic-package/internal/kibana"
+	"github.com/elastic/elastic-package/internal/profile"
+)
+
+type environmentProvider struct {
+	kibana        *kibana.Client
+	elasticsearch *elasticsearch.Client
+}
+
+func newEnvironmentProvider(profile *profile.Profile) (*environmentProvider, error) {
+	return &environmentProvider{}, nil
+}
+
+// BootUp configures the profile to use as stack the one indicated using environment variables.
+func (p *environmentProvider) BootUp(ctx context.Context, options Options) error {
+	config := Config{
+		Provider:              ProviderEnvironment,
+		ElasticsearchAPIKey:   os.Getenv(ElasticsearchAPIKeyEnv),
+		ElasticsearchHost:     os.Getenv(ElasticsearchHostEnv),
+		ElasticsearchUsername: os.Getenv(ElasticsearchUsernameEnv),
+		ElasticsearchPassword: os.Getenv(ElasticsearchPasswordEnv),
+		KibanaHost:            os.Getenv(KibanaHostEnv),
+		CACertFile:            os.Getenv(CACertificateEnv),
+
+		Parameters: make(map[string]string),
+	}
+	if err := requiredEnv(config.ElasticsearchHost, ElasticsearchHostEnv); err != nil {
+		return err
+	}
+	if err := requiredEnv(config.KibanaHost, KibanaHostEnv); err != nil {
+		return err
+	}
+
+	err := p.initClients()
+	if err != nil {
+		return err
+	}
+	// TODO: Migrate from serverless variables.
+	config.Parameters[ParamServerlessFleetURL], err = p.kibana.DefaultFleetServerURL(ctx)
+	if err != nil {
+		return fmt.Errorf("cannot discover default fleet server URL: %w", err)
+	}
+
+	localServices := &localServicesManager{
+		profile: options.Profile,
+	}
+	err = localServices.start(ctx, options, config)
+	if err != nil {
+		return fmt.Errorf("failed to start local services: %w", err)
+	}
+
+	err = storeConfig(options.Profile, config)
+	if err != nil {
+		return fmt.Errorf("failed to store config: %w", err)
+	}
+
+	return nil
+}
+
+func requiredEnv(value string, envVarName string) error {
+	if value == "" {
+		return fmt.Errorf("environment variable %s required", envVarName)
+	}
+	return nil
+}
+
+func (p *environmentProvider) initClients() error {
+	kibana, err := NewKibanaClient()
+	if err != nil {
+		return fmt.Errorf("cannot create Kibana client: %w", err)
+	}
+	p.kibana = kibana
+
+	elasticsearch, err := NewElasticsearchClient()
+	if err != nil {
+		return fmt.Errorf("cannot create Elasticsearch client: %w", err)
+	}
+	p.elasticsearch = elasticsearch
+
+	return nil
+}
+
+// TearDown stops and/or removes a stack.
+func (p *environmentProvider) TearDown(ctx context.Context, options Options) error {
+	localServices := &localServicesManager{
+		profile: options.Profile,
+	}
+	err := localServices.destroy(ctx)
+	if err != nil {
+		return fmt.Errorf("failed ot destroy local services: %v", err)
+	}
+	return nil
+}
+
+// Update updates resources associated to a stack.
+func (p *environmentProvider) Update(context.Context, Options) error {
+	return fmt.Errorf("not implemented")
+}
+
+// Dump dumps data for debug purpouses.
+func (p *environmentProvider) Dump(ctx context.Context, options DumpOptions) ([]DumpResult, error) {
+	for _, service := range options.Services {
+		if service != "elastic-agent" {
+			return nil, &ErrNotImplemented{
+				Operation: fmt.Sprintf("logs dump for service %s", service),
+				Provider:  ProviderServerless,
+			}
+		}
+	}
+	return Dump(ctx, options)
+}
+
+// Status obtains status information of the stack.
+func (p *environmentProvider) Status(context.Context, Options) ([]ServiceStatus, error) {
+	return nil, fmt.Errorf("not implemented")
+}

internal/stack/environment.go

@@ -19,10 +19,10 @@ import (
 )
 
 var (
-	serverlessStackResources = []resource.Resource{
+	localStackResources = []resource.Resource{
 		&resource.File{
 			Path:    ComposeFile,
-			Content: staticSource.Template("_static/serverless-docker-compose.yml.tmpl"),
+			Content: staticSource.Template("_static/local-services-docker-compose.yml.tmpl"),
 		},
 		&resource.File{
 			Path:    ElasticAgentEnvFile,
@@ -31,7 +31,9 @@ var (
 	}
 )
 
-func applyServerlessResources(profile *profile.Profile, stackVersion string, config Config) error {
+// applyLocalResources creates the local resources needed to run system tests when the stack
+// is not local.
+func applyLocalResources(profile *profile.Profile, stackVersion string, config Config) error {
 	appConfig, err := install.Configuration(install.OptionWithStackVersion(stackVersion))
 	if err != nil {
 		return fmt.Errorf("can't read application configuration: %w", err)
@@ -46,6 +48,7 @@ func applyServerlessResources(profile *profile.Profile, stackVersion string, con
 		"agent_image":        imageRefs.ElasticAgent,
 		"logstash_image":     imageRefs.Logstash,
 		"elasticsearch_host": esHostWithPort(config.ElasticsearchHost),
+		"api_key":            config.ElasticsearchAPIKey, // TODO: !! We will need to enroll with an enrollment token?
 		"username":           config.ElasticsearchUsername,
 		"password":           config.ElasticsearchPassword,
 		"kibana_host":        config.KibanaHost,
@@ -58,13 +61,13 @@ func applyServerlessResources(profile *profile.Profile, stackVersion string, con
 		Prefix: stackDir,
 	})
 
-	resources := append([]resource.Resource{}, serverlessStackResources...)
+	resources := append([]resource.Resource{}, localStackResources...)
 
 	// Keeping certificates in the profile directory for backwards compatibility reasons.
 	resourceManager.RegisterProvider("certs", &resource.FileProvider{
 		Prefix: profile.ProfilePath,
 	})
-	certResources, err := initTLSCertificates("certs", profile.ProfilePath, tlsServicesServerless)
+	certResources, err := initTLSCertificates("certs", profile.ProfilePath, tlsLocalServices)
 	if err != nil {
 		return fmt.Errorf("failed to create TLS files: %w", err)
 	}