Allow to run Elastic stack with other licenses (#2501)

This PR adds support into elastic-package to run an Elastic stack
with a different subscription. By default it keeps the trial subscription.
And in this PR it is added support to run an Elastic stack with basic
subscription.

Author: mrodm

.buildkite/pipeline.trigger.integration.tests.sh

@@ -22,6 +22,8 @@ STACK_COMMAND_TESTS=(
     test-stack-command-8x
     test-stack-command-9x
     test-stack-command-with-apm-server
+    test-stack-command-with-basic-subscription
+    test-stack-command-with-self-monitor
 )
 
 for test in "${STACK_COMMAND_TESTS[@]}"; do

Makefile

@@ -84,6 +84,9 @@ test-stack-command-with-apm-server:
 test-stack-command-with-self-monitor:
 	SELF_MONITOR_ENABLED=true ./scripts/test-stack-command.sh
 
+test-stack-command-with-basic-subscription:
+	ELASTIC_SUBSCRIPTION=basic ./scripts/test-stack-command.sh
+
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x test-stack-command-9x test-stack-command-with-apm-server
 
 test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash

README.md

@@ -652,6 +652,9 @@ The following settings are available per profile:
   the serverless stack provider.
 * `stack.serverless.region` can be used to select the region to use when starting
   serverless projects.
+* `stack.elastic_subscription` allows to select the Elastic subscription type to be used in the stack.
+  Currently, it is supported "basic" and "[trial](https://www.elastic.co/guide/en/elasticsearch/reference/current/start-trial.html)",
+  which enables all subscription features for 30 days.  Defaults to "trial".
 
 ## Useful environment variables
 

internal/profile/_static/config.yml.example

@@ -27,3 +27,5 @@
 # stack.agent.ports:
 # - 127.0.0.1:1514:1514/udp
 
+## Set license subscription
+# stack.elastic_subscription: "basic"

internal/profile/testdata/config.yml

@@ -2,6 +2,7 @@
 stack.geoip_dir: "/home/foo/Documents/ingest-geoip"
 stack.apm_enabled: true
 stack.logstash_enabled: true
+stack.elastic_subscription: basic
 
 # An empty string, should exist, but return empty.
 other.empty: ""

internal/stack/_static/elasticsearch.yml.tmpl

@@ -4,7 +4,8 @@ http.host: "0.0.0.0"
 
 indices.id_field_data.enabled: true
 
-xpack.license.self_generated.type: "trial"
+{{ $elastic_subscription := fact "elastic_subscription" }}
+xpack.license.self_generated.type: "{{ $elastic_subscription }}"
 xpack.security.enabled: true
 xpack.security.authc.api_key.enabled: true
 xpack.security.http.ssl.enabled: true

internal/stack/_static/kibana.yml.tmpl

@@ -161,3 +161,7 @@ xpack.fleet.outputs:
 {{- if eq $version "9.0.0-SNAPSHOT" }}
 xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false
 {{- end }}
+
+logging.loggers:
+  - name: plugins.fleet
+    level: debug

internal/stack/resources.go

@@ -11,6 +11,7 @@ import (
 	"html/template"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/Masterminds/semver/v3"
@@ -57,12 +58,13 @@ const (
 	elasticsearchUsername = "elastic"
 	elasticsearchPassword = "changeme"
 
-	configAPMEnabled         = "stack.apm_enabled"
-	configGeoIPDir           = "stack.geoip_dir"
-	configKibanaHTTP2Enabled = "stack.kibana_http2_enabled"
-	configLogsDBEnabled      = "stack.logsdb_enabled"
-	configLogstashEnabled    = "stack.logstash_enabled"
-	configSelfMonitorEnabled = "stack.self_monitor_enabled"
+	configAPMEnabled          = "stack.apm_enabled"
+	configGeoIPDir            = "stack.geoip_dir"
+	configKibanaHTTP2Enabled  = "stack.kibana_http2_enabled"
+	configLogsDBEnabled       = "stack.logsdb_enabled"
+	configLogstashEnabled     = "stack.logstash_enabled"
+	configSelfMonitorEnabled  = "stack.self_monitor_enabled"
+	configElasticSubscription = "stack.elastic_subscription"
 )
 
 var (
@@ -135,6 +137,11 @@ var (
 			Content: staticSource.File("_static/Dockerfile.logstash"),
 		},
 	}
+
+	elasticSubscriptionsSupported = []string{
+		"basic",
+		"trial",
+	}
 )
 
 func applyResources(profile *profile.Profile, stackVersion string) error {
@@ -145,6 +152,11 @@ func applyResources(profile *profile.Profile, stackVersion string) error {
 		return fmt.Errorf("failed to unmarshal stack.agent.ports: %w", err)
 	}
 
+	elasticSubscriptionProfile := profile.Config(configElasticSubscription, "trial")
+	if !slices.Contains(elasticSubscriptionsSupported, elasticSubscriptionProfile) {
+		return fmt.Errorf("unsupported Elastic subscription %q: supported subscriptions: %s", elasticSubscriptionProfile, strings.Join(elasticSubscriptionsSupported, ", "))
+	}
+
 	resourceManager := resource.NewManager()
 	resourceManager.AddFacter(resource.StaticFacter{
 		"registry_base_image":   PackageRegistryBaseImage,
@@ -168,6 +180,7 @@ func applyResources(profile *profile.Profile, stackVersion string) error {
 		"logsdb_enabled":       profile.Config(configLogsDBEnabled, "false"),
 		"logstash_enabled":     profile.Config(configLogstashEnabled, "false"),
 		"self_monitor_enabled": profile.Config(configSelfMonitorEnabled, "false"),
+		"elastic_subscription": elasticSubscriptionProfile,
 	})
 
 	if err := os.MkdirAll(stackDir, 0755); err != nil {

scripts/test-stack-command.sh

@@ -5,6 +5,7 @@ set -euxo pipefail
 VERSION=${1:-default}
 APM_SERVER_ENABLED=${APM_SERVER_ENABLED:-false}
 SELF_MONITOR_ENABLED=${SELF_MONITOR_ENABLED:-false}
+ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""}
 
 cleanup() {
   r=$?
@@ -23,6 +24,10 @@ cleanup() {
     elastic-package profiles delete with-self-monitor
   fi
 
+  if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then
+    elastic-package profiles delete with-elastic-subscription
+  fi
+
   exit $r
 }
 
@@ -71,6 +76,16 @@ stack.self_monitor_enabled: true
 EOF
 fi
 
+if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then
+  profile=with-elastic-subscription
+  elastic-package profiles create -v ${profile}
+  elastic-package profiles use ${profile}
+
+  cat ~/.elastic-package/profiles/${profile}/config.yml.example - <<EOF > ~/.elastic-package/profiles/${profile}/config.yml
+stack.elastic_subscription: ${ELASTIC_SUBSCRIPTION}
+EOF
+fi
+
 mkdir -p "${OUTPUT_PATH_STATUS}"
 
 # Initial status empty
@@ -115,6 +130,8 @@ elastic-package stack status -v 2> "${OUTPUT_PATH_STATUS}/running.txt"
 clean_status_output "${OUTPUT_PATH_STATUS}/expected_running.txt" > "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt"
 clean_status_output "${OUTPUT_PATH_STATUS}/running.txt" > "${OUTPUT_PATH_STATUS}/running_no_spaces.txt"
 
+diff -q "${OUTPUT_PATH_STATUS}/running_no_spaces.txt" "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt"
+
 if [ "${APM_SERVER_ENABLED}" = true ]; then
   curl http://localhost:8200/
 fi
@@ -127,4 +144,17 @@ if [ "${SELF_MONITOR_ENABLED}" = true ]; then
     -f "${ELASTIC_PACKAGE_ELASTICSEARCH_HOST}/metrics-system.*/_search?allow_no_indices=false&size=0"
 fi
 
-diff -q "${OUTPUT_PATH_STATUS}/running_no_spaces.txt" "${OUTPUT_PATH_STATUS}/expected_no_spaces.txt"
+subscription=$(curl -s -S \
+  -u "${ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME}:${ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD}" \
+  --cacert "${ELASTIC_PACKAGE_CA_CERT}" \
+  -f "${ELASTIC_PACKAGE_ELASTICSEARCH_HOST}/_license" |jq -r '.license.type')
+
+expected_subscription="trial"
+if [[ "${ELASTIC_SUBSCRIPTION}" != "" ]]; then
+    expected_subscription="${ELASTIC_SUBSCRIPTION}"
+fi
+
+if [[ "${subscription}" != "${expected_subscription}" ]]; then
+    echo "Unexpected \"${subscription}\" subscription found, but expected \"${expected_subscription}\""
+    exit 1
+fi

test/packages/parallel/apache_basic_license.stack_provider_settings

@@ -0,0 +1 @@
+stack.elastic_subscription=basic