Allow to define more than one service deployer and choose the deployer via configuration setting (#2638)

This PR allows us to define more than one service deployer in the
"_dev/deploy" folder (at package or data stream level). If there are
more than one service deployer defined, it must be added to the
test system configuration file a new setting deployer. This new setting
helps us to choose which specific specific service deployer will be
used by each test.

Author: mrodm

docs/howto/system_testing.md

@@ -387,6 +387,45 @@ wget -qO-  https://raw.github.com/elastic/elastic-package/main/script
 elastic-package test system --data-streams pod -v # start system tests for the "pod" data stream
 ```
 
+
+### Defining more than one service deployer
+
+Since `elastic-package` v0.113.0, it is allowed to define more than one service deployer in each `_dev/deploy` folder. And each system test
+configuration can choose which service deployer to use among them.
+For instance, a data stream could contain a definition for Docker Compose and a Terraform service deployers.
+
+First, `elastic-package` looks for the corresponding `_dev/folder` to use. It will follow this order, and the first one that exists has
+preference:
+- Deploy folder at Data Stream level: `packages/<package_name>/data_stream/<data_stream_name>/_dev/deploy/`
+- Deploy folder at Package level: `packages/<package_name>/data_stream/<data_stream_name>/_dev/deploy/`
+
+If there is more than one service deployer defined in the deploy folder found, the system test configuration files of the
+required tests must set the `deployer` field to choose which service deployer configure and start for that given test. If that setting
+is not defined and there are more than one service edployer, `elastic-package` will fail with an error since it is not supported
+to run several service deployers at the same time.
+
+Example of system test configuration including `deployer` setting:
+
+```yaml
+deployer: docker
+service: nginx
+vars: ~
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/access.log*"
+```
+
+In this example, `elastic-package` looks for a Docker Compose service deployer in the given `_dev/deploy` folder found previously.
+
+Each service deployer folder keep the same format and files as defined in previous sections.
+
+For instance, this allows to test one data stream using different inputs, each input with a different service deployer. One of them could be using
+the Docker Compose service deployer, and another input could be using terraform to create resources in AWS.
+
+You can find an example of a package using this in this [test package](../../test/packages/parallel/nginx_multiple_services/).
+
+
 ### Test case definition
 
 Next, we must define at least one configuration for each data stream that we
@@ -421,7 +460,11 @@ for system tests.
 | agent.provisioning_script.language | string | | Programming language of the provisioning script. Default: `sh`. |
 | agent.provisioning_script.contents | string | | Code to run as a provisioning script to customize the system where the agent will be run. |
 | agent.user | string | | User that runs the Elastic Agent process. |
+| assert.hit_count | integer |  | Exact number of documents to wait for being ingested. |
+| assert.min_count | integer |  | Minimum number of documents to wait for being ingested. |
+| assert.fields_present | []string|  | List of fields that must be present in the documents to stop waiting for new documents. |
 | data_stream.vars | dictionary |  | Data stream level variables to set (i.e. declared in `package_root/data_stream/$data_stream/manifest.yml`). If not specified the defaults from the manifest are used. |
+| deployer | string|  | Name of the service deployer to setup for this system test. Available values: docker, tf or k8s. |
 | ignore_service_error | boolean | no | If `true`, it will ignore any failures in the deployed test services. Defaults to `false`. |
 | input | string | yes | Input type to test (e.g. logfile, httpjson, etc). Defaults to the input used by the first stream in the data stream manifest. |
 | numeric_keyword_fields | []string |  | List of fields to ignore during validation that are mapped as `keyword` in Elasticsearch, but their JSON data type is a number. |
@@ -434,9 +477,6 @@ for system tests.
 | skip_transform_validation | boolean |  | Disable or enable the transforms validation performed in system tests. |
 | vars | dictionary |  | Package level variables to set (i.e. declared in `$package_root/manifest.yml`). If not specified the defaults from the manifest are used. |
 | wait_for_data_timeout | duration |  | Amount of time to wait for data to be present in Elasticsearch. Defaults to 10m. |
-| assert.hit_count | integer |  | Exact number of documents to wait for being ingested. |
-| assert.min_count | integer |  | Minimum number of documents to wait for being ingested. |
-| assert.fields_present | []string|  | List of fields that must be present in the documents to stop waiting for new documents. |
 
 For example, the `apache/access` data stream's `test-access-log-config.yml` is
 shown below.

internal/agentdeployer/factory.go

@@ -8,10 +8,11 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
+	"slices"
 
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/profile"
+	"github.com/elastic/elastic-package/internal/servicedeployer"
 )
 
 const (
@@ -30,6 +31,8 @@ type FactoryOptions struct {
 	StackVersion       string
 	PolicyName         string
 
+	DeployerName string
+
 	PackageName string
 	DataStream  string
 
@@ -81,27 +84,26 @@ func Factory(options FactoryOptions) (AgentDeployer, error) {
 }
 
 func selectAgentDeployerType(options FactoryOptions) (string, error) {
-	devDeployPath, err := FindDevDeployPath(options)
+	devDeployPath, err := servicedeployer.FindDevDeployPath(servicedeployer.FactoryOptions{
+		DataStreamRootPath: options.DataStreamRootPath,
+		DevDeployDir:       options.DevDeployDir,
+		PackageRootPath:    options.PackageRootPath,
+	})
 	if errors.Is(err, os.ErrNotExist) {
 		return "default", nil
 	}
 	if err != nil {
 		return "", fmt.Errorf("can't find \"%s\" directory: %w", options.DevDeployDir, err)
 	}
 
-	agentDeployerNames, err := findAgentDeployers(devDeployPath)
-	if errors.Is(err, os.ErrNotExist) || len(agentDeployerNames) == 0 {
+	agentDeployerName, err := findAgentDeployer(devDeployPath, options.DeployerName)
+	if errors.Is(err, os.ErrNotExist) || (err == nil && agentDeployerName == "") {
 		logger.Debugf("Not agent deployer found, using default one")
 		return "default", nil
 	}
 	if err != nil {
 		return "", fmt.Errorf("failed to find agent deployer: %w", err)
 	}
-	if len(agentDeployerNames) != 1 {
-		return "", fmt.Errorf("expected to find only one agent deployer in \"%s\"", devDeployPath)
-	}
-	agentDeployerName := agentDeployerNames[0]
-
 	// if package defines `_dev/deploy/docker` or `_dev/deploy/tf` folder to start their services,
 	// it should be using the default agent deployer`
 	if agentDeployerName == "docker" || agentDeployerName == "tf" {
@@ -111,43 +113,28 @@ func selectAgentDeployerType(options FactoryOptions) (string, error) {
 	return agentDeployerName, nil
 }
 
-// FindDevDeployPath function returns a path reference to the "_dev/deploy" directory.
-func FindDevDeployPath(options FactoryOptions) (string, error) {
-	dataStreamDevDeployPath := filepath.Join(options.DataStreamRootPath, options.DevDeployDir)
-	info, err := os.Stat(dataStreamDevDeployPath)
-	if err == nil && info.IsDir() {
-		return dataStreamDevDeployPath, nil
-	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return "", fmt.Errorf("stat failed for data stream (path: %s): %w", dataStreamDevDeployPath, err)
+func findAgentDeployer(devDeployPath, expectedDeployer string) (string, error) {
+	names, err := servicedeployer.FindAllServiceDeployers(devDeployPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to find service deployers in \"%s\": %w", devDeployPath, err)
 	}
+	deployers := slices.DeleteFunc(names, func(name string) bool {
+		return expectedDeployer != "" && name != expectedDeployer
+	})
 
-	packageDevDeployPath := filepath.Join(options.PackageRootPath, options.DevDeployDir)
-	info, err = os.Stat(packageDevDeployPath)
-	if err == nil && info.IsDir() {
-		return packageDevDeployPath, nil
-	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return "", fmt.Errorf("stat failed for package (path: %s): %w", packageDevDeployPath, err)
+	// If we have more than one agent deployer, we expect to find only one.
+	if expectedDeployer != "" && len(deployers) != 1 {
+		return "", fmt.Errorf("expected to find %q agent deployer in %q", expectedDeployer, devDeployPath)
 	}
 
-	return "", fmt.Errorf("\"%s\" %w", options.DevDeployDir, os.ErrNotExist)
-}
-
-func findAgentDeployers(devDeployPath string) ([]string, error) {
-	fis, err := os.ReadDir(devDeployPath)
-	if err != nil {
-		return nil, fmt.Errorf("can't read directory (path: %s): %w", devDeployPath, err)
+	// It is allowed to have no agent deployers
+	if len(deployers) == 0 {
+		return "", nil
 	}
 
-	var folders []os.DirEntry
-	for _, fi := range fis {
-		if fi.IsDir() {
-			folders = append(folders, fi)
-		}
+	if len(deployers) == 1 {
+		return deployers[0], nil
 	}
 
-	var names []string
-	for _, folder := range folders {
-		names = append(names, folder.Name())
-	}
-	return names, nil
+	return "", fmt.Errorf("expected to find only one agent deployer in \"%s\" (found %d agent deployers)", devDeployPath, len(deployers))
 }

internal/servicedeployer/factory.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 
 	"github.com/elastic/elastic-package/internal/profile"
 )
@@ -31,6 +32,8 @@ type FactoryOptions struct {
 
 	PolicyName string
 
+	DeployerName string
+
 	Variant string
 
 	RunTearDown  bool
@@ -46,10 +49,15 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) {
 		return nil, fmt.Errorf("can't find \"%s\" directory: %w", options.DevDeployDir, err)
 	}
 
-	serviceDeployerName, err := findServiceDeployer(devDeployPath)
+	serviceDeployerName, err := findServiceDeployer(devDeployPath, options.DeployerName)
 	if err != nil {
 		return nil, fmt.Errorf("can't find any valid service deployer: %w", err)
 	}
+	// It's allowed to not define a service deployer in system tests
+	// if deployerName is not defined in the test configuration.
+	if serviceDeployerName == "" {
+		return nil, nil
+	}
 
 	serviceDeployerPath := filepath.Join(devDeployPath, serviceDeployerName)
 
@@ -123,37 +131,62 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) {
 // FindDevDeployPath function returns a path reference to the "_dev/deploy" directory.
 func FindDevDeployPath(options FactoryOptions) (string, error) {
 	dataStreamDevDeployPath := filepath.Join(options.DataStreamRootPath, options.DevDeployDir)
-	if _, err := os.Stat(dataStreamDevDeployPath); err == nil {
+	info, err := os.Stat(dataStreamDevDeployPath)
+	if err == nil && info.IsDir() {
 		return dataStreamDevDeployPath, nil
-	} else if !errors.Is(err, os.ErrNotExist) {
+	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return "", fmt.Errorf("stat failed for data stream (path: %s): %w", dataStreamDevDeployPath, err)
 	}
 
 	packageDevDeployPath := filepath.Join(options.PackageRootPath, options.DevDeployDir)
-	if _, err := os.Stat(packageDevDeployPath); err == nil {
+	info, err = os.Stat(packageDevDeployPath)
+	if err == nil && info.IsDir() {
 		return packageDevDeployPath, nil
-	} else if !errors.Is(err, os.ErrNotExist) {
+	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return "", fmt.Errorf("stat failed for package (path: %s): %w", packageDevDeployPath, err)
 	}
 
 	return "", fmt.Errorf("\"%s\" %w", options.DevDeployDir, os.ErrNotExist)
 }
 
-func findServiceDeployer(devDeployPath string) (string, error) {
+func FindAllServiceDeployers(devDeployPath string) ([]string, error) {
 	fis, err := os.ReadDir(devDeployPath)
 	if err != nil {
-		return "", fmt.Errorf("can't read directory (path: %s): %w", devDeployPath, err)
+		return nil, fmt.Errorf("can't read directory (path: %s): %w", devDeployPath, err)
 	}
 
-	var folders []os.DirEntry
+	var names []string
 	for _, fi := range fis {
 		if fi.IsDir() {
-			folders = append(folders, fi)
+			names = append(names, fi.Name())
 		}
 	}
 
-	if len(folders) != 1 {
-		return "", fmt.Errorf("expected to find only one service deployer in \"%s\"", devDeployPath)
+	return names, nil
+}
+
+func findServiceDeployer(devDeployPath, expectedDeployer string) (string, error) {
+	names, err := FindAllServiceDeployers(devDeployPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to find service deployers in %q: %w", devDeployPath, err)
+	}
+	deployers := slices.DeleteFunc(names, func(name string) bool {
+		return expectedDeployer != "" && name != expectedDeployer
+	})
+
+	if len(deployers) == 1 {
+		return deployers[0], nil
+	}
+
+	if expectedDeployer != "" {
+		return "", fmt.Errorf("expected to find %q service deployer in %q", expectedDeployer, devDeployPath)
 	}
-	return folders[0].Name(), nil
+
+	// If "_dev/deploy" directory exists, but it is empty. It does not have any service deployer,
+	// package-spec does not disallow to be empty this folder.
+	if len(deployers) == 0 {
+		return "", nil
+	}
+
+	return "", fmt.Errorf("expected to find only one service deployer in %q (found %d service deployers)", devDeployPath, len(deployers))
 }

internal/testrunner/runners/system/test_config.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strings"
 	"time"
 
@@ -24,7 +25,10 @@ import (
 	"github.com/elastic/elastic-package/internal/testrunner"
 )
 
-var systemTestConfigFilePattern = regexp.MustCompile(`^test-([a-z0-9_.-]+)-config.yml$`)
+var (
+	systemTestConfigFilePattern = regexp.MustCompile(`^test-([a-z0-9_.-]+)-config.yml$`)
+	allowedDeployerNames        = []string{"docker", "k8s", "tf"}
+)
 
 type testConfig struct {
 	testrunner.SkippableConfig `config:",inline"`
@@ -37,6 +41,8 @@ type testConfig struct {
 	WaitForDataTimeout  time.Duration `config:"wait_for_data_timeout"`
 	SkipIgnoredFields   []string      `config:"skip_ignored_fields"`
 
+	Deployer string `config:"deployer"` // Name of the service deployer to use for this test.
+
 	Vars       common.MapStr `config:"vars"`
 	DataStream struct {
 		Vars common.MapStr `config:"vars"`
@@ -129,6 +135,11 @@ func newConfig(configFilePath string, svcInfo servicedeployer.ServiceInfo, servi
 		c.Agent.PreStartScript.Language = agentdeployer.DefaultAgentProgrammingLanguage
 	}
 
+	// Not included in package-spec validation for deployer name
+	if c.Deployer != "" && !slices.Contains(allowedDeployerNames, c.Deployer) {
+		return nil, fmt.Errorf("invalid deployer name %q in system test configuration file %q, allowed values are: %s", c.Deployer, configFilePath, strings.Join(allowedDeployerNames, ", "))
+	}
+
 	return &c, nil
 }