Add initial support to validate the mappings in system tests (#2214)

Add validation of the mappings when running system tests. Currently, this
validation does not take into account dynamic templates.

This process compares the mappings installed by Fleet (preview mappings)
with the ones obtained after ingesting new documents into Elasticsearch.
This new validation needs to be enabled by setting a new environment 
variable "ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD=mappings".
By default, elastic-package keeps using the same mechanism to validate that
all fields are documented.

Author: mrodm

.buildkite/pipeline.trigger.integration.tests.sh

@@ -44,14 +44,9 @@ CHECK_PACKAGES_TESTS=(
     test-check-packages-benchmarks
     test-check-packages-with-logstash
 )
-for independent_agent in false true ; do
 for test in "${CHECK_PACKAGES_TESTS[@]}"; do
-    label_suffix=""
-    if [[ "$independent_agent" == "false" ]]; then
-        label_suffix=" (stack agent)"
-    fi
     test_name=${test#"test-check-packages-"}
-    echo "      - label: \":go: Integration test: ${test_name}${label_suffix}\""
+    echo "      - label: \":go: Integration test: ${test_name}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
     echo "        agents:"
     echo "          provider: \"gcp\""
@@ -64,15 +59,10 @@ for test in "${CHECK_PACKAGES_TESTS[@]}"; do
     if [[ $test =~ with-kind$ ]]; then
         echo "          - build/kubectl-dump.txt"
     fi
-    if [[ "${independent_agent}" == "false" ]]; then
-        echo "        env:"
-        echo "          ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}"
-    fi
-done
 done
 
 pushd test/packages/false_positives > /dev/null
-for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
+while IFS= read -r -d '' package ; do
     package_name=$(basename "${package}")
     echo "      - label: \":go: Integration test (false positive): ${package_name}\""
     echo "        key: \"integration-false_positives-${package_name}\""
@@ -86,48 +76,56 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "          - build/test-results/*.xml"
     echo "          - build/test-results/*.xml.expected-errors.txt"  # these files are uploaded in case it is needed to review the xUnit files in case of CI reports success the step
     echo "          - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
-done
+done < <(find . -maxdepth 1 -mindepth 1 -type d -print0)
 popd > /dev/null
 
 pushd test/packages/parallel > /dev/null
-for independent_agent in false true; do
-for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
-    label_suffix=""
-    if [[ "$independent_agent" == "false" ]]; then
-        label_suffix=" (stack agent)"
-    fi
+while IFS= read -r -d '' package ; do
     package_name=$(basename "${package}")
-    if [[ "$independent_agent" == "false" && "$package_name" == "oracle" ]]; then
-        echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing required software)."
-        continue
-    fi
-
-    if [[ "$independent_agent" == "false" && "$package_name" == "auditd_manager" ]]; then
-        echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing capabilities)."
-        continue
-    fi
+    echo "      - label: \":go: Integration test: ${package_name}\""
+    echo "        key: \"integration-parallel-${package_name}-agent\""
+    echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
+    echo "        env:"
+    echo "          UPLOAD_SAFE_LOGS: 1"
+    echo "        agents:"
+    echo "          provider: \"gcp\""
+    echo "          image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
+    echo "        artifact_paths:"
+    echo "          - build/test-results/*.xml"
+    echo "          - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
+done < <(find . -maxdepth 1 -mindepth 1 -type d -print0)
 
-    if [[ "$independent_agent" == "false" && "$package_name" == "custom_entrypoint" ]]; then
-        echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing required files deployed in provisioning)."
-        continue
-    fi
+# Run system tests with the Elastic Agent from the Elastic stack just for one package
+package_name="apache"
+echo "      - label: \":go: Integration test: ${package_name} (stack agent)\""
+echo "        key: \"integration-parallel-${package_name}-stack-agent\""
+echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
+echo "        env:"
+echo "          UPLOAD_SAFE_LOGS: 1"
+echo "          ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: false"
+echo "        agents:"
+echo "          provider: \"gcp\""
+echo "          image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
+echo "        artifact_paths:"
+echo "          - build/test-results/*.xml"
+echo "          - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
 
-    echo "      - label: \":go: Integration test: ${package_name}${label_suffix}\""
-    echo "        key: \"integration-parallel-${package_name}-agent-${independent_agent}\""
+# Add steps to test validation method mappings
+while IFS= read -r -d '' package ; do
+    package_name=$(basename "${package}")
+    echo "      - label: \":go: Integration test: ${package_name} (just validate mappings)\""
+    echo "        key: \"integration-parallel-${package_name}-agent-validate-mappings\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"
     echo "        env:"
     echo "          UPLOAD_SAFE_LOGS: 1"
-    if [[ "${independent_agent}" == "false" ]]; then
-        echo "          ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}"
-    fi
+    echo "          ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD: mappings"
     echo "        agents:"
     echo "          provider: \"gcp\""
     echo "          image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
     echo "          - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package
-done
-done
+done < <(find . -maxdepth 1 -mindepth 1 -type d -print0)
 popd > /dev/null
 
 # TODO: Missing docker & docker-compose in MACOS ARM agent image, skip installation of packages in the meantime.
@@ -166,19 +164,11 @@ echo "          image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
 echo "        artifact_paths:"
 echo "          - build/elastic-stack-dump/install-zip-shellinit/logs/*.log"
 
-for independent_agent in false true; do
-    label_suffix=""
-    if [[ "$independent_agent" == "false" ]]; then
-        label_suffix=" (stack agent)"
-    fi
-    echo "      - label: \":go: Integration test: system-flags${label_suffix}\""
-    echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-system-test-flags"
-    echo "        agents:"
-    echo "          provider: \"gcp\""
-    echo "          image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
-    echo "        env:"
-    echo "          ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: ${independent_agent}"
-done
+echo "      - label: \":go: Integration test: system-flags\""
+echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-system-test-flags"
+echo "        agents:"
+echo "          provider: \"gcp\""
+echo "          image: \"${UBUNTU_X86_64_AGENT_IMAGE}\""
 
 echo "      - label: \":go: Integration test: profiles-command\""
 echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command"

.buildkite/scripts/integration_tests.sh

@@ -122,6 +122,10 @@ if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSIT
             package_folder="${package_folder}-stack_agent"
         fi
 
+        if [[ "${ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD:-""}" != "" ]]; then
+            package_folder="${package_folder}-${ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD}"
+        fi
+
         if [[ "${retry_count}" -ne 0 ]]; then
             package_folder="${package_folder}_retry_${retry_count}"
         fi

README.md

@@ -677,9 +677,16 @@ There are available some environment variables that could be used to change some
     - `ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI`: If set to `true`, the Elastic Agent image used for running agents will be using the Ubuntu docker images
       (e.g. `docker.elastic.co/elastic-agent/elastic-agent-complete`). If set to `false`, the Elastic Agent image used for the running agents will be based on the wolfi
       images (e.g. `docker.elastic.co/elastic-agent/elastic-agent-wolfi`). Default: `false`.
-    - `ELASTIC_PACKAGE_TEST_DUMP_SCENARIO_DOCS. If the variable is set, elastic-package will dump to a file the documents generated
+    - `ELASTIC_PACKAGE_TEST_DUMP_SCENARIO_DOCS`. If the variable is set, elastic-package will dump to a file the documents generated
       by system tests before they are verified. This is useful to know exactly what fields are being verified when investigating
       issues on this step. Documents are dumped to a file in the system temporary directory. It is disabled by default.
+    - `ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT`. If the variable is set to false, all system tests defined in the package will use
+      the Elastic Agent started along with the stack. If set to true, a new Elastic Agent will be started and enrolled for each test defined in the
+      package (and unenrolled at the end of each test). Default: `true`.
+    - `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `all`, `mappings` or `fields`. If this
+      variable is set to `fields`, then validation of fields will be based on the documents ingested into Elasticsearch. If this is set to
+      `mappings`, then validation of fields will be based on the mappings generated when the documents are ingested into Elasticsearch. If
+      set to `all`, then validation will be based on both methods mentioned previously. Default option: `fields`.
 
 - To configure the Elastic stack to be used by `elastic-package`:
     - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)

internal/elasticsearch/client.go

@@ -279,3 +279,85 @@ func (client *Client) redHealthCause(ctx context.Context) (string, error) {
 	}
 	return strings.Join(causes, ", "), nil
 }
+
+func (c *Client) SimulateIndexTemplate(ctx context.Context, indexTemplateName string) (json.RawMessage, json.RawMessage, error) {
+	resp, err := c.Indices.SimulateTemplate(
+		c.Indices.SimulateTemplate.WithContext(ctx),
+		c.Indices.SimulateTemplate.WithName(indexTemplateName),
+	)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to get field mapping for data stream %q: %w", indexTemplateName, err)
+	}
+	defer resp.Body.Close()
+	if resp.IsError() {
+		return nil, nil, fmt.Errorf("error getting mapping: %s", resp)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error reading mapping body: %w", err)
+	}
+
+	type mappingsIndexTemplate struct {
+		DynamicTemplates json.RawMessage `json:"dynamic_templates"`
+		Properties       json.RawMessage `json:"properties"`
+	}
+
+	type indexTemplateSimulated struct {
+		// Settings json.RawMessage       `json:"settings"`
+		Mappings mappingsIndexTemplate `json:"mappings"`
+	}
+
+	type previewTemplate struct {
+		Template indexTemplateSimulated `json:"template"`
+	}
+
+	var preview previewTemplate
+
+	if err := json.Unmarshal(body, &preview); err != nil {
+		return nil, nil, fmt.Errorf("error unmarshaling mappings: %w", err)
+	}
+
+	return preview.Template.Mappings.DynamicTemplates, preview.Template.Mappings.Properties, nil
+}
+
+func (c *Client) DataStreamMappings(ctx context.Context, dataStreamName string) (json.RawMessage, json.RawMessage, error) {
+	mappingResp, err := c.Indices.GetMapping(
+		c.Indices.GetMapping.WithContext(ctx),
+		c.Indices.GetMapping.WithIndex(dataStreamName),
+	)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to get field mapping for data stream %q: %w", dataStreamName, err)
+	}
+	defer mappingResp.Body.Close()
+	if mappingResp.IsError() {
+		return nil, nil, fmt.Errorf("error getting mapping: %s", mappingResp)
+	}
+	body, err := io.ReadAll(mappingResp.Body)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error reading mapping body: %w", err)
+	}
+
+	type mappings struct {
+		DynamicTemplates json.RawMessage `json:"dynamic_templates"`
+		Properties       json.RawMessage `json:"properties"`
+	}
+
+	mappingsRaw := map[string]struct {
+		Mappings mappings `json:"mappings"`
+	}{}
+
+	if err := json.Unmarshal(body, &mappingsRaw); err != nil {
+		return nil, nil, fmt.Errorf("error unmarshaling mappings: %w", err)
+	}
+
+	if len(mappingsRaw) != 1 {
+		return nil, nil, fmt.Errorf("exactly 1 mapping was expected, got %d", len(mappingsRaw))
+	}
+
+	var mappingsDefinition mappings
+	for _, v := range mappingsRaw {
+		mappingsDefinition = v.Mappings
+	}
+
+	return mappingsDefinition.DynamicTemplates, mappingsDefinition.Properties, nil
+}

internal/fields/dependency_manager.go

@@ -134,9 +134,31 @@ func parseECSFieldsSchema(content []byte) ([]FieldDefinition, error) {
 		return nil, fmt.Errorf("unmarshalling field body failed: %w", err)
 	}
 
+	// Force to set External as "ecs" in all these fields to be able to distinguish
+	// which fields come from ECS and which ones are loaded from the package directory
+	fields = setExternalAsECS(fields)
+
 	return fields, nil
 }
 
+func setExternalAsECS(fields []FieldDefinition) []FieldDefinition {
+	for i := 0; i < len(fields); i++ {
+		f := &fields[i]
+		f.External = "ecs"
+		if len(f.MultiFields) > 0 {
+			for j := 0; j < len(f.MultiFields); j++ {
+				mf := &f.MultiFields[j]
+				mf.External = "ecs"
+			}
+		}
+		if len(f.Fields) > 0 {
+			f.Fields = setExternalAsECS(f.Fields)
+		}
+	}
+
+	return fields
+}
+
 func asGitReference(reference string) (string, error) {
 	if !strings.HasPrefix(reference, gitReferencePrefix) {
 		return "", errors.New(`invalid Git reference ("git@" prefix expected)`)

internal/fields/dependency_manager_test.go

@@ -785,3 +785,55 @@ func TestDependencyManagerWithECS(t *testing.T) {
 		})
 	}
 }
+
+func TestValidate_SetExternalECS(t *testing.T) {
+	finder := packageRootTestFinder{"../../test/packages/other/imported_mappings_tests"}
+
+	validator, err := createValidatorForDirectoryAndPackageRoot("../../test/packages/other/imported_mappings_tests/data_stream/first",
+		finder,
+		WithSpecVersion("2.3.0"),
+		WithEnabledImportAllECSSChema(true))
+	require.NoError(t, err)
+	require.NotNil(t, validator)
+
+	require.NotEmpty(t, validator.Schema)
+
+	cases := []struct {
+		title    string
+		field    string
+		external string
+		exists   bool
+	}{
+		{
+			title:    "field defined just in ECS",
+			field:    "ecs.version",
+			external: "ecs",
+			exists:   true,
+		},
+		{
+			title:    "field defined fields directory package",
+			field:    "service.status.duration.histogram",
+			external: "",
+			exists:   true,
+		},
+		{
+			title:    "undefined field",
+			field:    "foo",
+			external: "",
+			exists:   false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			found := FindElementDefinition(c.field, validator.Schema)
+			if !c.exists {
+				assert.Nil(t, found)
+				return
+			}
+
+			require.NotNil(t, found)
+			assert.Equal(t, c.external, found.External)
+		})
+	}
+}