From cd24f7f8f62581e318b405dbe6e5d64325c21abc Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 14 Feb 2025 14:01:59 +0100 Subject: [PATCH 1/5] Set by default validation based on mappings --- .buildkite/pipeline.trigger.integration.tests.sh | 16 ---------------- internal/testrunner/runners/system/tester.go | 4 ++-- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 2d9276f604..9ec2bffbe1 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -110,22 +110,6 @@ echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package -# Add steps to test validation method mappings -while IFS= read -r -d '' package ; do - package_name=$(basename "${package}") - echo " - label: \":go: Integration test: ${package_name} (validate mappings)\"" - echo " key: \"integration-parallel-${package_name}-agent-validate-mappings\"" - echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" - echo " env:" - echo " UPLOAD_SAFE_LOGS: 1" - echo " ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD: mappings" - echo " agents:" - echo " provider: \"gcp\"" - echo " image: \"${UBUNTU_X86_64_AGENT_IMAGE}\"" - echo " artifact_paths:" - echo " - build/test-results/*.xml" - echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package -done < <(find . -maxdepth 1 -mindepth 1 -type d -print0) popd > /dev/null # TODO: Missing docker & docker-compose in MACOS ARM agent image, skip installation of packages in the meantime. diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index 6ac51a25bf..a5424a987d 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -288,8 +288,8 @@ func NewSystemTester(options SystemTesterOptions) (*tester, error) { r.runIndependentElasticAgent = strings.ToLower(v) == "true" } - // default method using just fields - r.fieldValidationMethod = fieldsMethod + // default method to validate using mappings (along with fields) + r.fieldValidationMethod = mappingsMethod v, ok = os.LookupEnv(fieldValidationTestMethodEnv) if ok { method, ok := validationMethods[v] From 2ff0a93af8dd02699b17628cc7b087646dde4d03 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 14 Feb 2025 17:52:10 +0100 Subject: [PATCH 2/5] Remove technical preview --- internal/testrunner/runners/system/tester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index a5424a987d..265f368e18 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -1646,7 +1646,7 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re } if r.fieldValidationMethod == mappingsMethod { - logger.Warn("Validation based on mappings enabled (technical preview)") + logger.Debug("Performing validation based on mappings") exceptionFields := listExceptionFields(scenario.docs, fieldsValidator) mappingsValidator, err := fields.CreateValidatorForMappings(r.esClient, From 1d00e536720b86d76613ebdad623378316effc17 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 14 Feb 2025 17:54:28 +0100 Subject: [PATCH 3/5] Update docs env. var --- tools/readme/readme.md.tmpl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl index 8070afd434..0bbbea880b 100644 --- a/tools/readme/readme.md.tmpl +++ b/tools/readme/readme.md.tmpl @@ -243,10 +243,11 @@ There are available some environment variables that could be used to change some - `ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT`. If the variable is set to false, all system tests defined in the package will use the Elastic Agent started along with the stack. If set to true, a new Elastic Agent will be started and enrolled for each test defined in the package (and unenrolled at the end of each test). Default: `true`. - - `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `all`, `mappings` or `fields`. If this - variable is set to `fields`, then validation of fields will be based on the documents ingested into Elasticsearch. If this is set to - `mappings`, then validation of fields will be based on the mappings generated when the documents are ingested into Elasticsearch. If - set to `all`, then validation will be based on both methods mentioned previously. Default option: `fields`. + - `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `mappings` or `fields`. If this + variable is set to `fields`, then validation of fields will be based on the contents of the documents ingested into Elasticsearch. If this is set to + `mappings`, then validation of fields will be based on their mappings generated when the documents are ingested into Elasticsearch as well as + the contents of the documents ingested into Elasticsearch. + Default option: `mappings`. - To configure the Elastic stack to be used by `elastic-package`: - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200) From c7a5ee7bcb4066893540ca5c7ab7e3afea155728 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 14 Feb 2025 18:00:37 +0100 Subject: [PATCH 4/5] Update system testing docs --- docs/howto/system_testing.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index c626e53053..8383dbed2f 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -18,6 +18,8 @@ Conceptually, running a system test involves the following steps: 1. Validate mappings are defined for the fields contained in the indexed documents. 1. Validate that the JSON data types contained `_source` are compatible with mappings declared for the field. +1. Validate mappings generated after ingesting documents are valid according to the definitions installed by the package. + - Requires `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD` to be unset or set to `mappings`. 1. If the Elastic Agent from the stack is not used, unenroll and remove the Elastic Agent as well as the test policies created. 1. Delete test artifacts and tear down the instance of the package's integration service. 1. Once the data stream have been system tested, unenroll and remove the Elastic Agent From 8e7fc3bf99792bbd411f85cb758ce29ffcb9e5b0 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 14 Feb 2025 18:14:54 +0100 Subject: [PATCH 5/5] Re-generate README --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d6d7fa4bb6..7af32d452f 100644 --- a/README.md +++ b/README.md @@ -689,10 +689,11 @@ There are available some environment variables that could be used to change some - `ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT`. If the variable is set to false, all system tests defined in the package will use the Elastic Agent started along with the stack. If set to true, a new Elastic Agent will be started and enrolled for each test defined in the package (and unenrolled at the end of each test). Default: `true`. - - `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `all`, `mappings` or `fields`. If this - variable is set to `fields`, then validation of fields will be based on the documents ingested into Elasticsearch. If this is set to - `mappings`, then validation of fields will be based on the mappings generated when the documents are ingested into Elasticsearch. If - set to `all`, then validation will be based on both methods mentioned previously. Default option: `fields`. + - `ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD`. This variable can take one of these values: `mappings` or `fields`. If this + variable is set to `fields`, then validation of fields will be based on the contents of the documents ingested into Elasticsearch. If this is set to + `mappings`, then validation of fields will be based on their mappings generated when the documents are ingested into Elasticsearch as well as + the contents of the documents ingested into Elasticsearch. + Default option: `mappings`. - To configure the Elastic stack to be used by `elastic-package`: - `ELASTIC_PACKAGE_ELASTICSEARCH_HOST`: Host of the elasticsearch (e.g. https://127.0.0.1:9200)