Merge branch '9.1' into mergify/bp/9.1/pr-45093

Author: mauri870

.buildkite/libbeat/pipeline.libbeat.yml

@@ -9,6 +9,7 @@ env:
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
+  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-beats-ubuntu-2204-fips"
 
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
@@ -88,21 +89,22 @@ steps:
           - github_commit_status:
               context: "libbeat: Ubuntu x86_64 Unit Tests"
 
-      - label: ":ubuntu: Libbeat: Ubuntu x86_64 Unit Tests with requirefips build tag"
+      - label: ":ubuntu: Libbeat: Ubuntu x86_64 Go Unit Tests with fips provider and requirefips build tag"
         key: "mandatory-linux-unit-test-fips-tag"
         command: |
-          set -euo pipefail
           cd libbeat
-          mage unitTest
+          mage goUnitTest
         retry:
           automatic:
             - limit: 1
         agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_X86_64_FIPS}"
+          instanceType: "m5.xlarge"
         env:
           FIPS: "true"
+          GOEXPERIMENT: "systemcrypto"
+          ASDF_PYTHON_VERSION: "3.9.13"
         artifact_paths:
           - "libbeat/build/*.xml"
           - "libbeat/build/*.json"
@@ -114,7 +116,7 @@ steps:
               debug: true
         notify:
           - github_commit_status:
-              context: "libbeat: Ubuntu x86_64 Unit Tests with requirefips build tag"
+              context: "libbeat: Ubuntu x86_64 Go Unit Tests with fips provider and requirefips build tag"
 
       - label: ":ubuntu: Libbeat: Ubuntu x86_64 fips140=only Unit Tests"
         key: "mandatory-linux-unit-test-fips-only"

CHANGELOG.next.asciidoc

@@ -474,6 +474,8 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Add support for relationship expansion to EntraID entity analytics provider. {issue}43324[43324] {pull}44761[44761]
 - Update CEL mito extensions to v1.21.0. {issue}40762[40762] {pull}45107[45107]
 - Update CEL mito extensions to v1.22.0. {pull}45245[45245]
+- Allow empty HTTPJSON cursor template value evaluations to be ignored by Fleet health status updates. {pull}45361[45361]
+- Add support for generalized token authentication to CEL input. {pull}45359[45359]
 
 *Auditbeat*
 

docs/extend/contributing-docs.md

@@ -2,45 +2,16 @@
 mapped_pages:
   - https://www.elastic.co/guide/en/beats/devguide/current/contributing-docs.html
 applies_to:
-  stack: discontinued 8.18
+  stack: ga 9.0
 ---
 
-# Contributing to the docs [contributing-docs]
-
-The Beats documentation follows the tagging guidelines described in the [Docs HOWTO](https://github.com/elastic/docs/blob/master/README.asciidoc). However it extends these capabilities in a couple ways:
-
-* The documentation makes extensive use of [AsciiDoc conditionals](https://docs.asciidoctor.org/asciidoc/latest/directives/conditionals/) to provide content that is reused across multiple books. This means that there might not be a single source file for each published HTML page. Some files are shared across multiple books, either as complete pages or snippets. For more details, refer to [Where to find the Beats docs source](#where-to-find-files).
-* The documentation includes some files that are generated from YAML source or pieced together from content that lives in `_meta` directories under the code (for example, the module and exported fields documentation). For more details, refer to [Generated docs](#generated-docs).
-
-
-## Where to find the Beats docs source [where-to-find-files]
-
-Because the Beats documentation makes use of shared content, doc generation scripts, and componentization, the source files are located in several places:
-
-| Documentation | Location of source files |
-| --- | --- |
-| Main docs for the Beat, including index files | `<beatname>/docs` |
-| Shared docs and Beats Platform Reference | `libbeat/docs` |
-| Processor docs | `docs` folders under processors in `libbeat/processors/`,`x-pack/<beatname>/processors/`, and `x-pack/libbeat/processors/` |
-| Output docs | `docs` folders under outputs in `libbeat/outputs/` |
-| Module docs | `_meta` folders under modules and datasets in `libbeat/module/`,`<beatname>/module/`, and `x-pack/<beatname>/module/` |
-
-The [conf.yaml](https://github.com/elastic/docs/blob/master/conf.yaml) file in the `docs` repo shows all the resources used to build each book. This file is used to drive the classic docs build and is the source of truth for file locations.
-
-::::{tip}
-If you can’t find the source for a page you want to update, go to the published page at www.elastic.co and click the Edit link to navigate to the source.
-::::
-
-
-The Beats documentation build also has dependencies on the following files in the [docs](https://github.com/elastic/docs) repo:
-
-* `shared/versions/stack/<version>.asciidoc`
-* `shared/attributes.asciidoc`
+# Contributing to the docs
 
+The Beats documentation is written in Markdown and is built using [elastic/docs-builder](https://github.com/elastic/docs-builder). Most Markdown files should be edited directly, but some Markdown files are generated.
 
 ## Generated docs [generated-docs]
 
-After updating `docs.asciidoc` files in `_meta` directories, you must run the doc collector scripts to regenerate the docs.
+After updating `docs.md` files in `_meta` directories, you must run the doc collector scripts to regenerate the docs.
 
 Make sure you [set up your Beats development environment](./index.md#setting-up-dev-environment) and use the correct Go version. The Go version is listed in the `version.asciidoc` file for the branch you want to update.
 
@@ -52,7 +23,6 @@ To run the docs collector scripts, change to the beats directory and run:
 The `make update` command overwrites files in the `docs` directories **without warning**. If you accidentally update a generated file and run `make update`, your changes will be overwritten.
 ::::
 
-
 To format your files, you might also need to run this command:
 
 `make fmt`
@@ -61,24 +31,24 @@ The make command calls the following scripts to generate the docs:
 
 [auditbeat/scripts/docs_collector.py](https://github.com/elastic/beats/blob/main/auditbeat/scripts/docs_collector.py) generates:
 
-* `auditbeat/docs/modules_list.asciidoc`
-* `auditbeat/docs/modules/*.asciidoc`
+* `docs/reference/auditbeat/auditbeat-modules.md`
+* `docs/reference/auditbeat/auditbeat-module-*.md`
 
 [filebeat/scripts/docs_collector.py](https://github.com/elastic/beats/blob/main/filebeat/scripts/docs_collector.py) generates:
 
-* `filebeat/docs/modules_list.asciidoc`
-* `filebeat/docs/modules/*.asciidoc`
+* `docs/reference/filebeat/filebeat-modules.md`
+* `docs/reference/filebeat/filebeat-module-*.md`
 
 [metricbeat/scripts/mage/docs_collector.go](https://github.com/elastic/beats/blob/main/metricbeat/scripts/mage/docs_collector.go) generates:
 
-* `metricbeat/docs/modules_list.asciidoc`
-* `metricbeat/docs/modules/*.asciidoc`
+* `docs/reference/metricbeat/metricbeat-modules.md`
+* `docs/reference/metricbeat/metricbeat-module-*.md`
 
-[libbeat/scripts/generate_fields_docs.py](https://github.com/elastic/beats/blob/main/libbeat/scripts/generate_fields_docs.py) generates
+[libbeat/scripts/generate_fields_docs.py](https://github.com/elastic/beats/blob/main/libbeat/scripts/generate_fields_docs.py) generates:
 
-* `auditbeat/docs/fields.asciidoc`
-* `filebeat/docs/fields.asciidoc`
-* `heartbeat/docs/fields.asciidoc`
-* `metricbeat/docs/fields.asciidoc`
-* `packetbeat/docs/fields.asciidoc`
-* `winlogbeat/docs/fields.asciidoc`
+* `docs/reference/auditbeat/exported-fields.md`
+* `docs/reference/filebeat/exported-fields.md`
+* `docs/reference/heartbeat/exported-fields.md`
+* `docs/reference/metricbeat/exported-fields.md`
+* `docs/reference/packetbeat/exported-fields.md`
+* `docs/reference/winlogbeat/exported-fields.md`

docs/reference/filebeat/filebeat-input-cel.md

@@ -250,7 +250,7 @@ Host environment variables are made available via the global map `env`. Only env
 
 The CEL environment enables the [optional types](https://pkg.go.dev/github.com/google/cel-go/cel#OptionalTypes) library using the version defined [here](https://pkg.go.dev/github.com/elastic/mito@v1.22.0/lib#OptionalTypesVersion) and the [two-variable comprehensions extensions](https://pkg.go.dev/github.com/google/cel-go/ext#TwoVarComprehensions) library using the version defined [here](https://pkg.go.dev/github.com/elastic/mito@v1.22.0/lib#TwoVarComprehensionVersion).
 
-Additionally, it supports authentication via Basic Authentication, Digest Authentication or OAuth2.
+Additionally, it supports authentication via Basic Authentication, Digest Authentication or OAuth2, or token authentication.
 
 Example configurations with authentication:
 
@@ -294,6 +294,23 @@ filebeat.inputs:
   resource.url: http://localhost
 ```
 
+```yaml
+filebeat.inputs:
+- type: cel
+  auth.token:
+    type: Bearer
+    value: supersecret_bearer_token
+  resource.url: http://localhost
+```
+
+```yaml
+filebeat.inputs:
+- type: cel
+  auth.token:
+    type: Token
+    value: supersecret_token
+  resource.url: http://localhost
+```
 
 ## Input state [input-state-cel]
 
@@ -617,6 +634,27 @@ Only one of the credentials settings can be set at once. For more information pl
 
 
 
+### `auth.token.enabled` [_auth_token_enabled]
+
+When set to `false`, disables the token authentication configuration. Default: `true`.
+
+::::{note}
+Token authentication settings are disabled if either `enabled` is set to `false` or the `auth.token` section is missing.
+::::
+
+
+
+### `auth.token.type` [_auth_token_type]
+
+The type of token to authenticate with, for example "Token" or "Bearer".
+
+
+### `auth.token.value` [_auth_token_value]
+
+The token value to use.
+
+
+
 ### `resource.url` [resource-parameters]
 
 The URL of the HTTP API. Required.

libbeat/api/server.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"sync"
 
 	"github.com/gorilla/mux"
 
@@ -37,6 +38,7 @@ type Server struct {
 	mux    *mux.Router
 	l      net.Listener
 	config Config
+	wg     sync.WaitGroup
 }
 
 // New creates a new API Server with no routes attached.
@@ -63,16 +65,23 @@ func New(log *logp.Logger, config *config.C) (*Server, error) {
 // Start starts the HTTP server and accepting new connection.
 func (s *Server) Start() {
 	s.log.Info("Starting stats endpoint")
+	s.wg.Add(1)
 	go func(l net.Listener) {
+		defer s.wg.Done()
 		s.log.Infof("Metrics endpoint listening on: %s (configured: %s)", l.Addr().String(), s.config.Host)
-		err := http.Serve(l, s.mux)
+		err := http.Serve(l, s.mux) //nolint:gosec // Keep original behavior
 		s.log.Infof("Stats endpoint (%s) finished: %v", l.Addr().String(), err)
 	}(s.l)
 }
 
 // Stop stops the API server and free any resource associated with the process like unix sockets.
 func (s *Server) Stop() error {
-	return s.l.Close()
+	err := s.l.Close()
+	if err != nil {
+		return fmt.Errorf("error stopping monitoring server: %w", err)
+	}
+	s.wg.Wait()
+	return nil
 }
 
 // AttachHandler will attach a handler at the specified route. Routes are

libbeat/esleg/eslegclient/bulkapi.go

@@ -63,10 +63,12 @@ type bulkRequest struct {
 type BulkResponse json.RawMessage
 
 // Bulk performs many index/delete operations in a single API call.
+// `header` is an additional set of custom HTTP headers that will be set to the HTTP request
 // Implements: http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
 func (conn *Connection) Bulk(
 	ctx context.Context,
 	index, docType string,
+	header http.Header,
 	params map[string]string, body []interface{},
 ) (int, BulkResponse, error) {
 	if len(body) == 0 {
@@ -80,14 +82,18 @@ func (conn *Connection) Bulk(
 		return 0, nil, err
 	}
 
-	mergedParams := mergeParams(conn.ConnectionSettings.Parameters, params)
+	mergedParams := mergeParams(conn.Parameters, params)
 
 	requ, err := newBulkRequest(conn.URL, index, docType, mergedParams, enc)
 	if err != nil {
 		apm.CaptureError(ctx, err).Send()
 		return 0, nil, err
 	}
 	requ.requ = apmHttpV2.RequestWithContext(ctx, requ.requ)
+	// multiple values per header are not supported
+	for name := range header {
+		requ.requ.Header.Set(name, header.Get(name))
+	}
 
 	return conn.sendBulkRequest(requ)
 }

libbeat/esleg/eslegclient/bulkapi_integration_test.go

@@ -24,13 +24,9 @@ import (
 	"fmt"
 	"os"
 	"testing"
-
-	"github.com/elastic/elastic-agent-libs/logp"
 )
 
 func TestBulk(t *testing.T) {
-	logp.TestingSetup(logp.WithSelectors("elasticsearch"))
-
 	client := getTestingElasticsearch(t)
 	index := fmt.Sprintf("packetbeat-unittest-%d", os.Getpid())
 
@@ -54,7 +50,7 @@ func TestBulk(t *testing.T) {
 	params := map[string]string{
 		"refresh": "true",
 	}
-	_, _, err := client.Bulk(context.Background(), index, "", params, body)
+	_, _, err := client.Bulk(context.Background(), index, "", nil, params, body)
 	if err != nil {
 		t.Fatalf("Bulk() returned error: %s", err)
 	}
@@ -77,8 +73,6 @@ func TestBulk(t *testing.T) {
 }
 
 func TestEmptyBulk(t *testing.T) {
-	logp.TestingSetup(logp.WithSelectors("elasticsearch"))
-
 	client := getTestingElasticsearch(t)
 	index := fmt.Sprintf("packetbeat-unittest-%d", os.Getpid())
 
@@ -87,7 +81,7 @@ func TestEmptyBulk(t *testing.T) {
 	params := map[string]string{
 		"refresh": "true",
 	}
-	_, resp, err := client.Bulk(context.Background(), index, "", params, body)
+	_, resp, err := client.Bulk(context.Background(), index, "", nil, params, body)
 	if err != nil {
 		t.Fatalf("Bulk() returned error: %s", err)
 	}
@@ -97,8 +91,6 @@ func TestEmptyBulk(t *testing.T) {
 }
 
 func TestBulkMoreOperations(t *testing.T) {
-	logp.TestingSetup(logp.WithSelectors("elasticsearch"))
-
 	client := getTestingElasticsearch(t)
 	index := fmt.Sprintf("packetbeat-unittest-%d", os.Getpid())
 
@@ -151,7 +143,7 @@ func TestBulkMoreOperations(t *testing.T) {
 	params := map[string]string{
 		"refresh": "true",
 	}
-	_, resp, err := client.Bulk(context.Background(), index, "", params, body)
+	_, resp, err := client.Bulk(context.Background(), index, "", nil, params, body)
 	if err != nil {
 		t.Fatalf("Bulk() returned error: %s [%s]", err, resp)
 	}