[8.19](backport #44205) capture extra vsphere metrics (cpu %, disk average, disk rate, disk n… (#44495)

* capture extra vsphere metrics (cpu %, disk average, disk rate, disk n… (#44205)

* capture extra vsphere metrics (cpu %, disk average, disk rate, disk number, memory usage %)

* add missing license headers

* mage fmt

* convert percentage values from fixed point integers

* add CHANGELOG.next.asciidoc entry

* refactor performance data fetching for datastore

* document new fields

* remove unused function

* remove debug log

* add unit tests

* mage fmt

* fixed precision issue for percentage performance manager values

* correct percentage property names

* converts kilobytes per sec to bytes per sec

* update fields.go

* fix linter

* fix type assertion pabic

* fix unit test failure

* address PR comments

* changed percentage fields to scaled_float type

* make field names same between fields.go and fields.yml

---------

Co-authored-by: Ishleen Kaur <102962586+ishleenk17@users.noreply.github.com>
(cherry picked from commit 14ac057)

# Conflicts:
#	auditbeat/module/file_integrity/mock_performance.go
#	auditbeat/module/file_integrity/performance.go
#	auditbeat/module/file_integrity/performance_test.go

* resolve merge

* clean up changelog

* fix missing imports

* update docs/fields.asciidoc

* fix linter

---------

Co-authored-by: stefans-elastic <stefan.stas@elastic.co>

Author: mergify[bot]

CHANGELOG.next.asciidoc

@@ -454,6 +454,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Add a warning log to metricbeat.vsphere in case vSphere connection has been configured as insecure. {pull}43104[43104]
 - Upgrade Prometheus Library to v0.300.1. {pull}43540[43540]
 - Updated Meraki API endpoint for Channel Utilization data. Switched to `GetOrganizationWirelessDevicesChannelUtilizationByDevice`. {pull}43485[43485]
+- Add new metrics to vSphere Virtual Machine dataset (CPU usage percentage, disk average usage, disk read/write rate, number of disk reads/writes, memory usage percentage). {pull}44205[44205]
 - Added checks for the Resty response object in all Meraki module API calls to ensure proper handling of nil responses. {pull}44193[44193]
 - Add latency config option to Azure Monitor module. {pull}44366[44366]
 

metricbeat/docs/fields.asciidoc

@@ -69629,6 +69629,16 @@ type: long
 
 --
 
+*`vsphere.virtualmachine.cpu.usage.percent`*::
++
+--
+CPU usage as a percentage.
+
+
+type: scaled_float
+
+--
+
 *`vsphere.virtualmachine.memory.used.guest.bytes`*::
 +
 --
@@ -69677,6 +69687,16 @@ format: bytes
 
 --
 
+*`vsphere.virtualmachine.memory.usage.percent`*::
++
+--
+Memory usage as percent of total configured or available memory.
+
+
+type: scaled_float
+
+--
+
 *`vsphere.virtualmachine.custom_fields`*::
 +
 --
@@ -69788,6 +69808,57 @@ type: object
 
 --
 
+
+*`vsphere.virtualmachine.disk.average.bytes`*::
++
+--
+Aggregated disk I/O rate.
+
+
+type: long
+
+--
+
+*`vsphere.virtualmachine.disk.read.average.bytes`*::
++
+--
+Rate at which data is read from each virtual disk on the virtual machine.
+
+
+type: long
+
+--
+
+*`vsphere.virtualmachine.disk.write.average.bytes`*::
++
+--
+Rate at which data is written to each virtual disk on the virtual machine.
+
+
+type: long
+
+--
+
+*`vsphere.virtualmachine.disk.numberRead.count`*::
++
+--
+Number of times data was read.
+
+
+type: long
+
+--
+
+*`vsphere.virtualmachine.disk.numberWrite.count`*::
++
+--
+Number of disk writes.
+
+
+type: long
+
+--
+
 [[exported-fields-windows]]
 == Windows fields
 

metricbeat/module/vsphere/client/mock_performance.go

@@ -0,0 +1,127 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package client
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/vmware/govmomi/performance"
+	"github.com/vmware/govmomi/vim25/types"
+
+	"github.com/elastic/elastic-agent-libs/logp"
+)
+
+type PerfManager interface {
+	AvailableMetric(ctx context.Context, entity types.ManagedObjectReference, interval int32) (performance.MetricList, error)
+	Query(ctx context.Context, spec []types.PerfQuerySpec) ([]types.BasePerfEntityMetricBase, error)
+	ToMetricSeries(ctx context.Context, series []types.BasePerfEntityMetricBase) ([]performance.EntityMetric, error)
+}
+
+type PerformanceDataFetcher struct {
+	perfManager PerfManager
+	logger      *logp.Logger
+}
+
+func NewPerformanceDataFetcher(logger *logp.Logger, perfManager PerfManager) *PerformanceDataFetcher {
+	return &PerformanceDataFetcher{
+		logger:      logger,
+		perfManager: perfManager,
+	}
+}
+
+func (p *PerformanceDataFetcher) GetPerfMetrics(ctx context.Context,
+	period int32,
+	objectType string,
+	objectName string,
+	objectReference types.ManagedObjectReference,
+	metrics map[string]*types.PerfCounterInfo,
+	metricSet map[string]struct{}) (metricMap map[string]interface{}, err error) {
+
+	metricMap = make(map[string]interface{})
+
+	availableMetric, err := p.perfManager.AvailableMetric(ctx, objectReference, period)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get available metrics: %w", err)
+	}
+
+	availableMetricByKey := availableMetric.ByKey()
+
+	// Filter for required metrics
+	var metricIDs []types.PerfMetricId
+	for key, metric := range metricSet {
+		if counter, ok := metrics[key]; ok {
+			if _, exists := availableMetricByKey[counter.Key]; exists {
+				metricIDs = append(metricIDs, types.PerfMetricId{
+					CounterId: counter.Key,
+					Instance:  "*",
+				})
+			}
+		} else {
+			p.logger.Warnf("Metric %s not found", metric)
+		}
+	}
+
+	spec := types.PerfQuerySpec{
+		Entity:     objectReference,
+		MetricId:   metricIDs,
+		MaxSample:  1,
+		IntervalId: period,
+	}
+
+	// Query performance data
+	samples, err := p.perfManager.Query(ctx, []types.PerfQuerySpec{spec})
+	if err != nil {
+		if strings.Contains(err.Error(), "ServerFaultCode: A specified parameter was not correct: querySpec.interval") {
+			return metricMap, fmt.Errorf("failed to query performance data: use one of the system's supported interval. consider adjusting period: %w", err)
+		}
+
+		return metricMap, fmt.Errorf("failed to query performance data: %w", err)
+	}
+
+	if len(samples) == 0 {
+		p.logger.Debug("No samples returned from performance manager")
+		return metricMap, nil
+	}
+
+	results, err := p.perfManager.ToMetricSeries(ctx, samples)
+	if err != nil {
+		return metricMap, fmt.Errorf("failed to convert performance data to metric series: %w", err)
+	}
+
+	if len(results) == 0 {
+		p.logger.Debug("No results returned from metric series conversion")
+		return metricMap, nil
+	}
+
+	for _, result := range results[0].Value {
+		if len(result.Value) > 0 {
+			value := result.Value[0]
+			if result.Unit == string(types.PerformanceManagerUnitPercent) {
+				metricMap[result.Name] = float64(value) / 100.0
+			} else {
+				metricMap[result.Name] = value
+			}
+			continue
+		}
+		p.logger.Debugf("For %s %s, Metric %s: No result found", objectType, objectName, result.Name)
+	}
+
+	return metricMap, nil
+}