From 4ec69210a44e68eea71fe98a4e6b6871a25ea0db Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 10:12:29 +0900 Subject: [PATCH 01/48] add rally subcommand in benchmark --- cmd/benchmark.go | 146 +++++++++++++++++- internal/benchrunner/runners/common/env.go | 10 ++ .../benchrunner/runners/system/metrics.go | 8 - 3 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 internal/benchrunner/runners/common/env.go diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 7849f25a60..bd8b77ba89 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -25,7 +25,9 @@ import ( "github.com/elastic/elastic-package/internal/benchrunner" "github.com/elastic/elastic-package/internal/benchrunner/reporters" "github.com/elastic/elastic-package/internal/benchrunner/reporters/outputs" + bench_common "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "github.com/elastic/elastic-package/internal/benchrunner/runners/pipeline" + "github.com/elastic/elastic-package/internal/benchrunner/runners/rally" "github.com/elastic/elastic-package/internal/benchrunner/runners/system" "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/common" @@ -48,6 +50,12 @@ These benchmarks allow you to benchmark any Ingest Node Pipelines defined by you For details on how to configure pipeline benchmarks for a package, review the [HOWTO guide](./docs/howto/pipeline_benchmarking.md). +#### Rally Benchmarks + +These benchmarks allow you to benchmark an integration corpus with rally. + +For details on how to configure rally benchmarks for a package, review the [HOWTO guide](./docs/howto/rally_benchmarking.md). + #### System Benchmarks These benchmarks allow you to benchmark an integration end to end. @@ -66,6 +74,9 @@ func setupBenchmarkCommand() *cobraext.Command { pipelineCmd := getPipelineCommand() cmd.AddCommand(pipelineCmd) + rallyCmd := getRallyCommand() + cmd.AddCommand(rallyCmd) + systemCmd := getSystemCommand() cmd.AddCommand(systemCmd) @@ -213,6 +224,133 @@ func pipelineCommandAction(cmd *cobra.Command, args []string) error { return nil } +func getRallyCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "rally", + Short: "Run rally benchmarks", + Long: "Run rally benchmarks for the package", + Args: cobra.NoArgs, + RunE: rallyCommandAction, + } + + cmd.Flags().StringP(cobraext.BenchNameFlagName, "", "", cobraext.BenchNameFlagDescription) + cmd.Flags().BoolP(cobraext.BenchReindexToMetricstoreFlagName, "", false, cobraext.BenchReindexToMetricstoreFlagDescription) + cmd.Flags().DurationP(cobraext.BenchMetricsIntervalFlagName, "", time.Second, cobraext.BenchMetricsIntervalFlagDescription) + cmd.Flags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) + cmd.Flags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) + + return cmd +} + +func rallyCommandAction(cmd *cobra.Command, args []string) error { + cmd.Println("Run rally benchmarks for the package") + + variant, err := cmd.Flags().GetString(cobraext.VariantFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.VariantFlagName) + } + + benchName, err := cmd.Flags().GetString(cobraext.BenchNameFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchNameFlagName) + } + + deferCleanup, err := cmd.Flags().GetDuration(cobraext.DeferCleanupFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.DeferCleanupFlagName) + } + + metricsInterval, err := cmd.Flags().GetDuration(cobraext.BenchMetricsIntervalFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchMetricsIntervalFlagName) + } + + dataReindex, err := cmd.Flags().GetBool(cobraext.BenchReindexToMetricstoreFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchReindexToMetricstoreFlagName) + } + + packageRootPath, found, err := packages.FindPackageRoot() + if !found { + return errors.New("package root not found") + } + if err != nil { + return fmt.Errorf("locating package root failed: %w", err) + } + + profile, err := cobraext.GetProfileFlag(cmd) + if err != nil { + return err + } + + signal.Enable() + + esClient, err := stack.NewElasticsearchClientFromProfile(profile) + if err != nil { + return fmt.Errorf("can't create Elasticsearch client: %w", err) + } + err = esClient.CheckHealth(cmd.Context()) + if err != nil { + return err + } + + kc, err := stack.NewKibanaClientFromProfile(profile) + if err != nil { + return fmt.Errorf("can't create Kibana client: %w", err) + } + + withOpts := []rally.OptionFunc{ + rally.WithVariant(variant), + rally.WithBenchmarkName(benchName), + rally.WithDeferCleanup(deferCleanup), + rally.WithMetricsInterval(metricsInterval), + rally.WithDataReindexing(dataReindex), + rally.WithPackageRootPath(packageRootPath), + rally.WithESAPI(esClient.API), + rally.WithKibanaClient(kc), + rally.WithProfile(profile), + } + + esMetricsClient, err := initializeESMetricsClient(cmd.Context()) + if err != nil { + return fmt.Errorf("can't create Elasticsearch metrics client: %w", err) + } + if esMetricsClient != nil { + withOpts = append(withOpts, rally.WithESMetricsAPI(esMetricsClient.API)) + } + + runner := rally.NewRallyBenchmark(rally.NewOptions(withOpts...)) + + r, err := benchrunner.Run(runner) + if err != nil { + return fmt.Errorf("error running package rally benchmarks: %w", err) + } + + multiReport, ok := r.(reporters.MultiReportable) + if !ok { + return fmt.Errorf("rally benchmark is expected to return multiple reports") + } + + reports := multiReport.Split() + if len(reports) != 2 { + return fmt.Errorf("rally benchmark is expected to return a human an a file report") + } + + // human report will always be the first + human := reports[0] + if err := reporters.WriteReportable(reporters.Output(outputs.ReportOutputSTDOUT), human); err != nil { + return fmt.Errorf("error writing benchmark report: %w", err) + } + + // file report will always be the second + file := reports[1] + if err := reporters.WriteReportable(reporters.Output(outputs.ReportOutputFile), file); err != nil { + return fmt.Errorf("error writing benchmark report: %w", err) + } + + return nil +} + func getSystemCommand() *cobra.Command { cmd := &cobra.Command{ Use: "system", @@ -410,10 +548,10 @@ func generateDataStreamCorpusCommandAction(cmd *cobra.Command, _ []string) error } func initializeESMetricsClient(ctx context.Context) (*elasticsearch.Client, error) { - address := os.Getenv(system.ESMetricstoreHostEnv) - user := os.Getenv(system.ESMetricstoreUsernameEnv) - pass := os.Getenv(system.ESMetricstorePasswordEnv) - cacert := os.Getenv(system.ESMetricstoreCACertificateEnv) + address := os.Getenv(bench_common.ESMetricstoreHostEnv) + user := os.Getenv(bench_common.ESMetricstoreUsernameEnv) + pass := os.Getenv(bench_common.ESMetricstorePasswordEnv) + cacert := os.Getenv(bench_common.ESMetricstoreCACertificateEnv) if address == "" || user == "" || pass == "" { logger.Debugf("can't initialize metricstore, missing environment configuration") return nil, nil diff --git a/internal/benchrunner/runners/common/env.go b/internal/benchrunner/runners/common/env.go new file mode 100644 index 0000000000..6b009e14e7 --- /dev/null +++ b/internal/benchrunner/runners/common/env.go @@ -0,0 +1,10 @@ +package common + +import "github.com/elastic/elastic-package/internal/environment" + +var ( + ESMetricstoreHostEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_HOST") + ESMetricstoreUsernameEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_USERNAME") + ESMetricstorePasswordEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_PASSWORD") + ESMetricstoreCACertificateEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_CA_CERT") +) diff --git a/internal/benchrunner/runners/system/metrics.go b/internal/benchrunner/runners/system/metrics.go index 3743d10d99..2b48f227d7 100644 --- a/internal/benchrunner/runners/system/metrics.go +++ b/internal/benchrunner/runners/system/metrics.go @@ -16,18 +16,10 @@ import ( "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/elasticsearch/ingest" - "github.com/elastic/elastic-package/internal/environment" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/servicedeployer" ) -var ( - ESMetricstoreHostEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_HOST") - ESMetricstoreUsernameEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_USERNAME") - ESMetricstorePasswordEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_PASSWORD") - ESMetricstoreCACertificateEnv = environment.WithElasticPackagePrefix("ESMETRICSTORE_CA_CERT") -) - type collector struct { ctxt servicedeployer.ServiceContext metadata benchMeta From 5f14e8c429ab81f0cf0a5ff2db0ccee5e312128e Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 10:13:20 +0900 Subject: [PATCH 02/48] add rally corpus output dir --- internal/configuration/locations/locations.go | 6 ++++++ internal/servicedeployer/terraform.go | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/internal/configuration/locations/locations.go b/internal/configuration/locations/locations.go index 18888f7e6e..680832704f 100644 --- a/internal/configuration/locations/locations.go +++ b/internal/configuration/locations/locations.go @@ -30,6 +30,7 @@ var ( elasticPackageDataHome = environment.WithElasticPackagePrefix("DATA_HOME") serviceLogsDir = filepath.Join(temporaryDir, "service_logs") + rallyCorpusDir = filepath.Join(temporaryDir, "rally_corpus") kubernetesDeployerDir = filepath.Join(deployerDir, "kubernetes") serviceOutputDir = filepath.Join(temporaryDir, "output") ) @@ -85,6 +86,11 @@ func (loc LocationManager) KubernetesDeployerDir() string { return filepath.Join(loc.stackPath, kubernetesDeployerDir) } +// RallyCorpusDir returns the rally coprus directory +func (loc LocationManager) RallyCorpusDir() string { + return filepath.Join(loc.stackPath, rallyCorpusDir) +} + // ServiceLogDir returns the log directory func (loc LocationManager) ServiceLogDir() string { return filepath.Join(loc.stackPath, serviceLogsDir) diff --git a/internal/servicedeployer/terraform.go b/internal/servicedeployer/terraform.go index 8bbb0c65d4..e7ee666ad6 100644 --- a/internal/servicedeployer/terraform.go +++ b/internal/servicedeployer/terraform.go @@ -218,4 +218,12 @@ func CreateOutputDir(locationManager *locations.LocationManager, runId string) ( return outputDir, nil } +func CreateRallyTrackDir(locationManager *locations.LocationManager, runId string) error { + outputDir := filepath.Join(locationManager.RallyCorpusDir(), runId) + if err := os.MkdirAll(outputDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + return nil +} + var _ ServiceDeployer = new(TerraformServiceDeployer) From 787276d289491c595ed1a4c00893cd80013bbe31 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 10:13:49 +0900 Subject: [PATCH 03/48] export GenerateRallyTrack --- internal/corpusgenerator/rally.go | 2 +- internal/corpusgenerator/utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/corpusgenerator/rally.go b/internal/corpusgenerator/rally.go index ae23432bf4..3faed5fba2 100644 --- a/internal/corpusgenerator/rally.go +++ b/internal/corpusgenerator/rally.go @@ -49,7 +49,7 @@ const ( ` ) -func generateRallyTrack(dataStream string, corpusFile *os.File, corpusDocsCount uint64) ([]byte, error) { +func GenerateRallyTrack(dataStream string, corpusFile *os.File, corpusDocsCount uint64) ([]byte, error) { t := template.New("rallytrack") parsedTpl, err := t.Delims("[[", "]]").Parse(rallyTrackTemplate) diff --git a/internal/corpusgenerator/utils.go b/internal/corpusgenerator/utils.go index 511f47954d..f55b3c8dab 100644 --- a/internal/corpusgenerator/utils.go +++ b/internal/corpusgenerator/utils.go @@ -56,7 +56,7 @@ func RunGenerator(generator genlib.Generator, dataStream, rallyTrackOutputDir st if len(rallyTrackOutputDir) > 0 { corpusFile := f.(*os.File) - rallyTrackContent, err := generateRallyTrack(dataStream, corpusFile, corpusDocsCount) + rallyTrackContent, err := GenerateRallyTrack(dataStream, corpusFile, corpusDocsCount) if err != nil { return err } From 9557ce43c4b10ca4cb9b784760af67a97d34a6e1 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 10:14:03 +0900 Subject: [PATCH 04/48] add rally runner --- internal/benchrunner/runners/rally/metrics.go | 370 ++++++++ .../runners/rally/metrics_index.json | 24 + internal/benchrunner/runners/rally/options.go | 103 ++ internal/benchrunner/runners/rally/report.go | 237 +++++ internal/benchrunner/runners/rally/runner.go | 880 ++++++++++++++++++ .../benchrunner/runners/rally/scenario.go | 119 +++ 6 files changed, 1733 insertions(+) create mode 100644 internal/benchrunner/runners/rally/metrics.go create mode 100644 internal/benchrunner/runners/rally/metrics_index.json create mode 100644 internal/benchrunner/runners/rally/options.go create mode 100644 internal/benchrunner/runners/rally/report.go create mode 100644 internal/benchrunner/runners/rally/runner.go create mode 100644 internal/benchrunner/runners/rally/scenario.go diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go new file mode 100644 index 0000000000..431e41d5dd --- /dev/null +++ b/internal/benchrunner/runners/rally/metrics.go @@ -0,0 +1,370 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package rally + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "io" + "sync" + "sync/atomic" + "time" + + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/elasticsearch/ingest" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/servicedeployer" +) + +type collector struct { + ctxt servicedeployer.ServiceContext + metadata benchMeta + scenario scenario + + interval time.Duration + esAPI *elasticsearch.API + metricsAPI *elasticsearch.API + datastream string + pipelinePrefix string + + wg sync.WaitGroup + stopped atomic.Bool + stopC chan struct{} + tick *time.Ticker + + startIngestMetrics map[string]ingest.PipelineStatsMap + endIngestMetrics map[string]ingest.PipelineStatsMap + startMetrics metrics + endMetrics metrics + diskUsage map[string]ingest.DiskUsage + startTotalHits int + endTotalHits int +} + +type metrics struct { + ts int64 + dsMetrics *ingest.DataStreamStats + nMetrics *ingest.NodesStats +} + +type metricsSummary struct { + ClusterName string + Nodes int + RunID string + CollectionStartTs int64 + CollectionEndTs int64 + DataStreamStats *ingest.DataStreamStats + IngestPipelineStats map[string]ingest.PipelineStatsMap + DiskUsage map[string]ingest.DiskUsage + TotalHits int + NodesStats map[string]ingest.NodeStats +} + +func newCollector( + ctxt servicedeployer.ServiceContext, + benchName string, + scenario scenario, + esAPI, metricsAPI *elasticsearch.API, + interval time.Duration, + datastream, pipelinePrefix string, +) *collector { + meta := benchMeta{Parameters: scenario} + meta.Info.Benchmark = benchName + meta.Info.RunID = ctxt.Test.RunID + return &collector{ + ctxt: ctxt, + interval: interval, + scenario: scenario, + metadata: meta, + esAPI: esAPI, + metricsAPI: metricsAPI, + datastream: datastream, + pipelinePrefix: pipelinePrefix, + stopC: make(chan struct{}), + } +} + +func (c *collector) start() { + c.tick = time.NewTicker(c.interval) + c.createMetricsIndex() + var once sync.Once + + c.wg.Add(1) + go func() { + defer c.tick.Stop() + defer c.wg.Done() + for { + select { + case <-c.stopC: + // last collect before stopping + c.collectMetricsPreviousToStop() + c.publish(c.createEventsFromMetrics(c.endMetrics)) + return + case <-c.tick.C: + once.Do(func() { + c.waitUntilReady() + c.startIngestMetrics = c.collectIngestMetrics() + c.startTotalHits = c.collectTotalHits() + c.startMetrics = c.collect() + c.publish(c.createEventsFromMetrics(c.startMetrics)) + }) + m := c.collect() + c.publish(c.createEventsFromMetrics(m)) + } + } + }() +} + +func (c *collector) stop() { + if !c.stopped.CompareAndSwap(false, true) { + return + } + close(c.stopC) + c.wg.Wait() +} + +func (c *collector) collect() metrics { + m := metrics{ + ts: time.Now().Unix(), + } + + nstats, err := ingest.GetNodesStats(c.esAPI) + if err != nil { + logger.Debug(err) + } else { + m.nMetrics = nstats + } + + dsstats, err := ingest.GetDataStreamStats(c.esAPI, c.datastream) + if err != nil { + logger.Debug(err) + } else { + m.dsMetrics = dsstats + } + + return m +} + +func (c *collector) publish(events [][]byte) { + if c.metricsAPI == nil { + return + } + for _, e := range events { + reqBody := bytes.NewReader(e) + resp, err := c.metricsAPI.Index(c.indexName(), reqBody) + if err != nil { + logger.Debugf("error indexing event: %v", err) + continue + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.Errorf("failed to read index response body: %v", err) + } + resp.Body.Close() + + if resp.StatusCode != 201 { + logger.Errorf("error indexing event (%d): %s: %v", resp.StatusCode, resp.Status(), elasticsearch.NewError(body)) + } + } +} + +//go:embed metrics_index.json +var metricsIndexBytes []byte + +func (c *collector) createMetricsIndex() { + if c.metricsAPI == nil { + return + } + + reader := bytes.NewReader(metricsIndexBytes) + + logger.Debugf("creating %s index in metricstore...", c.indexName()) + + createRes, err := c.metricsAPI.Indices.Create( + c.indexName(), + c.metricsAPI.Indices.Create.WithBody(reader), + ) + if err != nil { + logger.Debugf("could not create index: %v", err) + return + } + createRes.Body.Close() + + if createRes.IsError() { + logger.Debug("got a response error while creating index") + } +} + +func (c *collector) indexName() string { + return fmt.Sprintf("bench-metrics-%s-%s", c.datastream, c.ctxt.Test.RunID) +} + +func (c *collector) summarize() (*metricsSummary, error) { + sum := metricsSummary{ + RunID: c.ctxt.Test.RunID, + IngestPipelineStats: make(map[string]ingest.PipelineStatsMap), + NodesStats: make(map[string]ingest.NodeStats), + DiskUsage: c.diskUsage, + TotalHits: c.endTotalHits - c.startTotalHits, + } + + sum.ClusterName = c.startMetrics.nMetrics.ClusterName + sum.CollectionStartTs = c.startMetrics.ts + sum.CollectionEndTs = c.endMetrics.ts + sum.DataStreamStats = c.endMetrics.dsMetrics + sum.Nodes = len(c.endMetrics.nMetrics.Nodes) + + for node, endPStats := range c.endIngestMetrics { + startPStats, found := c.startIngestMetrics[node] + if !found { + logger.Debugf("node %s not found in initial metrics", node) + continue + } + sumStats := make(ingest.PipelineStatsMap) + for pname, endStats := range endPStats { + startStats, found := startPStats[pname] + if !found { + logger.Debugf("pipeline %s not found in node %s initial metrics", pname, node) + continue + } + sumStats[pname] = ingest.PipelineStats{ + StatsRecord: ingest.StatsRecord{ + Count: endStats.Count - startStats.Count, + Failed: endStats.Failed - startStats.Failed, + TimeInMillis: endStats.TimeInMillis - startStats.TimeInMillis, + }, + Processors: make([]ingest.ProcessorStats, len(endStats.Processors)), + } + for i, endPr := range endStats.Processors { + startPr := startStats.Processors[i] + sumStats[pname].Processors[i] = ingest.ProcessorStats{ + Type: endPr.Type, + Extra: endPr.Extra, + Conditional: endPr.Conditional, + Stats: ingest.StatsRecord{ + Count: endPr.Stats.Count - startPr.Stats.Count, + Failed: endPr.Stats.Failed - startPr.Stats.Failed, + TimeInMillis: endPr.Stats.TimeInMillis - startPr.Stats.TimeInMillis, + }, + } + } + } + sum.IngestPipelineStats[node] = sumStats + } + + return &sum, nil +} + +func (c *collector) waitUntilReady() { + logger.Debug("waiting for datastream to be created...") + + waitTick := time.NewTicker(time.Second) + defer waitTick.Stop() + +readyLoop: + for { + select { + case <-c.stopC: + return + case <-waitTick.C: + } + dsstats, err := ingest.GetDataStreamStats(c.esAPI, c.datastream) + if err != nil { + logger.Debug(err) + } + if dsstats != nil { + break readyLoop + } + } + + if c.scenario.WarmupTimePeriod > 0 { + logger.Debugf("waiting %s for warmup period", c.scenario.WarmupTimePeriod) + select { + case <-c.stopC: + return + case <-time.After(c.scenario.WarmupTimePeriod): + } + } + logger.Debug("metric collection starting...") +} + +func (c *collector) collectIngestMetrics() map[string]ingest.PipelineStatsMap { + ipMetrics, err := ingest.GetPipelineStatsByPrefix(c.esAPI, c.pipelinePrefix) + if err != nil { + logger.Debugf("could not get ingest pipeline metrics: %v", err) + return nil + } + return ipMetrics +} + +func (c *collector) collectDiskUsage() map[string]ingest.DiskUsage { + du, err := ingest.GetDiskUsage(c.esAPI, c.datastream) + if err != nil { + logger.Debugf("could not get disk usage metrics: %v", err) + return nil + } + return du +} + +func (c *collector) collectMetricsPreviousToStop() { + c.endIngestMetrics = c.collectIngestMetrics() + c.diskUsage = c.collectDiskUsage() + c.endTotalHits = c.collectTotalHits() + c.endMetrics = c.collect() +} + +func (c *collector) collectTotalHits() int { + totalHits, err := getTotalHits(c.esAPI, c.datastream) + if err != nil { + logger.Debugf("could not total hits: %w", err) + } + return totalHits +} + +func (c *collector) createEventsFromMetrics(m metrics) [][]byte { + dsEvent := struct { + Timestamp int64 `json:"@timestamp"` + *ingest.DataStreamStats + Meta benchMeta `json:"benchmark_metadata"` + }{ + Timestamp: m.ts * 1000, // ms to s + DataStreamStats: m.dsMetrics, + Meta: c.metadata, + } + + type nEvent struct { + Ts int64 `json:"@timestamp"` + ClusterName string `json:"cluster_name"` + NodeName string `json:"node_name"` + *ingest.NodeStats + Meta benchMeta `json:"benchmark_metadata"` + } + + var nEvents []interface{} + + for node, stats := range m.nMetrics.Nodes { + nEvents = append(nEvents, nEvent{ + Ts: m.ts * 1000, // ms to s + ClusterName: m.nMetrics.ClusterName, + NodeName: node, + NodeStats: &stats, + Meta: c.metadata, + }) + } + + var events [][]byte + for _, e := range append(nEvents, dsEvent) { + b, err := json.Marshal(e) + if err != nil { + logger.Debugf("error marhsaling metrics event: %w", err) + continue + } + events = append(events, b) + } + return events +} diff --git a/internal/benchrunner/runners/rally/metrics_index.json b/internal/benchrunner/runners/rally/metrics_index.json new file mode 100644 index 0000000000..5d4e724da0 --- /dev/null +++ b/internal/benchrunner/runners/rally/metrics_index.json @@ -0,0 +1,24 @@ +{ + "settings": { + "number_of_replicas": 0 + }, + "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "match_mapping_type": "string", + "mapping": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + } + } + } +} \ No newline at end of file diff --git a/internal/benchrunner/runners/rally/options.go b/internal/benchrunner/runners/rally/options.go new file mode 100644 index 0000000000..5dd466bb54 --- /dev/null +++ b/internal/benchrunner/runners/rally/options.go @@ -0,0 +1,103 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package rally + +import ( + "time" + + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/profile" +) + +// Options contains benchmark runner options. +type Options struct { + ESAPI *elasticsearch.API + KibanaClient *kibana.Client + DeferCleanup time.Duration + MetricsInterval time.Duration + ReindexData bool + ESMetricsAPI *elasticsearch.API + BenchName string + PackageRootPath string + Variant string + Profile *profile.Profile + RallyClientOptions ClientOptions +} + +type ClientOptions struct { + Host string + Username string + Password string +} +type OptionFunc func(*Options) + +func NewOptions(fns ...OptionFunc) Options { + var opts Options + for _, fn := range fns { + fn(&opts) + } + return opts +} + +func WithESAPI(api *elasticsearch.API) OptionFunc { + return func(opts *Options) { + opts.ESAPI = api + } +} + +func WithKibanaClient(c *kibana.Client) OptionFunc { + return func(opts *Options) { + opts.KibanaClient = c + } +} + +func WithPackageRootPath(path string) OptionFunc { + return func(opts *Options) { + opts.PackageRootPath = path + } +} + +func WithBenchmarkName(name string) OptionFunc { + return func(opts *Options) { + opts.BenchName = name + } +} + +func WithDeferCleanup(d time.Duration) OptionFunc { + return func(opts *Options) { + opts.DeferCleanup = d + } +} + +func WithMetricsInterval(d time.Duration) OptionFunc { + return func(opts *Options) { + opts.MetricsInterval = d + } +} + +func WithDataReindexing(b bool) OptionFunc { + return func(opts *Options) { + opts.ReindexData = b + } +} + +func WithESMetricsAPI(api *elasticsearch.API) OptionFunc { + return func(opts *Options) { + opts.ESMetricsAPI = api + } +} + +func WithVariant(name string) OptionFunc { + return func(opts *Options) { + opts.Variant = name + } +} + +func WithProfile(p *profile.Profile) OptionFunc { + return func(opts *Options) { + opts.Profile = p + } +} diff --git a/internal/benchrunner/runners/rally/report.go b/internal/benchrunner/runners/rally/report.go new file mode 100644 index 0000000000..44e7e9cde2 --- /dev/null +++ b/internal/benchrunner/runners/rally/report.go @@ -0,0 +1,237 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package rally + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/jedib0t/go-pretty/table" + "github.com/jedib0t/go-pretty/text" + + "github.com/elastic/elastic-package/internal/benchrunner/reporters" + "github.com/elastic/elastic-package/internal/elasticsearch/ingest" +) + +type report struct { + Info struct { + Benchmark string + Description string + RunID string + Package string + StartTs int64 + EndTs int64 + Duration time.Duration + GeneratedCorporaFile string + } + Parameters struct { + PackageVersion string + DataStream dataStream + Input string + Vars map[string]interface{} + WarmupTimePeriod time.Duration + Corpora corpora + } + ClusterName string + Nodes int + DataStreamStats *ingest.DataStreamStats + IngestPipelineStats map[string]ingest.PipelineStatsMap + DiskUsage map[string]ingest.DiskUsage + TotalHits int + RallyStats []rallyStat +} + +func createReport(benchName, corporaFile string, s *scenario, sum *metricsSummary, stats []rallyStat) (reporters.Reportable, error) { + r := newReport(benchName, corporaFile, s, sum, stats) + human := reporters.NewReport(s.Package, reportHumanFormat(r)) + + jsonBytes, err := reportJSONFormat(r) + if err != nil { + return nil, fmt.Errorf("rendering JSON report: %w", err) + } + + jsonFile := reporters.NewFileReport(s.Package, fmt.Sprintf("system/%s/report.json", sum.RunID), jsonBytes) + + mr := reporters.NewMultiReport(s.Package, []reporters.Reportable{human, jsonFile}) + + return mr, nil +} + +func newReport(benchName, corporaFile string, s *scenario, sum *metricsSummary, stats []rallyStat) *report { + var report report + report.Info.Benchmark = benchName + report.Info.Description = s.Description + report.Info.RunID = sum.RunID + report.Info.Package = s.Package + report.Info.StartTs = sum.CollectionStartTs + report.Info.EndTs = sum.CollectionEndTs + report.Info.Duration = time.Duration(sum.CollectionEndTs-sum.CollectionStartTs) * time.Second + report.Info.GeneratedCorporaFile = corporaFile + report.Parameters.PackageVersion = s.Version + report.Parameters.Input = s.Input + report.Parameters.Vars = s.Vars + report.Parameters.DataStream = s.DataStream + report.Parameters.WarmupTimePeriod = s.WarmupTimePeriod + report.Parameters.Corpora = s.Corpora + report.ClusterName = sum.ClusterName + report.Nodes = sum.Nodes + report.DataStreamStats = sum.DataStreamStats + report.IngestPipelineStats = sum.IngestPipelineStats + report.DiskUsage = sum.DiskUsage + report.TotalHits = sum.TotalHits + report.RallyStats = stats + return &report +} + +func reportJSONFormat(r *report) ([]byte, error) { + b, err := json.MarshalIndent(r, "", "\t") + if err != nil { + return nil, err + } + return b, nil +} + +func reportHumanFormat(r *report) []byte { + var report strings.Builder + report.WriteString(renderBenchmarkTable( + "info", + "benchmark", r.Info.Benchmark, + "description", r.Info.Description, + "run ID", r.Info.RunID, + "package", r.Info.Package, + "start ts (s)", r.Info.StartTs, + "end ts (s)", r.Info.EndTs, + "duration", r.Info.Duration, + "generated corpora file", r.Info.GeneratedCorporaFile, + ) + "\n") + + pkvs := []interface{}{ + "package version", r.Parameters.PackageVersion, + "input", r.Parameters.Input, + } + + for k, v := range r.Parameters.Vars { + pkvs = append(pkvs, fmt.Sprintf("vars.%s", k), v) + } + + pkvs = append(pkvs, "data_stream.name", r.Parameters.DataStream.Name) + + for k, v := range r.Parameters.DataStream.Vars { + pkvs = append(pkvs, fmt.Sprintf("data_stream.vars.%s", k), v) + } + + pkvs = append(pkvs, + "warmup time period", r.Parameters.WarmupTimePeriod, + ) + + if r.Parameters.Corpora.Generator != nil { + pkvs = append(pkvs, + "corpora.generator.total_events", r.Parameters.Corpora.Generator.TotalEvents, + "corpora.generator.template.path", r.Parameters.Corpora.Generator.Template.Path, + "corpora.generator.template.raw", r.Parameters.Corpora.Generator.Template.Raw, + "corpora.generator.template.type", r.Parameters.Corpora.Generator.Template.Type, + "corpora.generator.config.path", r.Parameters.Corpora.Generator.Config.Path, + "corpora.generator.config.raw", r.Parameters.Corpora.Generator.Config.Raw, + "corpora.generator.fields.path", r.Parameters.Corpora.Generator.Fields.Path, + "corpora.generator.fields.raw", r.Parameters.Corpora.Generator.Fields.Raw, + ) + } + + report.WriteString(renderBenchmarkTable( + "cluster info", + "name", r.ClusterName, + "nodes", r.Nodes, + ) + "\n") + + report.WriteString(renderBenchmarkTable( + "data stream stats", + "data stream", r.DataStreamStats.DataStream, + "approx total docs ingested", r.TotalHits, + "backing indices", r.DataStreamStats.BackingIndices, + "store size bytes", r.DataStreamStats.StoreSizeBytes, + "maximum ts (ms)", r.DataStreamStats.MaximumTimestamp, + ) + "\n") + + for index, du := range r.DiskUsage { + adu := du.AllFields + report.WriteString(renderBenchmarkTable( + fmt.Sprintf("disk usage for index %s (for all fields)", index), + "total", humanize.Bytes(adu.TotalInBytes), + "inverted_index.total", humanize.Bytes(adu.InvertedIndex.TotalInBytes), + "inverted_index.stored_fields", humanize.Bytes(adu.StoredFieldsInBytes), + "inverted_index.doc_values", humanize.Bytes(adu.DocValuesInBytes), + "inverted_index.points", humanize.Bytes(adu.PointsInBytes), + "inverted_index.norms", humanize.Bytes(adu.NormsInBytes), + "inverted_index.term_vectors", humanize.Bytes(adu.TermVectorsInBytes), + "inverted_index.knn_vectors", humanize.Bytes(adu.KnnVectorsInBytes), + ) + "\n") + } + + for node, pStats := range r.IngestPipelineStats { + for pipeline, stats := range pStats { + if stats.Count == 0 { + continue + } + kvs := []interface{}{ + "Totals", + fmt.Sprintf( + "Count: %d | Failed: %d | Time: %s", + stats.Count, + stats.Failed, + time.Duration(stats.TimeInMillis)*time.Millisecond, + ), + } + for _, procStats := range stats.Processors { + str := fmt.Sprintf( + "Count: %d | Failed: %d | Time: %s", + procStats.Stats.Count, + procStats.Stats.Failed, + time.Duration(procStats.Stats.TimeInMillis)*time.Millisecond, + ) + kvs = append(kvs, fmt.Sprintf("%s (%s)", procStats.Type, procStats.Extra), str) + } + report.WriteString(renderBenchmarkTable( + fmt.Sprintf("pipeline %s stats in node %s", pipeline, node), + kvs..., + ) + "\n") + } + } + + rsvs := make([]interface{}, 0, len(r.RallyStats)) + for _, rs := range r.RallyStats { + if rs.Metric == "Metric" { // let's skip the header + continue + } + + value := rs.Value + if len(rs.Unit) > 0 { + value = fmt.Sprintf("%v %s", rs.Value, rs.Unit) + } + rsvs = append(rsvs, rs.Metric, value) + } + + report.WriteString(renderBenchmarkTable("rally stats", rsvs...) + "\n") + + return []byte(report.String()) +} + +func renderBenchmarkTable(title string, kv ...interface{}) string { + t := table.NewWriter() + t.SetStyle(table.StyleRounded) + t.SetTitle(title) + t.SetColumnConfigs([]table.ColumnConfig{ + { + Number: 2, + Align: text.AlignRight, + }, + }) + for i := 0; i < len(kv)-1; i += 2 { + t.AppendRow(table.Row{kv[i], kv[i+1]}) + } + return t.Render() +} diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go new file mode 100644 index 0000000000..086e97e6e6 --- /dev/null +++ b/internal/benchrunner/runners/rally/runner.go @@ -0,0 +1,880 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package rally + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" + "github.com/elastic/elastic-package/internal/corpusgenerator" + "github.com/elastic/elastic-package/internal/stack" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib" + "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" + "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/fields" + + "github.com/elastic/elastic-package/internal/benchrunner" + "github.com/elastic/elastic-package/internal/benchrunner/reporters" + "github.com/elastic/elastic-package/internal/configuration/locations" + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/multierror" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/servicedeployer" + "github.com/elastic/elastic-package/internal/signal" +) + +const ( + // RallyCorpusAgentDir is folder path where rally corporsa files produced by the service + // are stored on the Rally container's filesystem. + RallyCorpusAgentDir = "/tmp/rally_corpus" + devDeployDir = "_dev/benchmark/rally/deploy" + + // BenchType defining system benchmark + BenchType benchrunner.Type = "system" +) + +type rallyStat struct { + Metric string + Task string + Value any + Unit string +} + +type runner struct { + options Options + scenario *scenario + + ctxt servicedeployer.ServiceContext + benchPolicy *kibana.Policy + runtimeDataStream string + pipelinePrefix string + generator genlib.Generator + mcollector *collector + + corporaFile string + trackFile string + reportFile string + + // Execution order of following handlers is defined in runner.TearDown() method. + deletePolicyHandler func() error + resetAgentPolicyHandler func() error + shutdownServiceHandler func() error + wipeDataStreamHandler func() error + clearCorporaHandler func() error +} + +func NewRallyBenchmark(opts Options) benchrunner.Runner { + return &runner{options: opts} +} + +func (r *runner) SetUp() error { + return r.setUp() +} + +// Run runs the system benchmarks defined under the given folder +func (r *runner) Run() (reporters.Reportable, error) { + return r.run() +} + +func (r *runner) TearDown() error { + if r.options.DeferCleanup > 0 { + logger.Debugf("waiting for %s before tearing down...", r.options.DeferCleanup) + signal.Sleep(r.options.DeferCleanup) + } + + var merr multierror.Error + + if r.resetAgentPolicyHandler != nil { + if err := r.resetAgentPolicyHandler(); err != nil { + merr = append(merr, err) + } + r.resetAgentPolicyHandler = nil + } + + if r.deletePolicyHandler != nil { + if err := r.deletePolicyHandler(); err != nil { + merr = append(merr, err) + } + r.deletePolicyHandler = nil + } + + if r.shutdownServiceHandler != nil { + if err := r.shutdownServiceHandler(); err != nil { + merr = append(merr, err) + } + r.shutdownServiceHandler = nil + } + + if r.wipeDataStreamHandler != nil { + if err := r.wipeDataStreamHandler(); err != nil { + merr = append(merr, err) + } + r.wipeDataStreamHandler = nil + } + + if r.clearCorporaHandler != nil { + if err := r.clearCorporaHandler(); err != nil { + merr = append(merr, err) + } + r.clearCorporaHandler = nil + } + + if len(merr) == 0 { + return nil + } + return merr +} + +func (r *runner) setUp() error { + locationManager, err := locations.NewLocationManager() + if err != nil { + return fmt.Errorf("reading service logs directory failed: %w", err) + } + + rallyCorpusDir := locationManager.RallyCorpusDir() + r.ctxt.Logs.Folder.Local = rallyCorpusDir + r.ctxt.Logs.Folder.Agent = RallyCorpusAgentDir + r.ctxt.Test.RunID = createRunID() + + outputDir, err := servicedeployer.CreateOutputDir(locationManager, r.ctxt.Test.RunID) + if err != nil { + return fmt.Errorf("could not create output dir for terraform deployer %w", err) + } + r.ctxt.OutputDir = outputDir + + err = servicedeployer.CreateRallyTrackDir(locationManager, r.ctxt.Test.RunID) + if err != nil { + return fmt.Errorf("could not create local rally track dir %w", err) + } + + scenario, err := readConfig(r.options.PackageRootPath, r.options.BenchName, r.ctxt) + if err != nil { + return err + } + r.scenario = scenario + + if r.scenario.Corpora.Generator != nil { + var err error + r.generator, err = r.initializeGenerator() + if err != nil { + return fmt.Errorf("can't initialize generator: %w", err) + } + } + + pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + if err != nil { + return fmt.Errorf("reading package manifest failed: %w", err) + } + + policy, err := r.createBenchmarkPolicy(pkgManifest) + if err != nil { + return err + } + r.benchPolicy = policy + + // Delete old data + logger.Debug("deleting old data in data stream...") + dataStreamManifest, err := packages.ReadDataStreamManifest( + filepath.Join( + getDataStreamPath(r.options.PackageRootPath, r.scenario.DataStream.Name), + packages.DataStreamManifestFile, + ), + ) + if err != nil { + return fmt.Errorf("reading data stream manifest failed: %w", err) + } + + r.runtimeDataStream = fmt.Sprintf( + "%s-%s.%s-%s", + dataStreamManifest.Type, + pkgManifest.Name, + dataStreamManifest.Name, + policy.Namespace, + ) + r.pipelinePrefix = fmt.Sprintf( + "%s-%s.%s-%s", + dataStreamManifest.Type, + pkgManifest.Name, + dataStreamManifest.Name, + r.scenario.Version, + ) + + r.wipeDataStreamHandler = func() error { + logger.Debugf("deleting data in data stream...") + if err := r.deleteDataStreamDocs(r.runtimeDataStream); err != nil { + return fmt.Errorf("error deleting data in data stream: %w", err) + } + return nil + } + + if err := r.deleteDataStreamDocs(r.runtimeDataStream); err != nil { + return fmt.Errorf("error deleting old data in data stream: %s: %w", r.runtimeDataStream, err) + } + + cleared, err := waitUntilTrue(func() (bool, error) { + if signal.SIGINT() { + return true, errors.New("SIGINT: cancel clearing data") + } + + hits, err := getTotalHits(r.options.ESAPI, r.runtimeDataStream) + return hits == 0, err + }, 2*time.Minute) + if err != nil || !cleared { + if err == nil { + err = errors.New("unable to clear previous data") + } + return err + } + + return nil +} + +func (r *runner) run() (report reporters.Reportable, err error) { + r.startMetricsColletion() + defer r.mcollector.stop() + + // if there is a generator config, generate the data + if r.generator != nil { + logger.Debugf("generating corpus data to %s...", r.ctxt.Logs.Folder.Local) + if err := r.runGenerator(r.ctxt.Logs.Folder.Local); err != nil { + return nil, fmt.Errorf("can't generate benchmarks data corpus for data stream: %w", err) + } + } + + rallyStats, err := r.runRally() + if err != nil { + return nil, err + } + + msum, err := r.collectAndSummarizeMetrics() + if err != nil { + return nil, fmt.Errorf("can't summarize metrics: %w", err) + } + + if err := r.reindexData(); err != nil { + return nil, fmt.Errorf("can't reindex data: %w", err) + } + + return createReport(r.options.BenchName, r.corporaFile, r.scenario, msum, rallyStats) +} + +func (r *runner) startMetricsColletion() { + // TODO collect agent hosts metrics using system integration + r.mcollector = newCollector( + r.ctxt, + r.options.BenchName, + *r.scenario, + r.options.ESAPI, + r.options.ESMetricsAPI, + r.options.MetricsInterval, + r.runtimeDataStream, + r.pipelinePrefix, + ) + r.mcollector.start() +} + +func (r *runner) collectAndSummarizeMetrics() (*metricsSummary, error) { + r.mcollector.stop() + sum, err := r.mcollector.summarize() + return sum, err +} + +func (r *runner) deleteDataStreamDocs(dataStream string) error { + body := strings.NewReader(`{ "query": { "match_all": {} } }`) + _, err := r.options.ESAPI.DeleteByQuery([]string{dataStream}, body) + if err != nil { + return err + } + return nil +} + +func (r *runner) createBenchmarkPolicy(pkgManifest *packages.PackageManifest) (*kibana.Policy, error) { + // Configure package (single data stream) via Ingest Manager APIs. + logger.Debug("creating benchmark policy...") + benchTime := time.Now().Format("20060102T15:04:05Z") + p := kibana.Policy{ + Name: fmt.Sprintf("ep-bench-%s-%s", r.options.BenchName, benchTime), + Description: fmt.Sprintf("policy created by elastic-package for benchmark %s", r.options.BenchName), + Namespace: "ep", + MonitoringEnabled: []string{"logs", "metrics"}, + } + + policy, err := r.options.KibanaClient.CreatePolicy(p) + if err != nil { + return nil, err + } + + packagePolicy, err := r.createPackagePolicy(pkgManifest, policy) + if err != nil { + return nil, err + } + + r.deletePolicyHandler = func() error { + var merr multierror.Error + + logger.Debug("deleting benchmark package policy...") + if err := r.options.KibanaClient.DeletePackagePolicy(*packagePolicy); err != nil { + merr = append(merr, fmt.Errorf("error cleaning up benchmark package policy: %w", err)) + } + + logger.Debug("deleting benchmark policy...") + if err := r.options.KibanaClient.DeletePolicy(*policy); err != nil { + merr = append(merr, fmt.Errorf("error cleaning up benchmark policy: %w", err)) + } + + if len(merr) > 0 { + return merr + } + + return nil + } + + return policy, nil +} + +func (r *runner) createPackagePolicy(pkgManifest *packages.PackageManifest, p *kibana.Policy) (*kibana.PackagePolicy, error) { + logger.Debug("creating package policy...") + + if r.scenario.Version == "" { + r.scenario.Version = pkgManifest.Version + } + + if r.scenario.Package == "" { + r.scenario.Package = pkgManifest.Name + } + + if r.scenario.PolicyTemplate == "" { + r.scenario.PolicyTemplate = pkgManifest.PolicyTemplates[0].Name + } + + pp := kibana.PackagePolicy{ + Namespace: "ep", + PolicyID: p.ID, + Force: true, + Inputs: map[string]kibana.PackagePolicyInput{ + fmt.Sprintf("%s-%s", r.scenario.PolicyTemplate, r.scenario.Input): { + Enabled: true, + Vars: r.scenario.Vars, + Streams: map[string]kibana.PackagePolicyStream{ + fmt.Sprintf("%s.%s", pkgManifest.Name, r.scenario.DataStream.Name): { + Enabled: true, + Vars: r.scenario.DataStream.Vars, + }, + }, + }, + }, + } + pp.Package.Name = pkgManifest.Name + pp.Package.Version = r.scenario.Version + + policy, err := r.options.KibanaClient.CreatePackagePolicy(pp) + if err != nil { + return nil, err + } + + return policy, nil +} + +func (r *runner) initializeGenerator() (genlib.Generator, error) { + totEvents := r.scenario.Corpora.Generator.TotalEvents + + config, err := r.getGeneratorConfig() + if err != nil { + return nil, err + } + + fields, err := r.getGeneratorFields() + if err != nil { + return nil, err + } + + tpl, err := r.getGeneratorTemplate() + if err != nil { + return nil, err + } + + genlib.InitGeneratorTimeNow(time.Now()) + genlib.InitGeneratorRandSeed(time.Now().UnixNano()) + + var generator genlib.Generator + switch r.scenario.Corpora.Generator.Template.Type { + default: + logger.Debugf("unknown generator template type %q, defaulting to \"placeholder\"", r.scenario.Corpora.Generator.Template.Type) + fallthrough + case "", "placeholder": + generator, err = genlib.NewGeneratorWithCustomTemplate(tpl, *config, fields, totEvents) + case "gotext": + generator, err = genlib.NewGeneratorWithTextTemplate(tpl, *config, fields, totEvents) + } + + if err != nil { + return nil, err + } + + return generator, nil +} + +func (r *runner) getGeneratorConfig() (*config.Config, error) { + var ( + data []byte + err error + ) + + if r.scenario.Corpora.Generator.Config.Path != "" { + configPath := filepath.Clean(filepath.Join(devPath, r.scenario.Corpora.Generator.Config.Path)) + configPath = os.ExpandEnv(configPath) + if _, err := os.Stat(configPath); err != nil { + return nil, fmt.Errorf("can't find config file %s: %w", configPath, err) + } + data, err = os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("can't read config file %s: %w", configPath, err) + } + } else if len(r.scenario.Corpora.Generator.Config.Raw) > 0 { + data, err = yaml.Marshal(r.scenario.Corpora.Generator.Config.Raw) + if err != nil { + return nil, fmt.Errorf("can't parse raw generator config: %w", err) + } + } + + cfg, err := config.LoadConfigFromYaml(data) + if err != nil { + return nil, fmt.Errorf("can't get generator config: %w", err) + } + + return &cfg, nil +} + +func (r *runner) getGeneratorFields() (fields.Fields, error) { + var ( + data []byte + err error + ) + + if r.scenario.Corpora.Generator.Fields.Path != "" { + fieldsPath := filepath.Clean(filepath.Join(devPath, r.scenario.Corpora.Generator.Fields.Path)) + fieldsPath = os.ExpandEnv(fieldsPath) + if _, err := os.Stat(fieldsPath); err != nil { + return nil, fmt.Errorf("can't find fields file %s: %w", fieldsPath, err) + } + + data, err = os.ReadFile(fieldsPath) + if err != nil { + return nil, fmt.Errorf("can't read fields file %s: %w", fieldsPath, err) + } + } else if len(r.scenario.Corpora.Generator.Fields.Raw) > 0 { + data, err = yaml.Marshal(r.scenario.Corpora.Generator.Config.Raw) + if err != nil { + return nil, fmt.Errorf("can't parse raw generator config: %w", err) + } + } + + fields, err := fields.LoadFieldsWithTemplateFromString(context.Background(), string(data)) + if err != nil { + return nil, fmt.Errorf("could not load fields yaml: %w", err) + } + + return fields, nil +} + +func (r *runner) getGeneratorTemplate() ([]byte, error) { + var ( + data []byte + err error + ) + + if r.scenario.Corpora.Generator.Template.Path != "" { + tplPath := filepath.Clean(filepath.Join(devPath, r.scenario.Corpora.Generator.Template.Path)) + tplPath = os.ExpandEnv(tplPath) + if _, err := os.Stat(tplPath); err != nil { + return nil, fmt.Errorf("can't find template file %s: %w", tplPath, err) + } + + data, err = os.ReadFile(tplPath) + if err != nil { + return nil, fmt.Errorf("can't read template file %s: %w", tplPath, err) + } + } else if len(r.scenario.Corpora.Generator.Template.Raw) > 0 { + data = []byte(r.scenario.Corpora.Generator.Template.Raw) + } + + return data, nil +} + +func (r *runner) runGenerator(destDir string) error { + corporaFile, err := os.CreateTemp(destDir, "corpus-*") + if err != nil { + return err + } + defer corporaFile.Close() + + if err := corporaFile.Chmod(os.ModePerm); err != nil { + return err + } + + buf := bytes.NewBufferString("") + var corpusDocsCount uint64 + for { + err := r.generator.Emit(buf) + if err == io.EOF { + break + } + + if err != nil { + return err + } + + // TODO: this should be taken care of by the corpus generator tool, once it will be done let's remove this + replacer := strings.NewReplacer("\n", "") + event := replacer.Replace(buf.String()) + if _, err = corporaFile.Write([]byte(event)); err != nil { + return err + } + + if _, err = corporaFile.Write([]byte("\n")); err != nil { + return err + } + + buf.Reset() + corpusDocsCount += 1 + } + + r.corporaFile = corporaFile.Name() + corporaFileForTrack, err := os.Open(r.corporaFile) + if err != nil { + return err + } + + trackFile, err := os.CreateTemp(destDir, "track-*.json") + if err != nil { + return err + } + r.trackFile = trackFile.Name() + rallyTrackContent, err := corpusgenerator.GenerateRallyTrack(r.runtimeDataStream, corporaFileForTrack, corpusDocsCount) + err = os.WriteFile(r.trackFile, rallyTrackContent, os.ModePerm) + if err != nil { + return err + } + defer trackFile.Close() + + reportFile, err := os.CreateTemp(destDir, "report-*.csv") + if err != nil { + return err + } + defer reportFile.Close() + + r.reportFile = reportFile.Name() + + r.clearCorporaHandler = func() error { + return errors.Join(os.Remove(r.corporaFile), os.Remove(r.reportFile), os.Remove(r.trackFile)) + } + + return r.generator.Close() +} + +func (r *runner) runRally() ([]rallyStat, error) { + logger.Debug("running rally...") + profileConfig, err := stack.StackInitConfig(r.options.Profile) + if err != nil { + return nil, fmt.Errorf("failed to load config from profile: %w", err) + } + + elasticsearchHost, found := os.LookupEnv(common.ESMetricstoreHostEnv) + if !found { + status, err := stack.Status(stack.Options{Profile: r.options.Profile}) + if err != nil { + return nil, fmt.Errorf("failed to check status of stack in current profile: %w", err) + } + if len(status) == 0 { + return nil, stack.ErrUnavailableStack + } + + elasticsearchHost = profileConfig.ElasticsearchHostPort + logger.Debugf("Configuring rally with Elasticsearch host from current profile (profile: %s, host: %q)", r.options.Profile.ProfileName, elasticsearchHost) + } + + elasticsearchPassword, found := os.LookupEnv(stack.ElasticsearchPasswordEnv) + if !found { + elasticsearchPassword = profileConfig.ElasticsearchPassword + } + elasticsearchUsername, found := os.LookupEnv(stack.ElasticsearchUsernameEnv) + if !found { + elasticsearchUsername = profileConfig.ElasticsearchUsername + } + + cmd := exec.Command("esrally", "race", "--race-id="+r.ctxt.Test.RunID, "--report-format=csv", fmt.Sprintf(`--report-file=%s`, r.reportFile), fmt.Sprintf(`--target-hosts={"default":["%s"]}`, elasticsearchHost), fmt.Sprintf(`--track-path=%s`, r.trackFile), fmt.Sprintf(`--client-options={"default":{"basic_auth_user":"%s","basic_auth_password":"%s","use_ssl":true,"verify_certs":false}}`, elasticsearchUsername, elasticsearchPassword), "--pipeline=benchmark-only") + errOutput := new(bytes.Buffer) + cmd.Stderr = errOutput + + logger.Debugf("output command: %s", cmd) + _, err = cmd.Output() + if err != nil { + return nil, fmt.Errorf("could not run esrally track in path: %s (stderr=%q): %w", r.ctxt.Logs.Folder.Local, errOutput.String(), err) + } + + reportCSV, err := os.Open(r.reportFile) + if err != nil { + return nil, fmt.Errorf("could not open esrally report in path: %s (stderr=%q): %w", r.ctxt.Logs.Folder.Local, errOutput.String(), err) + } + + reader := csv.NewReader(reportCSV) + + stats := make([]rallyStat, 0) + for { + record, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("could not read esrally report in path: %s (stderr=%q): %w", r.ctxt.Logs.Folder.Local, errOutput.String(), err) + } + + stats = append(stats, rallyStat{Metric: record[0], Task: record[1], Value: record[2], Unit: record[3]}) + } + + return stats, nil +} + +// reindexData will read all data generated during the benchmark and will reindex it to the metrisctore +func (r *runner) reindexData() error { + if !r.options.ReindexData { + return nil + } + if r.options.ESMetricsAPI == nil { + return errors.New("the option to reindex data is set, but the metricstore was not initialized") + } + + logger.Debug("starting reindexing of data...") + + logger.Debug("getting orignal mappings...") + // Get the mapping from the source data stream + mappingRes, err := r.options.ESAPI.Indices.GetMapping( + r.options.ESAPI.Indices.GetMapping.WithIndex(r.runtimeDataStream), + ) + if err != nil { + return fmt.Errorf("error getting mapping: %w", err) + } + defer mappingRes.Body.Close() + + body, err := io.ReadAll(mappingRes.Body) + if err != nil { + return fmt.Errorf("error reading mapping body: %w", err) + } + + mappings := map[string]struct { + Mappings json.RawMessage + }{} + + if err := json.Unmarshal(body, &mappings); err != nil { + return fmt.Errorf("error unmarshaling mappings: %w", err) + } + + if len(mappings) != 1 { + return fmt.Errorf("exactly 1 mapping was expected, got %d", len(mappings)) + } + + var mapping string + for _, v := range mappings { + mapping = string(v.Mappings) + } + + reader := bytes.NewReader( + []byte(fmt.Sprintf(`{ + "settings": {"number_of_replicas":0}, + "mappings": %s + }`, mapping)), + ) + + indexName := fmt.Sprintf("bench-reindex-%s-%s", r.runtimeDataStream, r.ctxt.Test.RunID) + + logger.Debugf("creating %s index in metricstore...", indexName) + + createRes, err := r.options.ESMetricsAPI.Indices.Create( + indexName, + r.options.ESMetricsAPI.Indices.Create.WithBody(reader), + ) + if err != nil { + return fmt.Errorf("could not create index: %w", err) + } + defer createRes.Body.Close() + + if createRes.IsError() { + return errors.New("got a response error while creating index") + } + + bodyReader := strings.NewReader(`{"query":{"match_all":{}}}`) + + logger.Debug("starting scrolling of events...") + res, err := r.options.ESAPI.Search( + r.options.ESAPI.Search.WithIndex(r.runtimeDataStream), + r.options.ESAPI.Search.WithBody(bodyReader), + r.options.ESAPI.Search.WithScroll(time.Minute), + r.options.ESAPI.Search.WithSize(10000), + ) + if err != nil { + return fmt.Errorf("error executing search: %w", err) + } + defer res.Body.Close() + + type searchRes struct { + Error *struct { + Reason string `json:"reson"` + } `json:"error"` + ScrollID string `json:"_scroll_id"` + Hits []struct { + ID string `json:"_id"` + Source map[string]interface{} `json:"_source"` + } `json:"hits"` + } + + // Iterate through the search results using the Scroll API + for { + var sr searchRes + if err := json.NewDecoder(res.Body).Decode(&sr); err != nil { + return fmt.Errorf("error decoding search response: %w", err) + } + + if sr.Error != nil { + return fmt.Errorf("error searching for documents: %s", sr.Error.Reason) + } + + if len(sr.Hits) == 0 { + break + } + + var bulkBodyBuilder strings.Builder + for _, hit := range sr.Hits { + bulkBodyBuilder.WriteString(fmt.Sprintf("{\"index\":{\"_index\":\"%s\",\"_id\":\"%s\"}}\n", indexName, hit.ID)) + enriched := r.enrichEventWithBenchmarkMetadata(hit.Source) + src, err := json.Marshal(enriched) + if err != nil { + return fmt.Errorf("error decoding _source: %w", err) + } + bulkBodyBuilder.WriteString(fmt.Sprintf("%s\n", string(src))) + } + + logger.Debugf("bulk request of %d events...", len(sr.Hits)) + + bulkRes, err := r.options.ESMetricsAPI.Bulk(strings.NewReader(bulkBodyBuilder.String())) + if err != nil { + return fmt.Errorf("error performing the bulk index request: %w", err) + } + bulkRes.Body.Close() + + if sr.ScrollID == "" { + return errors.New("error getting scroll ID") + } + + res, err = r.options.ESAPI.Scroll( + r.options.ESAPI.Scroll.WithScrollID(sr.ScrollID), + r.options.ESAPI.Scroll.WithScroll(time.Minute), + ) + if err != nil { + return fmt.Errorf("error executing scroll: %s", err) + } + res.Body.Close() + } + + logger.Debug("reindexing operation finished") + return nil +} + +type benchMeta struct { + Info struct { + Benchmark string `json:"benchmark"` + RunID string `json:"run_id"` + } `json:"info"` + Parameters scenario `json:"parameter"` +} + +func (r *runner) enrichEventWithBenchmarkMetadata(e map[string]interface{}) map[string]interface{} { + var m benchMeta + m.Info.Benchmark = r.options.BenchName + m.Info.RunID = r.ctxt.Test.RunID + m.Parameters = *r.scenario + e["benchmark_metadata"] = m + return e +} + +func getTotalHits(esapi *elasticsearch.API, dataStream string) (int, error) { + resp, err := esapi.Count( + esapi.Count.WithIndex(dataStream), + ) + if err != nil { + return 0, fmt.Errorf("could not search data stream: %w", err) + } + defer resp.Body.Close() + + var results struct { + Count int + Error *struct { + Type string + Reason string + } + Status int + } + + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return 0, fmt.Errorf("could not decode search results response: %w", err) + } + + numHits := results.Count + if results.Error != nil { + logger.Debugf("found %d hits in %s data stream: %s: %s Status=%d", + numHits, dataStream, results.Error.Type, results.Error.Reason, results.Status) + } else { + logger.Debugf("found %d hits in %s data stream", numHits, dataStream) + } + + return numHits, nil +} + +func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) { + timeoutTicker := time.NewTicker(timeout) + defer timeoutTicker.Stop() + + retryTicker := time.NewTicker(5 * time.Second) + defer retryTicker.Stop() + + for { + result, err := fn() + if err != nil { + return false, err + } + if result { + return true, nil + } + + select { + case <-retryTicker.C: + continue + case <-timeoutTicker.C: + return false, nil + } + } +} + +func createRunID() string { + return uuid.New().String() +} + +func getDataStreamPath(packageRoot, dataStream string) string { + return filepath.Join(packageRoot, "data_stream", dataStream) +} diff --git a/internal/benchrunner/runners/rally/scenario.go b/internal/benchrunner/runners/rally/scenario.go new file mode 100644 index 0000000000..59e9f3cbe7 --- /dev/null +++ b/internal/benchrunner/runners/rally/scenario.go @@ -0,0 +1,119 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package rally + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/aymerick/raymond" + "github.com/elastic/go-ucfg" + "github.com/elastic/go-ucfg/yaml" + + "github.com/elastic/elastic-package/internal/servicedeployer" +) + +const devPath = "_dev/benchmark/rally" + +type scenario struct { + Package string `config:"package" json:"package"` + Description string `config:"description" json:"description"` + Version string `config:"version" json:"version"` + PolicyTemplate string `config:"policy_template" json:"policy_template"` + Input string `config:"input" json:"input"` + Vars map[string]interface{} `config:"vars" json:"vars"` + DataStream dataStream `config:"data_stream" json:"data_stream"` + WarmupTimePeriod time.Duration `config:"warmup_time_period" json:"warmup_time_period"` + Corpora corpora `config:"corpora" json:"corpora"` +} + +type dataStream struct { + Name string `config:"name" json:"name"` + Vars map[string]interface{} `config:"vars" json:"vars"` +} + +type corpora struct { + Generator *generator `config:"generator" json:"generator"` +} + +type generator struct { + TotalEvents uint64 `config:"total_events" json:"total_events"` + Template corporaTemplate `config:"template" json:"template"` + Config corporaConfig `config:"config" json:"config"` + Fields corporaFields `config:"fields" json:"fields"` +} + +type corporaTemplate struct { + Raw string `config:"raw" json:"raw"` + Path string `config:"path" json:"path"` + Type string `config:"type" json:"type"` +} + +type corporaConfig struct { + Raw map[string]interface{} `config:"raw" json:"raw"` + Path string `config:"path" json:"path"` +} + +type corporaFields struct { + Raw map[string]interface{} `config:"raw" json:"raw"` + Path string `config:"path" json:"path"` +} + +func defaultConfig() *scenario { + return &scenario{} +} + +func readConfig(path, scenario string, ctxt servicedeployer.ServiceContext) (*scenario, error) { + configPath := filepath.Join(path, devPath, fmt.Sprintf("%s.yml", scenario)) + data, err := os.ReadFile(configPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("unable to find system benchmark configuration file: %s: %w", configPath, err) + } + return nil, fmt.Errorf("could not load system benchmark configuration file: %s: %w", configPath, err) + } + + data, err = applyContext(data, ctxt) + if err != nil { + return nil, fmt.Errorf("could not apply context to benchmark configuration file: %s: %w", configPath, err) + } + + cfg, err := yaml.NewConfig(data, ucfg.PathSep(".")) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + configPath = filepath.Join(path, devPath, fmt.Sprintf("%s.yaml", scenario)) + cfg, err = yaml.NewConfigWithFile(configPath) + } + if err != nil { + return nil, fmt.Errorf("can't load scenario: %s: %w", configPath, err) + } + } + + c := defaultConfig() + if err := cfg.Unpack(c); err != nil { + return nil, fmt.Errorf("can't unpack scenario configuration: %s: %w", configPath, err) + } + return c, nil +} + +// applyContext takes the given system benchmark configuration (data) and replaces any placeholder variables in +// it with values from the given context (ctxt). The context may be populated from various sources but usually the +// most interesting context values will be set by a ServiceDeployer in its SetUp method. +func applyContext(data []byte, ctxt servicedeployer.ServiceContext) ([]byte, error) { + tmpl, err := raymond.Parse(string(data)) + if err != nil { + return data, fmt.Errorf("parsing template body failed: %w", err) + } + tmpl.RegisterHelpers(ctxt.Aliases()) + + result, err := tmpl.Exec(ctxt) + if err != nil { + return data, fmt.Errorf("could not render data with context: %w", err) + } + return []byte(result), nil +} From c9c3cc791a25f660486891d3610192d382deab3b Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 10:14:37 +0900 Subject: [PATCH 05/48] fix generator config yaml for system benchmark --- .../system/logs-benchmark/config.yml | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml index af365d3c94..bbceb2681c 100644 --- a/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml +++ b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml @@ -1,40 +1,39 @@ -- name: IP - cardinality: - numerator: 1 - denominator: 100 -- name: Day - range: - min: 1 - max: 28 -- name: H - range: - min: 10 - max: 23 -- name: MS - range: - min: 10 - max: 59 -- name: Mon - enum: - - "Jan" - - "Feb" - - "Mar" - - "Apr" - - "May" - - "Jun" - - "Jul" - - "Aug" - - "Sep" - - "Oct" - - "Nov" - - "Dec" -- name: StatusCode - enum: ["200", "400", "404"] -- name: Size - range: - min: 1 - max: 1000 -- name: Port - range: - min: 8000 - max: 8080 +fields: + - name: IP + cardinality: 100 + - name: Day + range: + min: 1 + max: 28 + - name: H + range: + min: 10 + max: 23 + - name: MS + range: + min: 10 + max: 59 + - name: Mon + enum: + - "Jan" + - "Feb" + - "Mar" + - "Apr" + - "May" + - "Jun" + - "Jul" + - "Aug" + - "Sep" + - "Oct" + - "Nov" + - "Dec" + - name: StatusCode + enum: ["200", "400", "404"] + - name: Size + range: + min: 1 + max: 1000 + - name: Port + range: + min: 8000 + max: 8080 From 8dda99dcf828e0bb53fea4eb337228a846cca920 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 10:15:39 +0900 Subject: [PATCH 06/48] add rally benchmark test files --- .../_dev/benchmark/rally/logs-benchmark.yml | 21 ++++++++++ .../benchmark/rally/logs-benchmark/config.yml | 22 +++++++++++ .../benchmark/rally/logs-benchmark/fields.yml | 23 +++++++++++ .../rally/logs-benchmark/template.ndjson | 14 +++++++ .../benchmarks/rally_benchmark/changelog.yml | 6 +++ .../_dev/benchmark/pipeline/access-raw.log | 1 + .../testds/_dev/benchmark/pipeline/config.yml | 1 + .../testds/agent/stream/filestream.yml.hbs | 4 ++ .../elasticsearch/ingest_pipeline/default.yml | 23 +++++++++++ .../data_stream/testds/fields/base-fields.yml | 38 +++++++++++++++++++ .../data_stream/testds/manifest.yml | 16 ++++++++ .../benchmarks/rally_benchmark/docs/README.md | 2 + .../benchmarks/rally_benchmark/manifest.yml | 21 ++++++++++ 13 files changed, 192 insertions(+) create mode 100644 test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml create mode 100644 test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/config.yml create mode 100644 test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/fields.yml create mode 100644 test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/template.ndjson create mode 100644 test/packages/benchmarks/rally_benchmark/changelog.yml create mode 100644 test/packages/benchmarks/rally_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log create mode 100644 test/packages/benchmarks/rally_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml create mode 100644 test/packages/benchmarks/rally_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs create mode 100644 test/packages/benchmarks/rally_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/benchmarks/rally_benchmark/data_stream/testds/fields/base-fields.yml create mode 100644 test/packages/benchmarks/rally_benchmark/data_stream/testds/manifest.yml create mode 100644 test/packages/benchmarks/rally_benchmark/docs/README.md create mode 100644 test/packages/benchmarks/rally_benchmark/manifest.yml diff --git a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml new file mode 100644 index 0000000000..79db6f1384 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml @@ -0,0 +1,21 @@ +--- +description: Benchmark 20000 events ingested +input: filestream +vars: ~ +data_stream: + name: testds + vars: + paths: + - "dummy path" +warmup_time_period: 10s +wait_for_data_timeout: 10m +corpora: + generator: + total_events: 20000 + template: + type: gotext + path: ./logs-benchmark/template.ndjson + config: + path: ./logs-benchmark/config.yml + fields: + path: ./logs-benchmark/fields.yml diff --git a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/config.yml b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/config.yml new file mode 100644 index 0000000000..03b8ad10a3 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/config.yml @@ -0,0 +1,22 @@ +fields: + - name: '@timestamp' + period: 1s + - name: container.id + - name: log.flags + type: keyword + - name: log.offset + cardinality: 10000 + - name: tags + enum: ["production", "env2"] + - name: IP + cardinality: 100 + - name: StatusCode + enum: ["200", "400", "404"] + - name: Size + range: + min: 1 + max: 1000 + - name: Port + range: + min: 8000 + max: 8080 diff --git a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/fields.yml b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/fields.yml new file mode 100644 index 0000000000..32a82b632d --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/fields.yml @@ -0,0 +1,23 @@ +- name: timestamp + type: date +- name: container.id + type: keyword +- name: log.file.path + example: /var/log/fun-times.log + type: keyword +- name: log.flags + type: keyword +- name: log.offset + type: long +- name: tags + type: keyword +- name: IP + type: ip +- name: StatusCode + type: keyword +- name: Size + type: long +- name: Hostname + type: keyword +- name: Port + type: long diff --git a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/template.ndjson b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/template.ndjson new file mode 100644 index 0000000000..3567b2f28a --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark/template.ndjson @@ -0,0 +1,14 @@ +{{- $timestamp := generate "timestamp" -}} +{ +"@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }}", +"data_stream.type": "logs", +"data_stream.dataset": "rally_benchmarks.testds", +"data_stream.namespace": "ep", +"container.id": "{{ generate "container.id" }}", +"input.type": "filestream", +"log.file.path": "{{ generate "log.file.path" }}", +"log.flags": "{{ generate "log.flags" }}", +"log.offset": {{ generate "log.offset" }}, +"tags": ["rally_benchmark.testds", "forwarded", "{{ generate "tags" }}" ], +"message": "{{ generate "IP" }} - - [{{ $timestamp.Format "02/Jan/2006:15:04:05.999999 -0700" }}] \"GET /favicon.ico HTTP/1.1\" {{ generate "StatusCode" }} {{ generate "Size" }} \"http://{{ generate "Hostname" }}:{{ generate "Port" }}/\" \"skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"" +} \ No newline at end of file diff --git a/test/packages/benchmarks/rally_benchmark/changelog.yml b/test/packages/benchmarks/rally_benchmark/changelog.yml new file mode 100644 index 0000000000..69f3bf2e15 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "999.999.999" + changes: + - description: initial release + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/elastic-package/pull/1 diff --git a/test/packages/benchmarks/rally_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log b/test/packages/benchmarks/rally_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log new file mode 100644 index 0000000000..c8c9ffe960 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log @@ -0,0 +1 @@ +1.2.3.4 - - [25/Oct/2016:14:49:34 +0200] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" \ No newline at end of file diff --git a/test/packages/benchmarks/rally_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml b/test/packages/benchmarks/rally_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml new file mode 100644 index 0000000000..30a2b50cf6 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml @@ -0,0 +1 @@ +num_docs: 10000 diff --git a/test/packages/benchmarks/rally_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs b/test/packages/benchmarks/rally_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs new file mode 100644 index 0000000000..cc801fea22 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} diff --git a/test/packages/benchmarks/rally_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml b/test/packages/benchmarks/rally_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..f39b8ee231 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,23 @@ +--- +description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent + plugins. +processors: + - grok: + field: message + patterns: + - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) + - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" + %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} + "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" + pattern_definitions: + NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? + NGINX_NOTSEPARATOR: "[^\t ,:]+" + NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + ignore_missing: true + - user_agent: + field: user_agent.original + ignore_missing: true +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/benchmarks/rally_benchmark/data_stream/testds/fields/base-fields.yml b/test/packages/benchmarks/rally_benchmark/data_stream/testds/fields/base-fields.yml new file mode 100644 index 0000000000..0ec2cc7e01 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/data_stream/testds/fields/base-fields.yml @@ -0,0 +1,38 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: container.id + description: Unique container id. + ignore_above: 1024 + type: keyword +- name: input.type + description: Type of Filebeat input. + type: keyword +- name: log.file.path + description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + type: keyword +- name: log.source.address + description: Source address from which the log event was read / sent from. + type: keyword +- name: log.flags + description: Flags for the log file. + type: keyword +- name: log.offset + description: Offset of the entry in the log file. + type: long +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword diff --git a/test/packages/benchmarks/rally_benchmark/data_stream/testds/manifest.yml b/test/packages/benchmarks/rally_benchmark/data_stream/testds/manifest.yml new file mode 100644 index 0000000000..b7c55dea91 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/data_stream/testds/manifest.yml @@ -0,0 +1,16 @@ +title: Test +release: experimental +type: logs +streams: + - input: filestream + enabled: false + title: Logs + description: Collect logs + template_path: filestream.yml.hbs + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true diff --git a/test/packages/benchmarks/rally_benchmark/docs/README.md b/test/packages/benchmarks/rally_benchmark/docs/README.md new file mode 100644 index 0000000000..e0ef7b4a18 --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/docs/README.md @@ -0,0 +1,2 @@ +# Test integration + diff --git a/test/packages/benchmarks/rally_benchmark/manifest.yml b/test/packages/benchmarks/rally_benchmark/manifest.yml new file mode 100644 index 0000000000..579c27a30f --- /dev/null +++ b/test/packages/benchmarks/rally_benchmark/manifest.yml @@ -0,0 +1,21 @@ +format_version: 3.0.0 +name: rally_benchmarks +title: Rally benchmarks +version: 999.999.999 +description: Test for rally benchmark runner +categories: ["network"] +type: integration +conditions: + kibana: + version: '^8.0.0' +policy_templates: + - name: testpo + title: Test + description: Description + inputs: + - type: filestream + title: Collect logs + description: Collecting logs +owner: + github: elastic/integrations + type: elastic From dfd63bdf82cafac8091c0e6b2f31f515f2318718 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 10:37:21 +0900 Subject: [PATCH 07/48] fix from CI --- internal/benchrunner/runners/rally/report.go | 2 ++ internal/benchrunner/runners/rally/runner.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/internal/benchrunner/runners/rally/report.go b/internal/benchrunner/runners/rally/report.go index 44e7e9cde2..7795d31114 100644 --- a/internal/benchrunner/runners/rally/report.go +++ b/internal/benchrunner/runners/rally/report.go @@ -129,6 +129,8 @@ func reportHumanFormat(r *report) []byte { "warmup time period", r.Parameters.WarmupTimePeriod, ) + report.WriteString(renderBenchmarkTable("parameters", pkvs...) + "\n") + if r.Parameters.Corpora.Generator != nil { pkvs = append(pkvs, "corpora.generator.total_events", r.Parameters.Corpora.Generator.TotalEvents, diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 086e97e6e6..b775f61d79 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -568,6 +568,9 @@ func (r *runner) runGenerator(destDir string) error { } r.trackFile = trackFile.Name() rallyTrackContent, err := corpusgenerator.GenerateRallyTrack(r.runtimeDataStream, corporaFileForTrack, corpusDocsCount) + if err != nil { + return err + } err = os.WriteFile(r.trackFile, rallyTrackContent, os.ModePerm) if err != nil { return err From 08acf94011f4fb5dd386ce45d9833fda46323e1b Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 12:22:43 +0900 Subject: [PATCH 08/48] fix from CI --- README.md | 12 ++++++++++++ internal/benchrunner/runners/common/env.go | 4 ++++ internal/benchrunner/runners/rally/report.go | 2 ++ internal/benchrunner/runners/rally/runner.go | 7 ++++--- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ebb94c6dfa..585f3dbcb2 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,12 @@ These benchmarks allow you to benchmark any Ingest Node Pipelines defined by you For details on how to configure pipeline benchmarks for a package, review the [HOWTO guide](./docs/howto/pipeline_benchmarking.md). +#### Rally Benchmarks + +These benchmarks allow you to benchmark an integration corpus with rally. + +For details on how to configure rally benchmarks for a package, review the [HOWTO guide](./docs/howto/rally_benchmarking.md). + #### System Benchmarks These benchmarks allow you to benchmark an integration end to end. @@ -175,6 +181,12 @@ _Context: package_ Run pipeline benchmarks for the package. +### `elastic-package benchmark rally` + +_Context: package_ + +Run rally benchmarks for the package. + ### `elastic-package benchmark system` _Context: package_ diff --git a/internal/benchrunner/runners/common/env.go b/internal/benchrunner/runners/common/env.go index 6b009e14e7..b084c3ed92 100644 --- a/internal/benchrunner/runners/common/env.go +++ b/internal/benchrunner/runners/common/env.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package common import "github.com/elastic/elastic-package/internal/environment" diff --git a/internal/benchrunner/runners/rally/report.go b/internal/benchrunner/runners/rally/report.go index 7795d31114..b1aaad652a 100644 --- a/internal/benchrunner/runners/rally/report.go +++ b/internal/benchrunner/runners/rally/report.go @@ -144,6 +144,8 @@ func reportHumanFormat(r *report) []byte { ) } + report.WriteString(renderBenchmarkTable("parameters", pkvs...) + "\n") + report.WriteString(renderBenchmarkTable( "cluster info", "name", r.ClusterName, diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index b775f61d79..3f74113bb8 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -11,9 +11,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/elastic/elastic-package/internal/benchrunner/runners/common" - "github.com/elastic/elastic-package/internal/corpusgenerator" - "github.com/elastic/elastic-package/internal/stack" "io" "os" "os/exec" @@ -21,6 +18,10 @@ import ( "strings" "time" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" + "github.com/elastic/elastic-package/internal/corpusgenerator" + "github.com/elastic/elastic-package/internal/stack" + "github.com/google/uuid" "gopkg.in/yaml.v3" From d320aa5ae2cb3ad9eed1b8c608dce8c8fdfe1efc Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 12:25:23 +0900 Subject: [PATCH 09/48] fix repeated print of paramters --- internal/benchrunner/runners/rally/report.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/benchrunner/runners/rally/report.go b/internal/benchrunner/runners/rally/report.go index b1aaad652a..0cd4b86134 100644 --- a/internal/benchrunner/runners/rally/report.go +++ b/internal/benchrunner/runners/rally/report.go @@ -129,8 +129,6 @@ func reportHumanFormat(r *report) []byte { "warmup time period", r.Parameters.WarmupTimePeriod, ) - report.WriteString(renderBenchmarkTable("parameters", pkvs...) + "\n") - if r.Parameters.Corpora.Generator != nil { pkvs = append(pkvs, "corpora.generator.total_events", r.Parameters.Corpora.Generator.TotalEvents, From 298b97f57d2244466f28d74e94bbb4488e8345d2 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 24 Oct 2023 12:44:50 +0900 Subject: [PATCH 10/48] fix ES host env variable for rally --- internal/benchrunner/runners/rally/runner.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 3f74113bb8..171f3a3a54 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -18,7 +18,6 @@ import ( "strings" "time" - "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "github.com/elastic/elastic-package/internal/corpusgenerator" "github.com/elastic/elastic-package/internal/stack" @@ -600,7 +599,7 @@ func (r *runner) runRally() ([]rallyStat, error) { return nil, fmt.Errorf("failed to load config from profile: %w", err) } - elasticsearchHost, found := os.LookupEnv(common.ESMetricstoreHostEnv) + elasticsearchHost, found := os.LookupEnv(stack.ElasticsearchHostEnv) if !found { status, err := stack.Status(stack.Options{Profile: r.options.Profile}) if err != nil { From 373dba82661bfba694a06b62357ce45013af88df Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 25 Oct 2023 15:19:17 +0900 Subject: [PATCH 11/48] remove wait_for_data_timeout --- .../rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml index 79db6f1384..1112923371 100644 --- a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml +++ b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml @@ -8,7 +8,6 @@ data_stream: paths: - "dummy path" warmup_time_period: 10s -wait_for_data_timeout: 10m corpora: generator: total_events: 20000 From 918601f4ea4aea6a7a802da1052510593713b963 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 25 Oct 2023 15:19:24 +0900 Subject: [PATCH 12/48] changelog --- test/packages/benchmarks/rally_benchmark/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/packages/benchmarks/rally_benchmark/changelog.yml b/test/packages/benchmarks/rally_benchmark/changelog.yml index 69f3bf2e15..dde678acef 100644 --- a/test/packages/benchmarks/rally_benchmark/changelog.yml +++ b/test/packages/benchmarks/rally_benchmark/changelog.yml @@ -3,4 +3,4 @@ changes: - description: initial release type: enhancement # can be one of: enhancement, bugfix, breaking-change - link: https://github.com/elastic/elastic-package/pull/1 + link: https://github.com/elastic/elastic-package/pull/1522 From dccc3c7ce4e2a53002fed4aaf8a9927b19e47fe6 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 25 Oct 2023 15:19:50 +0900 Subject: [PATCH 13/48] spec reference --- test/packages/benchmarks/rally_benchmark/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/packages/benchmarks/rally_benchmark/manifest.yml b/test/packages/benchmarks/rally_benchmark/manifest.yml index 579c27a30f..40b31e1445 100644 --- a/test/packages/benchmarks/rally_benchmark/manifest.yml +++ b/test/packages/benchmarks/rally_benchmark/manifest.yml @@ -1,4 +1,4 @@ -format_version: 3.0.0 +format_version: 3.0.1 name: rally_benchmarks title: Rally benchmarks version: 999.999.999 From ddf91d6217ab89da8b8c040f8d2406cd2463d297 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Mon, 30 Oct 2023 19:18:35 +0900 Subject: [PATCH 14/48] cr fixes --- cmd/benchmark.go | 18 +++++ internal/benchrunner/runners/rally/options.go | 35 +++++++--- internal/benchrunner/runners/rally/runner.go | 69 +++++++++++++------ internal/cobraext/flags.go | 6 ++ 4 files changed, 97 insertions(+), 31 deletions(-) diff --git a/cmd/benchmark.go b/cmd/benchmark.go index bd8b77ba89..998325916e 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -238,6 +238,8 @@ func getRallyCommand() *cobra.Command { cmd.Flags().DurationP(cobraext.BenchMetricsIntervalFlagName, "", time.Second, cobraext.BenchMetricsIntervalFlagDescription) cmd.Flags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) cmd.Flags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) + cmd.Flags().StringP(cobraext.BenchCorpusRallyTrackOutputDirFlagName, "", "", cobraext.BenchCorpusRallyTrackOutputDirFlagDescription) + cmd.Flags().BoolP(cobraext.BenchCorpusRallyDryRunFlagName, "", false, cobraext.BenchCorpusRallyDryRunFlagDescription) return cmd } @@ -270,6 +272,16 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { return cobraext.FlagParsingError(err, cobraext.BenchReindexToMetricstoreFlagName) } + rallyTrackOutputDir, err := cmd.Flags().GetString(cobraext.BenchCorpusRallyTrackOutputDirFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchCorpusRallyTrackOutputDirFlagName) + } + + rallyDryRun, err := cmd.Flags().GetBool(cobraext.BenchCorpusRallyDryRunFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchCorpusRallyDryRunFlagName) + } + packageRootPath, found, err := packages.FindPackageRoot() if !found { return errors.New("package root not found") @@ -309,6 +321,8 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { rally.WithESAPI(esClient.API), rally.WithKibanaClient(kc), rally.WithProfile(profile), + rally.WithRallyTrackOutputDir(rallyTrackOutputDir), + rally.WithRallyDryRun(rallyDryRun), } esMetricsClient, err := initializeESMetricsClient(cmd.Context()) @@ -322,6 +336,10 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { runner := rally.NewRallyBenchmark(rally.NewOptions(withOpts...)) r, err := benchrunner.Run(runner) + if errors.Is(err, rally.DryRunError) { + return nil + } + if err != nil { return fmt.Errorf("error running package rally benchmarks: %w", err) } diff --git a/internal/benchrunner/runners/rally/options.go b/internal/benchrunner/runners/rally/options.go index 5dd466bb54..02aa8e6d7d 100644 --- a/internal/benchrunner/runners/rally/options.go +++ b/internal/benchrunner/runners/rally/options.go @@ -14,17 +14,18 @@ import ( // Options contains benchmark runner options. type Options struct { - ESAPI *elasticsearch.API - KibanaClient *kibana.Client - DeferCleanup time.Duration - MetricsInterval time.Duration - ReindexData bool - ESMetricsAPI *elasticsearch.API - BenchName string - PackageRootPath string - Variant string - Profile *profile.Profile - RallyClientOptions ClientOptions + ESAPI *elasticsearch.API + KibanaClient *kibana.Client + DeferCleanup time.Duration + MetricsInterval time.Duration + ReindexData bool + ESMetricsAPI *elasticsearch.API + BenchName string + PackageRootPath string + Variant string + Profile *profile.Profile + RallyTrackOutputDir string + DryRun bool } type ClientOptions struct { @@ -101,3 +102,15 @@ func WithProfile(p *profile.Profile) OptionFunc { opts.Profile = p } } + +func WithRallyTrackOutputDir(r string) OptionFunc { + return func(opts *Options) { + opts.RallyTrackOutputDir = r + } +} + +func WithRallyDryRun(d bool) OptionFunc { + return func(opts *Options) { + opts.DryRun = d + } +} diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 171f3a3a54..c9c1ff437e 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/magefile/mage/sh" "io" "os" "os/exec" @@ -44,12 +45,13 @@ const ( // RallyCorpusAgentDir is folder path where rally corporsa files produced by the service // are stored on the Rally container's filesystem. RallyCorpusAgentDir = "/tmp/rally_corpus" - devDeployDir = "_dev/benchmark/rally/deploy" - // BenchType defining system benchmark - BenchType benchrunner.Type = "system" + // BenchType defining rally benchmark + BenchType benchrunner.Type = "rally" ) +var DryRunError = errors.New("dry run: rally benchmark not executed") + type rallyStat struct { Metric string Task string @@ -73,11 +75,11 @@ type runner struct { reportFile string // Execution order of following handlers is defined in runner.TearDown() method. - deletePolicyHandler func() error - resetAgentPolicyHandler func() error - shutdownServiceHandler func() error - wipeDataStreamHandler func() error - clearCorporaHandler func() error + persistRallyTrackHandler func() error + deletePolicyHandler func() error + shutdownServiceHandler func() error + wipeDataStreamHandler func() error + clearCorporaHandler func() error } func NewRallyBenchmark(opts Options) benchrunner.Runner { @@ -101,11 +103,11 @@ func (r *runner) TearDown() error { var merr multierror.Error - if r.resetAgentPolicyHandler != nil { - if err := r.resetAgentPolicyHandler(); err != nil { + if r.persistRallyTrackHandler != nil { + if err := r.persistRallyTrackHandler(); err != nil { merr = append(merr, err) } - r.resetAgentPolicyHandler = nil + r.persistRallyTrackHandler = nil } if r.deletePolicyHandler != nil { @@ -115,13 +117,6 @@ func (r *runner) TearDown() error { r.deletePolicyHandler = nil } - if r.shutdownServiceHandler != nil { - if err := r.shutdownServiceHandler(); err != nil { - merr = append(merr, err) - } - r.shutdownServiceHandler = nil - } - if r.wipeDataStreamHandler != nil { if err := r.wipeDataStreamHandler(); err != nil { merr = append(merr, err) @@ -258,6 +253,11 @@ func (r *runner) run() (report reporters.Reportable, err error) { } } + if r.options.DryRun { + dummy := reporters.NewReport(r.scenario.Package, nil) + return dummy, DryRunError + } + rallyStats, err := r.runRally() if err != nil { return nil, err @@ -585,6 +585,30 @@ func (r *runner) runGenerator(destDir string) error { r.reportFile = reportFile.Name() + if r.options.RallyTrackOutputDir != "" { + r.persistRallyTrackHandler = func() error { + err := os.MkdirAll(r.options.RallyTrackOutputDir, os.ModeDir) + if err != nil { + return err + } + + persistedRallyTrack := filepath.Join(r.options.RallyTrackOutputDir, fmt.Sprintf("track-%s.json", r.runtimeDataStream)) + err = sh.Copy(persistedRallyTrack, trackFile.Name()) + if err != nil { + return err + } + + persistedCorpus := filepath.Join(r.options.RallyTrackOutputDir, filepath.Base(corporaFile.Name())) + err = sh.Copy(persistedCorpus, corporaFile.Name()) + if err != nil { + return errors.Join(os.Remove(persistedRallyTrack), err) + } + + logger.Infof("rally track and corpus saved at: %s", r.options.RallyTrackOutputDir) + return nil + } + } + r.clearCorporaHandler = func() error { return errors.Join(os.Remove(r.corporaFile), os.Remove(r.reportFile), os.Remove(r.trackFile)) } @@ -622,14 +646,19 @@ func (r *runner) runRally() ([]rallyStat, error) { elasticsearchUsername = profileConfig.ElasticsearchUsername } + _, err = exec.LookPath("esrally") + if err != nil { + return nil, errors.New("could not run esrally track in path: esrally not found, please follow instruction at https://esrally.readthedocs.io/en/stable/install.html") + } + cmd := exec.Command("esrally", "race", "--race-id="+r.ctxt.Test.RunID, "--report-format=csv", fmt.Sprintf(`--report-file=%s`, r.reportFile), fmt.Sprintf(`--target-hosts={"default":["%s"]}`, elasticsearchHost), fmt.Sprintf(`--track-path=%s`, r.trackFile), fmt.Sprintf(`--client-options={"default":{"basic_auth_user":"%s","basic_auth_password":"%s","use_ssl":true,"verify_certs":false}}`, elasticsearchUsername, elasticsearchPassword), "--pipeline=benchmark-only") errOutput := new(bytes.Buffer) cmd.Stderr = errOutput logger.Debugf("output command: %s", cmd) - _, err = cmd.Output() + output, err := cmd.Output() if err != nil { - return nil, fmt.Errorf("could not run esrally track in path: %s (stderr=%q): %w", r.ctxt.Logs.Folder.Local, errOutput.String(), err) + return nil, fmt.Errorf("could not run esrally track in path: %s (stdout=%s): %w", r.ctxt.Logs.Folder.Local, output, err) } reportCSV, err := os.Open(r.reportFile) diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 277598a435..38c6005932 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -53,6 +53,12 @@ const ( BenchWithTestSamplesFlagName = "use-test-samples" BenchWithTestSamplesFlagDescription = "use test samples for the benchmarks" + BenchCorpusRallyTrackOutputDirFlagName = "rally-track-output-dir" + BenchCorpusRallyTrackOutputDirFlagDescription = "output dir of the rally track: if present the command will save the generated rally track" + + BenchCorpusRallyDryRunFlagName = "dry-run" + BenchCorpusRallyDryRunFlagDescription = "Do not run rally bust just generate the rally track" + BuildSkipValidationFlagName = "skip-validation" BuildSkipValidationFlagDescription = "skip validation of the built package, use only if all validation issues have been acknowledged" From 45d5659792f7cff65985056ff06f3e2cf2a96a91 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 31 Oct 2023 15:49:56 +0900 Subject: [PATCH 15/48] fix check-static --- cmd/benchmark.go | 2 +- internal/benchrunner/runners/rally/runner.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 998325916e..e6a715ff5c 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -336,7 +336,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { runner := rally.NewRallyBenchmark(rally.NewOptions(withOpts...)) r, err := benchrunner.Run(runner) - if errors.Is(err, rally.DryRunError) { + if errors.Is(err, rally.ErrDryRun) { return nil } diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index c9c1ff437e..b0d14c4bae 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -11,7 +11,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/magefile/mage/sh" "io" "os" "os/exec" @@ -19,6 +18,8 @@ import ( "strings" "time" + "github.com/magefile/mage/sh" + "github.com/elastic/elastic-package/internal/corpusgenerator" "github.com/elastic/elastic-package/internal/stack" @@ -50,7 +51,7 @@ const ( BenchType benchrunner.Type = "rally" ) -var DryRunError = errors.New("dry run: rally benchmark not executed") +var ErrDryRun = errors.New("dry run: rally benchmark not executed") type rallyStat struct { Metric string @@ -77,7 +78,6 @@ type runner struct { // Execution order of following handlers is defined in runner.TearDown() method. persistRallyTrackHandler func() error deletePolicyHandler func() error - shutdownServiceHandler func() error wipeDataStreamHandler func() error clearCorporaHandler func() error } @@ -255,7 +255,7 @@ func (r *runner) run() (report reporters.Reportable, err error) { if r.options.DryRun { dummy := reporters.NewReport(r.scenario.Package, nil) - return dummy, DryRunError + return dummy, ErrDryRun } rallyStats, err := r.runRally() From e1ab9b68e5a9cae2ba5e1ed43fa1ee9496c7e0b7 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 31 Oct 2023 19:33:32 +0900 Subject: [PATCH 16/48] remove creation of benchmark policy, get rid of input and vars --- internal/benchrunner/runners/rally/report.go | 13 --- internal/benchrunner/runners/rally/runner.go | 98 +------------------ .../benchrunner/runners/rally/scenario.go | 18 ++-- .../_dev/benchmark/rally/logs-benchmark.yml | 3 - 4 files changed, 8 insertions(+), 124 deletions(-) diff --git a/internal/benchrunner/runners/rally/report.go b/internal/benchrunner/runners/rally/report.go index 0cd4b86134..4abbfeeea6 100644 --- a/internal/benchrunner/runners/rally/report.go +++ b/internal/benchrunner/runners/rally/report.go @@ -32,8 +32,6 @@ type report struct { Parameters struct { PackageVersion string DataStream dataStream - Input string - Vars map[string]interface{} WarmupTimePeriod time.Duration Corpora corpora } @@ -73,8 +71,6 @@ func newReport(benchName, corporaFile string, s *scenario, sum *metricsSummary, report.Info.Duration = time.Duration(sum.CollectionEndTs-sum.CollectionStartTs) * time.Second report.Info.GeneratedCorporaFile = corporaFile report.Parameters.PackageVersion = s.Version - report.Parameters.Input = s.Input - report.Parameters.Vars = s.Vars report.Parameters.DataStream = s.DataStream report.Parameters.WarmupTimePeriod = s.WarmupTimePeriod report.Parameters.Corpora = s.Corpora @@ -112,19 +108,10 @@ func reportHumanFormat(r *report) []byte { pkvs := []interface{}{ "package version", r.Parameters.PackageVersion, - "input", r.Parameters.Input, - } - - for k, v := range r.Parameters.Vars { - pkvs = append(pkvs, fmt.Sprintf("vars.%s", k), v) } pkvs = append(pkvs, "data_stream.name", r.Parameters.DataStream.Name) - for k, v := range r.Parameters.DataStream.Vars { - pkvs = append(pkvs, fmt.Sprintf("data_stream.vars.%s", k), v) - } - pkvs = append(pkvs, "warmup time period", r.Parameters.WarmupTimePeriod, ) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index b0d14c4bae..77de80d969 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -34,7 +34,6 @@ import ( "github.com/elastic/elastic-package/internal/benchrunner/reporters" "github.com/elastic/elastic-package/internal/configuration/locations" "github.com/elastic/elastic-package/internal/elasticsearch" - "github.com/elastic/elastic-package/internal/kibana" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" @@ -65,7 +64,6 @@ type runner struct { scenario *scenario ctxt servicedeployer.ServiceContext - benchPolicy *kibana.Policy runtimeDataStream string pipelinePrefix string generator genlib.Generator @@ -178,12 +176,6 @@ func (r *runner) setUp() error { return fmt.Errorf("reading package manifest failed: %w", err) } - policy, err := r.createBenchmarkPolicy(pkgManifest) - if err != nil { - return err - } - r.benchPolicy = policy - // Delete old data logger.Debug("deleting old data in data stream...") dataStreamManifest, err := packages.ReadDataStreamManifest( @@ -197,11 +189,10 @@ func (r *runner) setUp() error { } r.runtimeDataStream = fmt.Sprintf( - "%s-%s.%s-%s", + "%s-%s.%s-ep", dataStreamManifest.Type, pkgManifest.Name, dataStreamManifest.Name, - policy.Namespace, ) r.pipelinePrefix = fmt.Sprintf( "%s-%s.%s-%s", @@ -305,93 +296,6 @@ func (r *runner) deleteDataStreamDocs(dataStream string) error { return nil } -func (r *runner) createBenchmarkPolicy(pkgManifest *packages.PackageManifest) (*kibana.Policy, error) { - // Configure package (single data stream) via Ingest Manager APIs. - logger.Debug("creating benchmark policy...") - benchTime := time.Now().Format("20060102T15:04:05Z") - p := kibana.Policy{ - Name: fmt.Sprintf("ep-bench-%s-%s", r.options.BenchName, benchTime), - Description: fmt.Sprintf("policy created by elastic-package for benchmark %s", r.options.BenchName), - Namespace: "ep", - MonitoringEnabled: []string{"logs", "metrics"}, - } - - policy, err := r.options.KibanaClient.CreatePolicy(p) - if err != nil { - return nil, err - } - - packagePolicy, err := r.createPackagePolicy(pkgManifest, policy) - if err != nil { - return nil, err - } - - r.deletePolicyHandler = func() error { - var merr multierror.Error - - logger.Debug("deleting benchmark package policy...") - if err := r.options.KibanaClient.DeletePackagePolicy(*packagePolicy); err != nil { - merr = append(merr, fmt.Errorf("error cleaning up benchmark package policy: %w", err)) - } - - logger.Debug("deleting benchmark policy...") - if err := r.options.KibanaClient.DeletePolicy(*policy); err != nil { - merr = append(merr, fmt.Errorf("error cleaning up benchmark policy: %w", err)) - } - - if len(merr) > 0 { - return merr - } - - return nil - } - - return policy, nil -} - -func (r *runner) createPackagePolicy(pkgManifest *packages.PackageManifest, p *kibana.Policy) (*kibana.PackagePolicy, error) { - logger.Debug("creating package policy...") - - if r.scenario.Version == "" { - r.scenario.Version = pkgManifest.Version - } - - if r.scenario.Package == "" { - r.scenario.Package = pkgManifest.Name - } - - if r.scenario.PolicyTemplate == "" { - r.scenario.PolicyTemplate = pkgManifest.PolicyTemplates[0].Name - } - - pp := kibana.PackagePolicy{ - Namespace: "ep", - PolicyID: p.ID, - Force: true, - Inputs: map[string]kibana.PackagePolicyInput{ - fmt.Sprintf("%s-%s", r.scenario.PolicyTemplate, r.scenario.Input): { - Enabled: true, - Vars: r.scenario.Vars, - Streams: map[string]kibana.PackagePolicyStream{ - fmt.Sprintf("%s.%s", pkgManifest.Name, r.scenario.DataStream.Name): { - Enabled: true, - Vars: r.scenario.DataStream.Vars, - }, - }, - }, - }, - } - pp.Package.Name = pkgManifest.Name - pp.Package.Version = r.scenario.Version - - policy, err := r.options.KibanaClient.CreatePackagePolicy(pp) - if err != nil { - return nil, err - } - - return policy, nil -} - func (r *runner) initializeGenerator() (genlib.Generator, error) { totEvents := r.scenario.Corpora.Generator.TotalEvents diff --git a/internal/benchrunner/runners/rally/scenario.go b/internal/benchrunner/runners/rally/scenario.go index 59e9f3cbe7..ed914cc8cf 100644 --- a/internal/benchrunner/runners/rally/scenario.go +++ b/internal/benchrunner/runners/rally/scenario.go @@ -21,20 +21,16 @@ import ( const devPath = "_dev/benchmark/rally" type scenario struct { - Package string `config:"package" json:"package"` - Description string `config:"description" json:"description"` - Version string `config:"version" json:"version"` - PolicyTemplate string `config:"policy_template" json:"policy_template"` - Input string `config:"input" json:"input"` - Vars map[string]interface{} `config:"vars" json:"vars"` - DataStream dataStream `config:"data_stream" json:"data_stream"` - WarmupTimePeriod time.Duration `config:"warmup_time_period" json:"warmup_time_period"` - Corpora corpora `config:"corpora" json:"corpora"` + Package string `config:"package" json:"package"` + Description string `config:"description" json:"description"` + Version string `config:"version" json:"version"` + DataStream dataStream `config:"data_stream" json:"data_stream"` + WarmupTimePeriod time.Duration `config:"warmup_time_period" json:"warmup_time_period"` + Corpora corpora `config:"corpora" json:"corpora"` } type dataStream struct { - Name string `config:"name" json:"name"` - Vars map[string]interface{} `config:"vars" json:"vars"` + Name string `config:"name" json:"name"` } type corpora struct { diff --git a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml index 1112923371..4eebf30979 100644 --- a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml +++ b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml @@ -4,9 +4,6 @@ input: filestream vars: ~ data_stream: name: testds - vars: - paths: - - "dummy path" warmup_time_period: 10s corpora: generator: From 0508a044bf280183c8f02277766b6b9df6d582d5 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 07:58:38 +0900 Subject: [PATCH 17/48] Update cmd/benchmark.go Co-authored-by: Jaime Soriano Pastor --- cmd/benchmark.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/benchmark.go b/cmd/benchmark.go index e6a715ff5c..b3aec19cbb 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -25,7 +25,7 @@ import ( "github.com/elastic/elastic-package/internal/benchrunner" "github.com/elastic/elastic-package/internal/benchrunner/reporters" "github.com/elastic/elastic-package/internal/benchrunner/reporters/outputs" - bench_common "github.com/elastic/elastic-package/internal/benchrunner/runners/common" + benchcommon "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "github.com/elastic/elastic-package/internal/benchrunner/runners/pipeline" "github.com/elastic/elastic-package/internal/benchrunner/runners/rally" "github.com/elastic/elastic-package/internal/benchrunner/runners/system" From ab63709df6511074a489fac54bef1556f50109f6 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 07:58:48 +0900 Subject: [PATCH 18/48] Update cmd/benchmark.go Co-authored-by: Jaime Soriano Pastor --- cmd/benchmark.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/benchmark.go b/cmd/benchmark.go index b3aec19cbb..f857368c6e 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -351,7 +351,7 @@ func rallyCommandAction(cmd *cobra.Command, args []string) error { reports := multiReport.Split() if len(reports) != 2 { - return fmt.Errorf("rally benchmark is expected to return a human an a file report") + return fmt.Errorf("rally benchmark is expected to return a human and a file report") } // human report will always be the first From 755a435ae7308b33603c4ff4a4999365db9bbdff Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 07:59:46 +0900 Subject: [PATCH 19/48] Update internal/benchrunner/runners/rally/metrics.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index 431e41d5dd..130d382c84 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -89,7 +89,7 @@ func newCollector( } func (c *collector) start() { - c.tick = time.NewTicker(c.interval) + tick := time.NewTicker(c.interval) c.createMetricsIndex() var once sync.Once From a145c72fa6de8625c94d7e2963baea88a3a469f1 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 08:00:15 +0900 Subject: [PATCH 20/48] Update internal/benchrunner/runners/rally/runner.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 77de80d969..635a71ceea 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -783,8 +783,8 @@ func getTotalHits(esapi *elasticsearch.API, dataStream string) (int, error) { } func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) { - timeoutTicker := time.NewTicker(timeout) - defer timeoutTicker.Stop() + timeout := time.NewTimer(timeout) + defer timeout.Stop() retryTicker := time.NewTicker(5 * time.Second) defer retryTicker.Stop() From 145c6ae77e84c9da619842fc9cd0cea6d75f6c50 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 10:05:23 +0900 Subject: [PATCH 21/48] Update internal/benchrunner/runners/rally/metrics.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/metrics.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index 130d382c84..58c8fe3787 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -95,7 +95,8 @@ func (c *collector) start() { c.wg.Add(1) go func() { - defer c.tick.Stop() + tick := time.NewTicker(c.interval) + defer tick.Stop() defer c.wg.Done() for { select { From 6e8effcccc4fc7f0cede960633a1afd4cc72d767 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 10:35:08 +0900 Subject: [PATCH 22/48] fix cr suggestions merge --- cmd/benchmark.go | 8 +-- internal/benchrunner/runners/rally/metrics.go | 18 +++--- internal/benchrunner/runners/rally/runner.go | 18 +++--- .../benchrunner/runners/rally/scenario.go | 56 ++++--------------- 4 files changed, 32 insertions(+), 68 deletions(-) diff --git a/cmd/benchmark.go b/cmd/benchmark.go index f857368c6e..6b6b73ff00 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -566,10 +566,10 @@ func generateDataStreamCorpusCommandAction(cmd *cobra.Command, _ []string) error } func initializeESMetricsClient(ctx context.Context) (*elasticsearch.Client, error) { - address := os.Getenv(bench_common.ESMetricstoreHostEnv) - user := os.Getenv(bench_common.ESMetricstoreUsernameEnv) - pass := os.Getenv(bench_common.ESMetricstorePasswordEnv) - cacert := os.Getenv(bench_common.ESMetricstoreCACertificateEnv) + address := os.Getenv(benchcommon.ESMetricstoreHostEnv) + user := os.Getenv(benchcommon.ESMetricstoreUsernameEnv) + pass := os.Getenv(benchcommon.ESMetricstorePasswordEnv) + cacert := os.Getenv(benchcommon.ESMetricstoreCACertificateEnv) if address == "" || user == "" || pass == "" { logger.Debugf("can't initialize metricstore, missing environment configuration") return nil, nil diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index 58c8fe3787..e66524f787 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -89,15 +89,20 @@ func newCollector( } func (c *collector) start() { - tick := time.NewTicker(c.interval) c.createMetricsIndex() - var once sync.Once c.wg.Add(1) go func() { tick := time.NewTicker(c.interval) defer tick.Stop() defer c.wg.Done() + + c.waitUntilReady() + c.startIngestMetrics = c.collectIngestMetrics() + c.startTotalHits = c.collectTotalHits() + c.startMetrics = c.collect() + c.publish(c.createEventsFromMetrics(c.startMetrics)) + for { select { case <-c.stopC: @@ -105,14 +110,7 @@ func (c *collector) start() { c.collectMetricsPreviousToStop() c.publish(c.createEventsFromMetrics(c.endMetrics)) return - case <-c.tick.C: - once.Do(func() { - c.waitUntilReady() - c.startIngestMetrics = c.collectIngestMetrics() - c.startTotalHits = c.collectTotalHits() - c.startMetrics = c.collect() - c.publish(c.createEventsFromMetrics(c.startMetrics)) - }) + case <-tick.C: m := c.collect() c.publish(c.createEventsFromMetrics(m)) } diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 635a71ceea..0d91b2b398 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -157,7 +157,12 @@ func (r *runner) setUp() error { return fmt.Errorf("could not create local rally track dir %w", err) } - scenario, err := readConfig(r.options.PackageRootPath, r.options.BenchName, r.ctxt) + pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + if err != nil { + return fmt.Errorf("reading package manifest failed: %w", err) + } + + scenario, err := readConfig(r.options.PackageRootPath, r.options.BenchName, pkgManifest.Name, pkgManifest.Version) if err != nil { return err } @@ -171,11 +176,6 @@ func (r *runner) setUp() error { } } - pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) - if err != nil { - return fmt.Errorf("reading package manifest failed: %w", err) - } - // Delete old data logger.Debug("deleting old data in data stream...") dataStreamManifest, err := packages.ReadDataStreamManifest( @@ -783,8 +783,8 @@ func getTotalHits(esapi *elasticsearch.API, dataStream string) (int, error) { } func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) { - timeout := time.NewTimer(timeout) - defer timeout.Stop() + timeoutTimer := time.NewTimer(timeout) + defer timeoutTimer.Stop() retryTicker := time.NewTicker(5 * time.Second) defer retryTicker.Stop() @@ -801,7 +801,7 @@ func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) select { case <-retryTicker.C: continue - case <-timeoutTicker.C: + case <-timeoutTimer.C: return false, nil } } diff --git a/internal/benchrunner/runners/rally/scenario.go b/internal/benchrunner/runners/rally/scenario.go index ed914cc8cf..b5750785d5 100644 --- a/internal/benchrunner/runners/rally/scenario.go +++ b/internal/benchrunner/runners/rally/scenario.go @@ -11,11 +11,7 @@ import ( "path/filepath" "time" - "github.com/aymerick/raymond" - "github.com/elastic/go-ucfg" "github.com/elastic/go-ucfg/yaml" - - "github.com/elastic/elastic-package/internal/servicedeployer" ) const devPath = "_dev/benchmark/rally" @@ -64,52 +60,22 @@ func defaultConfig() *scenario { return &scenario{} } -func readConfig(path, scenario string, ctxt servicedeployer.ServiceContext) (*scenario, error) { +func readConfig(path, scenario, packageName, packageVersion string) (*scenario, error) { configPath := filepath.Join(path, devPath, fmt.Sprintf("%s.yml", scenario)) - data, err := os.ReadFile(configPath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("unable to find system benchmark configuration file: %s: %w", configPath, err) - } - return nil, fmt.Errorf("could not load system benchmark configuration file: %s: %w", configPath, err) - } - - data, err = applyContext(data, ctxt) - if err != nil { - return nil, fmt.Errorf("could not apply context to benchmark configuration file: %s: %w", configPath, err) + c := defaultConfig() + cfg, err := yaml.NewConfigWithFile(configPath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("can't load benchmark configuration: %s: %w", configPath, err) } - cfg, err := yaml.NewConfig(data, ucfg.PathSep(".")) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - configPath = filepath.Join(path, devPath, fmt.Sprintf("%s.yaml", scenario)) - cfg, err = yaml.NewConfigWithFile(configPath) - } - if err != nil { - return nil, fmt.Errorf("can't load scenario: %s: %w", configPath, err) + if err == nil { + if err := cfg.Unpack(c); err != nil { + return nil, fmt.Errorf("can't unpack benchmark configuration: %s: %w", configPath, err) } } - c := defaultConfig() - if err := cfg.Unpack(c); err != nil { - return nil, fmt.Errorf("can't unpack scenario configuration: %s: %w", configPath, err) - } - return c, nil -} - -// applyContext takes the given system benchmark configuration (data) and replaces any placeholder variables in -// it with values from the given context (ctxt). The context may be populated from various sources but usually the -// most interesting context values will be set by a ServiceDeployer in its SetUp method. -func applyContext(data []byte, ctxt servicedeployer.ServiceContext) ([]byte, error) { - tmpl, err := raymond.Parse(string(data)) - if err != nil { - return data, fmt.Errorf("parsing template body failed: %w", err) - } - tmpl.RegisterHelpers(ctxt.Aliases()) + c.Package = packageName + c.Version = packageVersion - result, err := tmpl.Exec(ctxt) - if err != nil { - return data, fmt.Errorf("could not render data with context: %w", err) - } - return []byte(result), nil + return c, nil } From b423b59e6413ed1e6038bfc287ab7ed38cb1831e Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 10:35:22 +0900 Subject: [PATCH 23/48] remove input and vars reference in the package --- .../_dev/benchmark/rally/logs-benchmark.yml | 2 -- .../testds/agent/stream/filestream.yml.hbs | 4 ---- .../rally_benchmark/data_stream/testds/manifest.yml | 13 ------------- 3 files changed, 19 deletions(-) delete mode 100644 test/packages/benchmarks/rally_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs diff --git a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml index 4eebf30979..b2ae43313d 100644 --- a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml +++ b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml @@ -1,7 +1,5 @@ --- description: Benchmark 20000 events ingested -input: filestream -vars: ~ data_stream: name: testds warmup_time_period: 10s diff --git a/test/packages/benchmarks/rally_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs b/test/packages/benchmarks/rally_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs deleted file mode 100644 index cc801fea22..0000000000 --- a/test/packages/benchmarks/rally_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs +++ /dev/null @@ -1,4 +0,0 @@ -paths: -{{#each paths as |path i|}} - - {{path}} -{{/each}} diff --git a/test/packages/benchmarks/rally_benchmark/data_stream/testds/manifest.yml b/test/packages/benchmarks/rally_benchmark/data_stream/testds/manifest.yml index b7c55dea91..250726a37b 100644 --- a/test/packages/benchmarks/rally_benchmark/data_stream/testds/manifest.yml +++ b/test/packages/benchmarks/rally_benchmark/data_stream/testds/manifest.yml @@ -1,16 +1,3 @@ title: Test release: experimental type: logs -streams: - - input: filestream - enabled: false - title: Logs - description: Collect logs - template_path: filestream.yml.hbs - vars: - - name: paths - type: text - title: Paths - multi: true - required: true - show_user: true From 0191b17dd181c1820f3643af4841d8519addcb5d Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 10:52:25 +0900 Subject: [PATCH 24/48] fix check-static --- internal/benchrunner/runners/rally/metrics.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index e66524f787..0ae7f93e87 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -34,7 +34,6 @@ type collector struct { wg sync.WaitGroup stopped atomic.Bool stopC chan struct{} - tick *time.Ticker startIngestMetrics map[string]ingest.PipelineStatsMap endIngestMetrics map[string]ingest.PipelineStatsMap From f7f98ca2f39c35750e91e109f7942b03c3e81543 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 12:13:07 +0900 Subject: [PATCH 25/48] Update internal/benchrunner/runners/rally/metrics.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index 0ae7f93e87..f0f13d783c 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -276,7 +276,7 @@ readyLoop: logger.Debug(err) } if dsstats != nil { - break readyLoop + break } } From 3447828bdc977a7e43d597a1e4331dcbe2a96b48 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 12:14:36 +0900 Subject: [PATCH 26/48] Update internal/benchrunner/runners/rally/metrics.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index f0f13d783c..eefad0f5d3 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -319,7 +319,7 @@ func (c *collector) collectMetricsPreviousToStop() { func (c *collector) collectTotalHits() int { totalHits, err := getTotalHits(c.esAPI, c.datastream) if err != nil { - logger.Debugf("could not total hits: %w", err) + logger.Debugf("could not get total hits: %w", err) } return totalHits } From 9ab7a892233b5540d62921a2077a8198a645485a Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 12:15:20 +0900 Subject: [PATCH 27/48] Update internal/benchrunner/runners/rally/runner.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 0d91b2b398..e7288081bc 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -169,11 +169,11 @@ func (r *runner) setUp() error { r.scenario = scenario if r.scenario.Corpora.Generator != nil { - var err error - r.generator, err = r.initializeGenerator() + generator, err := r.initializeGenerator() if err != nil { return fmt.Errorf("can't initialize generator: %w", err) } + r.generator = generator } // Delete old data From b9c98a4f43f144d09b18cf067047b0a80fc74f21 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 12:16:53 +0900 Subject: [PATCH 28/48] Update internal/benchrunner/runners/rally/runner.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index e7288081bc..68ae0329ae 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -433,7 +433,7 @@ func (r *runner) runGenerator(destDir string) error { return err } - buf := bytes.NewBufferString("") + var buf bytes.Buffer var corpusDocsCount uint64 for { err := r.generator.Emit(buf) From 2dd4c65e82745d36a8b8f0ddf71210252f5a8666 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 12:17:18 +0900 Subject: [PATCH 29/48] Update internal/benchrunner/runners/rally/metrics.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index eefad0f5d3..863289d1cc 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -359,7 +359,7 @@ func (c *collector) createEventsFromMetrics(m metrics) [][]byte { for _, e := range append(nEvents, dsEvent) { b, err := json.Marshal(e) if err != nil { - logger.Debugf("error marhsaling metrics event: %w", err) + logger.Debugf("error marshalling metrics event: %w", err) continue } events = append(events, b) From 92b7014ff3f159ebdbaba549f7eab377bfb4e109 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 12:18:05 +0900 Subject: [PATCH 30/48] Update internal/benchrunner/runners/rally/runner.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/runner.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 68ae0329ae..e07600d22e 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -446,8 +446,7 @@ func (r *runner) runGenerator(destDir string) error { } // TODO: this should be taken care of by the corpus generator tool, once it will be done let's remove this - replacer := strings.NewReplacer("\n", "") - event := replacer.Replace(buf.String()) + event := strings.Replace(buf.String(), "\n", "") if _, err = corporaFile.Write([]byte(event)); err != nil { return err } From 9ba06f538332e1106204dc959526434c2012e0a2 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 12:31:33 +0900 Subject: [PATCH 31/48] cr and merge from github fixes --- README.md | 2 +- cmd/benchmark.go | 2 +- internal/benchrunner/runners/rally/metrics.go | 1 - internal/benchrunner/runners/rally/runner.go | 79 ++++++++++--------- internal/corpusgenerator/rally.go | 7 +- .../benchmarks/rally_benchmark/manifest.yml | 8 -- 6 files changed, 49 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 585f3dbcb2..306c5bd928 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ Run pipeline benchmarks for the package. _Context: package_ -Run rally benchmarks for the package. +Run rally benchmarks for the package (esrally needs to be installed in the path of the system). ### `elastic-package benchmark system` diff --git a/cmd/benchmark.go b/cmd/benchmark.go index 6b6b73ff00..402ec29c79 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -228,7 +228,7 @@ func getRallyCommand() *cobra.Command { cmd := &cobra.Command{ Use: "rally", Short: "Run rally benchmarks", - Long: "Run rally benchmarks for the package", + Long: "Run rally benchmarks for the package (esrally needs to be installed in the path of the system)", Args: cobra.NoArgs, RunE: rallyCommandAction, } diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index 863289d1cc..f7aa4a9cd5 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -264,7 +264,6 @@ func (c *collector) waitUntilReady() { waitTick := time.NewTicker(time.Second) defer waitTick.Stop() -readyLoop: for { select { case <-c.stopC: diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index e07600d22e..50342a60ed 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -69,9 +69,9 @@ type runner struct { generator genlib.Generator mcollector *collector - corporaFile string - trackFile string - reportFile string + corpusFile string + trackFile string + reportFile string // Execution order of following handlers is defined in runner.TearDown() method. persistRallyTrackHandler func() error @@ -169,11 +169,11 @@ func (r *runner) setUp() error { r.scenario = scenario if r.scenario.Corpora.Generator != nil { - generator, err := r.initializeGenerator() + var err error + r.generator, err = r.initializeGenerator() if err != nil { return fmt.Errorf("can't initialize generator: %w", err) } - r.generator = generator } // Delete old data @@ -263,7 +263,7 @@ func (r *runner) run() (report reporters.Reportable, err error) { return nil, fmt.Errorf("can't reindex data: %w", err) } - return createReport(r.options.BenchName, r.corporaFile, r.scenario, msum, rallyStats) + return createReport(r.options.BenchName, r.corpusFile, r.scenario, msum, rallyStats) } func (r *runner) startMetricsColletion() { @@ -384,9 +384,9 @@ func (r *runner) getGeneratorFields() (fields.Fields, error) { return nil, fmt.Errorf("can't read fields file %s: %w", fieldsPath, err) } } else if len(r.scenario.Corpora.Generator.Fields.Raw) > 0 { - data, err = yaml.Marshal(r.scenario.Corpora.Generator.Config.Raw) + data, err = yaml.Marshal(r.scenario.Corpora.Generator.Fields.Raw) if err != nil { - return nil, fmt.Errorf("can't parse raw generator config: %w", err) + return nil, fmt.Errorf("can't parse raw generator fields: %w", err) } } @@ -423,17 +423,17 @@ func (r *runner) getGeneratorTemplate() ([]byte, error) { } func (r *runner) runGenerator(destDir string) error { - corporaFile, err := os.CreateTemp(destDir, "corpus-*") + corpusFile, err := os.CreateTemp(destDir, "corpus-*") if err != nil { - return err + return fmt.Errorf("cannot not create rally corpus file: %w", err) } - defer corporaFile.Close() + defer corpusFile.Close() - if err := corporaFile.Chmod(os.ModePerm); err != nil { - return err + if err := corpusFile.Chmod(os.ModePerm); err != nil { + return fmt.Errorf("cannot not set permission to rally corpus file: %w", err) } - var buf bytes.Buffer + buf := bytes.NewBufferString("") var corpusDocsCount uint64 for { err := r.generator.Emit(buf) @@ -442,47 +442,43 @@ func (r *runner) runGenerator(destDir string) error { } if err != nil { - return err + return fmt.Errorf("error while generating content for the rally corpus file: %w", err) } // TODO: this should be taken care of by the corpus generator tool, once it will be done let's remove this - event := strings.Replace(buf.String(), "\n", "") - if _, err = corporaFile.Write([]byte(event)); err != nil { - return err + event := strings.Replace(buf.String(), "\n", "", -1) + if _, err = corpusFile.Write([]byte(event)); err != nil { + return fmt.Errorf("error while saving content to the rally corpus file: %w", err) } - if _, err = corporaFile.Write([]byte("\n")); err != nil { - return err + if _, err = corpusFile.Write([]byte("\n")); err != nil { + return fmt.Errorf("error while saving newline to the rally corpus file: %w", err) } buf.Reset() corpusDocsCount += 1 } - r.corporaFile = corporaFile.Name() - corporaFileForTrack, err := os.Open(r.corporaFile) - if err != nil { - return err - } + r.corpusFile = corpusFile.Name() trackFile, err := os.CreateTemp(destDir, "track-*.json") if err != nil { - return err + return fmt.Errorf("cannot not create rally track file: %w", err) } r.trackFile = trackFile.Name() - rallyTrackContent, err := corpusgenerator.GenerateRallyTrack(r.runtimeDataStream, corporaFileForTrack, corpusDocsCount) + rallyTrackContent, err := corpusgenerator.GenerateRallyTrack(r.runtimeDataStream, corpusFile, corpusDocsCount) if err != nil { - return err + return fmt.Errorf("cannot not generate rally track content: %w", err) } err = os.WriteFile(r.trackFile, rallyTrackContent, os.ModePerm) if err != nil { - return err + return fmt.Errorf("cannot not save rally track content to file: %w", err) } defer trackFile.Close() reportFile, err := os.CreateTemp(destDir, "report-*.csv") if err != nil { - return err + return fmt.Errorf("cannot not save rally report file: %w", err) } defer reportFile.Close() @@ -492,18 +488,19 @@ func (r *runner) runGenerator(destDir string) error { r.persistRallyTrackHandler = func() error { err := os.MkdirAll(r.options.RallyTrackOutputDir, os.ModeDir) if err != nil { - return err + return fmt.Errorf("cannot not create rally track output dir: %w", err) } persistedRallyTrack := filepath.Join(r.options.RallyTrackOutputDir, fmt.Sprintf("track-%s.json", r.runtimeDataStream)) err = sh.Copy(persistedRallyTrack, trackFile.Name()) if err != nil { - return err + return fmt.Errorf("cannot not copy rally track to file in output dir: %w", err) } - persistedCorpus := filepath.Join(r.options.RallyTrackOutputDir, filepath.Base(corporaFile.Name())) - err = sh.Copy(persistedCorpus, corporaFile.Name()) + persistedCorpus := filepath.Join(r.options.RallyTrackOutputDir, filepath.Base(corpusFile.Name())) + err = sh.Copy(persistedCorpus, corpusFile.Name()) if err != nil { + err = fmt.Errorf("cannot not copy rally corpus to file in output dir: %w", err) return errors.Join(os.Remove(persistedRallyTrack), err) } @@ -513,7 +510,7 @@ func (r *runner) runGenerator(destDir string) error { } r.clearCorporaHandler = func() error { - return errors.Join(os.Remove(r.corporaFile), os.Remove(r.reportFile), os.Remove(r.trackFile)) + return errors.Join(os.Remove(r.corpusFile), os.Remove(r.reportFile), os.Remove(r.trackFile)) } return r.generator.Close() @@ -554,7 +551,17 @@ func (r *runner) runRally() ([]rallyStat, error) { return nil, errors.New("could not run esrally track in path: esrally not found, please follow instruction at https://esrally.readthedocs.io/en/stable/install.html") } - cmd := exec.Command("esrally", "race", "--race-id="+r.ctxt.Test.RunID, "--report-format=csv", fmt.Sprintf(`--report-file=%s`, r.reportFile), fmt.Sprintf(`--target-hosts={"default":["%s"]}`, elasticsearchHost), fmt.Sprintf(`--track-path=%s`, r.trackFile), fmt.Sprintf(`--client-options={"default":{"basic_auth_user":"%s","basic_auth_password":"%s","use_ssl":true,"verify_certs":false}}`, elasticsearchUsername, elasticsearchPassword), "--pipeline=benchmark-only") + cmd := exec.Command( + "esrally", + "race", + "--race-id="+r.ctxt.Test.RunID, + "--report-format=csv", + fmt.Sprintf(`--report-file=%s`, r.reportFile), + fmt.Sprintf(`--target-hosts={"default":["%s"]}`, elasticsearchHost), + fmt.Sprintf(`--track-path=%s`, r.trackFile), + fmt.Sprintf(`--client-options={"default":{"basic_auth_user":"%s","basic_auth_password":"%s","use_ssl":true,"verify_certs":false}}`, elasticsearchUsername, elasticsearchPassword), + "--pipeline=benchmark-only", + ) errOutput := new(bytes.Buffer) cmd.Stderr = errOutput diff --git a/internal/corpusgenerator/rally.go b/internal/corpusgenerator/rally.go index 3faed5fba2..0c2e5986ef 100644 --- a/internal/corpusgenerator/rally.go +++ b/internal/corpusgenerator/rally.go @@ -6,6 +6,7 @@ package corpusgenerator import ( "bytes" + "fmt" "os" "path/filepath" "text/template" @@ -54,12 +55,12 @@ func GenerateRallyTrack(dataStream string, corpusFile *os.File, corpusDocsCount parsedTpl, err := t.Delims("[[", "]]").Parse(rallyTrackTemplate) if err != nil { - return nil, err + return nil, fmt.Errorf("erro while parsing rally track template: %w", err) } fi, err := corpusFile.Stat() if err != nil { - return nil, err + return nil, fmt.Errorf("error with stat on rally corpus file: %w", err) } corpusSizeInBytes := fi.Size() @@ -74,7 +75,7 @@ func GenerateRallyTrack(dataStream string, corpusFile *os.File, corpusDocsCount err = parsedTpl.Execute(buf, templateData) if err != nil { - return nil, err + return nil, fmt.Errorf("error on parsin on rally track template: %w", err) } return buf.Bytes(), nil diff --git a/test/packages/benchmarks/rally_benchmark/manifest.yml b/test/packages/benchmarks/rally_benchmark/manifest.yml index 40b31e1445..7cbe79f7a0 100644 --- a/test/packages/benchmarks/rally_benchmark/manifest.yml +++ b/test/packages/benchmarks/rally_benchmark/manifest.yml @@ -8,14 +8,6 @@ type: integration conditions: kibana: version: '^8.0.0' -policy_templates: - - name: testpo - title: Test - description: Description - inputs: - - type: filestream - title: Collect logs - description: Collecting logs owner: github: elastic/integrations type: elastic From e3612deb68f49f328f1364737355eb9408b27017 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Wed, 1 Nov 2023 19:21:32 +0900 Subject: [PATCH 32/48] move CreateRallyTrackDir to rally package and handle package installation --- internal/benchrunner/runners/rally/runner.go | 43 ++++++++++++++++--- .../benchrunner/runners/rally/scenario.go | 18 +++----- internal/servicedeployer/terraform.go | 8 ---- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 50342a60ed..a71e990689 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -75,7 +75,7 @@ type runner struct { // Execution order of following handlers is defined in runner.TearDown() method. persistRallyTrackHandler func() error - deletePolicyHandler func() error + removePackageHandler func() error wipeDataStreamHandler func() error clearCorporaHandler func() error } @@ -108,11 +108,11 @@ func (r *runner) TearDown() error { r.persistRallyTrackHandler = nil } - if r.deletePolicyHandler != nil { - if err := r.deletePolicyHandler(); err != nil { + if r.removePackageHandler != nil { + if err := r.removePackageHandler(); err != nil { merr = append(merr, err) } - r.deletePolicyHandler = nil + r.removePackageHandler = nil } if r.wipeDataStreamHandler != nil { @@ -135,6 +135,14 @@ func (r *runner) TearDown() error { return merr } +func (r *runner) createRallyTrackDir(locationManager *locations.LocationManager) error { + outputDir := filepath.Join(locationManager.RallyCorpusDir(), r.ctxt.Test.RunID) + if err := os.MkdirAll(outputDir, 0755); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + return nil +} + func (r *runner) setUp() error { locationManager, err := locations.NewLocationManager() if err != nil { @@ -152,7 +160,7 @@ func (r *runner) setUp() error { } r.ctxt.OutputDir = outputDir - err = servicedeployer.CreateRallyTrackDir(locationManager, r.ctxt.Test.RunID) + err = r.createRallyTrackDir(locationManager) if err != nil { return fmt.Errorf("could not create local rally track dir %w", err) } @@ -168,6 +176,11 @@ func (r *runner) setUp() error { } r.scenario = scenario + err = r.installPackage(r.scenario.Package, r.scenario.Version) + if err != nil { + return fmt.Errorf("error installing package: %w", err) + } + if r.scenario.Corpora.Generator != nil { var err error r.generator, err = r.initializeGenerator() @@ -266,6 +279,26 @@ func (r *runner) run() (report reporters.Reportable, err error) { return createReport(r.options.BenchName, r.corpusFile, r.scenario, msum, rallyStats) } +func (r *runner) installPackage(packageName, packageVersion string) error { + // POST /epm/packages/{pkgName}/{pkgVersion} + // Configure package (single data stream) via Ingest Manager APIs. + logger.Debug("installing package...") + _, err := r.options.KibanaClient.InstallPackage(packageName, packageVersion) + if err != nil { + return fmt.Errorf("cannot install package %s@%s: %w", packageName, packageVersion, err) + } + + r.removePackageHandler = func() error { + logger.Debug("removing benchmark package...") + if _, err := r.options.KibanaClient.RemovePackage(packageName, packageVersion); err != nil { + return fmt.Errorf("error removing benchmark package: %w", err) + } + return nil + } + + return nil +} + func (r *runner) startMetricsColletion() { // TODO collect agent hosts metrics using system integration r.mcollector = newCollector( diff --git a/internal/benchrunner/runners/rally/scenario.go b/internal/benchrunner/runners/rally/scenario.go index b5750785d5..b91be203d2 100644 --- a/internal/benchrunner/runners/rally/scenario.go +++ b/internal/benchrunner/runners/rally/scenario.go @@ -36,26 +36,20 @@ type corpora struct { type generator struct { TotalEvents uint64 `config:"total_events" json:"total_events"` Template corporaTemplate `config:"template" json:"template"` - Config corporaConfig `config:"config" json:"config"` - Fields corporaFields `config:"fields" json:"fields"` + Config corporaAsset `config:"config" json:"config"` + Fields corporaAsset `config:"fields" json:"fields"` } +type corporaAsset struct { + Raw map[string]interface{} `config:"raw" json:"raw"` + Path string `config:"path" json:"path"` +} type corporaTemplate struct { Raw string `config:"raw" json:"raw"` Path string `config:"path" json:"path"` Type string `config:"type" json:"type"` } -type corporaConfig struct { - Raw map[string]interface{} `config:"raw" json:"raw"` - Path string `config:"path" json:"path"` -} - -type corporaFields struct { - Raw map[string]interface{} `config:"raw" json:"raw"` - Path string `config:"path" json:"path"` -} - func defaultConfig() *scenario { return &scenario{} } diff --git a/internal/servicedeployer/terraform.go b/internal/servicedeployer/terraform.go index e7ee666ad6..8bbb0c65d4 100644 --- a/internal/servicedeployer/terraform.go +++ b/internal/servicedeployer/terraform.go @@ -218,12 +218,4 @@ func CreateOutputDir(locationManager *locations.LocationManager, runId string) ( return outputDir, nil } -func CreateRallyTrackDir(locationManager *locations.LocationManager, runId string) error { - outputDir := filepath.Join(locationManager.RallyCorpusDir(), runId) - if err := os.MkdirAll(outputDir, 0755); err != nil { - return fmt.Errorf("failed to create output directory: %w", err) - } - return nil -} - var _ ServiceDeployer = new(TerraformServiceDeployer) From 00402c7ed8b941a6448167f479ab22d6f733efa8 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 2 Nov 2023 10:10:10 +0900 Subject: [PATCH 33/48] use package installer --- internal/benchrunner/runners/rally/runner.go | 28 +++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index a71e990689..38e59c4de7 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -18,6 +18,8 @@ import ( "strings" "time" + "github.com/elastic/elastic-package/internal/packages/installer" + "github.com/magefile/mage/sh" "github.com/elastic/elastic-package/internal/corpusgenerator" @@ -176,7 +178,7 @@ func (r *runner) setUp() error { } r.scenario = scenario - err = r.installPackage(r.scenario.Package, r.scenario.Version) + err = r.installPackage() if err != nil { return fmt.Errorf("error installing package: %w", err) } @@ -279,20 +281,28 @@ func (r *runner) run() (report reporters.Reportable, err error) { return createReport(r.options.BenchName, r.corpusFile, r.scenario, msum, rallyStats) } -func (r *runner) installPackage(packageName, packageVersion string) error { - // POST /epm/packages/{pkgName}/{pkgVersion} - // Configure package (single data stream) via Ingest Manager APIs. - logger.Debug("installing package...") - _, err := r.options.KibanaClient.InstallPackage(packageName, packageVersion) +func (r *runner) installPackage() error { + logger.Debug("Installing package...") + installer, err := installer.NewForPackage(installer.Options{ + Kibana: r.options.KibanaClient, + RootPath: r.options.PackageRootPath, + SkipValidation: true, + }) + + if err != nil { + return fmt.Errorf("failed to initialize package installer: %w", err) + } + + _, err = installer.Install() if err != nil { - return fmt.Errorf("cannot install package %s@%s: %w", packageName, packageVersion, err) + return fmt.Errorf("failed to install package: %w", err) } r.removePackageHandler = func() error { - logger.Debug("removing benchmark package...") - if _, err := r.options.KibanaClient.RemovePackage(packageName, packageVersion); err != nil { + if err := installer.Uninstall(); err != nil { return fmt.Errorf("error removing benchmark package: %w", err) } + return nil } From c8a65027cb79d5e554b9bff478537d306984deff Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 2 Nov 2023 10:11:42 +0900 Subject: [PATCH 34/48] Update internal/corpusgenerator/rally.go Co-authored-by: Jaime Soriano Pastor --- internal/corpusgenerator/rally.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/corpusgenerator/rally.go b/internal/corpusgenerator/rally.go index 0c2e5986ef..c50e55bf43 100644 --- a/internal/corpusgenerator/rally.go +++ b/internal/corpusgenerator/rally.go @@ -55,7 +55,7 @@ func GenerateRallyTrack(dataStream string, corpusFile *os.File, corpusDocsCount parsedTpl, err := t.Delims("[[", "]]").Parse(rallyTrackTemplate) if err != nil { - return nil, fmt.Errorf("erro while parsing rally track template: %w", err) + return nil, fmt.Errorf("error while parsing rally track template: %w", err) } fi, err := corpusFile.Stat() From b1adc98b141489bfecc0b4ad2b8731300728ba35 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 2 Nov 2023 10:11:52 +0900 Subject: [PATCH 35/48] Update internal/benchrunner/runners/rally/runner.go Co-authored-by: Jaime Soriano Pastor --- internal/benchrunner/runners/rally/runner.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 38e59c4de7..2acfc1765e 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -553,7 +553,11 @@ func (r *runner) runGenerator(destDir string) error { } r.clearCorporaHandler = func() error { - return errors.Join(os.Remove(r.corpusFile), os.Remove(r.reportFile), os.Remove(r.trackFile)) + return errors.Join( + os.Remove(r.corpusFile), + os.Remove(r.reportFile), + os.Remove(r.trackFile), + ) } return r.generator.Close() From b28b55d171c9fb53f968b9e9eef3b7b6db690ad5 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 2 Nov 2023 11:48:33 +0900 Subject: [PATCH 36/48] handle error logging in metrics.go, use bulk API --- internal/benchrunner/runners/rally/metrics.go | 48 ++++++++++--------- internal/benchrunner/runners/rally/runner.go | 26 +++++----- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index f7aa4a9cd5..971c7734a2 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -151,23 +151,28 @@ func (c *collector) publish(events [][]byte) { if c.metricsAPI == nil { return } - for _, e := range events { - reqBody := bytes.NewReader(e) - resp, err := c.metricsAPI.Index(c.indexName(), reqBody) - if err != nil { - logger.Debugf("error indexing event: %v", err) - continue - } + eventsForBulk := bytes.Join(events, []byte("\n")) + reqBody := bytes.NewReader(eventsForBulk) + resp, err := c.metricsAPI.Bulk(reqBody, c.metricsAPI.Bulk.WithIndex(c.indexName())) + if err != nil { + logger.Errorf("error indexing event in metricstore: %w", err) + return + } - body, err := io.ReadAll(resp.Body) - if err != nil { - logger.Errorf("failed to read index response body: %v", err) - } - resp.Body.Close() + if resp.Body == nil { + logger.Errorf("empty index response body from metricstore: %w", err) + return + } - if resp.StatusCode != 201 { - logger.Errorf("error indexing event (%d): %s: %v", resp.StatusCode, resp.Status(), elasticsearch.NewError(body)) - } + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.Errorf("failed to read index response body from metricstore: %w", err) + } + + resp.Body.Close() + + if resp.StatusCode != 201 { + logger.Errorf("error indexing event in metricstore (%d): %s: %v", resp.StatusCode, resp.Status(), elasticsearch.NewError(body)) } } @@ -188,13 +193,13 @@ func (c *collector) createMetricsIndex() { c.metricsAPI.Indices.Create.WithBody(reader), ) if err != nil { - logger.Debugf("could not create index: %v", err) + logger.Errorf("could not create index: %w", err) return } createRes.Body.Close() if createRes.IsError() { - logger.Debug("got a response error while creating index") + logger.Errorf("got a response error while creating index") } } @@ -270,10 +275,7 @@ func (c *collector) waitUntilReady() { return case <-waitTick.C: } - dsstats, err := ingest.GetDataStreamStats(c.esAPI, c.datastream) - if err != nil { - logger.Debug(err) - } + dsstats, _ := ingest.GetDataStreamStats(c.esAPI, c.datastream) if dsstats != nil { break } @@ -318,7 +320,7 @@ func (c *collector) collectMetricsPreviousToStop() { func (c *collector) collectTotalHits() int { totalHits, err := getTotalHits(c.esAPI, c.datastream) if err != nil { - logger.Debugf("could not get total hits: %w", err) + logger.Debugf("could not get total hits: %v", err) } return totalHits } @@ -358,7 +360,7 @@ func (c *collector) createEventsFromMetrics(m metrics) [][]byte { for _, e := range append(nEvents, dsEvent) { b, err := json.Marshal(e) if err != nil { - logger.Debugf("error marshalling metrics event: %w", err) + logger.Debugf("error marshalling metrics event: %v", err) continue } events = append(events, b) diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 2acfc1765e..1fae90bc92 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -191,8 +191,6 @@ func (r *runner) setUp() error { } } - // Delete old data - logger.Debug("deleting old data in data stream...") dataStreamManifest, err := packages.ReadDataStreamManifest( filepath.Join( getDataStreamPath(r.options.PackageRootPath, r.scenario.DataStream.Name), @@ -217,15 +215,7 @@ func (r *runner) setUp() error { r.scenario.Version, ) - r.wipeDataStreamHandler = func() error { - logger.Debugf("deleting data in data stream...") - if err := r.deleteDataStreamDocs(r.runtimeDataStream); err != nil { - return fmt.Errorf("error deleting data in data stream: %w", err) - } - return nil - } - - if err := r.deleteDataStreamDocs(r.runtimeDataStream); err != nil { + if err := r.wipeDataStreamOnSetup(); err != nil { return fmt.Errorf("error deleting old data in data stream: %s: %w", r.runtimeDataStream, err) } @@ -247,6 +237,20 @@ func (r *runner) setUp() error { return nil } +func (r *runner) wipeDataStreamOnSetup() error { + // Delete old data + logger.Debug("deleting old data in data stream...") + r.wipeDataStreamHandler = func() error { + logger.Debugf("deleting data in data stream...") + if err := r.deleteDataStreamDocs(r.runtimeDataStream); err != nil { + return fmt.Errorf("error deleting data in data stream: %w", err) + } + return nil + } + + return r.deleteDataStreamDocs(r.runtimeDataStream) +} + func (r *runner) run() (report reporters.Reportable, err error) { r.startMetricsColletion() defer r.mcollector.stop() From b8bd588c513a0a01984be423f2bb692aa55e9ef2 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 2 Nov 2023 11:48:39 +0900 Subject: [PATCH 37/48] docs --- docs/howto/rally_benchmarking.md | 268 +++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 docs/howto/rally_benchmarking.md diff --git a/docs/howto/rally_benchmarking.md b/docs/howto/rally_benchmarking.md new file mode 100644 index 0000000000..644995ae9b --- /dev/null +++ b/docs/howto/rally_benchmarking.md @@ -0,0 +1,268 @@ +# HOWTO: Writing system benchmarks for a package + +## Introduction +Elastic Packages are comprised of data streams. A rally benchmark runs `esrally` track with a corpus of data into an Elasticsearch data stream, and reports rally stats as well as retrieving performance metrics from the Elasticsearch nodes. + +## Conceptual process + +Conceptually, running a rally benchmark involves the following steps: + +1. Deploy the Elastic Stack, including Elasticsearch, Kibana, and the Elastic Agent(s). This step takes time so it should typically be done once as a pre-requisite to running a system benchmark scenario. +1. Install a package that configures its assets for every data stream in the package. +1. Metrics collections from the cluster starts. (**TODO**: record metrics from all Elastic Agents involved using the `system` integration.) +1. Send the collected metrics to the ES Metricstore if set. +1. Generate data (it uses the [corpus-generator-tool](https://github.com/elastic/elastic-integration-corpus-generator-tool)) +1. Run an `esrally` track with the corpus of generated data. `esrally` must be installed on the system where the `elastic-package` is run and available in the `PATH`. +1. Wait for the `esrally` track to be executed. +1. Metrics collection ends and a summary report is created. +1. Delete test artifacts. +1. Optionally reindex all ingested data into the ES Metricstore for further analysis. +1. **TODO**: Optionally compare results against another benchmark run. + +### Benchmark scenario definition + +We must define at least one configuration for the package that we +want to benchmark. There can be multiple scenarios defined for the same package. + +``` +/ + _dev/ + benchmark/ + rally/ + .yml +``` + +The `.yml` files allow you to define various settings for the benchmark scenario +along with values for package and data stream-level variables. These are the available configuration options for system benchmarks. + +| Option | Type | Required | Description | +|---------------------------------|------------|---|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| package | string | | The name of the package. If omitted will pick the current package, this is to allow for future definition of benchmarks outside of the packages folders. | +| description | string | | A description for the scenario. | +| version | string | | The version of the package to benchmark. If omitted will pick the current version of the package. | +| data_stream.name | string | yes | The data stream to benchmark. | +| warmup_time_period | duration | | Warmup time period. All data prior to this period will be ignored in the benchmark results. | +| corpora.generator.total_events | uint64 | | Number of total events to generate. Example: `20000` | +| corpora.generator.template.raw | string | | Raw template for the corpus generator. | +| corpora.generator.template.path | string | | Path to the template for the corpus generator. If a `path` is defined, it will override any `raw` template definition. | +| corpora.generator.template.type | string | | Type of the template for the corpus generator. Default `placeholder`. | +| corpora.generator.config.raw | dictionary | | Raw config for the corpus generator. | +| corpora.generator.config.path | string | | Path to the config for the corpus generator. If a `path` is defined, it will override any `raw` config definition. | +| corpora.generator.fields.raw | dictionary | | Raw fields for the corpus generator. | +| corpora.generator.fields.path | string | | Path to the fields for the corpus generator. If a `path` is defined, it will override any `raw` fields definition. | + +Example: + +`logs-benchmark.yml` +```yaml +--- +--- +description: Benchmark 20000 events ingested +data_stream: + name: testds +warmup_time_period: 10s +corpora: + generator: + total_events: 900000 + template: + type: gotext + path: ./logs-benchmark/template.ndjson + config: + path: ./logs-benchmark/config.yml + fields: + path: ./logs-benchmark/fields.yml +``` + +There is no need to define an `input` and `vars` for the package, +since we don't create any agent policy, and we don't enroll any agent. + +## Running a rally benchmark + +Once the configuration is defined as described in the previous section, you are ready to run rally benchmarks for a package. + +First you must deploy the Elastic Stack. + +``` +elastic-package stack up -d +``` + +For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`. + +Next, you must invoke the system benchmark runner. + +``` +elastic-package benchmark rally --benchmark logs-benchmark -v +# ... debug output +--- Benchmark results for package: rally_benchmarks - START --- +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ info │ +├────────────────────────┬─────────────────────────────────────────────────────────────────────────┤ +│ benchmark │ logs-benchmark │ +│ description │ Benchmark 20000 events ingested │ +│ run ID │ cb62ba92-14b9-4562-98ce-9251e0936a5e │ +│ package │ rally_benchmarks │ +│ start ts (s) │ 1698892087 │ +│ end ts (s) │ 1698892134 │ +│ duration │ 47s │ +│ generated corpora file │ /Users/andreaspacca/.elastic-package/tmp/rally_corpus/corpus-3633691003 │ +╰────────────────────────┴─────────────────────────────────────────────────────────────────────────╯ +╭────────────────────────────────────────────────────────────────────╮ +│ parameters │ +├─────────────────────────────────┬──────────────────────────────────┤ +│ package version │ 999.999.999 │ +│ data_stream.name │ testds │ +│ warmup time period │ 10s │ +│ corpora.generator.total_events │ 900000 │ +│ corpora.generator.template.path │ ./logs-benchmark/template.ndjson │ +│ corpora.generator.template.raw │ │ +│ corpora.generator.template.type │ gotext │ +│ corpora.generator.config.path │ ./logs-benchmark/config.yml │ +│ corpora.generator.config.raw │ map[] │ +│ corpora.generator.fields.path │ ./logs-benchmark/fields.yml │ +│ corpora.generator.fields.raw │ map[] │ +╰─────────────────────────────────┴──────────────────────────────────╯ +╭───────────────────────╮ +│ cluster info │ +├───────┬───────────────┤ +│ name │ elasticsearch │ +│ nodes │ 1 │ +╰───────┴───────────────╯ +╭──────────────────────────────────────────────────────────────╮ +│ data stream stats │ +├────────────────────────────┬─────────────────────────────────┤ +│ data stream │ logs-rally_benchmarks.testds-ep │ +│ approx total docs ingested │ 900000 │ +│ backing indices │ 1 │ +│ store size bytes │ 440617124 │ +│ maximum ts (ms) │ 1698892076196 │ +╰────────────────────────────┴─────────────────────────────────╯ +╭───────────────────────────────────────╮ +│ disk usage for index .ds-logs-rally_b │ +│ enchmarks.testds-ep-2023.11.01-000001 │ +│ (for all fields) │ +├──────────────────────────────┬────────┤ +│ total │ 262 MB │ +│ inverted_index.total │ 84 MB │ +│ inverted_index.stored_fields │ 96 MB │ +│ inverted_index.doc_values │ 72 MB │ +│ inverted_index.points │ 9.7 MB │ +│ inverted_index.norms │ 0 B │ +│ inverted_index.term_vectors │ 0 B │ +│ inverted_index.knn_vectors │ 0 B │ +╰──────────────────────────────┴────────╯ +╭────────────────────────────────────────────────────────────────────────────────────────────╮ +│ pipeline logs-rally_benchmarks.testds-999.999.999 stats in node 7AYCd2EXQaCSOf-0fKxFBg │ +├────────────────────────────────────────────────┬───────────────────────────────────────────┤ +│ Totals │ Count: 900000 | Failed: 0 | Time: 38.993s │ +│ grok () │ Count: 900000 | Failed: 0 | Time: 36.646s │ +│ user_agent () │ Count: 900000 | Failed: 0 | Time: 1.368s │ +│ pipeline (logs-rally_benchmarks.testds@custom) │ Count: 900000 | Failed: 0 | Time: 102ms │ +╰────────────────────────────────────────────────┴───────────────────────────────────────────╯ +╭────────────────────────────────────────────────────────────────────────────────────────────╮ +│ rally stats │ +├────────────────────────────────────────────────────────────────┬───────────────────────────┤ +│ Cumulative indexing time of primary shards │ 3.3734666666666664 min │ +│ Min cumulative indexing time across primary shards │ 0 min │ +│ Median cumulative indexing time across primary shards │ 0.046950000000000006 min │ +│ Max cumulative indexing time across primary shards │ 1.7421333333333335 min │ +│ Cumulative indexing throttle time of primary shards │ 0 min │ +│ Min cumulative indexing throttle time across primary shards │ 0 min │ +│ Median cumulative indexing throttle time across primary shards │ 0.0 min │ +│ Max cumulative indexing throttle time across primary shards │ 0 min │ +│ Cumulative merge time of primary shards │ 0.8019666666666667 min │ +│ Cumulative merge count of primary shards │ 449 │ +│ Min cumulative merge time across primary shards │ 0 min │ +│ Median cumulative merge time across primary shards │ 0.009525 min │ +│ Max cumulative merge time across primary shards │ 0.5432 min │ +│ Cumulative merge throttle time of primary shards │ 0.21998333333333334 min │ +│ Min cumulative merge throttle time across primary shards │ 0 min │ +│ Median cumulative merge throttle time across primary shards │ 0.0 min │ +│ Max cumulative merge throttle time across primary shards │ 0.21998333333333334 min │ +│ Cumulative refresh time of primary shards │ 0.41008333333333336 min │ +│ Cumulative refresh count of primary shards │ 14095 │ +│ Min cumulative refresh time across primary shards │ 0 min │ +│ Median cumulative refresh time across primary shards │ 0.011966666666666665 min │ +│ Max cumulative refresh time across primary shards │ 0.2056333333333333 min │ +│ Cumulative flush time of primary shards │ 8.79825 min │ +│ Cumulative flush count of primary shards │ 13859 │ +│ Min cumulative flush time across primary shards │ 6.666666666666667e-05 min │ +│ Median cumulative flush time across primary shards │ 0.45098333333333335 min │ +│ Max cumulative flush time across primary shards │ 0.6979833333333333 min │ +│ Total Young Gen GC time │ 0.646 s │ +│ Total Young Gen GC count │ 147 │ +│ Total Old Gen GC time │ 0 s │ +│ Total Old Gen GC count │ 0 │ +│ Store size │ 0.23110682144761086 GB │ +│ Translog size │ 0.5021391455084085 GB │ +│ Heap used for segments │ 0 MB │ +│ Heap used for doc values │ 0 MB │ +│ Heap used for terms │ 0 MB │ +│ Heap used for norms │ 0 MB │ +│ Heap used for points │ 0 MB │ +│ Heap used for stored fields │ 0 MB │ +│ Segment count │ 325 │ +│ Total Ingest Pipeline count │ 900063 │ +│ Total Ingest Pipeline time │ 51.994 s │ +│ Total Ingest Pipeline failed │ 0 │ +│ Min Throughput │ 23987.18 docs/s │ +│ Mean Throughput │ 46511.27 docs/s │ +│ Median Throughput │ 49360.50 docs/s │ +│ Max Throughput │ 51832.58 docs/s │ +│ 50th percentile latency │ 645.8979995000007 ms │ +│ 90th percentile latency │ 896.532670700001 ms │ +│ 99th percentile latency │ 1050.0004142499988 ms │ +│ 100th percentile latency │ 1064.915250000002 ms │ +│ 50th percentile service time │ 645.8979995000007 ms │ +│ 90th percentile service time │ 896.532670700001 ms │ +│ 99th percentile service time │ 1050.0004142499988 ms │ +│ 100th percentile service time │ 1064.915250000002 ms │ +│ error rate │ 0.00 % │ +╰────────────────────────────────────────────────────────────────┴───────────────────────────╯ + +--- Benchmark results for package: rally_benchmarks - END --- +Done +``` + +Finally, when you are done running the benchmark, bring down the Elastic Stack. + +``` +elastic-package stack down +``` + +## Setting up an external metricstore + +A metricstore can be set up to send metrics collected during the benchmark execution. + +An external metricstore might be useful for: + +- Store monitoring data of the benchmark scenario for all its execution time. +- Analyse the data generated during a benchmark. This is possible when using the `reindex-to-metricstore` flag. +- **TODO**: Store benchmark results for various benchmark runs permanently for later comparison. + +In order to initialize it, you need to set up the following environment variables: + +```bash +export ELASTIC_PACKAGE_ESMETRICSTORE_HOST=https://127.0.0.1:9200 +export ELASTIC_PACKAGE_ESMETRICSTORE_USERNAME=elastic +export ELASTIC_PACKAGE_ESMETRICSTORE_PASSWORD=changeme +export ELASTIC_PACKAGE_ESMETRICSTORE_CA_CERT="$HOME/.elastic-package/profiles/default/certs/ca-cert.pem" +``` + +The only one that is optional is `ELASTIC_PACKAGE_ESMETRICSTORE_CA_CERT`. + +When these are detected, metrics will be automatically collected every second and sent to a new index called `bench-metrics-{dataset}-{testRunID}"`. + +The collected metrics include the following node stats: `nodes.*.breakers`, `nodes.*.indices`, `nodes.*.jvm.mem`, `nodes.*.jvm.gc`, `nodes.*.jvm.buffer_pools`, `nodes.*.os.mem`, `nodes.*.process.cpu`, `nodes.*.thread_pool`, and `nodes.*.transport`. + +Ingest pipelines metrics are only collected at the end since its own collection would affect the benchmark results. + +You can see a sample collected metric [here](./sample_metric.json) + +Additionally, if the `reindex-to-metricstore` flag is used, the data generated during the benchmark will be sent to the metricstore into an index called `bench-reindex-{datastream}-{testRunID}` for further analysis. The events will be enriched with metadata related to the benchmark run. + +## Persisting rally tracks and dry-run + +If the `rally-track-output-dir` flag is used, the track and the corpus generated during the benchmark will be saved in the directory passed as value of the flag. +Additionally, if the `dry-run` flag is used, the command will exits before running `esrally`. +If both the flags above are used at the same time, the command will just generate the track and corpus and save them, without running any benchmark. +If the `dry-run` flag only is used, the command will just wipe the data stream returning no report. From f6532d9b86833c01e57fe7e1737179a366b53399 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 2 Nov 2023 17:53:30 +0900 Subject: [PATCH 38/48] add refresh index in metrics, collect only start and end --- internal/benchrunner/runners/rally/metrics.go | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index 971c7734a2..d3971a5344 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -90,30 +90,14 @@ func newCollector( func (c *collector) start() { c.createMetricsIndex() + c.collectMetricsBeforeRallyRun() + c.wg.Add(1) go func() { - tick := time.NewTicker(c.interval) - defer tick.Stop() - defer c.wg.Done() - - c.waitUntilReady() - c.startIngestMetrics = c.collectIngestMetrics() - c.startTotalHits = c.collectTotalHits() - c.startMetrics = c.collect() - c.publish(c.createEventsFromMetrics(c.startMetrics)) - - for { - select { - case <-c.stopC: - // last collect before stopping - c.collectMetricsPreviousToStop() - c.publish(c.createEventsFromMetrics(c.endMetrics)) - return - case <-tick.C: - m := c.collect() - c.publish(c.createEventsFromMetrics(m)) - } - } + <-c.stopC + // last collect before stopping + c.collectMetricsAfterRallyRun() + c.publish(c.createEventsFromMetrics(c.endMetrics)) }() } @@ -125,6 +109,21 @@ func (c *collector) stop() { c.wg.Wait() } +func (c *collector) collectMetricsBeforeRallyRun() { + c.waitUntilReady() + + _, err := c.esAPI.Indices.Refresh(c.esAPI.Indices.Refresh.WithIndex(c.datastream)) + if err == nil { + c.startTotalHits = c.collectTotalHits() + c.startMetrics = c.collect() + c.diskUsage = c.collectDiskUsage() + c.startIngestMetrics = c.collectIngestMetrics() + c.publish(c.createEventsFromMetrics(c.startMetrics)) + } else { + logger.Errorf("unable to refresh data stream at the beginning of rally run") + } +} + func (c *collector) collect() metrics { m := metrics{ ts: time.Now().Unix(), @@ -310,11 +309,19 @@ func (c *collector) collectDiskUsage() map[string]ingest.DiskUsage { return du } -func (c *collector) collectMetricsPreviousToStop() { - c.endIngestMetrics = c.collectIngestMetrics() +func (c *collector) collectMetricsAfterRallyRun() { + _, err := c.esAPI.Indices.Refresh(c.esAPI.Indices.Refresh.WithIndex(c.datastream)) + if err != nil { + logger.Errorf("unable to refresh data stream at the end of rally run") + return + } + c.diskUsage = c.collectDiskUsage() - c.endTotalHits = c.collectTotalHits() c.endMetrics = c.collect() + c.endIngestMetrics = c.collectIngestMetrics() + c.endTotalHits = c.collectTotalHits() + + c.publish(c.createEventsFromMetrics(c.endMetrics)) } func (c *collector) collectTotalHits() int { From 592cdcba7837d91f56290acd4b2f2cad3341384d Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 2 Nov 2023 17:58:20 +0900 Subject: [PATCH 39/48] wait only for warmup --- internal/benchrunner/runners/rally/metrics.go | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index d3971a5344..e10cb34cf5 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -110,18 +110,18 @@ func (c *collector) stop() { } func (c *collector) collectMetricsBeforeRallyRun() { - c.waitUntilReady() + c.waitForWarmup() _, err := c.esAPI.Indices.Refresh(c.esAPI.Indices.Refresh.WithIndex(c.datastream)) - if err == nil { - c.startTotalHits = c.collectTotalHits() - c.startMetrics = c.collect() - c.diskUsage = c.collectDiskUsage() - c.startIngestMetrics = c.collectIngestMetrics() - c.publish(c.createEventsFromMetrics(c.startMetrics)) - } else { + if err != nil { logger.Errorf("unable to refresh data stream at the beginning of rally run") + return } + + c.startTotalHits = c.collectTotalHits() + c.startMetrics = c.collect() + c.startIngestMetrics = c.collectIngestMetrics() + c.publish(c.createEventsFromMetrics(c.startMetrics)) } func (c *collector) collect() metrics { @@ -262,24 +262,7 @@ func (c *collector) summarize() (*metricsSummary, error) { return &sum, nil } -func (c *collector) waitUntilReady() { - logger.Debug("waiting for datastream to be created...") - - waitTick := time.NewTicker(time.Second) - defer waitTick.Stop() - - for { - select { - case <-c.stopC: - return - case <-waitTick.C: - } - dsstats, _ := ingest.GetDataStreamStats(c.esAPI, c.datastream) - if dsstats != nil { - break - } - } - +func (c *collector) waitForWarmup() { if c.scenario.WarmupTimePeriod > 0 { logger.Debugf("waiting %s for warmup period", c.scenario.WarmupTimePeriod) select { From 671867c5471bb9f0573aa69e2aaa3a09698de6d0 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 2 Nov 2023 18:15:31 +0900 Subject: [PATCH 40/48] remove warmup, include unloaded segment in pipeline stats --- internal/benchrunner/runners/rally/metrics.go | 14 -------------- internal/benchrunner/runners/rally/report.go | 12 +++--------- internal/benchrunner/runners/rally/scenario.go | 12 +++++------- internal/elasticsearch/ingest/nodestats.go | 6 ++++-- 4 files changed, 12 insertions(+), 32 deletions(-) diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index e10cb34cf5..842419d03e 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -110,8 +110,6 @@ func (c *collector) stop() { } func (c *collector) collectMetricsBeforeRallyRun() { - c.waitForWarmup() - _, err := c.esAPI.Indices.Refresh(c.esAPI.Indices.Refresh.WithIndex(c.datastream)) if err != nil { logger.Errorf("unable to refresh data stream at the beginning of rally run") @@ -262,18 +260,6 @@ func (c *collector) summarize() (*metricsSummary, error) { return &sum, nil } -func (c *collector) waitForWarmup() { - if c.scenario.WarmupTimePeriod > 0 { - logger.Debugf("waiting %s for warmup period", c.scenario.WarmupTimePeriod) - select { - case <-c.stopC: - return - case <-time.After(c.scenario.WarmupTimePeriod): - } - } - logger.Debug("metric collection starting...") -} - func (c *collector) collectIngestMetrics() map[string]ingest.PipelineStatsMap { ipMetrics, err := ingest.GetPipelineStatsByPrefix(c.esAPI, c.pipelinePrefix) if err != nil { diff --git a/internal/benchrunner/runners/rally/report.go b/internal/benchrunner/runners/rally/report.go index 4abbfeeea6..2e3bbc811c 100644 --- a/internal/benchrunner/runners/rally/report.go +++ b/internal/benchrunner/runners/rally/report.go @@ -30,10 +30,9 @@ type report struct { GeneratedCorporaFile string } Parameters struct { - PackageVersion string - DataStream dataStream - WarmupTimePeriod time.Duration - Corpora corpora + PackageVersion string + DataStream dataStream + Corpora corpora } ClusterName string Nodes int @@ -72,7 +71,6 @@ func newReport(benchName, corporaFile string, s *scenario, sum *metricsSummary, report.Info.GeneratedCorporaFile = corporaFile report.Parameters.PackageVersion = s.Version report.Parameters.DataStream = s.DataStream - report.Parameters.WarmupTimePeriod = s.WarmupTimePeriod report.Parameters.Corpora = s.Corpora report.ClusterName = sum.ClusterName report.Nodes = sum.Nodes @@ -112,10 +110,6 @@ func reportHumanFormat(r *report) []byte { pkvs = append(pkvs, "data_stream.name", r.Parameters.DataStream.Name) - pkvs = append(pkvs, - "warmup time period", r.Parameters.WarmupTimePeriod, - ) - if r.Parameters.Corpora.Generator != nil { pkvs = append(pkvs, "corpora.generator.total_events", r.Parameters.Corpora.Generator.TotalEvents, diff --git a/internal/benchrunner/runners/rally/scenario.go b/internal/benchrunner/runners/rally/scenario.go index b91be203d2..89186575dd 100644 --- a/internal/benchrunner/runners/rally/scenario.go +++ b/internal/benchrunner/runners/rally/scenario.go @@ -9,7 +9,6 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/elastic/go-ucfg/yaml" ) @@ -17,12 +16,11 @@ import ( const devPath = "_dev/benchmark/rally" type scenario struct { - Package string `config:"package" json:"package"` - Description string `config:"description" json:"description"` - Version string `config:"version" json:"version"` - DataStream dataStream `config:"data_stream" json:"data_stream"` - WarmupTimePeriod time.Duration `config:"warmup_time_period" json:"warmup_time_period"` - Corpora corpora `config:"corpora" json:"corpora"` + Package string `config:"package" json:"package"` + Description string `config:"description" json:"description"` + Version string `config:"version" json:"version"` + DataStream dataStream `config:"data_stream" json:"data_stream"` + Corpora corpora `config:"corpora" json:"corpora"` } type dataStream struct { diff --git a/internal/elasticsearch/ingest/nodestats.go b/internal/elasticsearch/ingest/nodestats.go index 4b9da276bf..30bb0ce79d 100644 --- a/internal/elasticsearch/ingest/nodestats.go +++ b/internal/elasticsearch/ingest/nodestats.go @@ -118,8 +118,10 @@ func GetPipelineStatsByPrefix(esClient *elasticsearch.API, pipelinePrefix string } func requestPipelineStats(esClient *elasticsearch.API) ([]byte, error) { - statsReq := esClient.Nodes.Stats.WithFilterPath("nodes.*.ingest.pipelines") - resp, err := esClient.Nodes.Stats(statsReq) + filterPathReq := esClient.Nodes.Stats.WithFilterPath("nodes.*.ingest.pipelines") + includeUnloadedSegmentReq := esClient.Nodes.Stats.WithIncludeUnloadedSegments(true) + + resp, err := esClient.Nodes.Stats(filterPathReq, includeUnloadedSegmentReq) if err != nil { return nil, fmt.Errorf("node stats API call failed: %w", err) } From 86a6b2f4011ea30f75d83c75883cb3d07c53614a Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Thu, 2 Nov 2023 18:43:11 +0900 Subject: [PATCH 41/48] Update internal/cobraext/flags.go Co-authored-by: Nicolas Ruflin --- internal/cobraext/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 38c6005932..f02bf6e661 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -57,7 +57,7 @@ const ( BenchCorpusRallyTrackOutputDirFlagDescription = "output dir of the rally track: if present the command will save the generated rally track" BenchCorpusRallyDryRunFlagName = "dry-run" - BenchCorpusRallyDryRunFlagDescription = "Do not run rally bust just generate the rally track" + BenchCorpusRallyDryRunFlagDescription = "Do not run rally but just generate the rally track" BuildSkipValidationFlagName = "skip-validation" BuildSkipValidationFlagDescription = "skip validation of the built package, use only if all validation issues have been acknowledged" From 13672e27b41ec5bc6b097fba34bd7554228a5568 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Fri, 3 Nov 2023 17:10:36 +0900 Subject: [PATCH 42/48] docs about replay of rally tracks, bugfixes on metrics --- docs/howto/rally_benchmarking.md | 16 +++++++++++++++- internal/benchrunner/runners/rally/metrics.go | 14 +++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/howto/rally_benchmarking.md b/docs/howto/rally_benchmarking.md index 644995ae9b..c60500440c 100644 --- a/docs/howto/rally_benchmarking.md +++ b/docs/howto/rally_benchmarking.md @@ -265,4 +265,18 @@ Additionally, if the `reindex-to-metricstore` flag is used, the data generated d If the `rally-track-output-dir` flag is used, the track and the corpus generated during the benchmark will be saved in the directory passed as value of the flag. Additionally, if the `dry-run` flag is used, the command will exits before running `esrally`. If both the flags above are used at the same time, the command will just generate the track and corpus and save them, without running any benchmark. -If the `dry-run` flag only is used, the command will just wipe the data stream returning no report. +If the `dry-run` flag only is used, the command will just wipe the data stream returning no report. + +## Replaying a persisted rally track +In the directory of the `rally-track-output-dir` flag two files are saved: +1. The rally track: `track-%data_stream.type%-%data_stream.dataset%-%data_stream.namespace%.json` +2. The track corpus: `corpus-%unix_timestamp%` + +Both files are required to replay the rally benchmark. The first file references the second in its content. +The command to run for replaying the track is the following: +```shell +rally --target-hosts='{"default":["%es_cluster_host:es_cluster_port%"]}' --track-path=%path/to/saved-track-json% --client-options='{"default":{"basic_auth_user":"%es_user%","basic_auth_password":"%es_user%","use_ssl":true,"verify_certs":false}}' --pipeline=benchmark-only +``` + +Please refer to [esrally CLI reference](https://esrally.readthedocs.io/en/stable/command_line_reference.html) for more details. + diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index 842419d03e..11f5d405f6 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -94,6 +94,8 @@ func (c *collector) start() { c.wg.Add(1) go func() { + defer c.wg.Done() + <-c.stopC // last collect before stopping c.collectMetricsAfterRallyRun() @@ -213,11 +215,17 @@ func (c *collector) summarize() (*metricsSummary, error) { TotalHits: c.endTotalHits - c.startTotalHits, } - sum.ClusterName = c.startMetrics.nMetrics.ClusterName + if c.startMetrics.nMetrics != nil { + sum.ClusterName = c.startMetrics.nMetrics.ClusterName + } sum.CollectionStartTs = c.startMetrics.ts sum.CollectionEndTs = c.endMetrics.ts - sum.DataStreamStats = c.endMetrics.dsMetrics - sum.Nodes = len(c.endMetrics.nMetrics.Nodes) + if c.endMetrics.dsMetrics != nil { + sum.DataStreamStats = c.endMetrics.dsMetrics + } + if c.endMetrics.nMetrics != nil { + sum.Nodes = len(c.endMetrics.nMetrics.Nodes) + } for node, endPStats := range c.endIngestMetrics { startPStats, found := c.startIngestMetrics[node] From 3dc1a58ca7b1dd4b42a7a4be2d5fc2348021b566 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Fri, 3 Nov 2023 17:15:36 +0900 Subject: [PATCH 43/48] temporary: alias github.com/elastic/package-spec/v3 to github.com/aspacca/package-spec/v3 --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 5ce44e1ca4..f05919d17e 100644 --- a/go.mod +++ b/go.mod @@ -179,3 +179,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/elastic/package-spec/v3 => github.com/aspacca/package-spec/v3 v3.0.0-20231103004050-58dd45594c9c diff --git a/go.sum b/go.sum index 019a502a7b..7f4dfc03ef 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aspacca/package-spec/v3 v3.0.0-20231103004050-58dd45594c9c h1:qIX9EkadQZWBHUJ9EfzAMCZpKRwHc5xD/3WrY3gpWuQ= +github.com/aspacca/package-spec/v3 v3.0.0-20231103004050-58dd45594c9c/go.mod h1:proxSLyZsrgNOnWMqg01ROXcnU8b8DRa0OiY4T/dN90= github.com/aymerick/raymond v2.0.2+incompatible h1:VEp3GpgdAnv9B2GFyTvqgcKvY+mfKMjPOA3SbKLtnU0= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bitfield/gotestdox v0.2.1 h1:Zj8IMLAO5/oiAKoMmtN96eyFiPZraJRTH2p0zDgtxc0= @@ -144,8 +146,6 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= github.com/elastic/kbncontent v0.1.1 h1:X6ZXKPTW3MwFZPLMZcPAbpUIsKQhFImhBGKOd83+UFo= github.com/elastic/kbncontent v0.1.1/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= -github.com/elastic/package-spec/v3 v3.0.0 h1:t4domWWtALwms0pnWKHLceNY5LZe2EKPgRzdfBPLcas= -github.com/elastic/package-spec/v3 v3.0.0/go.mod h1:proxSLyZsrgNOnWMqg01ROXcnU8b8DRa0OiY4T/dN90= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= From 95822069d0d2e67ad827bd431d3b300ff98c73c6 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Fri, 3 Nov 2023 17:42:05 +0900 Subject: [PATCH 44/48] remove warmup_time_period --- .../rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml index b2ae43313d..16e45f7782 100644 --- a/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml +++ b/test/packages/benchmarks/rally_benchmark/_dev/benchmark/rally/logs-benchmark.yml @@ -2,7 +2,6 @@ description: Benchmark 20000 events ingested data_stream: name: testds -warmup_time_period: 10s corpora: generator: total_events: 20000 From bb18dc8ec9057853529a57e7bb4c67ce4eb5e072 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Fri, 3 Nov 2023 18:07:41 +0900 Subject: [PATCH 45/48] temporary: format version --- test/packages/benchmarks/rally_benchmark/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/packages/benchmarks/rally_benchmark/manifest.yml b/test/packages/benchmarks/rally_benchmark/manifest.yml index 7cbe79f7a0..6744f6547a 100644 --- a/test/packages/benchmarks/rally_benchmark/manifest.yml +++ b/test/packages/benchmarks/rally_benchmark/manifest.yml @@ -1,4 +1,4 @@ -format_version: 3.0.1 +format_version: 3.0.0 name: rally_benchmarks title: Rally benchmarks version: 999.999.999 From 5804ef93faf9ddf117f27ffd0002c2000614506b Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Mon, 6 Nov 2023 18:17:04 +0900 Subject: [PATCH 46/48] remove replace in go mod. update to latest commit --- go.mod | 4 +--- go.sum | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f05919d17e..e45fbcb6b4 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-resource v0.1.1 github.com/elastic/go-ucfg v0.8.6 - github.com/elastic/package-spec/v3 v3.0.0 + github.com/elastic/package-spec/v3 v3.0.1-0.20231106091150-1a7d2cbd6e65 github.com/fatih/color v1.15.0 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.9.0 @@ -179,5 +179,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) - -replace github.com/elastic/package-spec/v3 => github.com/aspacca/package-spec/v3 v3.0.0-20231103004050-58dd45594c9c diff --git a/go.sum b/go.sum index 7f4dfc03ef..553e27d1cf 100644 --- a/go.sum +++ b/go.sum @@ -146,6 +146,7 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= github.com/elastic/kbncontent v0.1.1 h1:X6ZXKPTW3MwFZPLMZcPAbpUIsKQhFImhBGKOd83+UFo= github.com/elastic/kbncontent v0.1.1/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= +github.com/elastic/package-spec/v3 v3.0.1-0.20231106091150-1a7d2cbd6e65/go.mod h1:proxSLyZsrgNOnWMqg01ROXcnU8b8DRa0OiY4T/dN90= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= From cc39585379424e580f5632279367d1db57c03da7 Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 7 Nov 2023 12:02:35 +0900 Subject: [PATCH 47/48] make check-static --- go.sum | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 553e27d1cf..9dbfdf0304 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,6 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aspacca/package-spec/v3 v3.0.0-20231103004050-58dd45594c9c h1:qIX9EkadQZWBHUJ9EfzAMCZpKRwHc5xD/3WrY3gpWuQ= -github.com/aspacca/package-spec/v3 v3.0.0-20231103004050-58dd45594c9c/go.mod h1:proxSLyZsrgNOnWMqg01ROXcnU8b8DRa0OiY4T/dN90= github.com/aymerick/raymond v2.0.2+incompatible h1:VEp3GpgdAnv9B2GFyTvqgcKvY+mfKMjPOA3SbKLtnU0= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bitfield/gotestdox v0.2.1 h1:Zj8IMLAO5/oiAKoMmtN96eyFiPZraJRTH2p0zDgtxc0= @@ -146,6 +144,7 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= github.com/elastic/kbncontent v0.1.1 h1:X6ZXKPTW3MwFZPLMZcPAbpUIsKQhFImhBGKOd83+UFo= github.com/elastic/kbncontent v0.1.1/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= +github.com/elastic/package-spec/v3 v3.0.1-0.20231106091150-1a7d2cbd6e65 h1:wXKMGvGwjY9OWhqZVUvZYf22c9M8EHdJM6NmJ+IXxTw= github.com/elastic/package-spec/v3 v3.0.1-0.20231106091150-1a7d2cbd6e65/go.mod h1:proxSLyZsrgNOnWMqg01ROXcnU8b8DRa0OiY4T/dN90= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= From 943367a33444935e40822a747234d4b3b610968f Mon Sep 17 00:00:00 2001 From: Andrea Spacca Date: Tue, 7 Nov 2023 19:15:39 +0900 Subject: [PATCH 48/48] github.com/elastic/package-spec/v3@v3.0.1 --- go.mod | 2 +- go.sum | 4 ++-- test/packages/benchmarks/rally_benchmark/manifest.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index e45fbcb6b4..375285962d 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-resource v0.1.1 github.com/elastic/go-ucfg v0.8.6 - github.com/elastic/package-spec/v3 v3.0.1-0.20231106091150-1a7d2cbd6e65 + github.com/elastic/package-spec/v3 v3.0.1 github.com/fatih/color v1.15.0 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.9.0 diff --git a/go.sum b/go.sum index 9dbfdf0304..cd50996da2 100644 --- a/go.sum +++ b/go.sum @@ -144,8 +144,8 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= github.com/elastic/kbncontent v0.1.1 h1:X6ZXKPTW3MwFZPLMZcPAbpUIsKQhFImhBGKOd83+UFo= github.com/elastic/kbncontent v0.1.1/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= -github.com/elastic/package-spec/v3 v3.0.1-0.20231106091150-1a7d2cbd6e65 h1:wXKMGvGwjY9OWhqZVUvZYf22c9M8EHdJM6NmJ+IXxTw= -github.com/elastic/package-spec/v3 v3.0.1-0.20231106091150-1a7d2cbd6e65/go.mod h1:proxSLyZsrgNOnWMqg01ROXcnU8b8DRa0OiY4T/dN90= +github.com/elastic/package-spec/v3 v3.0.1 h1:bYEW9Kyrm4pQXnIX6IVmWgOVQV/bjIjUVN85+j5HsZE= +github.com/elastic/package-spec/v3 v3.0.1/go.mod h1:proxSLyZsrgNOnWMqg01ROXcnU8b8DRa0OiY4T/dN90= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= diff --git a/test/packages/benchmarks/rally_benchmark/manifest.yml b/test/packages/benchmarks/rally_benchmark/manifest.yml index 6744f6547a..7cbe79f7a0 100644 --- a/test/packages/benchmarks/rally_benchmark/manifest.yml +++ b/test/packages/benchmarks/rally_benchmark/manifest.yml @@ -1,4 +1,4 @@ -format_version: 3.0.0 +format_version: 3.0.1 name: rally_benchmarks title: Rally benchmarks version: 999.999.999