Rally benchmark aws.ec2 logs

packages/aws/_dev/benchmark/rally/ec2logs-benchmark.yml

@@ -0,0 +1,14 @@
+---
+description: Benchmark 20000 aws.ec2_logs events ingested
+data_stream:
+  name: ec2_logs
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      type: gotext
+      path: ./ec2logs-benchmark/template.ndjson
+    config:
+      path: ./ec2logs-benchmark/config.yml
+    fields:
+      path: ./ec2logs-benchmark/fields.yml

packages/aws/_dev/benchmark/rally/ec2logs-benchmark/config.yml

@@ -0,0 +1,16 @@
+fields:
+  - name: 'timestamp'
+    period: -24h # one day
+  - name: process.name
+    enum: ["journal", "kernel", "systemd"]
+  - name: agent.id
+    value: "da6cb4c8-c84c-4c5f-97c7-f8586a098af4"
+  - name: cloud.region
+    enum: ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "sa-east-1", "af-south-1", "ap-east-1", "ap-south-2", "ap-southeast-3", "eu-south-2", "eu-central-2", "me-south-1", "me-central-1"]
+    cardinality: 100
+  - name: aws.cloudwatch.log_stream
+    cardinality: 100
+  - name: host.name
+    cardinality: 100
+  - name: aws.ec2.ip_address
+    cardinality: 100

packages/aws/_dev/benchmark/rally/ec2logs-benchmark/fields.yml

@@ -0,0 +1,20 @@
+- name: timestamp
+  type: date
+- name: process.name
+  type: keyword
+- name: event.ingested
+  type: date
+- name: aws.ec2.ip_address
+  type: ip
+- name: message
+  type: text
+- name: aws.cloudwatch.log_stream
+  type: keyword
+- name: host.name
+  type: keyword
+- name: agent.id
+  type: keyword
+- name: event.id
+  type: keyword
+- name: cloud.region
+  type: keyword

packages/aws/_dev/benchmark/rally/ec2logs-benchmark/template.ndjson

@@ -0,0 +1,48 @@
+{{- $timestamp := generate "timestamp" }}
+{{- $ip := generate "aws.ec2.ip_address" }}
+{{- $pname := generate "process.name" }}
+{{- $logstream := generate "aws.cloudwatch.log_stream" }}
+{{- $hostname := generate "host.name" }}
+{{- $agentId := generate "agent.id" }}
+{
+  "@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }}",
+  "aws.cloudwatch": {
+    "log_stream": "{{$logstream}}",
+    "ingestion_time": "{{ $timestamp | date "2006-01-02T15:04:05.000Z" }}",
+    "log_group": "/var/log/messages"
+  },
+  "cloud": {
+    "region": "{{ generate "cloud.region" }}"
+  },
+  "log.file.path": "/var/log/messages/{{$logstream}}",
+  "input": {
+    "type": "aws-cloudwatch"
+  },
+  "data_stream": {
+    "namespace": "ep",
+    "type": "logs",
+    "dataset": "aws.ec2_logs"
+  },
+  "process": {
+    "name": "{{ $pname }}"
+  },
+  "message": "{{$timestamp | date "2006-01-02T15:04:05.000Z"}} {{$timestamp | date "Jan"}} {{$timestamp | date "02"}} {{$timestamp | date "15:04:05"}} {{printf "ip-%s" ($ip | splitList "." | join "-")}} {{$pname}}: {{generate "message"}}",
+  "event": {
+    "id": "{{ generate "event.id" }}",
+    "ingested": "{{ generate "event.ingested" | date "2006-01-02T15:04:05.000000000Z" }}",
+    "dataset": "aws.ec2_logs"
+  },
+  "host": {
+    "name": "{{$hostname}}"
+  },
+  "agent": {
+    "id": "{{$agentId}}",
+    "name": "{{$hostname}}",
+    "type": "filebeat",
+    "version": "8.8.0",
+    "ephemeral_id": "{{$agentId}}"
+  },
+  "tags": [
+    "preserve_original_event"
+  ]
+}