Skip to content

Commit d5342ea

Browse files
pickypgpickypg
committed
[AutoOps] Use ECS formatting for all field names
This changes the exported fields to use ECS formatting (snakecase) and also to add some standard ECS fields to the root or module.
1 parent 5c91c96 commit d5342ea

File tree

16 files changed

+138
-165
lines changed

16 files changed

+138
-165
lines changed

‎x-pack/metricbeat/module/autoops_es/auto_ops_testing/testing.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -351,23 +351,23 @@ func CheckEventWithTransactionId(t *testing.T, event mb.Event, info utils.Cluste
351351
CheckEvent(t, event, info)
352352

353353
// matching transaction ID
354-
require.Equal(t, transactionId, GetObjectValue(event.ModuleFields, "transactionId"))
354+
require.Equal(t, transactionId, GetObjectValue(event.ModuleFields, "transaction_id"))
355355
}
356356

357357
func CheckEventWithRandomTransactionId(t *testing.T, event mb.Event, info utils.ClusterInfo) {
358358
CheckEvent(t, event, info)
359359

360360
// valid, random UUID
361-
_, err := uuid.FromString(GetObjectValue(event.ModuleFields, "transactionId").(string))
361+
_, err := uuid.FromString(GetObjectValue(event.ModuleFields, "transaction_id").(string))
362362
require.NoError(t, err)
363363
}
364364

365365
func CheckAllEventsUseSameTransactionId(t *testing.T, events []mb.Event) {
366366
if len(events) > 1 {
367-
transactionId := GetObjectValue(events[0].ModuleFields, "transactionId")
367+
transactionId := GetObjectValue(events[0].ModuleFields, "transaction_id")
368368

369369
for _, event := range events {
370-
require.Equal(t, transactionId, GetObjectValue(event.ModuleFields, "transactionId"))
370+
require.Equal(t, transactionId, GetObjectValue(event.ModuleFields, "transaction_id"))
371371
}
372372
}
373373
}

‎x-pack/metricbeat/module/autoops_es/cat_shards/index_shards.go

Lines changed: 63 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,19 @@ type Shard struct {
3333
}
3434

3535
type AssignedShard struct {
36-
ShardNum int32 `json:"shardNum"`
36+
ShardNum int32 `json:"shard_num"`
3737
Primary bool `json:"primary"`
38-
SizeInBytes *int64 `json:"sizeInBytes"`
39-
DocsCount *int64 `json:"docsCount"`
40-
SegmentsCount *int64 `json:"segmentsCount"`
38+
SizeInBytes *int64 `json:"size_in_bytes"`
39+
DocsCount *int64 `json:"docs_count"`
40+
SegmentsCount *int64 `json:"segments_count"`
4141
State string `json:"state"`
4242
}
4343

4444
type UnassignedShard struct {
45-
ShardNum int32 `json:"shardNum"`
45+
ShardNum int32 `json:"shard_num"`
4646
Primary bool `json:"primary"`
47-
UnassignedReason *string `json:"unassignedReason"`
48-
UnassignedDetails *string `json:"unassignedDetails"`
47+
UnassignedReason *string `json:"unassigned_reason"`
48+
UnassignedDetails *string `json:"unassigned_details"`
4949
}
5050

5151
type IndexMetadata struct {
@@ -58,63 +58,63 @@ type IndexMetadata struct {
5858
}
5959

6060
type NodeIndexShards struct {
61-
TotalFractions int32 `json:"totalFractions"`
61+
TotalFractions int32 `json:"total_fractions"`
6262

6363
Index string `json:"index"`
64-
IndexNode string `json:"indexNode"`
65-
IndexStatus *string `json:"indexStatus"`
66-
IndexType *string `json:"indexType"`
64+
IndexNode string `json:"index_node"`
65+
IndexStatus *string `json:"index_status"`
66+
IndexType *string `json:"index_type"`
6767
Aliases []string `json:"aliases"`
6868
Attributes []string `json:"attributes"`
69-
IsHidden *bool `json:"isHidden"`
70-
IsOpen *bool `json:"isOpen"`
71-
IsSystem *bool `json:"isSystem"`
72-
NodeId string `json:"nodeId"`
73-
NodeName string `json:"nodeName"`
74-
AssignShards []AssignedShard `json:"assignShards"`
75-
InitializingShards []AssignedShard `json:"initializingShards"`
76-
RelocatingShards []AssignedShard `json:"relocatingShards"`
77-
UnassignedShards []UnassignedShard `json:"unAssignShards"`
78-
Shards int32 `json:"shardsCount"`
79-
PrimaryShards int32 `json:"primaryShardsCount"`
80-
ReplicaShards int32 `json:"replicaShardsCount"`
81-
Initializing int32 `json:"initializingShardsCount"`
82-
Relocating int32 `json:"relocatingShardsCount"`
83-
Unassigned int32 `json:"unassignedShardsCount"`
84-
UnassignedPrimaryShards int32 `json:"totalUnAssignedPrimaryShards"`
85-
UnassignedReplicasShards int32 `json:"totalUnAssignedReplicasShards"`
86-
SegmentsCount *int64 `json:"segmentsCount"`
87-
SizeInBytes *int64 `json:"sizeInBytes"`
88-
TotalSegmentsCount *int64 `json:"totalSegmentsCount"` // includes replicas
89-
TotalSizeInBytes *int64 `json:"totalSizeInBytes"` // includes replicas
90-
MaxShardSizeInBytes *int64 `json:"maxShardSizeInBytes"`
91-
MinShardSizeInBytes *int64 `json:"minShardSizeInBytes"`
92-
TotalMaxShardSizeInBytes *int64 `json:"totalMaxShardSizeInBytes"` // includes replicas
93-
TotalMinShardSizeInBytes *int64 `json:"totalMinShardSizeInBytes"` // includes replicas
69+
IsHidden *bool `json:"is_hidden"`
70+
IsOpen *bool `json:"is_open"`
71+
IsSystem *bool `json:"is_system"`
72+
NodeId string `json:"node_id"`
73+
NodeName string `json:"node_name"`
74+
AssignShards []AssignedShard `json:"assign_shards"`
75+
InitializingShards []AssignedShard `json:"initializing_shards"`
76+
RelocatingShards []AssignedShard `json:"relocating_shards"`
77+
UnassignedShards []UnassignedShard `json:"unassign_shards"`
78+
Shards int32 `json:"shards_count"`
79+
PrimaryShards int32 `json:"primary_shards_count"`
80+
ReplicaShards int32 `json:"replica_shards_count"`
81+
Initializing int32 `json:"initializing_shards_count"`
82+
Relocating int32 `json:"relocating_shards_count"`
83+
Unassigned int32 `json:"unassigned_shards_count"`
84+
UnassignedPrimaryShards int32 `json:"total_unassigned_primary_shards"`
85+
UnassignedReplicasShards int32 `json:"total_unassigned_replicas_shards"`
86+
SegmentsCount *int64 `json:"segments_count"`
87+
SizeInBytes *int64 `json:"size_in_bytes"`
88+
TotalSegmentsCount *int64 `json:"total_segments_count"` // includes replicas
89+
TotalSizeInBytes *int64 `json:"total_size_in_bytes"` // includes replicas
90+
MaxShardSizeInBytes *int64 `json:"max_shard_size_in_bytes"`
91+
MinShardSizeInBytes *int64 `json:"min_shard_size_in_bytes"`
92+
TotalMaxShardSizeInBytes *int64 `json:"total_max_shard_size_in_bytes"` // includes replicas
93+
TotalMinShardSizeInBytes *int64 `json:"total_min_shard_size_in_bytes"` // includes replicas
9494

9595
// indexing metrics only consider primary shards!
9696

97-
DocsCount *int64 `json:"docsCount"`
98-
IndexFailedRatePerSecond *float64 `json:"indexFailedRatePerSecond"`
99-
IndexLatencyInMillis *float64 `json:"indexLatencyInMillis"`
100-
IndexRatePerSecond *float64 `json:"indexRatePerSecond"`
101-
IndexingFailedIndexTotal *int64 `json:"indexingFailedIndexTotal"`
102-
IndexingIndexTotal *int64 `json:"indexingIndexTotal"`
103-
IndexingIndexTotalTime *int64 `json:"indexingTotalTime"`
104-
GetMissingDocTotal *int64 `json:"getMissingDocTotal"` // includes replicas
105-
GetMissingDocTotalTime *int64 `json:"getMissingDocTotalTime"` // includes replicas
106-
GetMissingDocRatePerSecond *float64 `json:"getDocMissingRatePerSecond"` // includes replicas
107-
MergeLatencyInMillis *float64 `json:"mergeLatencyInMillis"`
108-
MergeRatePerSecond *float64 `json:"mergeRatePerSecond"`
109-
MergesTotal *int64 `json:"mergesTotal"`
110-
MergesTotalTime *int64 `json:"mergesTotalTime"`
111-
SearchLatencyInMillis *float64 `json:"searchLatencyInMillis"` // includes replicas
112-
SearchQueryTime *int64 `json:"searchQueryTime"` // includes replicas
113-
SearchQueryTotal *int64 `json:"searchQueryTotal"` // includes replicas
114-
SearchRatePerSecond *float64 `json:"searchRatePerSecond"` // includes replicas
115-
TotalMergesTotal *int64 `json:"totalMergesTotal"` // includes replicas
116-
TotalMergesTotalTime *int64 `json:"totalMergesTotalTime"` // includes replicas
117-
TimestampDiff *int64 `json:"timestampDiff"`
97+
DocsCount *int64 `json:"docs_count"`
98+
IndexFailedRatePerSecond *float64 `json:"index_failed_rate_per_second"`
99+
IndexLatencyInMillis *float64 `json:"index_latency_in_millis"`
100+
IndexRatePerSecond *float64 `json:"index_rate_per_second"`
101+
IndexingFailedIndexTotal *int64 `json:"indexing_failed_index_total"`
102+
IndexingIndexTotal *int64 `json:"indexing_index_total"`
103+
IndexingIndexTotalTime *int64 `json:"indexing_index_total_time"`
104+
GetMissingDocTotal *int64 `json:"get_missing_doc_total"` // includes replicas
105+
GetMissingDocTotalTime *int64 `json:"get_missing_doc_total_time"` // includes replicas
106+
GetMissingDocRatePerSecond *float64 `json:"get_missing_doc_rate_per_second"` // includes replicas
107+
MergeLatencyInMillis *float64 `json:"merge_latency_in_millis"`
108+
MergeRatePerSecond *float64 `json:"merge_rate_per_second"`
109+
MergesTotal *int64 `json:"merges_total"`
110+
MergesTotalTime *int64 `json:"merges_total_time"`
111+
SearchLatencyInMillis *float64 `json:"search_latency_in_millis"` // includes replicas
112+
SearchQueryTime *int64 `json:"search_query_time"` // includes replicas
113+
SearchQueryTotal *int64 `json:"search_query_total"` // includes replicas
114+
SearchRatePerSecond *float64 `json:"search_rate_per_second"` // includes replicas
115+
TotalMergesTotal *int64 `json:"total_merges_total"` // includes replicas
116+
TotalMergesTotalTime *int64 `json:"total_merges_total_time"` // includes replicas
117+
TimestampDiff *int64 `json:"timestamp_diff"`
118118
}
119119

120120
type NodeShardCount struct {
@@ -132,13 +132,13 @@ type NodeShardCount struct {
132132
}
133133

134134
type ShardInfo struct {
135-
ShardNum string `json:"shardNum"`
136-
ShardId string `json:"shardId"`
135+
ShardNum string `json:"shard_num"`
136+
ShardId string `json:"shard_id"`
137137
Primary bool `json:"primary"`
138-
SizeInBytes uint64 `json:"sizeInBytes"`
139-
DocsCount uint64 `json:"docsCount"`
140-
UnAssignedReason string `json:"unassignedReason"`
141-
UnAssignedDetails string `json:"unassignedDetails"`
138+
SizeInBytes uint64 `json:"size_in_bytes"`
139+
DocsCount uint64 `json:"docs_count"`
140+
UnAssignedReason string `json:"unassigned_reason"`
141+
UnAssignedDetails string `json:"unassigned_details"`
142142
}
143143

144144
func toAssignedShard(shard Shard) AssignedShard {

‎x-pack/metricbeat/module/autoops_es/cat_template/data.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func getNamedTemplates(transactionId string, info *utils.ClusterInfo, templates
4343
} else if template, err := templateSchema.Apply(data); err != nil {
4444
errs = append(errs, fmt.Errorf("failed applying template schema for %v: %w", name, err))
4545
} else {
46-
template["templateName"] = name
46+
template["template_name"] = name
4747
reporter(transactionId, info, template)
4848
}
4949
}

‎x-pack/metricbeat/module/autoops_es/cat_template/data_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func expectValidParsedDataCheckingTemplateNames(t *testing.T, data metricset.Fet
4646
require.NotNil(t, auto_ops_testing.GetObjectValue(event.MetricSetFields, "template.settings"))
4747
require.NotNil(t, auto_ops_testing.GetObjectValue(event.MetricSetFields, "template.mappings"))
4848
require.NotNil(t, auto_ops_testing.GetObjectValue(event.MetricSetFields, "template.aliases"))
49-
templateName := auto_ops_testing.GetObjectAsString(t, event.MetricSetFields, "template.templateName")
49+
templateName := auto_ops_testing.GetObjectAsString(t, event.MetricSetFields, "template.template_name")
5050
require.True(t, slices.Contains(templateNames, templateName), "template '%s' is not in the expected values", templateName)
5151

5252
// mapper is expected to drop this field if it appears
@@ -87,8 +87,8 @@ func expectValidParsedDetailedTemplates(t *testing.T, data metricset.FetcherData
8787

8888
require.Equal(t, 2, len(events))
8989

90-
event1 := auto_ops_testing.GetEventByName(t, events, "template.templateName", "simple-response")
91-
event2 := auto_ops_testing.GetEventByName(t, events, "template.templateName", "detailed-response")
90+
event1 := auto_ops_testing.GetEventByName(t, events, "template.template_name", "simple-response")
91+
event2 := auto_ops_testing.GetEventByName(t, events, "template.template_name", "detailed-response")
9292

9393
auto_ops_testing.CheckEventWithRandomTransactionId(t, event2, data.ClusterInfo)
9494

@@ -109,7 +109,7 @@ func expectValidParsedDetailedTemplates(t *testing.T, data metricset.FetcherData
109109
// metrics exist
110110

111111
// event 1 (simple-response)
112-
require.Equal(t, "simple-response", auto_ops_testing.GetObjectValue(event1.MetricSetFields, "template.templateName"))
112+
require.Equal(t, "simple-response", auto_ops_testing.GetObjectValue(event1.MetricSetFields, "template.template_name"))
113113
require.EqualValues(t, 1, auto_ops_testing.GetObjectValue(event1.MetricSetFields, "template.order"))
114114
require.Nil(t, auto_ops_testing.GetObjectValue(event1.MetricSetFields, "template.version"))
115115
require.ElementsMatch(t, []string{"*"}, auto_ops_testing.GetObjectValue(event1.MetricSetFields, "template.index_patterns"))
@@ -118,7 +118,7 @@ func expectValidParsedDetailedTemplates(t *testing.T, data metricset.FetcherData
118118
require.Equal(t, auto_ops_testing.GetObjectAsJson(simpleTemplate, "template.aliases"), auto_ops_testing.GetObjectAsJson(event1.MetricSetFields, "template.aliases"))
119119

120120
// event 2 (detailed-response)
121-
require.Equal(t, "detailed-response", auto_ops_testing.GetObjectValue(event2.MetricSetFields, "template.templateName"))
121+
require.Equal(t, "detailed-response", auto_ops_testing.GetObjectValue(event2.MetricSetFields, "template.template_name"))
122122
require.EqualValues(t, 789, auto_ops_testing.GetObjectValue(event2.MetricSetFields, "template.order"))
123123
require.EqualValues(t, 123456, auto_ops_testing.GetObjectValue(event2.MetricSetFields, "template.version"))
124124
require.ElementsMatch(t, []string{"a", "b", "c", "d*"}, auto_ops_testing.GetObjectValue(event2.MetricSetFields, "template.index_patterns"))

‎x-pack/metricbeat/module/autoops_es/component_template/data.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func getNamedTemplates(transactionId string, info *utils.ClusterInfo, templates
4545
continue
4646
}
4747

48-
template["templateName"] = templateData.Name
48+
template["template_name"] = templateData.Name
4949

5050
reporter(transactionId, info, template)
5151
}

‎x-pack/metricbeat/module/autoops_es/component_template/data_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func expectValidParsedDataWithTemplateNames(t *testing.T, data metricset.Fetcher
5656

5757
var actualTemplateNames []string
5858
for _, event := range events {
59-
name := auto_ops_testing.GetObjectAsString(t, event.MetricSetFields, "template.templateName")
59+
name := auto_ops_testing.GetObjectAsString(t, event.MetricSetFields, "template.template_name")
6060
actualTemplateNames = append(actualTemplateNames, name)
6161
}
6262
require.Equal(t, len(templateNames), len(actualTemplateNames), "Wrong number of template names %s", actualTemplateNames)
@@ -82,8 +82,8 @@ func expectValidParsedDetailedTemplates(t *testing.T, data metricset.FetcherData
8282

8383
require.Equal(t, 2, len(events))
8484

85-
event1 := auto_ops_testing.GetEventByName(t, events, "template.templateName", "simple-response")
86-
event2 := auto_ops_testing.GetEventByName(t, events, "template.templateName", "detailed-response")
85+
event1 := auto_ops_testing.GetEventByName(t, events, "template.template_name", "simple-response")
86+
event2 := auto_ops_testing.GetEventByName(t, events, "template.template_name", "detailed-response")
8787

8888
auto_ops_testing.CheckEventWithRandomTransactionId(t, event2, data.ClusterInfo)
8989

@@ -104,13 +104,13 @@ func expectValidParsedDetailedTemplates(t *testing.T, data metricset.FetcherData
104104
// metrics exist
105105

106106
// event 1 (simple-response)
107-
require.Equal(t, "simple-response", auto_ops_testing.GetObjectValue(event1.MetricSetFields, "template.templateName"))
107+
require.Equal(t, "simple-response", auto_ops_testing.GetObjectValue(event1.MetricSetFields, "template.template_name"))
108108
require.Nil(t, auto_ops_testing.GetObjectValue(event1.MetricSetFields, "template.version"))
109109
require.Equal(t, auto_ops_testing.GetObjectAsJson(simpleTemplate, "template.template.settings"), auto_ops_testing.GetObjectAsJson(event1.MetricSetFields, "template.template.settings"))
110110
require.Nil(t, auto_ops_testing.GetObjectValue(event1.MetricSetFields, "template._meta"))
111111

112112
// event 2 (detailed-response)
113-
require.Equal(t, "detailed-response", auto_ops_testing.GetObjectValue(event2.MetricSetFields, "template.templateName"))
113+
require.Equal(t, "detailed-response", auto_ops_testing.GetObjectValue(event2.MetricSetFields, "template.template_name"))
114114
require.EqualValues(t, 123456, auto_ops_testing.GetObjectValue(event2.MetricSetFields, "template.version"))
115115
require.Equal(t, auto_ops_testing.GetObjectAsJson(detailedTemplate, "template.template"), auto_ops_testing.GetObjectAsJson(event2.MetricSetFields, "template.template"))
116116
require.EqualValues(t, 456, auto_ops_testing.GetObjectValue(event2.MetricSetFields, "template._meta.property1"))

‎x-pack/metricbeat/module/autoops_es/events/events.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,12 @@ func CreateEvent(info *utils.ClusterInfo, metricSetFields mapstr.M, transactionI
2525
"name": info.ClusterName,
2626
"version": info.Version.Number.String(),
2727
},
28-
"transactionId": transactionId,
28+
"transaction_id": transactionId,
2929
},
3030
RootFields: mapstr.M{
31+
"event": mapstr.M{
32+
"kind": "metric",
33+
},
3134
"orchestrator": mapstr.M{
3235
"resource": mapstr.M{
3336
"id": utils.GetResourceID(),
@@ -39,8 +42,8 @@ func CreateEvent(info *utils.ClusterInfo, metricSetFields mapstr.M, transactionI
3942

4043
// Report an event and mark the fraction and total fractions consistently
4144
func ReportEvent(r mb.ReporterV2, event mb.Event, index int, total int) {
42-
event.ModuleFields["fractionId"] = index
43-
event.ModuleFields["totalAmountOfFractions"] = total
45+
event.ModuleFields["fraction_id"] = index
46+
event.ModuleFields["total_amount_of_fractions"] = total
4447

4548
r.Event(event)
4649
}

‎x-pack/metricbeat/module/autoops_es/events/events_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ func TestCreateEvent(t *testing.T) {
6060
require.Equal(t, "value2", auto_ops_testing.GetObjectValue(event.MetricSetFields, "obj1.field1"))
6161
require.Equal(t, "value3", auto_ops_testing.GetObjectValue(event.MetricSetFields, "obj2.field1"))
6262

63+
// event
64+
require.Equal(t, "metric", auto_ops_testing.GetObjectValue(event.RootFields, "event.kind"))
65+
6366
// orchestrator
6467
require.Equal(t, "resource-id", auto_ops_testing.GetObjectValue(event.RootFields, "orchestrator.resource.id"))
6568
}

‎x-pack/metricbeat/module/autoops_es/index_template/data.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func getNamedTemplates(transactionId string, info *utils.ClusterInfo, templates
5353
continue
5454
}
5555

56-
template["templateName"] = templateData.Name
56+
template["template_name"] = templateData.Name
5757

5858
reporter(transactionId, info, template)
5959
}

0 commit comments

Comments
 (0)