[9.0](backport #43063) filebeat,libbeat,x-pack/filebeat: clean up state store types (#43366)

* filebeat,libbeat,x-pack/filebeat: clean up state store types (#43063)

This centralises the state store to simplify and clarify the use of persistent
stores.

(cherry picked from commit df62e10)

# Conflicts:
#	filebeat/beater/filebeat.go
#	filebeat/input/journald/environment_test.go

* fix conflicts

---------

Co-authored-by: Dan Kortschak <dan.kortschak@elastic.co>

Author: mergify[bot]

filebeat/beater/filebeat.go

@@ -24,7 +24,6 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/elastic/beats/v7/filebeat/backup"
 	"github.com/elastic/beats/v7/filebeat/channel"
@@ -79,14 +78,7 @@ type Filebeat struct {
 	pipeline       beat.PipelineConnector
 }
 
-type PluginFactory func(beat.Info, *logp.Logger, StateStore) []v2.Plugin
-
-type StateStore interface {
-	// Access returns the storage registry depending on the type. This is needed for the Elasticsearch state store which
-	// is guarded by the feature.IsElasticsearchStateStoreEnabledForInput(typ) check.
-	Access(typ string) (*statestore.Store, error)
-	CleanupInterval() time.Duration
-}
+type PluginFactory func(beat.Info, *logp.Logger, statestore.States) []v2.Plugin
 
 // New creates a new Filebeat pointer instance.
 func New(plugins PluginFactory) beat.Creator {
@@ -568,7 +560,7 @@ func newPipelineLoaderFactory(ctx context.Context, esConfig *conf.C) fileset.Pip
 
 // some of the filestreams might want to take over the loginput state
 // if their `take_over` flag is set to `true`.
-func processLogInputTakeOver(stateStore StateStore, config *cfg.Config) error {
+func processLogInputTakeOver(stateStore statestore.States, config *cfg.Config) error {
 	inputs, err := fetchInputConfiguration(config)
 	if err != nil {
 		return fmt.Errorf("Failed to fetch input configuration when attempting take over: %w", err)
@@ -577,7 +569,7 @@ func processLogInputTakeOver(stateStore StateStore, config *cfg.Config) error {
 		return nil
 	}
 
-	store, err := stateStore.Access("")
+	store, err := stateStore.StoreFor("")
 	if err != nil {
 		return fmt.Errorf("Failed to access state when attempting take over: %w", err)
 	}

filebeat/beater/store.go

@@ -32,6 +32,8 @@ import (
 	"github.com/elastic/elastic-agent-libs/paths"
 )
 
+var _ statestore.States = (*filebeatStore)(nil)
+
 type filebeatStore struct {
 	registry      *statestore.Registry
 	esRegistry    *statestore.Registry
@@ -84,8 +86,8 @@ func (s *filebeatStore) Close() {
 	s.registry.Close()
 }
 
-// Access returns the storage registry depending on the type. Default is the file store.
-func (s *filebeatStore) Access(typ string) (*statestore.Store, error) {
+// StoreFor returns the storage registry depending on the type. Default is the file store.
+func (s *filebeatStore) StoreFor(typ string) (*statestore.Store, error) {
 	if features.IsElasticsearchStateStoreEnabledForInput(typ) && s.esRegistry != nil {
 		return s.esRegistry.Get(s.storeName)
 	}

filebeat/input/default-inputs/inputs.go

@@ -18,25 +18,25 @@
 package inputs
 
 import (
-	"github.com/elastic/beats/v7/filebeat/beater"
 	"github.com/elastic/beats/v7/filebeat/input/filestream"
 	"github.com/elastic/beats/v7/filebeat/input/kafka"
 	"github.com/elastic/beats/v7/filebeat/input/tcp"
 	"github.com/elastic/beats/v7/filebeat/input/udp"
 	"github.com/elastic/beats/v7/filebeat/input/unix"
 	v2 "github.com/elastic/beats/v7/filebeat/input/v2"
 	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/statestore"
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
-func Init(info beat.Info, log *logp.Logger, components beater.StateStore) []v2.Plugin {
+func Init(info beat.Info, log *logp.Logger, components statestore.States) []v2.Plugin {
 	return append(
 		genericInputs(log, components),
 		osInputs(info, log, components)...,
 	)
 }
 
-func genericInputs(log *logp.Logger, components beater.StateStore) []v2.Plugin {
+func genericInputs(log *logp.Logger, components statestore.States) []v2.Plugin {
 	return []v2.Plugin{
 		filestream.Plugin(log, components),
 		kafka.Plugin(),

filebeat/input/default-inputs/inputs_linux.go

@@ -20,18 +20,14 @@ package inputs
 import (
 	"github.com/elastic/beats/v7/filebeat/input/journald"
 	v2 "github.com/elastic/beats/v7/filebeat/input/v2"
-	cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor"
 	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/statestore"
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
 // inputs that are only supported on linux
 
-type osComponents interface {
-	cursor.StateStore
-}
-
-func osInputs(info beat.Info, log *logp.Logger, components osComponents) []v2.Plugin {
+func osInputs(info beat.Info, log *logp.Logger, components statestore.States) []v2.Plugin {
 	var plugins []v2.Plugin
 
 	zeroPlugin := v2.Plugin{}

filebeat/input/default-inputs/inputs_windows.go

@@ -19,17 +19,13 @@ package inputs
 
 import (
 	v2 "github.com/elastic/beats/v7/filebeat/input/v2"
-	cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor"
 	"github.com/elastic/beats/v7/filebeat/input/winlog"
 	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/statestore"
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
-type osComponents interface {
-	cursor.StateStore
-}
-
-func osInputs(info beat.Info, log *logp.Logger, components osComponents) []v2.Plugin {
+func osInputs(info beat.Info, log *logp.Logger, components statestore.States) []v2.Plugin {
 	return []v2.Plugin{
 		winlog.Plugin(log, components),
 	}

filebeat/input/filestream/environment_test.go

@@ -48,7 +48,7 @@ import (
 type inputTestingEnvironment struct {
 	t          *testing.T
 	workingDir string
-	stateStore loginp.StateStore
+	stateStore statestore.States
 	pipeline   *mockPipelineConnector
 
 	pluginInitOnce sync.Once
@@ -194,7 +194,7 @@ func (e *inputTestingEnvironment) abspath(filename string) string {
 }
 
 func (e *inputTestingEnvironment) requireRegistryEntryCount(expectedCount int) {
-	inputStore, _ := e.stateStore.Access("")
+	inputStore, _ := e.stateStore.StoreFor("")
 
 	actual := 0
 	err := inputStore.Each(func(_ string, _ statestore.ValueDecoder) (bool, error) {
@@ -331,7 +331,7 @@ func (e *inputTestingEnvironment) requireNoEntryInRegistry(filename, inputID str
 		e.t.Fatalf("cannot stat file when cheking for offset: %+v", err)
 	}
 
-	inputStore, _ := e.stateStore.Access("")
+	inputStore, _ := e.stateStore.StoreFor("")
 	id := getIDFromPath(filepath, inputID, fi)
 
 	var entry registryEntry
@@ -352,7 +352,7 @@ func (e *inputTestingEnvironment) requireOffsetInRegistryByID(key string, expect
 }
 
 func (e *inputTestingEnvironment) getRegistryState(key string) (registryEntry, error) {
-	inputStore, _ := e.stateStore.Access("")
+	inputStore, _ := e.stateStore.StoreFor("")
 
 	var entry registryEntry
 	err := inputStore.Get(key, &entry)
@@ -539,11 +539,13 @@ func (e *inputTestingEnvironment) requireEventTimestamp(nr int, ts string) {
 	require.True(e.t, selectedEvent.Timestamp.Equal(tm), "got: %s, expected: %s", selectedEvent.Timestamp.String(), tm.String())
 }
 
+var _ statestore.States = (*testInputStore)(nil)
+
 type testInputStore struct {
 	registry *statestore.Registry
 }
 
-func openTestStatestore() loginp.StateStore {
+func openTestStatestore() statestore.States {
 	return &testInputStore{
 		registry: statestore.NewRegistry(storetest.NewMemoryStoreBackend()),
 	}
@@ -553,7 +555,7 @@ func (s *testInputStore) Close() {
 	s.registry.Close()
 }
 
-func (s *testInputStore) Access(_ string) (*statestore.Store, error) {
+func (s *testInputStore) StoreFor(string) (*statestore.Store, error) {
 	return s.registry.Get("filebeat")
 }
 

filebeat/input/filestream/input.go

@@ -40,6 +40,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/reader/parser"
 	"github.com/elastic/beats/v7/libbeat/reader/readfile"
 	"github.com/elastic/beats/v7/libbeat/reader/readfile/encoding"
+	"github.com/elastic/beats/v7/libbeat/statestore"
 	conf "github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
@@ -67,7 +68,7 @@ type filestream struct {
 }
 
 // Plugin creates a new filestream input plugin for creating a stateful input.
-func Plugin(log *logp.Logger, store loginp.StateStore) input.Plugin {
+func Plugin(log *logp.Logger, store statestore.States) input.Plugin {
 	return input.Plugin{
 		Name:       pluginName,
 		Stability:  feature.Stable,

filebeat/input/filestream/input_test.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile"
 	v2 "github.com/elastic/beats/v7/filebeat/input/v2"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/statestore"
@@ -235,10 +234,12 @@ func generateFile(t testing.TB, dir string, lineCount int) string {
 	return filename
 }
 
-func createTestStore(t testing.TB) loginp.StateStore {
+func createTestStore(t testing.TB) statestore.States {
 	return &testStore{registry: statestore.NewRegistry(storetest.NewMemoryStoreBackend())}
 }
 
+var _ statestore.States = (*testStore)(nil)
+
 type testStore struct {
 	registry *statestore.Registry
 }
@@ -247,7 +248,7 @@ func (s *testStore) Close() {
 	s.registry.Close()
 }
 
-func (s *testStore) Access(_ string) (*statestore.Store, error) {
+func (s *testStore) StoreFor(string) (*statestore.Store, error) {
 	return s.registry.Get("filestream-benchmark")
 }
 

filebeat/input/filestream/internal/input-logfile/manager.go

@@ -51,7 +51,7 @@ type InputManager struct {
 	Logger *logp.Logger
 
 	// StateStore gives the InputManager access to the persistent key value store.
-	StateStore StateStore
+	StateStore statestore.States
 
 	// Type must contain the name of the input type. It is used to create the key name
 	// for all sources the inputs collect from.
@@ -87,12 +87,6 @@ var errNoInputRunner = errors.New("no input runner available")
 // Deprecated: Inputs without an ID are not supported anymore.
 const globalInputID = ".global"
 
-// StateStore interface and configurations used to give the Manager access to the persistent store.
-type StateStore interface {
-	Access(typ string) (*statestore.Store, error)
-	CleanupInterval() time.Duration
-}
-
 func (cim *InputManager) init() error {
 	cim.initOnce.Do(func() {
 

filebeat/input/filestream/internal/input-logfile/store.go

@@ -141,10 +141,10 @@ type (
 // hook into store close for testing purposes
 var closeStore = (*store).close
 
-func openStore(log *logp.Logger, statestore StateStore, prefix string) (*store, error) {
+func openStore(log *logp.Logger, statestore statestore.States, prefix string) (*store, error) {
 	ok := false
 
-	persistentStore, err := statestore.Access("")
+	persistentStore, err := statestore.StoreFor("")
 	if err != nil {
 		return nil, err
 	}