server enable

Author: 李红

mgr/runner.go

@@ -274,6 +274,12 @@ func NewLogExportRunner(rc RunnerConfig, cleanChan chan<- cleaner.CleanSignal, r
 	if err != nil {
 		return nil, err
 	}
+	var serverConfigs = make([]map[string]interface{}, 0, len(transformers))
+	for _, transform := range transformers {
+		if serverTransformer, ok := transform.(transforms.ServerTansformer); ok {
+			serverConfigs = append(serverConfigs, serverTransformer.ServerConfig())
+		}
+	}
 	senders := make([]sender.Sender, 0)
 	for i, senderConfig := range rc.SendersConfig {
 		if senderConfig[sender.KeySenderType] == sender.TypePandora {
@@ -289,6 +295,11 @@ func NewLogExportRunner(rc RunnerConfig, cleanChan chan<- cleaner.CleanSignal, r
 		if err != nil {
 			return nil, err
 		}
+		if senderConfig[sender.KeySenderType] == sender.TypePandora {
+			if serverSender, ok := s.(sender.ServerSender); ok {
+				serverSender.SetServer(serverConfigs)
+			}
+		}
 		senders = append(senders, s)
 		delete(rc.SendersConfig[i], sender.InnerUserAgent)
 	}
@@ -305,7 +316,7 @@ func createTransformers(rc RunnerConfig) ([]transforms.Transformer, error) {
 	transformers := make([]transforms.Transformer, 0)
 	for idx := range rc.Transforms {
 		tConf := rc.Transforms[idx]
-		tp := tConf[transforms.KeyType]
+		tp := tConf[KeyType]
 		if tp == nil {
 			return nil, fmt.Errorf("transformer config type is empty %v", tConf)
 		}

sender/pandora/pandora.go

@@ -31,6 +31,8 @@ import (
 
 var osInfo = []string{KeyCore, KeyHostName, KeyOsInfo, KeyLocalIp}
 
+var _ sender.ServerSender = &Sender{}
+
 const (
 	SendTypeRaw    = "raw"
 	SendTypeNormal = "normal"
@@ -48,6 +50,7 @@ type Sender struct {
 	microsecondCounter uint64
 	extraInfo          map[string]string
 	sendType           string
+	EnbleServerIp      bool
 }
 
 // UserSchema was parsed pandora schema from user's raw schema
@@ -1126,3 +1129,14 @@ func (s *Sender) Name() string {
 func (s *Sender) Close() error {
 	return s.client.Close()
 }
+
+func (s *Sender) SetServer(serverConfigs []map[string]interface{}) error {
+	for _, serverConfig := range serverConfigs {
+		if transformType, ok := serverConfig[KeyType].(string); ok && transformType == KeyIP {
+			if localEnable, ok := serverConfig[LocalEnable].(bool); ok && !localEnable {
+				s.opt.autoCreate += fmt.Sprintf(",%v ip", KeyIP)
+			}
+		}
+	}
+	return nil
+}

sender/sender.go

@@ -220,6 +220,10 @@ type StatsSender interface {
 	Restore(*StatsInfo)
 }
 
+type ServerSender interface {
+	SetServer([]map[string]interface{}) error
+}
+
 // SenderRegistry sender 的工厂类。可以注册自定义sender
 type Registry struct {
 	senderTypeMap map[string]func(conf.MapConf) (Sender, error)

transforms/ip/ip.go

@@ -26,12 +26,14 @@ var (
 	_ transforms.StatsTransformer = &Transformer{}
 	_ transforms.Transformer      = &Transformer{}
 	_ transforms.Initializer      = &Transformer{}
+	_ transforms.ServerTansformer = &Transformer{}
 )
 
 type Transformer struct {
 	StageTime   string `json:"stage"`
 	Key         string `json:"key"`
 	DataPath    string `json:"data_path"`
+	LocalEnable bool   `json:"local_enable"`
 	KeyAsPrefix bool   `json:"key_as_prefix"`
 	Language    string `json:"language"`
 
@@ -91,6 +93,9 @@ func (_ *Transformer) RawTransform(datas []string) ([]string, error) {
 }
 
 func (t *Transformer) Transform(datas []Data) ([]Data, error) {
+	if !t.LocalEnable {
+		return datas, nil
+	}
 	var err, fmtErr error
 	errNum := 0
 	if t.loc == nil {
@@ -219,16 +224,28 @@ func (_ *Transformer) SampleConfig() string {
 
 func (_ *Transformer) ConfigOptions() []Option {
 	return []Option{
+		{
+			KeyName:       LocalEnable,
+			Element:       Radio,
+			ChooseOnly:    true,
+			ChooseOptions: []interface{}{true, false},
+			Default:       false,
+			Required:      true,
+			DefaultNoUse:  false,
+			Description:   "是否使用本地解析，默认为服务端解析",
+			Type:          transforms.TransformTypeBoolean,
+		},
 		transforms.KeyFieldName,
 		{
-			KeyName:      "data_path",
-			ChooseOnly:   false,
-			Default:      "",
-			Required:     true,
-			Placeholder:  "your/path/to/ip.dat(x)",
-			DefaultNoUse: true,
-			Description:  "IP数据库路径(data_path)",
-			Type:         transforms.TransformTypeString,
+			KeyName:       "data_path",
+			ChooseOnly:    false,
+			Default:       "",
+			Required:      true,
+			Placeholder:   "your/path/to/ip.dat(x)",
+			DefaultNoUse:  true,
+			Description:   "IP数据库路径(data_path)",
+			Type:          transforms.TransformTypeString,
+			AdvanceDepend: LocalEnable,
 		},
 		{
 			KeyName:       "key_as_prefix",
@@ -240,17 +257,19 @@ func (_ *Transformer) ConfigOptions() []Option {
 			Element:       Checkbox,
 			Description:   "字段名称作为前缀(key_as_prefix)",
 			Type:          transforms.TransformTypeString,
+			AdvanceDepend: LocalEnable,
 		},
 		{
-			KeyName:      "language",
-			ChooseOnly:   false,
-			Default:      "zh-CN",
-			Required:     true,
-			Placeholder:  "zh-CN",
-			DefaultNoUse: true,
-			Description:  "mmdb格式库使用的语种",
-			Advance:      true,
-			Type:         transforms.TransformTypeString,
+			KeyName:       "language",
+			ChooseOnly:    false,
+			Default:       "zh-CN",
+			Required:      true,
+			Placeholder:   "zh-CN",
+			DefaultNoUse:  true,
+			Description:   "mmdb格式库使用的语种",
+			Advance:       true,
+			Type:          transforms.TransformTypeString,
+			AdvanceDepend: LocalEnable,
 		},
 	}
 }
@@ -275,6 +294,13 @@ func (t *Transformer) Close() error {
 	return nil
 }
 
+func (t *Transformer) ServerConfig() map[string]interface{} {
+	config := make(map[string]interface{})
+	config[KeyType] = Name
+	config[LocalEnable] = t.LocalEnable
+	return config
+}
+
 func init() {
 	transforms.Add(Name, func() transforms.Transformer {
 		return &Transformer{}

transforms/registry.go

@@ -4,10 +4,6 @@ import (
 	. "github.com/qiniu/logkit/utils/models"
 )
 
-const (
-	KeyType = "type"
-)
-
 const (
 	TransformTypeString  = "string"
 	TransformTypeLong    = "long"
@@ -34,6 +30,10 @@ type Transformer interface {
 	Stats() StatsInfo
 }
 
+type ServerTansformer interface {
+	ServerConfig() map[string]interface{}
+}
+
 // DataReader 代表了一个可直接读取内存数据结构的读取器
 type StatsTransformer interface {
 	// ReadData 用于读取一条数据以及数据的实际读取字节

utils/models/models.go

@@ -36,6 +36,8 @@ const (
 
 	KeyPandoraStash      = "pandora_stash"       // 当只有一条数据且 sendError 时候，将其转化为 raw 发送到 pandora_stash 这个字段
 	KeyPandoraSeparateId = "pandora_separate_id" // 当一条数据大于2M且 sendError 时候，将其切片，切片记录到 pandora_separate_id 这个字段
+	KeyIP                = "ip"                  // schema ip
+	KeyType              = "type"
 
 	SchemaFreeTokensPrefix = "schema_free_tokens_"
 	LogDBTokensPrefix      = "logdb_tokens_"
@@ -59,6 +61,8 @@ const (
 	Checkbox    = "checkbox"
 	Radio       = "radio"
 	InputNumber = "inputNumber"
+
+	LocalEnable = "local_enable"
 )
 
 var (