misc: pg enable metrics

env_gen.md

@@ -200,9 +200,10 @@
  | PARALLELISM_LIMIT_FOR_TAG_PROCESSING |  |  | 
  | PG_ADDR | 127.0.0.1 |  | 
  | PG_DATABASE | orchestrator |  | 
- | PG_EXPORT_PROM_METRICS | false |  | 
+ | PG_EXPORT_PROM_METRICS | true |  | 
+ | PG_LOG_ALL_FAILURE_QUERIES | true |  | 
  | PG_LOG_ALL_QUERY | false |  | 
- | PG_LOG_QUERY | true |  | 
+ | PG_LOG_SLOW_QUERY | true |  | 
  | PG_PASSWORD | {password} |  | 
  | PG_PORT | 5432 |  | 
  | PG_QUERY_DUR_THRESHOLD | 5000 |  | 

go.mod

@@ -271,7 +271,7 @@ require (
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/kube-aggregator v0.29.6 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
-	mellium.im/sasl v0.3.1 // indirect
+	mellium.im/sasl v0.3.2 // indirect
 	oras.land/oras-go/v2 v2.3.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
@@ -288,8 +288,8 @@ require gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 
 replace (
 	github.com/argoproj/argo-workflows/v3 v3.5.10 => github.com/devtron-labs/argo-workflows/v3 v3.5.13
-	github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250111030602-92e4c3e15f98
-	github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250111030602-92e4c3e15f98
+	github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250112200957-a668cc3673da
+	github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250112200957-a668cc3673da
 	github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127
 	github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5
 	k8s.io/api => k8s.io/api v0.29.7

go.sum

@@ -792,10 +792,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq
 github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU=
 github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA=
-github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250111030602-92e4c3e15f98 h1:JsIuIbPBT0FmO+RxuNNyXEaxRpBDfWf0hjHvjuX57Q8=
-github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250111030602-92e4c3e15f98/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250111030602-92e4c3e15f98 h1:TP5AAayaxEyvbr1Yf9rHBQJpaBz0Tl/LStgnLJId+TQ=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250111030602-92e4c3e15f98/go.mod h1:GXADr0W94UqtDJe6/3possrPj6fD/3IUNISfYfevWqg=
+github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250112200957-a668cc3673da h1:xGvqLLRlPfoLq+mvIoN0tOLLkWYrsEh2vhCORb3EDfY=
+github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250112200957-a668cc3673da/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250112200957-a668cc3673da h1:hTYetUW3nH8bhiZu+s7GUg+Neh9P5CWWlZGigwCAzHA=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250112200957-a668cc3673da/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
 github.com/devtron-labs/protos v0.0.3-0.20240802105333-92ee9bb85d80 h1:xwbTeijNTf4/j1v+tSfwVqwLVnReas/NqEKeQHvSTys=
@@ -2493,8 +2493,8 @@ launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80
 launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA=
 lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
-mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo=
-mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
+mellium.im/sasl v0.3.2 h1:PT6Xp7ccn9XaXAnJ03FcEjmAn7kK1x7aoXV6F+Vmrl0=
+mellium.im/sasl v0.3.2/go.mod h1:NKXDi1zkr+BlMHLQjY3ofYuU4KSPFxknb8mfEu6SveY=
 modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
 modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
 modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=

pkg/sql/connection.go

@@ -17,7 +17,8 @@
 package sql
 
 import (
-	"github.com/devtron-labs/devtron/internal/middleware"
+	"github.com/devtron-labs/common-lib/utils"
+	"github.com/devtron-labs/common-lib/utils/bean"
 	"go.uber.org/zap"
 	"reflect"
 	"time"
@@ -27,24 +28,29 @@ import (
 )
 
 type Config struct {
-	Addr                   string `env:"PG_ADDR" envDefault:"127.0.0.1"`
-	Port                   string `env:"PG_PORT" envDefault:"5432"`
-	User                   string `env:"PG_USER" envDefault:""`
-	Password               string `env:"PG_PASSWORD" envDefault:"" secretData:"-"`
-	Database               string `env:"PG_DATABASE" envDefault:"orchestrator"`
-	CasbinDatabase         string `env:"CASBIN_DATABASE" envDefault:"casbin"`
+	Addr            string `env:"PG_ADDR" envDefault:"127.0.0.1"`
+	Port            string `env:"PG_PORT" envDefault:"5432"`
+	User            string `env:"PG_USER" envDefault:""`
+	Password        string `env:"PG_PASSWORD" envDefault:"" secretData:"-"`
+	Database        string `env:"PG_DATABASE" envDefault:"orchestrator"`
+	CasbinDatabase  string `env:"CASBIN_DATABASE" envDefault:"casbin"`
 	ApplicationName string `env:"APP" envDefault:"orchestrator" envDescription:"Application name"`
-	LogQuery               bool   `env:"PG_LOG_QUERY" envDefault:"true"`
-	LogAllQuery            bool   `env:"PG_LOG_ALL_QUERY" envDefault:"false"`
-	ExportPromMetrics      bool   `env:"PG_EXPORT_PROM_METRICS" envDefault:"false"`
-	QueryDurationThreshold int64  `env:"PG_QUERY_DUR_THRESHOLD" envDefault:"5000"`
-	ReadTimeout            int64  `env:"PG_READ_TIMEOUT" envDefault:"30"`
-	WriteTimeout           int64  `env:"PG_WRITE_TIMEOUT" envDefault:"30"`
+	ReadTimeout     int64  `env:"PG_READ_TIMEOUT" envDefault:"30"`
+	WriteTimeout    int64  `env:"PG_WRITE_TIMEOUT" envDefault:"30"`
+	bean.PgQueryMonitoringConfig
 }
 
 func GetConfig() (*Config, error) {
 	cfg := &Config{}
 	err := env.Parse(cfg)
+	if err != nil {
+		return cfg, err
+	}
+	monitoringCfg, err := bean.GetPgQueryMonitoringConfig(cfg.ApplicationName)
+	if err != nil {
+		return cfg, err
+	}
+	cfg.PgQueryMonitoringConfig = monitoringCfg
 	return cfg, err
 }
 
@@ -71,28 +77,7 @@ func NewDbConnection(cfg *Config, logger *zap.SugaredLogger) (*pg.DB, error) {
 	}
 
 	// --------------
-	dbConnection.OnQueryProcessed(func(event *pg.QueryProcessedEvent) {
-		queryDuration := time.Since(event.StartTime)
-
-		// Expose prom metrics
-		if cfg.ExportPromMetrics {
-			middleware.PgQueryDuration.WithLabelValues("value").Observe(queryDuration.Seconds())
-		}
-
-		query, err := event.FormattedQuery()
-		if err != nil {
-			logger.Errorw("Error formatting query",
-				"err", err)
-			return
-		}
-
-		// Log pg query if enabled
-		if cfg.LogAllQuery || (cfg.LogQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold) {
-			logger.Debugw("query time",
-				"duration", queryDuration.Seconds(),
-				"query", query)
-		}
-	})
+	dbConnection.OnQueryProcessed(utils.GetPGPostQueryProcessor(cfg.PgQueryMonitoringConfig))
 	return dbConnection, err
 }
 

vendor/mellium.im/sasl/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 All notable changes to this project will be documented in this file.
 
+## v0.3.2 — 2024-09-11
+
+### Added
+
+- Support for SASL ANONYMOUS
+- Support for the server side of SCRAM
+
+### Fixed
+
+- Support for fast XOR removed from the repo and now uses the upstream version
+  shipped with the Go tool chain (which supports more architectures)
+
 
 ##  v0.3.1 — 2022-12-28
 

vendor/mellium.im/sasl/anonymous.go

@@ -0,0 +1,19 @@
+// Copyright 2024 The Mellium Contributors.
+// Use of this source code is governed by the BSD 2-clause
+// license that can be found in the LICENSE file.
+
+package sasl
+
+var anonymous = Mechanism{
+	Name: "ANONYMOUS",
+	Start: func(*Negotiator) (bool, []byte, interface{}, error) {
+		// Per XEP-0175 we do not send any trace data.
+		return false, nil, nil, nil
+	},
+	Next: func(m *Negotiator, _ []byte, _ interface{}) (_ bool, _ []byte, _ interface{}, err error) {
+		if m.State()&Receiving != Receiving || m.State()&AuthTextSent != AuthTextSent {
+			err = ErrTooManySteps
+		}
+		return
+	},
+}

vendor/mellium.im/sasl/mechanism.go

@@ -43,6 +43,10 @@ var (
 	// ScramSha1 is a Mechanism that implements the SCRAM-SHA-1 authentication
 	// mechanism defined in RFC 5802.
 	ScramSha1 Mechanism = scram("SCRAM-SHA-1", sha1.New)
+
+	// Anonymous is a Mechanism that implements the ANONYMOUS
+	// authentication mechanism as defined by RFC 4505.
+	Anonymous Mechanism = anonymous
 )
 
 // Mechanism represents a SASL mechanism that can be used by a Client or Server

vendor/mellium.im/sasl/negotiator.go

@@ -7,6 +7,7 @@ package sasl
 import (
 	"crypto/rand"
 	"crypto/tls"
+	"fmt"
 	"strings"
 )
 
@@ -85,18 +86,28 @@ func NewServer(m Mechanism, permissions func(*Negotiator) bool, opts ...Option)
 	return machine
 }
 
+// SaltedCredentialsFetcher defines function that fetches user information using
+// Username and optional Identity from storage.
+//
+// Function should return saltedPassword as well as salt and iterations used
+// to generate saltedPassword for specified SCRAM MechanismName. If there is no
+// such Username or Username is not authorized to take Identity function should
+// return ErrAuthn as an error.
+type SaltedCredentialsFetcher func(Username, Identity []byte, MechanismName string) (salt []byte, saltedPassword []byte, iterations int64, err error)
+
 // A Negotiator represents a SASL client or server state machine that can
 // attempt to negotiate auth. Negotiators should not be used from multiple
 // goroutines, and must be reset between negotiation attempts.
 type Negotiator struct {
-	tlsState         *tls.ConnectionState
-	remoteMechanisms []string
-	credentials      func() (Username, Password, Identity []byte)
-	permissions      func(*Negotiator) bool
-	mechanism        Mechanism
-	state            State
-	nonce            []byte
-	cache            interface{}
+	tlsState          *tls.ConnectionState
+	remoteMechanisms  []string
+	credentials       func() (Username, Password, Identity []byte) // client only
+	saltedCredentials SaltedCredentialsFetcher                     // server only
+	permissions       func(*Negotiator) bool
+	mechanism         Mechanism
+	state             State
+	nonce             []byte
+	cache             interface{}
 }
 
 // Nonce returns a unique nonce that is reset for each negotiation attempt. It
@@ -159,14 +170,25 @@ func (c *Negotiator) Reset() {
 }
 
 // Credentials returns a username, and password for authentication and optional
-// identity for authorization.
+// identity for authorization. Used in client negotiator.
 func (c *Negotiator) Credentials() (username, password, identity []byte) {
 	if c.credentials != nil {
 		return c.credentials()
 	}
 	return
 }
 
+// SaltedCredentials returns a salt, saltedPassword and iteration count for a
+// given username and optional identity for client authorization. Refer to
+// SaltedCredentialsFetcher documentation for details.
+// Used in server negotiator.
+func (c *Negotiator) SaltedCredentials(username, identity []byte) (salt []byte, saltedPassword []byte, iterations int64, err error) {
+	if c.saltedCredentials != nil {
+		return c.saltedCredentials(username, identity, c.mechanism.Name)
+	}
+	return nil, nil, 0, fmt.Errorf("sasl: salted credentials not provided")
+}
+
 // Permissions is the callback used by the server to authenticate the user.
 func (c *Negotiator) Permissions(opts ...Option) bool {
 	if c.permissions != nil {

vendor/mellium.im/sasl/options.go

@@ -59,3 +59,11 @@ func Credentials(f func() (Username, Password, Identity []byte)) Option {
 		n.credentials = f
 	}
 }
+
+// SaltedCredentials provides the negotiator with a SaltedCredentialsFetcher
+// that used to fetch user information from storage.
+func SaltedCredentials(f SaltedCredentialsFetcher) Option {
+	return func(n *Negotiator) {
+		n.saltedCredentials = f
+	}
+}