Add NFR tests logging

Author: tataruty

tests/framework/crossplane.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	. "github.com/onsi/ginkgo/v2"
 	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -42,10 +43,13 @@ const crossplaneImageName = "nginx-crossplane:latest"
 
 // ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field,
 // and returns whether or not that field exists where it should.
-func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error {
+func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField, opts ...Option) error {
 	b, err := json.Marshal(conf)
 	if err != nil {
-		return fmt.Errorf("error marshaling nginx config: %w", err)
+		marshalErr := fmt.Errorf("error marshaling nginx config: %w", err)
+		GinkgoWriter.Printf("%v\n", marshalErr)
+
+		return marshalErr
 	}
 
 	for _, config := range conf.Config {
@@ -55,7 +59,7 @@ func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) err
 
 		for _, directive := range config.Parsed {
 			if expFieldCfg.Server == "" && expFieldCfg.Upstream == "" {
-				if expFieldCfg.fieldFound(directive) {
+				if expFieldCfg.fieldFound(directive, opts...) {
 					return nil
 				}
 				continue
@@ -65,13 +69,15 @@ func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) err
 				return nil
 			}
 
-			if expFieldCfg.Upstream != "" && fieldExistsInUpstream(expFieldCfg, *directive) {
+			if expFieldCfg.Upstream != "" && fieldExistsInUpstream(expFieldCfg, *directive, opts...) {
 				return nil
 			}
 		}
 	}
+	directiveErr := fmt.Errorf("directive %s not found in: nginx config %s", expFieldCfg.Directive, string(b))
+	GinkgoWriter.Printf("ERROR: %v\n", directiveErr)
 
-	return fmt.Errorf("directive %s not found in: nginx config %s", expFieldCfg.Directive, string(b))
+	return directiveErr
 }
 
 func fieldExistsInServer(
@@ -94,7 +100,16 @@ func fieldExistsInServer(
 func fieldExistsInUpstream(
 	expFieldCfg ExpectedNginxField,
 	directive Directive,
+	opts ...Option,
 ) bool {
+	options := LogOptions(opts...)
+	if options.logEnabled {
+		GinkgoWriter.Printf(
+			"Checking upstream for directive %q with value %q\n",
+			expFieldCfg.Directive,
+			expFieldCfg.Value,
+		)
+	}
 	if directive.Directive == "upstream" && directive.Args[0] == expFieldCfg.Upstream {
 		for _, directive := range directive.Block {
 			if expFieldCfg.fieldFound(directive) {
@@ -115,15 +130,29 @@ func getServerName(serverBlock Directives) string {
 	return ""
 }
 
-func (e ExpectedNginxField) fieldFound(directive *Directive) bool {
+func (e ExpectedNginxField) fieldFound(directive *Directive, opts ...Option) bool {
+	options := LogOptions(opts...)
 	arg := strings.Join(directive.Args, " ")
 
 	valueMatch := arg == e.Value
 	if e.ValueSubstringAllowed {
 		valueMatch = strings.Contains(arg, e.Value)
 	}
 
-	return directive.Directive == e.Directive && valueMatch
+	if directive.Directive == e.Directive && valueMatch {
+		if options.logEnabled {
+			GinkgoWriter.Printf(
+				"Found field %q with value %q in field %q with value %q\n",
+				e.Directive,
+				e.Value,
+				directive.Directive,
+				arg,
+			)
+		}
+		return true
+	}
+
+	return false
 }
 
 func fieldExistsInLocation(locationDirective *Directive, expFieldCfg ExpectedNginxField) bool {
@@ -201,7 +230,10 @@ func injectCrossplaneContainer(
 
 	podClient := k8sClient.CoreV1().Pods(namespace)
 	if _, err := podClient.UpdateEphemeralContainers(ctx, ngfPodName, pod, metav1.UpdateOptions{}); err != nil {
-		return fmt.Errorf("error adding ephemeral container: %w", err)
+		containerErr := fmt.Errorf("error adding ephemeral container: %w", err)
+		GinkgoWriter.Printf("%v\n", containerErr)
+
+		return containerErr
 	}
 
 	return nil
@@ -231,7 +263,10 @@ func createCrossplaneExecutor(
 
 	exec, err := remotecommand.NewSPDYExecutor(k8sConfig, http.MethodPost, req.URL())
 	if err != nil {
-		return nil, fmt.Errorf("error creating executor: %w", err)
+		executorErr := fmt.Errorf("error creating executor: %w", err)
+		GinkgoWriter.Printf("%v\n", executorErr)
+
+		return nil, executorErr
 	}
 
 	return exec, nil

tests/framework/generate_manifests.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"text/template"
 
-	. "github.com/onsi/ginkgo/v2"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -222,7 +221,6 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err
 }
 
 func generateSecrets(secrets []string) ([]client.Object, error) {
-	GinkgoWriter.Printf("Generating secrets\n")
 	objects := make([]client.Object, 0, len(secrets))
 
 	for _, secret := range secrets {
@@ -239,7 +237,6 @@ func generateSecrets(secrets []string) ([]client.Object, error) {
 
 		objects = append(objects, objs...)
 	}
-	GinkgoWriter.Printf("Generated %d secrets\n", len(objects))
 
 	return objects, nil
 }

tests/framework/info.go

@@ -84,11 +84,11 @@ func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) {
 }
 
 // AddNginxLogsAndEventsToReport adds nginx logs and events from the namespace to the report if the spec failed.
-func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string) {
+func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string, opts ...Option) {
 	if CurrentSpecReport().Failed() {
 		var returnLogs string
 
-		nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout)
+		nginxPodNames, _ := rm.GetReadyNginxPodNames(namespace, rm.TimeoutConfig.GetStatusTimeout, opts...)
 
 		for _, nginxPodName := range nginxPodNames {
 			returnLogs += fmt.Sprintf("Logs for Nginx Pod %s:\n", nginxPodName)

tests/framework/load.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"time"
 
+	. "github.com/onsi/ginkgo/v2"
 	vegeta "github.com/tsenart/vegeta/v12/lib"
 )
 
@@ -49,6 +50,7 @@ type Metrics struct {
 // RunLoadTest uses Vegeta to send traffic to the provided Targets at the given rate for the given duration and writes
 // the results to the provided file.
 func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) {
+	GinkgoWriter.Printf("Running load test: %s\n", cfg.Description)
 	vegTargets := convertTargetToVegetaTarget(cfg.Targets)
 	targeter := vegeta.NewStaticTargeter(vegTargets...)
 
@@ -61,7 +63,12 @@ func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) {
 		Timeout: vegeta.DefaultTimeout,
 		Transport: &http.Transport{
 			DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
-				return dialer.DialContext(ctx, network, cfg.Proxy)
+				conn, err := dialer.DialContext(ctx, network, cfg.Proxy)
+				if err != nil {
+					GinkgoWriter.Printf("ERROR occurred during dialing %q in %q network, error: %s\n", cfg.Proxy, network, err)
+				}
+
+				return conn, err
 			},
 			TLSClientConfig: &tls.Config{
 				InsecureSkipVerify: true, //nolint:gosec // self-signed cert for testing

tests/framework/logging.go

@@ -11,3 +11,12 @@ func WithLoggingDisabled() Option {
 		opts.logEnabled = false
 	}
 }
+
+func LogOptions(opts ...Option) *Options {
+	options := &Options{logEnabled: true}
+	for _, opt := range opts {
+		opt(options)
+	}
+
+	return options
+}

tests/framework/ngf.go

@@ -14,7 +14,6 @@ import (
 	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -101,7 +100,7 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
 }
 
 // CreateLicenseSecret creates the NGINX Plus JWT secret.
-func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) error {
+func CreateLicenseSecret(rm ResourceManager, namespace, filename string) error {
 	GinkgoWriter.Printf("Creating NGINX Plus license secret in namespace %q from file %q\n", namespace, filename)
 
 	conf, err := os.ReadFile(filename)
@@ -121,11 +120,8 @@ func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) er
 		},
 	}
 
-	if err := k8sClient.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) {
-		createNSErr := fmt.Errorf("error creating namespace: %w", err)
-		GinkgoWriter.Printf("%v\n", createNSErr)
-
-		return createNSErr
+	if err := rm.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) {
+		return fmt.Errorf("error creating namespace: %w", err)
 	}
 
 	secret := &core.Secret{
@@ -138,7 +134,7 @@ func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) er
 		},
 	}
 
-	if err := k8sClient.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) {
+	if err := rm.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) {
 		createSecretErr := fmt.Errorf("error creating secret: %w", err)
 		GinkgoWriter.Printf("%v\n", createSecretErr)
 
@@ -185,7 +181,7 @@ func UpgradeNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
 }
 
 // UninstallNGF uninstalls NGF.
-func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, error) {
+func UninstallNGF(cfg InstallationConfig, rm ResourceManager) ([]byte, error) {
 	args := []string{
 		"uninstall", cfg.ReleaseName, "--namespace", cfg.Namespace,
 	}
@@ -199,20 +195,20 @@ func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, erro
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
-	err = k8sClient.Delete(ctx, &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cfg.Namespace}})
+	err = rm.Delete(ctx, &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cfg.Namespace}}, nil)
 	if err != nil && !apierrors.IsNotFound(err) {
 		return nil, err
 	}
 
 	var crList apiext.CustomResourceDefinitionList
-	if err := k8sClient.List(ctx, &crList); err != nil {
+	if err := rm.List(ctx, &crList); err != nil {
 		return nil, err
 	}
 
 	for _, cr := range crList.Items {
 		if strings.Contains(cr.Spec.Group, "gateway.nginx.org") {
 			cr := cr
-			if err := k8sClient.Delete(ctx, &cr); err != nil && !apierrors.IsNotFound(err) {
+			if err := rm.Delete(ctx, &cr, nil); err != nil && !apierrors.IsNotFound(err) {
 				return nil, err
 			}
 		}