Avoid encoding html entities in JSON documents (#1464)

JSON encoders in Go standard library encode HTML special characters by
default, so the resulting JSON is safe to use in HTML documents.

In sample documents and test results in packages we want the raw JSON,
so we can see the actual content that would be ingested when using the
integration.

Author: jsoriano

internal/docs/readme_test.go

@@ -11,6 +11,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-package/internal/packages"
 )
 
 func TestGenerateReadme(t *testing.T) {
@@ -152,6 +154,9 @@ An example event for ` + "`example`" + ` looks as following:
 			err = createSampleEventFile(c.packageRoot, c.dataStreamName, c.sampleEventJsonContents)
 			require.NoError(t, err)
 
+			err = createManifestFile(c.packageRoot)
+			require.NoError(t, err)
+
 			rendered, err := renderReadme(filename, c.packageRoot, templatePath, linksMap)
 			require.NoError(t, err)
 
@@ -293,6 +298,13 @@ func createSampleEventFile(packageRoot, dataStreamName, contents string) error {
 	return nil
 }
 
+func createManifestFile(packageRoot string) error {
+	// Minimal content needed to render readme.
+	manifest := `format_version: 2.10.0`
+	manifestFile := filepath.Join(packageRoot, packages.PackageManifestFile)
+	return os.WriteFile(manifestFile, []byte(manifest), 0644)
+}
+
 func createDataStreamFolder(packageRoot, dataStreamName string) (string, error) {
 	if dataStreamName == "" {
 		return "", nil

internal/docs/sample_event.go

@@ -10,7 +10,10 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
+
 	"github.com/elastic/elastic-package/internal/formatter"
+	"github.com/elastic/elastic-package/internal/packages"
 )
 
 const sampleEventFile = "sample_event.json"
@@ -23,7 +26,17 @@ func renderSampleEvent(packageRoot, dataStreamName string) (string, error) {
 		return "", fmt.Errorf("reading sample event file failed (path: %s): %w", eventPath, err)
 	}
 
-	formatted, _, err := formatter.JSONFormatter(body)
+	manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
+	if err != nil {
+		return "", fmt.Errorf("reading package manifest failed: %w", err)
+	}
+	specVersion, err := semver.NewVersion(manifest.SpecVersion)
+	if err != nil {
+		return "", fmt.Errorf("parsing format version %q failed: %w", manifest.SpecVersion, err)
+	}
+
+	jsonFormatter := formatter.JSONFormatterBuilder(*specVersion)
+	formatted, _, err := jsonFormatter.Format(body)
 	if err != nil {
 		return "", fmt.Errorf("formatting sample event file failed (path: %s): %w", eventPath, err)
 	}

internal/formatter/formatter.go

@@ -8,19 +8,36 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+
+	"github.com/Masterminds/semver/v3"
+
+	"github.com/elastic/elastic-package/internal/packages"
 )
 
 type formatter func(content []byte) ([]byte, bool, error)
 
-var formatters = map[string]formatter{
-	".json": JSONFormatter,
-	".yaml": YAMLFormatter,
-	".yml":  YAMLFormatter,
+func newFormatter(specVersion semver.Version, ext string) formatter {
+	switch ext {
+	case ".json":
+		return JSONFormatterBuilder(specVersion).Format
+	case ".yaml", ".yml":
+		return YAMLFormatter
+	default:
+		return nil
+	}
 }
 
 // Format method formats files inside of the integration directory.
 func Format(packageRoot string, failFast bool) error {
-	err := filepath.Walk(packageRoot, func(path string, info os.FileInfo, err error) error {
+	manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
+	if err != nil {
+		return fmt.Errorf("failed to read package manifest: %w", err)
+	}
+	specVersion, err := semver.NewVersion(manifest.SpecVersion)
+	if err != nil {
+		return fmt.Errorf("failed to parse package format version %q: %w", manifest.SpecVersion, err)
+	}
+	err = filepath.Walk(packageRoot, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -31,7 +48,7 @@ func Format(packageRoot string, failFast bool) error {
 		if info.IsDir() {
 			return nil
 		}
-		err = formatFile(path, failFast)
+		err = formatFile(path, failFast, *specVersion)
 		if err != nil {
 			return fmt.Errorf("formatting file failed (path: %s): %w", path, err)
 		}
@@ -44,17 +61,15 @@ func Format(packageRoot string, failFast bool) error {
 	return nil
 }
 
-func formatFile(path string, failFast bool) error {
-	file := filepath.Base(path)
-	ext := filepath.Ext(file)
-
+func formatFile(path string, failFast bool, specVersion semver.Version) error {
 	content, err := os.ReadFile(path)
 	if err != nil {
 		return fmt.Errorf("reading file content failed: %w", err)
 	}
 
-	format, defined := formatters[ext]
-	if !defined {
+	ext := filepath.Ext(filepath.Base(path))
+	format := newFormatter(specVersion, ext)
+	if format == nil {
 		return nil // no errors returned as we have few files that will be never formatted (png, svg, log, etc.)
 	}
 

internal/formatter/json_formatter.go

@@ -5,13 +5,31 @@
 package formatter
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+
+	"github.com/Masterminds/semver/v3"
 )
 
-// JSONFormatter function is responsible for formatting the given JSON input.
-// The function is exposed, so it can be used by other internal packages, e.g. to format sample events in docs.
-func JSONFormatter(content []byte) ([]byte, bool, error) {
+type JSONFormatter interface {
+	Format([]byte) ([]byte, bool, error)
+	Encode(doc any) ([]byte, error)
+}
+
+func JSONFormatterBuilder(specVersion semver.Version) JSONFormatter {
+	if specVersion.LessThan(semver.MustParse("2.12.0")) {
+		return &jsonFormatterWithHTMLEncoding{}
+	}
+
+	return &jsonFormatter{}
+}
+
+// jsonFormatterWithHTMLEncoding function is responsible for formatting the given JSON input.
+// It encodes special HTML characters.
+type jsonFormatterWithHTMLEncoding struct{}
+
+func (jsonFormatterWithHTMLEncoding) Format(content []byte) ([]byte, bool, error) {
 	var rawMessage json.RawMessage
 	err := json.Unmarshal(content, &rawMessage)
 	if err != nil {
@@ -24,3 +42,35 @@ func JSONFormatter(content []byte) ([]byte, bool, error) {
 	}
 	return formatted, string(content) == string(formatted), nil
 }
+
+func (jsonFormatterWithHTMLEncoding) Encode(doc any) ([]byte, error) {
+	return json.MarshalIndent(doc, "", "    ")
+}
+
+// jsonFormatter function is responsible for formatting the given JSON input.
+type jsonFormatter struct{}
+
+func (jsonFormatter) Format(content []byte) ([]byte, bool, error) {
+	var formatted bytes.Buffer
+	err := json.Indent(&formatted, content, "", "    ")
+	if err != nil {
+		return nil, false, fmt.Errorf("formatting JSON document failed: %w", err)
+	}
+
+	return formatted.Bytes(), bytes.Equal(content, formatted.Bytes()), nil
+}
+
+func (jsonFormatter) Encode(doc any) ([]byte, error) {
+	var formatted bytes.Buffer
+	enc := json.NewEncoder(&formatted)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", "    ")
+
+	err := enc.Encode(doc)
+	if err != nil {
+		return nil, err
+	}
+
+	// Trimming to be consistent with MarshalIndent, that seems to trim the result.
+	return bytes.TrimSpace(formatted.Bytes()), nil
+}

internal/formatter/json_formatter_test.go

@@ -0,0 +1,106 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package formatter_test
+
+import (
+	"testing"
+
+	"github.com/Masterminds/semver/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-package/internal/formatter"
+)
+
+func TestJSONFormatterFormat(t *testing.T) {
+	cases := []struct {
+		title    string
+		version  *semver.Version
+		content  string
+		expected string
+		valid    bool
+	}{
+		{
+			title:   "invalid json 2.0",
+			version: semver.MustParse("2.0.0"),
+			content: `{"foo":}`,
+			valid:   false,
+		},
+		{
+			title:   "invalid json 3.0",
+			version: semver.MustParse("3.0.0"),
+			content: `{"foo":}`,
+			valid:   false,
+		},
+		{
+			title:   "encode html in old versions",
+			version: semver.MustParse("2.0.0"),
+			content: `{"a": "<script></script>"}`,
+			expected: `{
+    "a": "\u003cscript\u003e\u003c/script\u003e"
+}`,
+			valid: true,
+		},
+		{
+			title:   "don't encode html since 2.12.0",
+			version: semver.MustParse("2.12.0"),
+			content: `{"a": "<script></script>"}`,
+			expected: `{
+    "a": "<script></script>"
+}`,
+			valid: true,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			jsonFormatter := formatter.JSONFormatterBuilder(*c.version)
+			formatted, equal, err := jsonFormatter.Format([]byte(c.content))
+			if !c.valid {
+				assert.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			assert.Equal(t, c.expected, string(formatted))
+			assert.Equal(t, c.content == c.expected, equal)
+		})
+	}
+}
+
+func TestJSONFormatterEncode(t *testing.T) {
+	cases := []struct {
+		title    string
+		version  *semver.Version
+		object   any
+		expected string
+	}{
+		{
+			title:   "encode html in old versions",
+			version: semver.MustParse("2.0.0"),
+			object:  map[string]any{"a": "<script></script>"},
+			expected: `{
+    "a": "\u003cscript\u003e\u003c/script\u003e"
+}`,
+		},
+		{
+			title:   "don't encode html since 2.12.0",
+			version: semver.MustParse("2.12.0"),
+			object:  map[string]any{"a": "<script></script>"},
+			expected: `{
+    "a": "<script></script>"
+}`,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			jsonFormatter := formatter.JSONFormatterBuilder(*c.version)
+			formatted, err := jsonFormatter.Encode(c.object)
+			require.NoError(t, err)
+			assert.Equal(t, c.expected, string(formatted))
+		})
+	}
+}

internal/testrunner/runners/pipeline/runner.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Masterminds/semver/v3"
 	"gopkg.in/yaml.v3"
 
 	"github.com/elastic/elastic-package/internal/common"
@@ -283,10 +284,19 @@ func (r *runner) loadTestCaseFile(testCaseFile string) (*testCase, error) {
 func (r *runner) verifyResults(testCaseFile string, config *testConfig, result *testResult, fieldsValidator *fields.Validator) error {
 	testCasePath := filepath.Join(r.options.TestFolder.Path, testCaseFile)
 
+	manifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath)
+	if err != nil {
+		return fmt.Errorf("failed to read package manifest: %w", err)
+	}
+	specVersion, err := semver.NewVersion(manifest.SpecVersion)
+	if err != nil {
+		return fmt.Errorf("failed to parse package format version %q: %w", manifest.SpecVersion, err)
+	}
+
 	if r.options.GenerateTestResult {
 		// TODO: Add tests to cover regressive use of json.Unmarshal in writeTestResult.
 		// See https://github.com/elastic/elastic-package/pull/717.
-		err := writeTestResult(testCasePath, result)
+		err := writeTestResult(testCasePath, result, *specVersion)
 		if err != nil {
 			return fmt.Errorf("writing test result failed: %w", err)
 		}
@@ -301,7 +311,7 @@ func (r *runner) verifyResults(testCaseFile string, config *testConfig, result *
 	if stackConfig.Provider == stack.ProviderServerless {
 		skipGeoIP = true
 	}
-	err = compareResults(testCasePath, config, result, skipGeoIP)
+	err = compareResults(testCasePath, config, result, skipGeoIP, *specVersion)
 	if _, ok := err.(testrunner.ErrTestCaseFailed); ok {
 		return err
 	}