Skip to content

Commit 89bbf3d

Browse files
committed
added a YAML Merger PromotionStep
Signed-off-by: Prune <prune@lecentre.net>
1 parent 09a755b commit 89bbf3d

File tree

11 files changed

+680
-4
lines changed

11 files changed

+680
-4
lines changed
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
---
2+
sidebar_label: yaml-merge
3+
description: Merge multiple YAML file into a single file.
4+
---
5+
6+
# `yaml-merge`
7+
8+
`yaml-merge` merges multiple YAML files into a single file. This step
9+
most often used to merge multiple Helm values.yaml files into a single
10+
file and is commonly followed by a [`helm-template` step](helm-template.md).
11+
YAML files are merged in order, so the first one is the base, and all
12+
subsequent files are "overlays", modifying the default values.
13+
14+
## Configuration
15+
16+
| Name | Type | Required | Description |
17+
|------|------|----------|-------------|
18+
| `inPaths` | `[]string` | Y | Paths to a YAML files. This path is relative to the temporary workspace that Kargo provisions for use by the promotion process. |
19+
| `outPath` | `string` | Y | The path to the output file. This path is relative to the temporary workspace that Kargo provisions for use by the promotion process. |
20+
| `strict` | `bool` | N | Strict will cause the directive to fail if an input path does not exist. Defaults to `false`. |
21+
22+
## Output
23+
24+
| Name | Type | Description |
25+
|------|------|-------------|
26+
| `commitMessage` | `string` | A description of the change(s) applied by this step. Typically, a subsequent [`git-commit` step](git-commit.md) will reference this output and aggregate this commit message fragment with other like it to build a comprehensive commit message that describes all changes. |
27+
28+
## Examples
29+
30+
### Common Usage
31+
32+
In this example, two Helm values, one global, one more specific, are merged
33+
into a new single file, that is then commited.
34+
35+
This pattern is commonly used when you need to merge global values
36+
into a final `values.yaml` file, to be used by Helm.
37+
38+
```yaml
39+
vars:
40+
- name: gitRepo
41+
value: https://github.com/example/repo.git
42+
steps:
43+
- uses: git-clone
44+
config:
45+
repoURL: ${{ vars.gitRepo }}
46+
checkout:
47+
- commit: ${{ commitFrom(vars.gitRepo).ID }}
48+
path: ./src
49+
- branch: stage/${{ ctx.stage }}
50+
create: true
51+
path: ./out
52+
- uses: git-clear
53+
config:
54+
path: ./out
55+
- uses: yaml-merge
56+
config:
57+
inPaths:
58+
- ./src/charts/my-chart/values.yaml
59+
- ./src/charts/qa/values.yaml
60+
- ./src/charts/qa/cluster-a/values.yaml
61+
outPath: ./out/charts/my-chart/values.yaml
62+
# Render manifests to ./out, commit, push, etc...
63+
```

internal/promotion/runner/builtin/init.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ func Initialize(kargoClient, argocdClient client.Client, credsDB credentials.Dat
4545
newOutputComposer(),
4646
),
4747
newTarExtractor(),
48+
newYAMLMerger(),
4849
newYAMLParser(),
4950
newYAMLUpdater(),
5051
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"title": "YAMLMergeConfig",
4+
5+
"type": "object",
6+
"required": ["inPaths", "outPath"],
7+
"additionalProperties": false,
8+
"properties": {
9+
"inPaths": {
10+
"type": "array",
11+
"description": "InPaths is the list of paths to YAML files to process",
12+
"minItems": 1,
13+
"items": {
14+
"type": "string"
15+
}
16+
},
17+
"outPath": {
18+
"type": "string",
19+
"description": "OutPath is the path to the merged YAML file to create or update.",
20+
"minLength": 1
21+
},
22+
"strict": {
23+
"type": "boolean",
24+
"description": "Strict will cause the directive to fail if the input path does not exist.",
25+
"default": false
26+
}
27+
}
28+
}

internal/promotion/runner/builtin/schemas/yaml-update-config.json

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
{
22
"$schema": "https://json-schema.org/draft/2020-12/schema",
33
"title": "YAMLUpdateConfig",
4-
5-
"definitions": {
64

5+
"definitions": {
76
"yamlUpdate": {
87
"type": "object",
98
"additionalProperties": false,
@@ -19,9 +18,8 @@
1918
},
2019
"required": ["key", "value"]
2120
}
22-
2321
},
24-
22+
2523
"type": "object",
2624
"required": ["path", "updates"],
2725
"additionalProperties": false,
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package builtin
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"path/filepath"
8+
"strings"
9+
10+
securejoin "github.com/cyphar/filepath-securejoin"
11+
"github.com/xeipuuv/gojsonschema"
12+
13+
kargoapi "github.com/akuity/kargo/api/v1alpha1"
14+
"github.com/akuity/kargo/internal/yaml"
15+
"github.com/akuity/kargo/pkg/promotion"
16+
"github.com/akuity/kargo/pkg/x/promotion/runner/builtin"
17+
)
18+
19+
// yamlMerger is an implementation of the promotion.StepRunner interface that
20+
// updates do a merge of the multiple YAML files.
21+
type yamlMerger struct {
22+
schemaLoader gojsonschema.JSONLoader
23+
}
24+
25+
// newYAMLMerger returns an implementation of the promotion.StepRunner interface
26+
// that updates the values of specified keys in a YAML file.
27+
func newYAMLMerger() promotion.StepRunner {
28+
r := &yamlMerger{}
29+
r.schemaLoader = getConfigSchemaLoader(r.Name())
30+
return r
31+
}
32+
33+
// Name implements the promotion.StepRunner interface.
34+
func (y *yamlMerger) Name() string {
35+
return "yaml-merge"
36+
}
37+
38+
// Run implements the promotion.StepRunner interface.
39+
func (y *yamlMerger) Run(
40+
ctx context.Context,
41+
stepCtx *promotion.StepContext,
42+
) (promotion.StepResult, error) {
43+
failure := promotion.StepResult{Status: kargoapi.PromotionStepStatusErrored}
44+
45+
if err := y.validate(stepCtx.Config); err != nil {
46+
return failure, err
47+
}
48+
49+
// Convert the configuration into a typed struct
50+
cfg, err := promotion.ConfigToStruct[builtin.YAMLMergeConfig](stepCtx.Config)
51+
if err != nil {
52+
return failure, fmt.Errorf("could not convert config into %s config: %w", y.Name(), err)
53+
}
54+
55+
return y.run(ctx, stepCtx, cfg)
56+
}
57+
58+
// validate validates yamlMerger configuration against a JSON schema.
59+
func (y *yamlMerger) validate(cfg promotion.Config) error {
60+
return validate(y.schemaLoader, gojsonschema.NewGoLoader(cfg), y.Name())
61+
}
62+
63+
func (y *yamlMerger) run(
64+
_ context.Context,
65+
stepCtx *promotion.StepContext,
66+
cfg builtin.YAMLMergeConfig,
67+
) (promotion.StepResult, error) {
68+
69+
result := promotion.StepResult{Status: kargoapi.PromotionStepStatusSucceeded}
70+
failure := promotion.StepResult{Status: kargoapi.PromotionStepStatusErrored}
71+
72+
mergedFiles := []string{} // keep track of files actually merged
73+
74+
// Secure join the paths to prevent path traversal attacks.
75+
yamlData := []string{}
76+
for _, path := range cfg.InPaths {
77+
inPath, err := securejoin.SecureJoin(stepCtx.WorkDir, path)
78+
if err != nil {
79+
return promotion.StepResult{Status: kargoapi.PromotionStepStatusErrored},
80+
fmt.Errorf("could not secure join inPath %q: %w", path, err)
81+
}
82+
83+
inBytes, err := os.ReadFile(inPath)
84+
if err != nil {
85+
// we skip if file does not exist
86+
if !cfg.Strict && os.IsNotExist(err) {
87+
continue
88+
}
89+
return failure, fmt.Errorf(
90+
"error reading file %q: %w",
91+
inPath,
92+
err,
93+
)
94+
}
95+
96+
// we skip if file is empty
97+
if len(inBytes) == 0 {
98+
continue
99+
}
100+
101+
mergedFiles = append(mergedFiles, path)
102+
yamlData = append(yamlData, string(inBytes))
103+
}
104+
105+
// merge YAML files
106+
outYAML, err := yaml.MergeYAMLFiles(yamlData)
107+
108+
// write yaml file
109+
outPath, err := securejoin.SecureJoin(stepCtx.WorkDir, cfg.OutPath)
110+
if err != nil {
111+
return promotion.StepResult{Status: kargoapi.PromotionStepStatusErrored},
112+
fmt.Errorf("could not secure join outPath %q: %w", cfg.OutPath, err)
113+
}
114+
115+
if err := os.MkdirAll(filepath.Dir(outPath), 0o700); err != nil {
116+
return failure, fmt.Errorf("Error creating directory structure %s: %w", filepath.Dir(outPath), err)
117+
}
118+
if err = os.WriteFile(outPath, []byte(outYAML), 0o600); err != nil {
119+
return failure, fmt.Errorf("Error writing to file %s: %w", outPath, err)
120+
}
121+
122+
// add commit msg
123+
if commitMsg := y.generateCommitMessage(cfg.OutPath, mergedFiles); commitMsg != "" {
124+
result.Output = map[string]any{
125+
"commitMessage": commitMsg,
126+
}
127+
}
128+
return result, nil
129+
}
130+
131+
func (y *yamlMerger) generateCommitMessage(path string, fileList []string) string {
132+
if len(path) <= 1 {
133+
return "no YAML file merged"
134+
}
135+
136+
var commitMsg strings.Builder
137+
_, _ = commitMsg.WriteString(fmt.Sprintf("Merged YAML files to %s\n", path))
138+
for _, file := range fileList {
139+
_, _ = commitMsg.WriteString(
140+
fmt.Sprintf(
141+
"\n- %s",
142+
file,
143+
),
144+
)
145+
}
146+
147+
return commitMsg.String()
148+
}

0 commit comments

Comments
 (0)