From 7054f8589dced8e27f57916e388715678cd1c457 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 7 May 2025 12:12:55 +0200
Subject: [PATCH 1/9] Set the expected COMMAND when DRY_RUN is false

Use different DRY_RUN env. vars for each step.
---
 .buildkite/pipeline.cloud-cleanup.yml    | 10 +++++++++-
 .buildkite/scripts/cloud-cleanup-oidc.sh |  4 +---
 .buildkite/scripts/cloud-cleanup.sh      |  6 ++----
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/.buildkite/pipeline.cloud-cleanup.yml b/.buildkite/pipeline.cloud-cleanup.yml
index 26e759cbdd..418fa8f702 100644
--- a/.buildkite/pipeline.cloud-cleanup.yml
+++ b/.buildkite/pipeline.cloud-cleanup.yml
@@ -19,6 +19,14 @@ steps:
         - label: "False"
           value: "false"
       default: "true"
+    - select: "DRY_RUN (Deprecated step)"
+      key: "DRY_RUN_DEPRECATED"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "true"
     if: "build.source == 'ui'"
 
   - wait: ~
@@ -42,7 +50,7 @@ steps:
     command: ".buildkite/scripts/cloud-cleanup.sh"
     env:
       RESOURCE_RETENTION_PERIOD: "24 hours"
-      DRY_RUN: "${DRY_RUN:-true}"
+      DRY_RUN: "${DRY_RUN_DEPRECATED:-true}"
     agents:
       provider: "gcp" # this step requires docker
 
diff --git a/.buildkite/scripts/cloud-cleanup-oidc.sh b/.buildkite/scripts/cloud-cleanup-oidc.sh
index 25716c021d..b55e750d9e 100755
--- a/.buildkite/scripts/cloud-cleanup-oidc.sh
+++ b/.buildkite/scripts/cloud-cleanup-oidc.sh
@@ -31,9 +31,7 @@ resources_to_delete=0
 
 COMMAND="validate"
 if [[ "${DRY_RUN}" != "true" ]]; then
-    # TODO: to be changed to "destroy --confirm" once it can be tested
-    # that filters work as expected
-    COMMAND="plan"
+    COMMAND="destroy --confirm"
 else
     COMMAND="plan"
 fi
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 84b98c13ab..c828cb7915 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -25,15 +25,13 @@ export DELETE_RESOURCES_BEFORE_DATE
 
 CLOUD_REAPER_IMAGE="${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0"
 
-DRY_RUN="$(buildkite-agent meta-data get DRY_RUN --default "${DRY_RUN:-"true"}")"
+DRY_RUN="$(buildkite-agent meta-data get DRY_RUN_DEPRECATED --default "${DRY_RUN:-"true"}")"
 
 resources_to_delete=0
 
 COMMAND="validate"
 if [[ "${DRY_RUN}" != "true" ]]; then
-    # TODO: to be changed to "destroy --confirm" once it can be tested
-    # that filters work as expected
-    COMMAND="plan"
+    COMMAND="destroy --confirm"
 else
     COMMAND="plan"
 fi

From 04243b8158ecdae06ac32f6f6525f633364f9903 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 7 May 2025 12:26:56 +0200
Subject: [PATCH 2/9] Add annotations

---
 .buildkite/scripts/cloud-cleanup-oidc.sh | 5 +++++
 .buildkite/scripts/cloud-cleanup.sh      | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/.buildkite/scripts/cloud-cleanup-oidc.sh b/.buildkite/scripts/cloud-cleanup-oidc.sh
index b55e750d9e..98d5ebebfc 100755
--- a/.buildkite/scripts/cloud-cleanup-oidc.sh
+++ b/.buildkite/scripts/cloud-cleanup-oidc.sh
@@ -36,6 +36,11 @@ else
     COMMAND="plan"
 fi
 
+buildkite-agent annotate \
+  "Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"" \
+  --context "ctx-cloud-reaper-info" \
+  --style "info"
+
 any_resources_to_delete() {
     local file=$1
     local number=0
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index c828cb7915..b53a8048dc 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -36,6 +36,11 @@ else
     COMMAND="plan"
 fi
 
+buildkite-agent annotate \
+  "Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"" \
+  --context "ctx-cloud-reaper-info" \
+  --style "info"
+
 any_resources_to_delete() {
     local file=$1
     local number=0

From 7fb14def92386a63f83e616fe20194c2ee7848ed Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 7 May 2025 12:35:41 +0200
Subject: [PATCH 3/9] Update annotation with step key

---
 .buildkite/scripts/cloud-cleanup-oidc.sh | 2 +-
 .buildkite/scripts/cloud-cleanup.sh      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup-oidc.sh b/.buildkite/scripts/cloud-cleanup-oidc.sh
index 98d5ebebfc..a522a49459 100755
--- a/.buildkite/scripts/cloud-cleanup-oidc.sh
+++ b/.buildkite/scripts/cloud-cleanup-oidc.sh
@@ -37,7 +37,7 @@ else
 fi
 
 buildkite-agent annotate \
-  "Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"" \
+  "[${BUILDKITE_STEP_KEY}] Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"" \
   --context "ctx-cloud-reaper-info" \
   --style "info"
 
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index b53a8048dc..158732a7cd 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -37,7 +37,7 @@ else
 fi
 
 buildkite-agent annotate \
-  "Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"" \
+  "[${BUILDKITE_STEP_KEY}] Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"" \
   --context "ctx-cloud-reaper-info" \
   --style "info"
 

From e4d3ce11ecaf5a72118b13b13cc9a8df861c0b48 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 7 May 2025 12:41:55 +0200
Subject: [PATCH 4/9] Update context annotation

---
 .buildkite/scripts/cloud-cleanup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 158732a7cd..961d5ec74c 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -38,7 +38,7 @@ fi
 
 buildkite-agent annotate \
   "[${BUILDKITE_STEP_KEY}] Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"" \
-  --context "ctx-cloud-reaper-info" \
+  --context "ctx-cloud-reaper-info-deprecated" \
   --style "info"
 
 any_resources_to_delete() {

From 464b115661222d167f0f8a8581cd9db6a312dee1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 7 May 2025 13:19:18 +0200
Subject: [PATCH 5/9] Add exception

---
 .buildkite/scripts/cloud-cleanup-oidc.sh | 10 ++++++++++
 .buildkite/scripts/cloud-cleanup.sh      | 10 ++++++++++
 2 files changed, 20 insertions(+)

diff --git a/.buildkite/scripts/cloud-cleanup-oidc.sh b/.buildkite/scripts/cloud-cleanup-oidc.sh
index a522a49459..34e75ccb61 100755
--- a/.buildkite/scripts/cloud-cleanup-oidc.sh
+++ b/.buildkite/scripts/cloud-cleanup-oidc.sh
@@ -48,6 +48,16 @@ any_resources_to_delete() {
     # ⇒ Loading configuration...
     # ✓ Succeeded to load configuration
     # Scanning resources... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00
+    
+    # FIXME:: When running with DRY_RUN: false there could be more lines.
+    # In the case, there is nothing to delete, there is one more line:
+    # ⇒ Nothing to destroy !
+    # but there are no examples when resources are deleted to add the required logic
+    if [[ "${DRY_RUN}" == false ]] ; then
+        if tail -n 1 ${file} | grep "Nothing to destroy" 2> /dev/null ; then
+            return 1
+        fi
+    fi
     number=$(tail -n +4 "${file}" | wc -l)
     if [ "${number}" -eq 0 ]; then
         return 1
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 961d5ec74c..159cc06811 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -48,6 +48,16 @@ any_resources_to_delete() {
     # ⇒ Loading configuration...
     # ✓ Succeeded to load configuration
     # Scanning resources... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00
+
+    # FIXME:: When running with DRY_RUN: false there could be more lines.
+    # In the case, there is nothing to delete, there is one more line:
+    # ⇒ Nothing to destroy !
+    # but there are no examples when resources are deleted to add the required logic
+    if [[ "${DRY_RUN}" == false ]] ; then
+        if tail -n 1 ${file} | grep "Nothing to destroy" 2> /dev/null ; then
+            return 1
+        fi
+    fi
     number=$(tail -n +4 "${file}" | wc -l)
     if [ "${number}" -eq 0 ]; then
         return 1

From 479d8c481c2750d410fd16be905eab12eb729641 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 7 May 2025 13:27:32 +0200
Subject: [PATCH 6/9] Do not show output in grep command

---
 .buildkite/scripts/cloud-cleanup-oidc.sh | 2 +-
 .buildkite/scripts/cloud-cleanup.sh      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup-oidc.sh b/.buildkite/scripts/cloud-cleanup-oidc.sh
index 34e75ccb61..af92b0edd4 100755
--- a/.buildkite/scripts/cloud-cleanup-oidc.sh
+++ b/.buildkite/scripts/cloud-cleanup-oidc.sh
@@ -54,7 +54,7 @@ any_resources_to_delete() {
     # ⇒ Nothing to destroy !
     # but there are no examples when resources are deleted to add the required logic
     if [[ "${DRY_RUN}" == false ]] ; then
-        if tail -n 1 ${file} | grep "Nothing to destroy" 2> /dev/null ; then
+        if tail -n 1 ${file} | grep -q "Nothing to destroy" ; then
             return 1
         fi
     fi
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 159cc06811..8328523deb 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -54,7 +54,7 @@ any_resources_to_delete() {
     # ⇒ Nothing to destroy !
     # but there are no examples when resources are deleted to add the required logic
     if [[ "${DRY_RUN}" == false ]] ; then
-        if tail -n 1 ${file} | grep "Nothing to destroy" 2> /dev/null ; then
+        if tail -n 1 ${file} | grep -q "Nothing to destroy" ; then
             return 1
         fi
     fi

From 5bd8746fcb33112e2a641c2e6f0046fc40bda056 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 7 May 2025 13:29:12 +0200
Subject: [PATCH 7/9] Add header in buildkite output

---
 .buildkite/scripts/cloud-cleanup-oidc.sh | 4 ++--
 .buildkite/scripts/cloud-cleanup.sh      | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup-oidc.sh b/.buildkite/scripts/cloud-cleanup-oidc.sh
index af92b0edd4..030a5b0f98 100755
--- a/.buildkite/scripts/cloud-cleanup-oidc.sh
+++ b/.buildkite/scripts/cloud-cleanup-oidc.sh
@@ -99,7 +99,7 @@ cloud_reaper_aws() {
     echo "--- Configuring ephemeral user"
     create_aws_ephemeral_user
 
-    echo "Validating configuration"
+    echo "--- Validating configuration"
     docker run --rm -v "$(pwd)/.buildkite/configs/cleanup.aws.yml":/etc/cloud-reaper/config.yml \
       -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID_EPHEMERAL" \
       -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY_EPHEMERAL" \
@@ -111,7 +111,7 @@ cloud_reaper_aws() {
           --config /etc/cloud-reaper/config.yml \
           validate
 
-    echo "Scanning resources"
+    echo "--- Scanning resources"
     docker run --rm -v "$(pwd)/.buildkite/configs/cleanup.aws.yml":/etc/cloud-reaper/config.yml \
       -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID_EPHEMERAL" \
       -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY_EPHEMERAL" \
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 8328523deb..21d59adb21 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -66,7 +66,7 @@ any_resources_to_delete() {
 }
 
 cloud_reaper_aws() {
-    echo "Validating configuration"
+    echo "--- Validating configuration"
     docker run --rm -v "$(pwd)/.buildkite/configs/cleanup.aws.yml":/etc/cloud-reaper/config.yml \
       -e AWS_SECRET_ACCESS_KEY="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
       -e AWS_ACCESS_KEY_ID="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \
@@ -77,7 +77,7 @@ cloud_reaper_aws() {
           --config /etc/cloud-reaper/config.yml \
           validate
 
-    echo "Scanning resources"
+    echo "--- Scanning resources"
     docker run --rm -v "$(pwd)/.buildkite/configs/cleanup.aws.yml":/etc/cloud-reaper/config.yml \
       -e AWS_SECRET_ACCESS_KEY="${ELASTIC_PACKAGE_AWS_SECRET_KEY}" \
       -e AWS_ACCESS_KEY_ID="${ELASTIC_PACKAGE_AWS_ACCESS_KEY}" \

From bb930ec33ad0c552b788fd4ba9e1797568f5e413 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 7 May 2025 13:38:23 +0200
Subject: [PATCH 8/9] Restore plan for cloud-reaper

---
 .buildkite/scripts/cloud-cleanup-oidc.sh | 6 ++++--
 .buildkite/scripts/cloud-cleanup.sh      | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup-oidc.sh b/.buildkite/scripts/cloud-cleanup-oidc.sh
index 030a5b0f98..f84b070a75 100755
--- a/.buildkite/scripts/cloud-cleanup-oidc.sh
+++ b/.buildkite/scripts/cloud-cleanup-oidc.sh
@@ -31,7 +31,9 @@ resources_to_delete=0
 
 COMMAND="validate"
 if [[ "${DRY_RUN}" != "true" ]]; then
-    COMMAND="destroy --confirm"
+    # TODO: to be changed to "destroy --confirm" once it can be tested
+    # that filters work as expected
+    COMMAND="plan"
 else
     COMMAND="plan"
 fi
@@ -49,7 +51,7 @@ any_resources_to_delete() {
     # ✓ Succeeded to load configuration
     # Scanning resources... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00
     
-    # FIXME:: When running with DRY_RUN: false there could be more lines.
+    # FIXME:: When running with "destroy --confirm" there could be more lines.
     # In the case, there is nothing to delete, there is one more line:
     # ⇒ Nothing to destroy !
     # but there are no examples when resources are deleted to add the required logic
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 21d59adb21..5f03247ba5 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -31,7 +31,9 @@ resources_to_delete=0
 
 COMMAND="validate"
 if [[ "${DRY_RUN}" != "true" ]]; then
-    COMMAND="destroy --confirm"
+    # TODO: to be changed to "destroy --confirm" once it can be tested
+    # that filters work as expected
+    COMMAND="plan"
 else
     COMMAND="plan"
 fi
@@ -49,7 +51,7 @@ any_resources_to_delete() {
     # ✓ Succeeded to load configuration
     # Scanning resources... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00
 
-    # FIXME:: When running with DRY_RUN: false there could be more lines.
+    # FIXME:: When running with "destroy --confirm" command there could be more lines.
     # In the case, there is nothing to delete, there is one more line:
     # ⇒ Nothing to destroy !
     # but there are no examples when resources are deleted to add the required logic

From f30e89a04083fea0a5396934256372fe8c519142 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 7 May 2025 13:41:29 +0200
Subject: [PATCH 9/9] Update annotation

---
 .buildkite/scripts/cloud-cleanup-oidc.sh | 4 +++-
 .buildkite/scripts/cloud-cleanup.sh      | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/cloud-cleanup-oidc.sh b/.buildkite/scripts/cloud-cleanup-oidc.sh
index f84b070a75..0120eb9392 100755
--- a/.buildkite/scripts/cloud-cleanup-oidc.sh
+++ b/.buildkite/scripts/cloud-cleanup-oidc.sh
@@ -30,16 +30,18 @@ DRY_RUN="$(buildkite-agent meta-data get DRY_RUN --default "${DRY_RUN:-"true"}")
 resources_to_delete=0
 
 COMMAND="validate"
+redshift_message=""
 if [[ "${DRY_RUN}" != "true" ]]; then
     # TODO: to be changed to "destroy --confirm" once it can be tested
     # that filters work as expected
     COMMAND="plan"
+    redshift_message=" - stale redshift clusters will be deleted"
 else
     COMMAND="plan"
 fi
 
 buildkite-agent annotate \
-  "[${BUILDKITE_STEP_KEY}] Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"" \
+  "[${BUILDKITE_STEP_KEY}] Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"${redshift_message}" \
   --context "ctx-cloud-reaper-info" \
   --style "info"
 
diff --git a/.buildkite/scripts/cloud-cleanup.sh b/.buildkite/scripts/cloud-cleanup.sh
index 5f03247ba5..2aaa8a0ae2 100644
--- a/.buildkite/scripts/cloud-cleanup.sh
+++ b/.buildkite/scripts/cloud-cleanup.sh
@@ -30,16 +30,18 @@ DRY_RUN="$(buildkite-agent meta-data get DRY_RUN_DEPRECATED --default "${DRY_RUN
 resources_to_delete=0
 
 COMMAND="validate"
+redshift_message=""
 if [[ "${DRY_RUN}" != "true" ]]; then
     # TODO: to be changed to "destroy --confirm" once it can be tested
     # that filters work as expected
     COMMAND="plan"
+    redshift_message=" - stale redshift clusters will be deleted"
 else
     COMMAND="plan"
 fi
 
 buildkite-agent annotate \
-  "[${BUILDKITE_STEP_KEY}] Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"" \
+  "[${BUILDKITE_STEP_KEY}] Running DRY_RUN (${DRY_RUN}) using cloud-reaper command \"${COMMAND}\"${redshift_message}" \
   --context "ctx-cloud-reaper-info-deprecated" \
   --style "info"