Skip to content

Commit

Permalink
feat(RELEASE-1252): make run-file-updates task idempotent
Browse files Browse the repository at this point in the history
Signed-off-by: Jing Qi <[email protected]>

The PR is to make the run-file-updates task idempotent by ensuring that
 a new MR isn't created unnecessarily when one already exists with the
same content in the value of `seed` and `replacements` fields in `paths`
 parameter.
And it adds some logic to check all of the `key` in the `replacements`
 parameter. If all of the `key` exist but the content of the file does
 not require any update, return success.
  • Loading branch information
jinqi7 committed Feb 5, 2025
1 parent b9a01d9 commit cb50294
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 6 deletions.
3 changes: 0 additions & 3 deletions tasks/internal/process-file-updates-task/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,6 @@ replacements to a yaml file that already exists. It will attempt to create a Mer
| tempDir | temp dir for cloning and updates | Yes | /tmp/$(context.taskRun.uid)/file-updates |
| internalRequestPipelineRunName | name of the PipelineRun that called this task | No | - |

## Changes in 0.1.1
* Revert idempotent changes as they are not working

## Changes in 0.1.0
* make run-file-updates task idempotent

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ kind: Task
metadata:
name: process-file-updates-task
labels:
app.kubernetes.io/version: "0.1.1"
app.kubernetes.io/version: "0.1.0"
annotations:
tekton.dev/pipelines.minVersion: "0.12.1"
tekton.dev/tags: release
Expand Down Expand Up @@ -203,7 +203,7 @@ spec:
# only a single line was replaced and that the result
# block has the same number of lines as before
sed -ne "${startBlock},+${valueSize}p" "${targetFile}" > "${TEMP}/result.txt"
diff -u "${TEMP}/{found,result}.txt" > "${TEMP}/diff.txt" || true
diff -u "${TEMP}/found.txt" "${TEMP}/result.txt" > "${TEMP}/diff.txt" || true
replacedBlockLines=$(wc -l < "${TEMP}/result.txt")
if [[ $replacedBlockLines != $(( valueSize +1 )) ]]; then
Expand Down Expand Up @@ -251,7 +251,66 @@ spec:
fi
echo -e "\n*** START LOCAL CHANGES ***\n"
git diff
# compare the differences between the working directory and the staging area
git diff | tee "${TEMP}"/tempMRFile.diff
# compare the differences between the staging area and the latest commit
git diff --cached | tee "${TEMP}"/tempMRFile-cached.diff
if [[ -s "${TEMP}"/tempMRFile.diff ]]; then
# replacements exist
# It's to deal with the lines like "@@ -N1,N2 +N3,N4 @@",
# but it's possible to meet a line like "@@ -4,7 +4,7 @@ $schema: /app/app-settings.yml".
awk '/^@@/ {match($0, /@@ ([^@]+) @@/, arr); print arr[1]}' "${TEMP}"/tempMRFile.diff |\
tee "${TEMP}"/lineFile
elif [[ ! -s "${TEMP}"/tempMRFile-cached.diff ]]; then
# only seed exists and the file with the same content is there already
echo "the seeding file already exists" \
| tee -a "$(results.fileUpdatesInfo.path)"
echo -n "Success" |tee "$(results.fileUpdatesState.path)"
exit 0
fi
openMRList=$(glab mr list -R "${UPSTREAM_REPO}" --search "Konflux release" |grep "^!"|| true)
while IFS= read -r oneItem; do
mrNum=$(echo "$oneItem" | cut -f1 | tr -d '!')
foundMR=false
glab mr diff "${mrNum}" --repo "${UPSTREAM_REPO}" > "${TEMP}"/oneMR.diff
if [[ ! -s "${TEMP}"/lineFile ]]; then
# only seed exists and no replacements
grep '^[+][^+]' "${TEMP}"/oneMR.diff | sed -E 's/^[+]//' | tee "${TEMP}/mrFile"
# compare if the contents and the updated file names are the same
if diff -q "${TEMP}"/mrFile "${targetFile}" > /dev/null; then
grep "^diff --git" "${TEMP}"/oneMR.diff | tee "${TEMP}"/tmpFile
while read -r line ; do
changedFileName=$(echo "$line" | awk '{sub(/^b\//, "", $NF); print $NF}')
if [[ "${changedFileName}" != "${targetFile}" ]]; then
continue
fi
foundMR=true
break
done < "${TEMP}"/tmpFile
fi
else
# replacements exist
grep '^[-+@]' "${TEMP}"/oneMR.diff | sed -E 's/^[@]//; s/@@.*//' | tee "${TEMP}/mrFile"
grep '^[-+@]' "${TEMP}"/tempMRFile.diff | sed -E 's/^[@]//; s/@@.*//' | tee "${TEMP}/tmpFile"
if diff -q "${TEMP}"/mrFile "${TEMP}"/tmpFile > /dev/null; then
foundMR=true
fi
fi
# if all the lines are matched, the MR is the same one
# it should exit 0 if the same MR exists
if [[ "$foundMR" == true ]]; then
echo "there is an existing MR with the same updates in the repo"
echo "{\"merge_request\":\"${UPSTREAM_REPO}/-/merge_requests/${mrNum}\"}" \
| tee -a "$(results.fileUpdatesInfo.path)"
echo -n "Success" | tee "$(results.fileUpdatesState.path)"
exit 0
fi
done <<< "$openMRList"
echo -e "\n*** END LOCAL CHANGES ***\n"
WORKING_BRANCH=$(uuidgen |awk '{print substr($1, 1, 8)}')
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
---
apiVersion: tekton.dev/v1
kind: Pipeline
metadata:
name: test-process-file-updates-replacements-idempotent
spec:
description: |
Run the process-file-updates task with replacements. No commit
is created if there is a mocked MR with the same content.
workspaces:
- name: tests-workspace
tasks:
- name: setup
workspaces:
- name: pipeline
workspace: tests-workspace
taskSpec:
workspaces:
- name: pipeline
steps:
- name: setup-values
image: quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f
script: |
#!/usr/bin/env bash
set -eux
mkdir -p "$(workspaces.pipeline.path)/$(context.pipelineRun.uid)/file-updates"
cd "$(workspaces.pipeline.path)/$(context.pipelineRun.uid)/file-updates"
mkdir replace-idempotent
cd replace-idempotent
git config --global init.defaultBranch main
git init .
git config --global user.email "[email protected]"
git config --global user.name "tester"
mkdir addons
cat > "addons/my-addon2.yaml" << EOF
indexImage:
name: test
EOF
git add addons/my-addon2.yaml
git commit -m "prior commit"
- name: run-task
taskRef:
name: process-file-updates-task
params:
- name: upstream_repo
value: "https://some.gitlab/test/replace-idempotent"
- name: repo
value: "https://some.gitlab/test/replace-idempotent"
- name: ref
value: "main"
- name: paths
value: >-
[{"path":"addons/my-addon2.yaml","replacements":[{"key":".indexImage",
"replacement":"|indexImage.*|indexImage: Jack|"}]}]
- name: application
value: "scott"
- name: file_updates_secret
value: "file-updates-secret"
- name: tempDir
value: "$(workspaces.pipeline.path)/$(context.pipelineRun.uid)/file-updates"
- name: internalRequestPipelineRunName
value: $(context.pipelineRun.name)
workspaces:
- name: pipeline
workspace: tests-workspace
runAfter:
- setup
- name: check-result
runAfter:
- run-task
params:
- name: fileUpdatesInfo
value: $(tasks.run-task.results.fileUpdatesInfo)
- name: fileUpdatesState
value: $(tasks.run-task.results.fileUpdatesState)
- name: tempDir
value: "$(workspaces.pipeline.path)/$(context.pipelineRun.uid)/file-updates"
taskSpec:
params:
- name: fileUpdatesInfo
type: string
- name: fileUpdatesState
type: string
- name: tempDir
type: string
steps:
- name: check-result
image: quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f
script: |
#!/usr/bin/env bash
set -eux
cd "$(params.tempDir)/replace-idempotent"
commits=$(git log --oneline | wc -l)

echo "Test no more commit created except the one in setup step"
test "${commits}" == "1"
rm -rf "$(params.tempDir)"

workspaces:
- name: pipeline
workspace: tests-workspace

0 comments on commit cb50294

Please sign in to comment.