koji scripts: improve garbage collection

* Untag builds whose only tag is built-by-xcp-ng (script
  untag_lone_builds.py)
* Run koji-gc --delete to delete builds marked for deletion (trashcan
  tag) previously.
* Run koji-gc --trash to mark untagged builds for deletion (trashcan
  tag).

Signed-off-by: Samuel Verschelde <[email protected]>

scripts/koji/koji_garbage_collector.sh

@@ -1,10 +1,35 @@
 #!/bin/bash
 
-# remove old scratch builds
+# This script removes old scratch builds but also untags old builds whose only tag is built-by-xcp-ng,
+# then runs koji-gc so that they can be put in the trashcan if nothing references them anymore,
+# and then at next run be deleted.
+
+# *** remove old scratch builds ***
 
 TOPDIR=/mnt/koji
 TIMEARG="+90"
 IFS=$'\n'
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# koji-gc is able to flag builds for deletion, based on koji-gc policies defined in /etc/koji-gc/koji-gc.conf,
+# but the presence of the "built-by-xcp-ng" tag prevents it from doing so.
+# Consequently, we first look for such builds and untag them.
+echo
+echo "*** untag_lone_builds.py ***"
+su -l kojiadmin -c "$SCRIPT_DIR/untag_lone_builds.py"
+
+# delete builds in the trashcan tag in a previous pass
+echo
+echo "*** koji-gc --action=delete ***"
+su -l kojiadmin -c "koji-gc --action=delete"
+
+# mark new untagged builds for deletion (trashcan tag)
+echo
+echo "*** koji-gc --action=trash ***"
+su -l kojiadmin -c "koji-gc --action=trash"
+
+echo
+echo "*** Remove old scratch builds ***"
 
 # we completely remove those that are old enough
 # scratch directories are /mnt/koji/scratch/$username/task_$taskid/
@@ -14,6 +39,9 @@ for x in $(find $TOPDIR/scratch/ -mindepth 2 -type d -name 'task_*' -prune -mtim
     find "$x" -xdev "!" -type d "!" -name "*.deleted" -printf '%s\t %p\n' -delete -exec touch {}.deleted \; -exec chown apache.apache {}.deleted \;
 done
 
+echo
+echo "*** Remove old tasks ***"
+
 # for tasks, try to remove as a unit
 for x in $(find "$TOPDIR"/work/tasks/ -mindepth 2 -maxdepth 2 -type d -mtime $TIMEARG); do
     # delete broken symlinks (that will be links to deleted scratch builds) but leave other symlinks alone

scripts/koji/untag_lone_builds.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import koji
+
+config = koji.read_config("koji")
+s = koji.ClientSession('https://kojihub.xcp-ng.org', config)
+s.ssl_login(config['cert'], None, config['serverca'])
+
+# We look for builds tagged only with built-by-xcp-ng, and no other tag
+# These are builds that we don't need in our history and that we could
+# delete to recover some disk space.
+# To do so, we just untag them, so that koji-gc later marks them for deletion
+# if all conditions for this are met.
+tag = 'built-by-xcp-ng'
+tagged = s.listTagged(tag)
+result = []
+with s.multicall() as m:
+    result = [m.listTags(binfo) for binfo in tagged]
+loners = []
+for binfo, tinfos in zip(tagged, result):
+    tags = [tinfo['name'] for tinfo in tinfos.result]
+    if len(tags) == 1:
+        loners.append((binfo['id'], binfo['nvr']))
+
+print("The following packages built by XCP-ng don't belong to any tag other than %s" % tag)
+print("and will be removed from it so that they can be garbage-collected later.")
+for id, nvr in loners:
+    print("Untagging build {} ({}) from {}.".format(id, nvr, tag))
+    s.untagBuild(tag, id)