diff --git a/ci/eval/compare/generate-step-summary.jq b/ci/eval/compare/generate-step-summary.jq
index 28597eaec371f..961877cd27974 100644
--- a/ci/eval/compare/generate-step-summary.jq
+++ b/ci/eval/compare/generate-step-summary.jq
@@ -4,12 +4,27 @@ def truncate(xs; n):
   end;
 
 def itemize_packages(xs):
-  # we truncate the list to stay below the GitHub limit of 1MB per step summary.
-  truncate(xs; 3000) | map("- [\(.)](https://search.nixos.org/packages?channel=unstable&show=\(.)&from=0&size=50&sort=relevance&type=packages&query=\(.))") | join("\n");
+  truncate(xs; 2000) |
+    map("- [\(.)](https://search.nixos.org/packages?channel=unstable&show=\(.)&from=0&size=50&sort=relevance&type=packages&query=\(.))")  |
+    join("\n");
+
+def get_title(s; xs):
+  s + " (" + (xs | length | tostring) + ")";
 
 def section(title; xs):
-  "<details> <summary>" + title + " (" + (xs | length | tostring) + ")</summary>\n\n" + itemize_packages(xs) + "</details>";
+  "<details> <summary>" + get_title(title; xs) + "</summary>\n\n" + itemize_packages(xs) + "</details>";
+
+def fallback_document(content; n):
+  if content | utf8bytelength > n then
+    get_title("Added packages"; .attrdiff.added) + "\n\n" +
+    get_title("Removed packages"; .attrdiff.removed) + "\n\n" +
+    get_title("Changed packages"; .attrdiff.changed)
+  else content
+  end;
 
-section("Added packages"; .attrdiff.added) + "\n\n" +
-section("Removed packages"; .attrdiff.removed) + "\n\n" +
-section("Changed packages"; .attrdiff.changed)
+# we truncate the list to stay below the GitHub limit of 1MB per step summary.
+fallback_document(
+  section("Added packages"; .attrdiff.added) + "\n\n" +
+  section("Removed packages"; .attrdiff.removed) + "\n\n" +
+  section("Changed packages"; .attrdiff.changed); 1000 * 1000
+)