From c40c060c82fd94e89fdb63329fa44336f4af45f4 Mon Sep 17 00:00:00 2001
From: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com>
Date: Mon, 12 Feb 2024 12:39:54 -0500
Subject: [PATCH] Add known issue for stuck agent upgrade (8.12.0, 8.12.1)
 (#908)

* Add known issue for stuck agent upgrade (8.12.0, 8.12.1)

* Fix workaround steps

(cherry picked from commit c29d35233deb361da217ded7124029a09abec424)
---
 .../release-notes/release-notes-8.12.asciidoc | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)

diff --git a/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc b/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc
index 034d5494d..79806d67e 100644
--- a/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc
+++ b/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc
@@ -132,6 +132,50 @@ So, if you are relying on an integration level custom ingest pipeline introduced
 Refer to the <<data-streams-pipelines,Ingest pipelines>> documentation for details and examples.
 ====
 
+[discrete]
+[[known-issues-8.12.1]]
+=== Known issues
+
+[[known-issue-3263-8121]]
+.Agents upgraded to 8.12.0 are stuck in a non-upgradeable state
+[%collapsible]
+====
+
+*Details*
+
+An issue discovered in {fleet-server} prevents {agents} that have been upgraded to version 8.12.0 from being upgraded again, using the {fleet} UI, to version 8.12.1 or higher.
+
+*Impact* +
+
+As a workaround, we recommend you to use the {kib} {fleet} API to upgrade these agents using the `force` flag.
+
+To upgrade a single {agent}:
+
+[source,"shell"]
+----
+POST kbn:/api/fleet/agents/<agent_id>/upgrade
+{
+  "version": "8.12.1",
+  "force": true
+}
+----
+
+To bulk upgrade a set of {agents}:
+
+[source,"shell"]
+----
+POST kbn:/api/fleet/agents/bulk_upgrade
+  {
+    "agents": "agent.version:8.12.0",
+    "version": "8.12.1",
+    "force": true
+  }
+----
+
+This issue is planned to be fixed in versions 8.12.2 and 8.13.0.
+
+====
+
 [discrete]
 [[bug-fixes-8.12.1]]
 === Bug fixes
@@ -405,6 +449,73 @@ Note that according to our <<agent-policy-scaling-recommendations,policy scaling
 
 ====
 
+[[known-issue-3263-8120]]
+.Agents upgraded to 8.12.0 are stuck in a non-upgradeable state
+[%collapsible]
+====
+
+*Details*
+
+An issue discovered in {fleet-server} prevents {agents} that have been upgraded to version 8.12.0 from being upgraded again, using the {fleet} UI, to version 8.12.1 or higher.
+
+This issue is planned to be fixed in versions 8.12.2 and 8.13.0.
+
+*Impact* +
+
+As a workaround, we recommend you to use the {kib} {fleet} API to update any documents in which `upgrade_details` is either `null` or not defined. Note that these steps must be run as a superuser.
+
+[source,"shell"]
+----
+ POST _security/role/fleet_superuser
+ {
+    "indices": [
+        {
+            "names": [".fleet*",".kibana*"],
+            "privileges": ["all"],
+            "allow_restricted_indices": true
+        }
+    ]
+  }
+----
+
+[source,"shell"]
+----
+POST _security/user/fleet_superuser 
+ {
+    "password": "password",
+    "roles": ["superuser", "fleet_superuser"]
+ }
+----
+
+[source,"shell"]
+----
+curl -sk -XPOST --user fleet_superuser:password -H 'content-type:application/json' \
+  -H'x-elastic-product-origin:fleet' \
+  http://localhost:9200/.fleet-agents/_update_by_query \
+  -d '{
+  "script": {
+    "source": "ctx._source.remove(\"upgrade_details\")",
+    "lang": "painless"
+  },
+  "query": {
+    "bool": {
+        "must_not": {
+          "exists": {
+            "field": "upgrade_details"
+          }
+        }
+      }
+    }
+}'
+----
+
+[source,"shell"]
+----
+DELETE _security/user/fleet_superuser
+DELETE _security/role/fleet_superuser
+----
+====
+
 [discrete]
 [[new-features-8.12.0]]
 === New features