diff --git a/mergify_cli/__init__.py b/mergify_cli/__init__.py
index 0fc5723..e7453b6 100755
--- a/mergify_cli/__init__.py
+++ b/mergify_cli/__init__.py
@@ -1,5 +1,5 @@
 #
-#  Copyright © 2021-2023 Mergify SAS
+#  Copyright © 2021-2024 Mergify SAS
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -572,9 +572,7 @@ async def stack(
         pulls: list[PullRequest] = []
         continue_create_or_update = True
         for changeid, commit, title, message in changes:
-            depends_on = ""
-            if pulls:
-                depends_on = f"\n\nDepends-On: #{pulls[-1]['number']}"
+            depends_on = pulls[-1] if pulls else None
             stacked_dest_branch = f"{stack_prefix}/{changeid}"
             if continue_create_or_update:
                 pull, action = await create_or_update_stack(
@@ -584,7 +582,7 @@ async def stack(
                     changeid,
                     commit,
                     title,
-                    message + depends_on,
+                    format_pull_description(message, depends_on),
                     known_changeids,
                     create_as_draft,
                 )
@@ -630,6 +628,16 @@ async def stack(
         console.log("[green]Finished :tada:[/]")
 
 
+def format_pull_description(message: str, depends_on: PullRequest | None) -> str:
+    depends_on_header = ""
+    if depends_on is not None:
+        depends_on_header = f"\n\nDepends-On: #{depends_on['number']}"
+
+    message = CHANGEID_RE.sub("", message).rstrip("\n")
+
+    return message + depends_on_header
+
+
 def GitHubToken(v: str) -> str:
     if not v:
         raise ValueError
diff --git a/mergify_cli/tests/test_mergify_cli.py b/mergify_cli/tests/test_mergify_cli.py
index 31792af..ba71277 100644
--- a/mergify_cli/tests/test_mergify_cli.py
+++ b/mergify_cli/tests/test_mergify_cli.py
@@ -167,7 +167,7 @@ async def test_stack_create(
         "head": "/current-branch/I29617d37762fd69809c255d7e7073cb11f8fbf50",
         "base": "main",
         "title": "Title commit 1",
-        "body": "Message commit 1\n\nChange-Id: I29617d37762fd69809c255d7e7073cb11f8fbf50",
+        "body": "Message commit 1",
         "draft": False,
     }
 
@@ -177,7 +177,7 @@ async def test_stack_create(
         "head": "/current-branch/I29617d37762fd69809c255d7e7073cb11f8fbf51",
         "base": "/current-branch/I29617d37762fd69809c255d7e7073cb11f8fbf50",
         "title": "Title commit 2",
-        "body": "Message commit 2\n\nChange-Id: I29617d37762fd69809c255d7e7073cb11f8fbf51\n\nDepends-On: #1",
+        "body": "Message commit 2\n\nDepends-On: #1",
         "draft": False,
     }
 
@@ -272,7 +272,7 @@ async def test_stack_update(
         "head": "/current-branch/I29617d37762fd69809c255d7e7073cb11f8fbf50",
         "base": "main",
         "title": "Title",
-        "body": "Message\n\nChange-Id: I29617d37762fd69809c255d7e7073cb11f8fbf50",
+        "body": "Message",
     }