diff --git a/package-lock.json b/package-lock.json
index ed9b6651..0f0ea106 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6,10 +6,11 @@
   "packages": {
     "": {
       "name": "@ghost-fvtt/foundry-publish",
-      "version": "2.3.0",
+      "version": "2.4.0",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
+        "@playwright/test": "^1.39.0",
         "chalk": "^5.3.0",
         "commander": "^11.0.0",
         "fp-ts": "^2.12.2",
@@ -739,6 +740,20 @@
         "node": ">=14"
       }
     },
+    "node_modules/@playwright/test": {
+      "version": "1.39.0",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.39.0.tgz",
+      "integrity": "sha512-3u1iFqgzl7zr004bGPYiN/5EZpRUSFddQBra8Rqll5N0/vfpqlP9I9EXqAoGacuAbX6c9Ulg/Cjqglp5VkK6UQ==",
+      "dependencies": {
+        "playwright": "1.39.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.14",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.14.tgz",
@@ -3004,6 +3019,19 @@
       "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
       "dev": true
     },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "hasInstallScript": true,
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/function-bind": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
@@ -4940,6 +4968,23 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.39.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.39.0.tgz",
+      "integrity": "sha512-naE5QT11uC/Oiq0BwZ50gDmy8c8WLPRTEWuSSFVG2egBka/1qMoSqYQcROMT9zLwJ86oPofcTH2jBY/5wWOgIw==",
+      "dependencies": {
+        "playwright-core": "1.39.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
     "node_modules/playwright-chromium": {
       "version": "1.39.0",
       "resolved": "https://registry.npmjs.org/playwright-chromium/-/playwright-chromium-1.39.0.tgz",
diff --git a/package.json b/package.json
index 35c0c9b6..5ecdc28d 100644
--- a/package.json
+++ b/package.json
@@ -67,6 +67,7 @@
     "typescript": "5.2.2"
   },
   "dependencies": {
+    "@playwright/test": "^1.39.0",
     "chalk": "^5.3.0",
     "commander": "^11.0.0",
     "fp-ts": "^2.12.2",
diff --git a/src/publish/publish-flow.ts b/src/publish/publish-flow.ts
index 213a0c91..e6850b7b 100644
--- a/src/publish/publish-flow.ts
+++ b/src/publish/publish-flow.ts
@@ -3,7 +3,10 @@
 // SPDX-License-Identifier: MIT
 
 import path from 'node:path';
+
+import { expect } from '@playwright/test';
 import { chromium, type Dialog, type Page } from 'playwright-chromium';
+
 import type { Options } from '../options';
 
 const foundryBaseURL = 'https://foundryvtt.com/';
@@ -34,7 +37,7 @@ async function login(page: Page, { username, password }: Options) {
   await page.getByPlaceholder('Username').fill(username);
   await page.getByPlaceholder('Password').fill(password);
   await page.getByRole('button', { name: 'Log In' }).click();
-  await page.getByText(`You are now logged in as ${username}!`).isVisible();
+  await expect(page.getByText(`You are now logged in as ${username}!`)).toBeVisible();
   console.log('Login successful.');
 }
 
@@ -68,8 +71,17 @@ async function publishPackage(page: Page, options: Options) {
   await page.locator('#save').click();
   await page.waitForURL(getPackageURL(options));
 
+  const savePackageErrorText = 'Could not save package. Please correct errors below.';
+  if (!options.dryRun) {
+    await expect(
+      page.getByText(/Package .+ was saved successfully/).or(page.getByText(savePackageErrorText)),
+    ).toBeVisible();
+  }
+  const failedToSave = await page.getByText(savePackageErrorText).isVisible();
+
   const errors = await page.locator('ul.errorlist li').evaluateAll((li) => li.map((element) => element.textContent));
-  if (errors.length > 0) {
+
+  if (failedToSave || errors.length > 0) {
     throw new Error(`Errors while publishing package:\n${errors.join('\n')}`);
   }
 
@@ -93,13 +105,15 @@ async function deleteObsoleteVersions(page: Page, options: Options) {
             .map((e) => e.id),
         options,
       );
-    console.log(versionsToDelete);
 
     const accept = (dialog: Dialog) => dialog.accept();
     page.on('dialog', accept);
     for (const version of versionsToDelete) {
       console.log(`Deleting version ${version}…`);
       await page.locator(`[id="${version}"]`).locator('button[name=delete-version]').click();
+      if (!options.dryRun) {
+        await expect(page.getByText(/Package .+ was saved successfully/)).toBeVisible();
+      }
     }
     page.off('dialog', accept);
     console.log('Obsolete versions deleted successfully.');