diff --git a/src/FileAbstraction/ReplacingFile.php b/src/FileAbstraction/ReplacingFile.php
index 03c4eee4..f4aad08f 100644
--- a/src/FileAbstraction/ReplacingFile.php
+++ b/src/FileAbstraction/ReplacingFile.php
@@ -13,12 +13,18 @@
 class ReplacingFile extends File
 {
     protected bool $removeReplacedFile;
-
-    public function __construct(string $path, bool $checkPath = true, bool $removeReplacedFile = false)
-    {
+    protected bool $removeReplacedFileOnError;
+
+    public function __construct(
+        string $path,
+        bool $checkPath = true,
+        bool $removeReplacedFile = false,
+        bool $removeReplacedFileOnError = false
+    ) {
         parent::__construct($path, $checkPath);
 
         $this->removeReplacedFile = $removeReplacedFile;
+        $this->removeReplacedFileOnError = $removeReplacedFileOnError;
     }
 
     public function getClientOriginalName(): string
@@ -37,4 +43,16 @@ public function setRemoveReplacedFile(bool $removeReplacedFile): self
 
         return $this;
     }
+
+    public function isRemoveReplacedFileOnError(): bool
+    {
+        return $this->removeReplacedFileOnError;
+    }
+
+    public function setRemoveReplacedFileOnError(bool $removeReplacedFileOnError): self
+    {
+        $this->removeReplacedFileOnError = $removeReplacedFileOnError;
+
+        return $this;
+    }
 }
diff --git a/src/Storage/AbstractStorage.php b/src/Storage/AbstractStorage.php
index 98591474..94c66b40 100644
--- a/src/Storage/AbstractStorage.php
+++ b/src/Storage/AbstractStorage.php
@@ -46,7 +46,17 @@ public function upload(object $obj, PropertyMapping $mapping): void
 
         $dir = $mapping->getUploadDir($obj);
 
-        $this->doUpload($mapping, $file, $dir, $name);
+        try {
+            $this->doUpload($mapping, $file, $dir, $name);
+        } catch (\Exception $e) {
+            if ($file instanceof ReplacingFile
+                && $file->isRemoveReplacedFileOnError()
+            ) {
+                unlink($file->getPathname());
+            }
+
+            throw $e;
+        }
 
         if ($file instanceof ReplacingFile
             && $file->isRemoveReplacedFile()