diff --git a/.github/workflows/maven-compiler.yml b/.github/workflows/maven-compiler.yml
index 17a7a7a855..bd3763ff75 100644
--- a/.github/workflows/maven-compiler.yml
+++ b/.github/workflows/maven-compiler.yml
@@ -44,9 +44,3 @@ jobs:
 
     - name: Build with Maven
       run: mvn package --file pom.xml
-
-    - name: Upload the artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: slimefun-${{ github.event.number }}
-        path: 'target/Slimefun v4.9-UNOFFICIAL.jar'
diff --git a/.github/workflows/preview-builds.yml b/.github/workflows/preview-builds.yml
index 0d45c81b24..2b0b32b59f 100644
--- a/.github/workflows/preview-builds.yml
+++ b/.github/workflows/preview-builds.yml
@@ -2,7 +2,7 @@ name: Preview builds
 
 on:
   workflow_run:
-    workflows: ["Java CI"]
+    workflows: ["Pull Request"]
     types:
       - completed
 
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      # Kinda jank way to grab the PR and run ID and then download the artifact
+      # Kinda jank way to grab the PR and commit hash and then download the artifact
       # TODO: Move this code to our own mini-action
       - name: Grab PR & run ID and download the artifact
         uses: actions/github-script@v6
@@ -30,13 +30,13 @@ jobs:
             });
 
             for (const artifact of allArtifacts.data.artifacts) {
-              // Extract the PR number from the artifact name
-              const match = /^slimefun-(\d+)$/.exec(artifact.name);
+              // Extract the PR number and commit hash from the artifact name
+              const match = /^slimefun-(\d+)-([a-f0-9]{8})$/.exec(artifact.name);
               if (match) {
                 require("fs").appendFileSync(
                   process.env.GITHUB_ENV,
-                  `\nWORKFLOW_PR_ID=${match[1]}` +
-                    `\nWORKFLOW_RUN_ID=${context.payload.workflow_run.id}`
+                  `\nPR_NUMBER=${match[1]}` +
+                    `\nCOMMIT_HASH=${match[2]}`
                 );
 
                 const download = await github.rest.actions.downloadArtifact({
@@ -56,26 +56,27 @@ jobs:
         run: |
           unzip preview.zip
           rm preview.zip
-          mv 'Slimefun v4.9-UNOFFICIAL.jar' preview.jar
+          mv 'Slimefun vPreview Build #${{ env.PR_NUMBER }}-${{ env.COMMIT_HASH }}' preview.jar
 
       - name: Upload to preview service
         run: |
-          curl -v -X POST \
+          curl -X POST \
             -H 'Authorization: ${{ secrets.PUBLISH_TOKEN }}' \
             -H "X-Checksum: $(sha256sum 'preview.jar' | awk '{print $1}')" \
             --data-binary '@preview.jar' \
-            https://preview-builds.walshy.dev/upload/Slimefun/${{ env.WORKFLOW_PR_ID }}/${{ env.WORKFLOW_RUN_ID }}
+            https://preview-builds.walshy.dev/upload/Slimefun/${{ env.PR_NUMBER }}/${{ env.COMMIT_HASH }}
 
       - name: Post comment
         uses: marocchino/sticky-pull-request-comment@v2
         with:
-          number: ${{ env.WORKFLOW_PR_ID }}
+          number: ${{ env.PR_NUMBER }}
           message: |
             ### Slimefun preview build
 
             A Slimefun preview build is available for testing!
+            Commit: ${{ env.COMMIT_HASH }}
 
-            https://preview-builds.walshy.dev/download/Slimefun/${{ env.WORKFLOW_PR_ID }}/${{ env.WORKFLOW_RUN_ID }}
+            https://preview-builds.walshy.dev/download/Slimefun/${{ env.PR_NUMBER }}/${{ env.COMMIT_HASH }}
 
             > **Note**: This is not a supported build and is only here for the purposes of testing.
             > Do not run this on a live server and do not report bugs anywhere but this PR!
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
new file mode 100644
index 0000000000..2c3a31ab8c
--- /dev/null
+++ b/.github/workflows/pull-request.yml
@@ -0,0 +1,52 @@
+name: Pull Request
+
+on:
+  pull_request:
+    paths:
+    - '.github/workflows/**'
+    - 'src/**'
+    - 'pom.xml'
+
+permissions:
+  contents: read
+
+jobs:
+  setup-preview-build:
+    name: Preview build
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up JDK 17
+      uses: actions/setup-java@v3.11.0
+      with:
+        distribution: 'adopt'
+        java-version: '17'
+        java-package: jdk
+        architecture: x64
+
+    - name: Cache Maven packages
+      uses: actions/cache@v3
+      with:
+        path: ~/.m2
+        key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+        restore-keys: ${{ runner.os }}-m2
+
+    # Setup for the preview build
+    - run: |
+        SHORT_COMMIT_HASH=$(git rev-parse --short=8 ${{ github.sha }})
+        JAR_VERSION="Preview Build #${{ github.event.number }}-$SHORT_COMMIT_HASH"
+        echo "SHORT_COMMIT_HASH=$SHORT_COMMIT_HASH" >> "$GITHUB_ENV"
+        echo "JAR_VERSION=$JAR_VERSION" >> "$GITHUB_ENV"
+        sed -i "s/<version>4.9-UNOFFICIAL<\/version>/<version>$JAR_VERSION<\/version>/g" pom.xml
+
+    - name: Build with Maven
+      run: mvn package
+
+    - name: Upload the artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: slimefun-${{ github.event.number }}-${{ env.SHORT_COMMIT_HASH }}
+        path: 'target/Slimefun v${{ env.JAR_VERSION }}.jar'