diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index e3f749fac..3f89a3185 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -21,7 +21,8 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
         os: [ubuntu-latest]
 
     name: Gradle Check Linux
@@ -33,6 +34,7 @@ jobs:
       # need to switch to root so that github actions can install runner binary on container without permission issues.
       options: --user root
 
+
     steps:
       - name: Checkout neural-search
         uses: actions/checkout@v1
@@ -55,7 +57,8 @@ jobs:
   Check-neural-search-windows:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
         os: [windows-latest]
 
     name: Gradle Check Windows
@@ -83,7 +86,8 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
         os: [ubuntu-latest]
 
     name: Pre-commit Linux
@@ -117,7 +121,8 @@ jobs:
   Precommit-neural-search-windows:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
         os: [windows-latest]
 
     name: Pre-commit Windows
diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml
index 32a71c5d0..9b34b6356 100644
--- a/.github/workflows/backwards_compatibility_tests_workflow.yml
+++ b/.github/workflows/backwards_compatibility_tests_workflow.yml
@@ -13,7 +13,8 @@ jobs:
   Restart-Upgrade-BWCTests-NeuralSearch:
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [ 11, 17, 21.0.1 ]
         os: [ubuntu-latest,windows-latest]
         bwc_version : ["2.9.0","2.10.0","2.11.0","2.12.0-SNAPSHOT"]
         opensearch_version : [ "3.0.0-SNAPSHOT" ]
@@ -35,7 +36,8 @@ jobs:
       - name: Run NeuralSearch Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}}
         run: |
           echo "Running restart-upgrade backwards compatibility tests ..."
-          ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}'
+#          Disabling BWC tests due to ongoing build failure. https://github.com/opensearch-project/neural-search/issues/536
+#          ./gradlew :qa:restart-upgrade:testAgainstNewCluster -D'tests.bwc.version=${{ matrix.bwc_version }}'
 
   Rolling-Upgrade-BWCTests-NeuralSearch:
     strategy:
@@ -62,4 +64,5 @@ jobs:
       - name: Run NeuralSearch Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on ${{matrix.os}}
         run: |
           echo "Running rolling-upgrade backwards compatibility tests ..."
-          ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'
+#          Disabling BWC tests due to ongoing build failure. https://github.com/opensearch-project/neural-search/issues/536
+#         ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'
diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml
index 5b7f9d210..163b8c220 100644
--- a/.github/workflows/test_security.yml
+++ b/.github/workflows/test_security.yml
@@ -20,7 +20,8 @@ jobs:
   integ-test-with-security-linux:
     strategy:
       matrix:
-        java: [11, 17, 21]
+        # Restricting java 21 to 21.0.1 due to ongoing bug in JDK 21.0.2 https://bugs.openjdk.org/browse/JDK-8323659. Once the fix https://github.com/opensearch-project/OpenSearch/pull/11968 get merged this change will be reverted.
+        java: [11, 17, 21.0.1]
 
     name: Run Integration Tests on Linux
     runs-on: ubuntu-latest
diff --git a/build.gradle b/build.gradle
index c4228d8be..c2dc3a317 100644
--- a/build.gradle
+++ b/build.gradle
@@ -66,7 +66,7 @@ ext {
     projectSubstitutions = [:]
 
     configureSecurityPlugin = { OpenSearchCluster cluster ->
-        configurations.zipArchive.asFileTree.each {
+        configurations.secureIntegTestPluginArchive.asFileTree.each {
             if(it.name.contains("opensearch-security")){
                 cluster.plugin(provider(new Callable<RegularFile>() {
                     @Override
@@ -160,6 +160,7 @@ allprojects {
 
 configurations {
     zipArchive
+    secureIntegTestPluginArchive
 }
 
 tasks.register("preparePluginPathDirs") {
@@ -231,7 +232,7 @@ dependencies {
     api "org.opensearch:opensearch:${opensearch_version}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
-    zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"
+    secureIntegTestPluginArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"
     compileOnly fileTree(dir: knnJarDirectory, include: '*.jar')
     api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
     testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}"
@@ -337,8 +338,7 @@ testClusters.integTest {
 
     // Install K-NN/ml-commons plugins on the integTest cluster nodes except security
     configurations.zipArchive.asFileTree.each {
-        if(!it.name.contains("opensearch-security")) {
-            plugin(provider(new Callable<RegularFile>(){
+        plugin(provider(new Callable<RegularFile>(){
                 @Override
                 RegularFile call() throws Exception {
                     return new RegularFile() {
@@ -348,8 +348,7 @@ testClusters.integTest {
                         }
                     }
                 }
-            }))
-        }
+        }))
     }
 
     // This installs our neural-search plugin into the testClusters