diff --git a/build/images/Dockerfile.build.windows b/build/images/Dockerfile.build.windows
index 0fdc2735c76..3ab92df561c 100644
--- a/build/images/Dockerfile.build.windows
+++ b/build/images/Dockerfile.build.windows
@@ -51,4 +51,6 @@ COPY --from=antrea-build-windows /wins/wins.exe /k/antrea/utils/wins.exe
 COPY --from=windows-ovs /Windows/System32/vcruntime140.dll /Windows/System32/
 COPY --from=windows-ovs /Windows/System32/libeay32.dll /Windows/System32/
 COPY --from=windows-ovs /Windows/System32/ssleay32.dll /Windows/System32/
-COPY --from=windows-ovs /openvswitch/usr/bin /k/antrea/bin/
+
+RUN mkdir C:\openvswitch
+COPY --from=windows-ovs /openvswitch /openvswitch/
diff --git a/build/yamls/antrea-windows-containerd-with-ovs.yml b/build/yamls/antrea-windows-containerd-with-ovs.yml
index 15a7748de94..cca93bae5bd 100644
--- a/build/yamls/antrea-windows-containerd-with-ovs.yml
+++ b/build/yamls/antrea-windows-containerd-with-ovs.yml
@@ -18,7 +18,7 @@ data:
     $ErrorActionPreference = "Stop"
     $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
     $mountPath =  ($mountPath.Replace('\', '/')).TrimEnd('/')
-    $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/k/antrea/bin"
+    $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/k/antrea/bin;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin"
     & antrea-agent --config=$mountPath/etc/antrea/antrea-agent.conf --logtostderr=false --log_dir=c:/var/log/antrea --alsologtostderr --log_file_max_size=100 --log_file_max_num=4 --v=0
   Run-AntreaOVS-Containerd.ps1: |
     $ErrorActionPreference = "Stop"
@@ -73,7 +73,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-agent-windows-b848hc8m8h
+  name: antrea-agent-windows-47gftkh2c5
   namespace: kube-system
 ---
 apiVersion: v1
@@ -315,7 +315,7 @@ spec:
         - $env:CONTAINER_SANDBOX_MOUNT_POINT/var/lib/antrea-windows/Run-AntreaOVS-Containerd.ps1
         command:
         - powershell
-        image: antrea/windows-ovs:1809-3.0.5
+        image: antrea/antrea-windows:latest
         imagePullPolicy: IfNotPresent
         name: antrea-ovs
         volumeMounts:
@@ -359,7 +359,7 @@ spec:
         name: antrea-windows-config
       - configMap:
           defaultMode: 420
-          name: antrea-agent-windows-b848hc8m8h
+          name: antrea-agent-windows-47gftkh2c5
         name: antrea-agent-windows
       - hostPath:
           path: /var/log/antrea/
diff --git a/build/yamls/antrea-windows-containerd.yml b/build/yamls/antrea-windows-containerd.yml
index dfbeeeab039..dab1040d477 100644
--- a/build/yamls/antrea-windows-containerd.yml
+++ b/build/yamls/antrea-windows-containerd.yml
@@ -18,13 +18,13 @@ data:
     $ErrorActionPreference = "Stop"
     $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
     $mountPath =  ($mountPath.Replace('\', '/')).TrimEnd('/')
-    $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/k/antrea/bin"
+    $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/k/antrea/bin;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin"
     & antrea-agent --config=$mountPath/etc/antrea/antrea-agent.conf --logtostderr=false --log_dir=c:/var/log/antrea --alsologtostderr --log_file_max_size=100 --log_file_max_num=4 --v=0
 kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-agent-windows-c6tftd5ttg
+  name: antrea-agent-windows-75tg78f7k9
   namespace: kube-system
 ---
 apiVersion: v1
@@ -279,7 +279,7 @@ spec:
         name: antrea-windows-config
       - configMap:
           defaultMode: 420
-          name: antrea-agent-windows-c6tftd5ttg
+          name: antrea-agent-windows-75tg78f7k9
         name: antrea-agent-windows
       - hostPath:
           path: /var/log/antrea/
diff --git a/build/yamls/windows/containerd-with-ovs/containerd-with-ovs.yml b/build/yamls/windows/containerd-with-ovs/containerd-with-ovs.yml
index 97209958097..d4df7b9a5fa 100644
--- a/build/yamls/windows/containerd-with-ovs/containerd-with-ovs.yml
+++ b/build/yamls/windows/containerd-with-ovs/containerd-with-ovs.yml
@@ -15,7 +15,7 @@ spec:
               exec:
                 command: ["powershell", "-file", "$env:CONTAINER_SANDBOX_MOUNT_POINT/var/lib/antrea-windows/VMSwitchExtension-AntreaAgent-Containerd.ps1", "-VMSwitchExtension", "enable"]
         - name: antrea-ovs
-          image: antrea/windows-ovs:1809-3.0.5
+          image: antrea-windows
           imagePullPolicy: IfNotPresent
           args:
             - -file
diff --git a/build/yamls/windows/containerd/conf/Run-AntreaAgent-Containerd.ps1 b/build/yamls/windows/containerd/conf/Run-AntreaAgent-Containerd.ps1
index 49e9dcdb831..bf86ccb5c80 100644
--- a/build/yamls/windows/containerd/conf/Run-AntreaAgent-Containerd.ps1
+++ b/build/yamls/windows/containerd/conf/Run-AntreaAgent-Containerd.ps1
@@ -1,5 +1,5 @@
 $ErrorActionPreference = "Stop"
 $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
 $mountPath =  ($mountPath.Replace('\', '/')).TrimEnd('/')
-$env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/k/antrea/bin"
+$env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/k/antrea/bin;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin"
 & antrea-agent --config=$mountPath/etc/antrea/antrea-agent.conf --logtostderr=false --log_dir=c:/var/log/antrea --alsologtostderr --log_file_max_size=100 --log_file_max_num=4 --v=0