From 80ca2f168ed6189638ec9c463a23497fc9ef666d Mon Sep 17 00:00:00 2001
From: TOUSSAINT Briac <briac.toussaint@inria.fr>
Date: Tue, 13 Aug 2024 17:41:55 +0200
Subject: [PATCH] support for non-centered cameras

---
 gaussian_renderer/__init__.py                 |  4 ++
 scene/cameras.py                              | 16 ++++--
 scene/dataset_readers.py                      | 17 ++++--
 .../cuda_rasterizer/backward.cu               | 14 +++--
 .../cuda_rasterizer/backward.h                | 10 ++--
 .../cuda_rasterizer/forward.cu                | 54 +++++++++++--------
 .../cuda_rasterizer/forward.h                 | 24 +++++----
 .../cuda_rasterizer/rasterizer.h              | 15 +++---
 .../cuda_rasterizer/rasterizer_impl.cu        | 22 +++++---
 .../diff_gaussian_rasterization/__init__.py   |  8 +++
 .../rasterize_points.cu                       |  6 +++
 .../rasterize_points.h                        |  3 ++
 train.py                                      | 22 +++++++-
 utils/camera_utils.py                         |  1 +
 utils/depth_utils.py                          |  5 +-
 utils/graphics_utils.py                       | 35 ++++++++++++
 16 files changed, 192 insertions(+), 64 deletions(-)

diff --git a/gaussian_renderer/__init__.py b/gaussian_renderer/__init__.py
index d159583..b0ebe43 100644
--- a/gaussian_renderer/__init__.py
+++ b/gaussian_renderer/__init__.py
@@ -41,6 +41,8 @@ def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor,
         image_width=int(viewpoint_camera.image_width),
         tanfovx=tanfovx,
         tanfovy=tanfovy,
+        cx=viewpoint_camera.cx,
+        cy=viewpoint_camera.cy,
         kernel_size=kernel_size,
         subpixel_offset=subpixel_offset,
         bg=bg_color,
@@ -141,6 +143,8 @@ def integrate(points3D, viewpoint_camera, pc : GaussianModel, pipe, bg_color : t
         image_width=int(viewpoint_camera.image_width),
         tanfovx=tanfovx,
         tanfovy=tanfovy,
+        cx=viewpoint_camera.cx,
+        cy=viewpoint_camera.cy,
         kernel_size=kernel_size,
         subpixel_offset=subpixel_offset,
         bg=bg_color,
diff --git a/scene/cameras.py b/scene/cameras.py
index 81aef99..f15bbf7 100644
--- a/scene/cameras.py
+++ b/scene/cameras.py
@@ -12,10 +12,10 @@
 import torch
 from torch import nn
 import numpy as np
-from utils.graphics_utils import getWorld2View2, getProjectionMatrix
+from utils.graphics_utils import getWorld2View2, getProjectionMatrix, getProjectionMatrixShift
 
 class Camera(nn.Module):
-    def __init__(self, colmap_id, R, T, FoVx, FoVy, image, gt_alpha_mask,
+    def __init__(self, colmap_id, R, T, FoVx, FoVy, focal_x, focal_y, cx, cy, image, gt_alpha_mask,
                  image_name, uid,
                  trans=np.array([0.0, 0.0, 0.0]), scale=1.0, data_device = "cuda"
                  ):
@@ -27,6 +27,10 @@ def __init__(self, colmap_id, R, T, FoVx, FoVy, image, gt_alpha_mask,
         self.T = T
         self.FoVx = FoVx
         self.FoVy = FoVy
+        self.focal_x = focal_x
+        self.focal_y = focal_y
+        self.cx = cx
+        self.cy = cy
         self.image_name = image_name
 
         try:
@@ -54,7 +58,11 @@ def __init__(self, colmap_id, R, T, FoVx, FoVy, image, gt_alpha_mask,
         self.scale = scale
 
         self.world_view_transform = torch.tensor(getWorld2View2(R, T, trans, scale)).transpose(0, 1).cuda()
-        self.projection_matrix = getProjectionMatrix(znear=self.znear, zfar=self.zfar, fovX=self.FoVx, fovY=self.FoVy).transpose(0,1).cuda()
+        # self.projection_matrix = getProjectionMatrix(znear=self.znear, zfar=self.zfar, fovX=self.FoVx, fovY=self.FoVy).transpose(0,1).cuda()
+        self.projection_matrix = getProjectionMatrixShift(znear=self.znear, zfar=self.zfar, 
+                                                          focal_x=self.focal_x, focal_y=self.focal_y, cx=self.cx, cy=self.cy,
+                                                          width=self.image_width, height=self.image_height,
+                                                          fovX=self.FoVx, fovY=self.FoVy).transpose(0,1).cuda()
         self.full_proj_transform = (self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0)
         self.camera_center = self.world_view_transform.inverse()[3, :3]
         
@@ -69,6 +77,8 @@ def __init__(self, width, height, fovy, fovx, znear, zfar, world_view_transform,
         self.image_height = height    
         self.FoVy = fovy
         self.FoVx = fovx
+        self.cx = width / 2.0
+        self.cy = height / 2.0
         self.znear = znear
         self.zfar = zfar
         self.world_view_transform = world_view_transform
diff --git a/scene/dataset_readers.py b/scene/dataset_readers.py
index 012da33..4d74192 100644
--- a/scene/dataset_readers.py
+++ b/scene/dataset_readers.py
@@ -29,6 +29,10 @@ class CameraInfo(NamedTuple):
     T: np.array
     FovY: np.array
     FovX: np.array
+    focal_length_x: np.array
+    focal_length_y: np.array
+    cx: np.array
+    cy: np.array
     image: np.array
     image_path: str
     image_name: str
@@ -83,12 +87,16 @@ def readColmapCameras(cam_extrinsics, cam_intrinsics, images_folder):
         T = np.array(extr.tvec)
 
         if intr.model=="SIMPLE_PINHOLE":
-            focal_length_x = intr.params[0]
-            FovY = focal2fov(focal_length_x, height)
+            focal_length_y = focal_length_x = intr.params[0]
+            FovY = focal2fov(focal_length_y, height)
             FovX = focal2fov(focal_length_x, width)
+            cx = width / 2
+            cy = height / 2
         elif intr.model=="PINHOLE":
             focal_length_x = intr.params[0]
             focal_length_y = intr.params[1]
+            cx = intr.params[2]
+            cy = intr.params[3]
             FovY = focal2fov(focal_length_y, height)
             FovX = focal2fov(focal_length_x, width)
         else:
@@ -103,8 +111,9 @@ def readColmapCameras(cam_extrinsics, cam_intrinsics, images_folder):
         
         image = Image.open(image_path)
 
-        cam_info = CameraInfo(uid=uid, R=R, T=T, FovY=FovY, FovX=FovX, image=image,
-                              image_path=image_path, image_name=image_name, width=width, height=height)
+        cam_info = CameraInfo(uid=uid, R=R, T=T, FovY=FovY, FovX=FovX, 
+                              focal_length_x=focal_length_x, focal_length_y=focal_length_y, cx=cx, cy=cy, 
+                              image=image, image_path=image_path, image_name=image_name, width=width, height=height)
         cam_infos.append(cam_info)
     sys.stdout.write('\n')
     return cam_infos
diff --git a/submodules/diff-gaussian-rasterization/cuda_rasterizer/backward.cu b/submodules/diff-gaussian-rasterization/cuda_rasterizer/backward.cu
index 4e62e80..9c7f956 100644
--- a/submodules/diff-gaussian-rasterization/cuda_rasterizer/backward.cu
+++ b/submodules/diff-gaussian-rasterization/cuda_rasterizer/backward.cu
@@ -636,8 +636,9 @@ __global__ void __launch_bounds__(BLOCK_X * BLOCK_Y)
 renderCUDA(
 	const uint2* __restrict__ ranges,
 	const uint32_t* __restrict__ point_list,
-	int W, int H,
-	float focal_x, float focal_y,
+	const int W, const int H,
+	const float focal_x, const float focal_y,
+	const float cx, const float cy,
 	const float2* __restrict__ subpixel_offset,
 	const float* __restrict__ bg_color,
 	const float2* __restrict__ points_xy_image,
@@ -679,7 +680,7 @@ renderCUDA(
 	int toDo = range.y - range.x;
 
 	// create the ray
-	float2 ray = { (pixf.x - W/2.) / focal_x, (pixf.y - H/2.) / focal_y };
+	float2 ray = { (pixf.x - cx) / focal_x, (pixf.y - cy) / focal_y };
 
 	__shared__ int collected_id[BLOCK_SIZE];
 	__shared__ float2 collected_xy[BLOCK_SIZE];
@@ -968,6 +969,7 @@ void BACKWARD::preprocess(
 	const float* viewmatrix,
 	const float* projmatrix,
 	const float focal_x, float focal_y,
+	const float cx, const float cy,
 	const float tan_fovx, float tan_fovy,
 	const float kernel_size,
 	const glm::vec3* campos,
@@ -1036,8 +1038,9 @@ void BACKWARD::render(
 	const dim3 grid, const dim3 block,
 	const uint2* ranges,
 	const uint32_t* point_list,
-	int W, int H,
-	float focal_x, float focal_y,
+	const int W, const int H,
+	const float focal_x, const float focal_y,
+	const float cx, const float cy,
 	const float2* subpixel_offset,
 	const float* bg_color,
 	const float2* means2D,
@@ -1066,6 +1069,7 @@ void BACKWARD::render(
 		point_list,
 		W, H,
 		focal_x, focal_y,
+		cx, cy,
 		subpixel_offset,
 		bg_color,
 		means2D,
diff --git a/submodules/diff-gaussian-rasterization/cuda_rasterizer/backward.h b/submodules/diff-gaussian-rasterization/cuda_rasterizer/backward.h
index 9866d1f..51afef9 100644
--- a/submodules/diff-gaussian-rasterization/cuda_rasterizer/backward.h
+++ b/submodules/diff-gaussian-rasterization/cuda_rasterizer/backward.h
@@ -25,8 +25,9 @@ namespace BACKWARD
 		const dim3 grid, dim3 block,
 		const uint2* ranges,
 		const uint32_t* point_list,
-		int W, int H,
-		float focal_x, float focal_y,
+		const int W, const int H,
+		const float focal_x, const float focal_y,
+		const float cx, const float cy,
 		const float2* subpixel_offset,
 		const float* bg_color,
 		const float2* means2D,
@@ -63,8 +64,9 @@ namespace BACKWARD
 		const float* view2gaussian,
 		const float* view,
 		const float* proj,
-		const float focal_x, float focal_y,
-		const float tan_fovx, float tan_fovy,
+		const float focal_x, const float focal_y,
+		const float cx, const float cy,
+		const float tan_fovx, const float tan_fovy,
 		const float kernel_size,
 		const glm::vec3* campos,
 		float3* dL_dmean2D,
diff --git a/submodules/diff-gaussian-rasterization/cuda_rasterizer/forward.cu b/submodules/diff-gaussian-rasterization/cuda_rasterizer/forward.cu
index 50e6390..3280cfa 100644
--- a/submodules/diff-gaussian-rasterization/cuda_rasterizer/forward.cu
+++ b/submodules/diff-gaussian-rasterization/cuda_rasterizer/forward.cu
@@ -411,8 +411,9 @@ __global__ void __launch_bounds__(BLOCK_X * BLOCK_Y)
 renderCUDA(
 	const uint2* __restrict__ ranges,
 	const uint32_t* __restrict__ point_list,
-	int W, int H,
-	float focal_x, float focal_y,
+	const int W, const int H,
+	const float focal_x, const float focal_y,
+	const float cx, const float cy,
 	const float2* __restrict__ subpixel_offset,
 	const float2* __restrict__ points_xy_image,
 	const float* __restrict__ features,
@@ -445,7 +446,7 @@ renderCUDA(
 	bool done = !inside;
 
 	// create the ray
-	float2 ray = { (pixf.x - W/2.) / focal_x, (pixf.y - H/2.) / focal_y };
+	float2 ray = { (pixf.x - cx) / focal_x, (pixf.y - cy) / focal_y };
 
 	// Load start/end range of IDs to process in bit sorted list.
 	uint2 range = ranges[block.group_index().y * horizontal_blocks + block.group_index().x];
@@ -615,8 +616,9 @@ void FORWARD::render(
 	const dim3 grid, dim3 block,
 	const uint2* ranges,
 	const uint32_t* point_list,
-	int W, int H,
-	float focal_x, float focal_y,
+	const int W, const int H,
+	const float focal_x, const float focal_y,
+	const float cx, const float cy,
 	const float2* subpixel_offset,
 	const float2* means2D,
 	const float* colors,
@@ -639,6 +641,7 @@ void FORWARD::render(
 		point_list,
 		W, H,
 		focal_x, focal_y,
+		cx, cy,
 		subpixel_offset,
 		means2D,
 		colors,
@@ -671,9 +674,10 @@ void FORWARD::preprocess(int P, int D, int M,
 	const float* viewmatrix,
 	const float* projmatrix,
 	const glm::vec3* cam_pos,
-	const int W, int H,
-	const float focal_x, float focal_y,
-	const float tan_fovx, float tan_fovy,
+	const int W, const int H,
+	const float focal_x, const float focal_y,
+	const float cx, const float cy,
+	const float tan_fovx, const float tan_fovy,
 	const float kernel_size,
 	int* radii,
 	float2* means2D,
@@ -725,9 +729,10 @@ __global__ void preprocessPointsCUDA(int P, int D, int M,
 	const float* viewmatrix,
 	const float* projmatrix,
 	const glm::vec3* cam_pos,
-	const int W, int H,
-	const float tan_fovx, float tan_fovy,
-	const float focal_x, float focal_y,
+	const int W, const int H,
+	const float tan_fovx, const float tan_fovy,
+	const float focal_x, const float focal_y,
+	const float cx, const float cy,
 	float2* points2D,
 	float* depths,
 	const dim3 grid,
@@ -753,7 +758,7 @@ __global__ void preprocessPointsCUDA(int P, int D, int M,
 	float p_w = 1.0f / (p_hom.w + 0.0000001f);
 	float3 p_proj = { p_hom.x * p_w, p_hom.y * p_w, p_hom.z * p_w };
 
-	float2 point_image = {focal_x * p_view.x / (p_view.z + 0.0000001f) + W/2., focal_y * p_view.y / (p_view.z + 0.0000001f) + H/2.};
+	float2 point_image = {focal_x * p_view.x / (p_view.z + 0.0000001f) + cx, focal_y * p_view.y / (p_view.z + 0.0000001f) + cy};
 
 	// If the point is outside the image, quit.
 	if (point_image.x < 0 || point_image.x >= W || point_image.y < 0 || point_image.y >= H)
@@ -770,9 +775,10 @@ void FORWARD::preprocess_points(int PN, int D, int M,
 		const float* viewmatrix,
 		const float* projmatrix,
 		const glm::vec3* cam_pos,
-		const int W, int H,
-		const float focal_x, float focal_y,
-		const float tan_fovx, float tan_fovy,
+		const int W, const int H,
+		const float focal_x, const float focal_y,
+		const float cx, const float cy,
+		const float tan_fovx, const float tan_fovy,
 		float2* points2D,
 		float* depths,
 		const dim3 grid,
@@ -788,6 +794,7 @@ void FORWARD::preprocess_points(int PN, int D, int M,
 		W, H,
 		tan_fovx, tan_fovy,
 		focal_x, focal_y,
+		cx, cy,
 		points2D,
 		depths,
 		grid,
@@ -807,8 +814,9 @@ integrateCUDA(
 	const uint2* __restrict__ point_ranges,
 	const uint32_t* __restrict__ gaussian_list,
 	const uint32_t* __restrict__ point_list,
-	int W, int H,
-	float focal_x, float focal_y,
+	const int W, const int H,
+	const float focal_x, const float focal_y,
+	const float cx, const float cy,
 	const float2* __restrict__ subpixel_offset,
 	const float2* __restrict__ points2D,
 	const float* __restrict__ features,
@@ -845,7 +853,7 @@ integrateCUDA(
 	const float depth_input = inside ? subpixel_offset[pix_id].x : 0.0f;
 
 	// create the ray
-	float2 ray = { (pixf.x - W/2.) / focal_x, (pixf.y - H/2.) / focal_y };
+	float2 ray = { (pixf.x - cx) / focal_x, (pixf.y - cy) / focal_y };
 
 	// Load start/end range of IDs to process in bit sorted list.
 	uint2 range = gaussian_ranges[block.group_index().y * horizontal_blocks + block.group_index().x];
@@ -917,7 +925,7 @@ integrateCUDA(
 			
 			bool used = false;
 			for (int k = 0; k < 5; ++k){
-				float3 ray_point = { (pixf.x + offset_xs[k] - W/2.) / focal_x, (pixf.y + offset_ys[k] - H/2.) / focal_y, 1.0f };
+				float3 ray_point = { (pixf.x + offset_xs[k] - cx) / focal_x, (pixf.y + offset_ys[k] - cy) / focal_y, 1.0f };
 				
 				const float normal[3] = { view2gaussian_j[0] * ray_point.x + view2gaussian_j[1] * ray_point.y + view2gaussian_j[2], 
 									      view2gaussian_j[1] * ray_point.x + view2gaussian_j[3] * ray_point.y + view2gaussian_j[4],
@@ -1155,7 +1163,7 @@ integrateCUDA(
 				// iterate over all projected points
 				for (int k = 0; k < num_projected; k++){
 					// create the ray
-					float3 ray_point = { (projected_xy[k].x - W/2.) / focal_x, (projected_xy[k].y - H/2.) / focal_y, 1.0 };
+					float3 ray_point = { (projected_xy[k].x - cx) / focal_x, (projected_xy[k].y - cy) / focal_y, 1.0 };
 					float ray_depth = projected_depth[k];
 
 					const float normal[3] = { view2gaussian_j[0] * ray_point.x + view2gaussian_j[1] * ray_point.y + view2gaussian_j[2], 
@@ -1223,8 +1231,9 @@ void FORWARD::integrate(
 	const uint2* point_ranges,
 	const uint32_t* gaussian_list,
 	const uint32_t* point_list,
-	int W, int H,
-	float focal_x, float focal_y,
+	const int W, const int H,
+	const float focal_x, const float focal_y,
+	const float cx, const float cy,
 	const float2* subpixel_offset,
 	const float2* points2D,
 	const float* colors,
@@ -1251,6 +1260,7 @@ void FORWARD::integrate(
 		point_list,
 		W, H,
 		focal_x, focal_y,
+		cx, cy,
 		subpixel_offset,
 		points2D,
 		colors,
diff --git a/submodules/diff-gaussian-rasterization/cuda_rasterizer/forward.h b/submodules/diff-gaussian-rasterization/cuda_rasterizer/forward.h
index 7cc5831..7463955 100644
--- a/submodules/diff-gaussian-rasterization/cuda_rasterizer/forward.h
+++ b/submodules/diff-gaussian-rasterization/cuda_rasterizer/forward.h
@@ -35,9 +35,10 @@ namespace FORWARD
 		const float* viewmatrix,
 		const float* projmatrix,
 		const glm::vec3* cam_pos,
-		const int W, int H,
-		const float focal_x, float focal_y,
-		const float tan_fovx, float tan_fovy,
+		const int W, const int H,
+		const float focal_x, const float focal_y,
+		const float cx, const float cy,
+		const float tan_fovx, const float tan_fovy,
 		const float kernel_size,
 		int* radii,
 		float2* points_xy_image,
@@ -55,8 +56,9 @@ namespace FORWARD
 		const dim3 grid, dim3 block,
 		const uint2* ranges,
 		const uint32_t* point_list,
-		int W, int H,
-		float focal_x, float focal_y,
+		const int W, const int H,
+		const float focal_x, const float focal_y,
+		const float cx, const float cy,
 		const float2* subpixel_offset,
 		const float2* points_xy_image,
 		const float* features,
@@ -80,9 +82,10 @@ namespace FORWARD
 		const float* viewmatrix,
 		const float* projmatrix,
 		const glm::vec3* cam_pos,
-		const int W, int H,
-		const float focal_x, float focal_y,
-		const float tan_fovx, float tan_fovy,
+		const int W, const int H,
+		const float focal_x, const float focal_y,
+		const float cx, const float cy,
+		const float tan_fovx, const float tan_fovy,
 		float2* points2D,
 		float* depths,
 		const dim3 grid,
@@ -96,8 +99,9 @@ namespace FORWARD
 		const uint2* point_ranges,
 		const uint32_t* gaussian_list,
 		const uint32_t* point_list,
-		int W, int H,
-		float focal_x, float focal_y,
+		const int W, const int H,
+		const float focal_x, const float focal_y,
+		const float cx, const float cy,
 		const float2* subpixel_offset,
 		const float2* points2D,
 		const float* features,
diff --git a/submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer.h b/submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer.h
index f3c3d11..adc07ea 100644
--- a/submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer.h
+++ b/submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer.h
@@ -34,7 +34,7 @@ namespace CudaRasterizer
 			std::function<char* (size_t)> imageBuffer,
 			const int P, int D, int M,
 			const float* background,
-			const int width, int height,
+			const int width, const int height,
 			const float* means3D,
 			const float* shs,
 			const float* colors_precomp,
@@ -47,7 +47,8 @@ namespace CudaRasterizer
 			const float* viewmatrix,
 			const float* projmatrix,
 			const float* cam_pos,
-			const float tan_fovx, float tan_fovy,
+			const float tan_fovx, const float tan_fovy,
+			const float cx, const float cy,
 			const float kernel_size,
 			const float* subpixel_offset,
 			const bool prefiltered,
@@ -58,7 +59,7 @@ namespace CudaRasterizer
 		static void backward(
 			const int P, int D, int M, int R,
 			const float* background,
-			const int width, int height,
+			const int width, const int height,
 			const float* means3D,
 			const float* shs,
 			const float* colors_precomp,
@@ -70,7 +71,8 @@ namespace CudaRasterizer
 			const float* viewmatrix,
 			const float* projmatrix,
 			const float* campos,
-			const float tan_fovx, float tan_fovy,
+			const float tan_fovx, const float tan_fovy,
+			const float cx, const float cy,
 			const float kernel_size,
 			const float* subpixel_offset,
 			const int* radii,
@@ -98,7 +100,7 @@ namespace CudaRasterizer
 			std::function<char* (size_t)> point_binningBuffer,
 			const int PN, const int P, int D, int M,
 			const float* background,
-			const int width, int height,
+			const int width, const int height,
 			const float* points3D,
 			const float* means3D,
 			const float* shs,
@@ -112,7 +114,8 @@ namespace CudaRasterizer
 			const float* viewmatrix,
 			const float* projmatrix,
 			const float* cam_pos,
-			const float tan_fovx, float tan_fovy,
+			const float tan_fovx, const float tan_fovy,
+			const float cx, const float cy,
 			const float kernel_size,
 			const float* subpixel_offset,
 			const bool prefiltered,
diff --git a/submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer_impl.cu b/submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer_impl.cu
index 268a1d1..53478a3 100644
--- a/submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer_impl.cu
+++ b/submodules/diff-gaussian-rasterization/cuda_rasterizer/rasterizer_impl.cu
@@ -250,7 +250,7 @@ int CudaRasterizer::Rasterizer::forward(
 	std::function<char* (size_t)> imageBuffer,
 	const int P, int D, int M,
 	const float* background,
-	const int width, int height,
+	const int width, const int height,
 	const float* means3D,
 	const float* shs,
 	const float* colors_precomp,
@@ -263,7 +263,8 @@ int CudaRasterizer::Rasterizer::forward(
 	const float* viewmatrix,
 	const float* projmatrix,
 	const float* cam_pos,
-	const float tan_fovx, float tan_fovy,
+	const float tan_fovx, const float tan_fovy,
+	const float cx, const float cy,
 	const float kernel_size,
 	const float* subpixel_offset,
 	const bool prefiltered,
@@ -313,6 +314,7 @@ int CudaRasterizer::Rasterizer::forward(
 		(glm::vec3*)cam_pos,
 		width, height,
 		focal_x, focal_y,
+		cx, cy,
 		tan_fovx, tan_fovy,
 		kernel_size,
 		radii,
@@ -384,6 +386,7 @@ int CudaRasterizer::Rasterizer::forward(
 		binningState.point_list,
 		width, height,
 		focal_x, focal_y,
+		cx, cy,
 		(float2*)subpixel_offset,
 		geomState.means2D,
 		feature_ptr,
@@ -409,7 +412,7 @@ int CudaRasterizer::Rasterizer::forward(
 void CudaRasterizer::Rasterizer::backward(
 	const int P, int D, int M, int R,
 	const float* background,
-	const int width, int height,
+	const int width, const int height,
 	const float* means3D,
 	const float* shs,
 	const float* colors_precomp,
@@ -421,7 +424,8 @@ void CudaRasterizer::Rasterizer::backward(
 	const float* viewmatrix,
 	const float* projmatrix,
 	const float* campos,
-	const float tan_fovx, float tan_fovy,
+	const float tan_fovx, const float tan_fovy,
+	const float cx, const float cy,
 	const float kernel_size,
 	const float* subpixel_offset,
 	const int* radii,
@@ -473,6 +477,7 @@ void CudaRasterizer::Rasterizer::backward(
 		binningState.point_list,
 		width, height,
 		focal_x, focal_y,
+		cx, cy,
 		(float2*)subpixel_offset,
 		background,
 		geomState.means2D,
@@ -509,6 +514,7 @@ void CudaRasterizer::Rasterizer::backward(
 		viewmatrix,
 		projmatrix,
 		focal_x, focal_y,
+		cx, cy,
 		tan_fovx, tan_fovy,
 		kernel_size,
 		(glm::vec3*)campos,
@@ -535,7 +541,7 @@ int CudaRasterizer::Rasterizer::integrate(
 	std::function<char* (size_t)> point_binningBuffer,
 	const int PN, const int P, int D, int M,
 	const float* background,
-	const int width, int height,
+	const int width, const int height,
 	const float* points3D,
 	const float* means3D,
 	const float* shs,
@@ -549,7 +555,8 @@ int CudaRasterizer::Rasterizer::integrate(
 	const float* viewmatrix,
 	const float* projmatrix,
 	const float* cam_pos,
-	const float tan_fovx, float tan_fovy,
+	const float tan_fovx, const float tan_fovy,
+	const float cx, const float cy,
 	const float kernel_size,
 	const float* subpixel_offset,
 	const bool prefiltered,
@@ -605,6 +612,7 @@ int CudaRasterizer::Rasterizer::integrate(
 		(glm::vec3*)cam_pos,
 		width, height,
 		focal_x, focal_y,
+		cx, cy,
 		tan_fovx, tan_fovy,
 		kernel_size,
 		radii,
@@ -685,6 +693,7 @@ int CudaRasterizer::Rasterizer::integrate(
 		(glm::vec3*)cam_pos,
 		width, height,
 		focal_x, focal_y,
+		cx, cy,
 		tan_fovx, tan_fovy,
 		pointState.points2D,
 		pointState.depths,
@@ -769,6 +778,7 @@ int CudaRasterizer::Rasterizer::integrate(
 		point_binningState.point_list,
 		width, height,
 		focal_x, focal_y,
+		cx, cy,
 		(float2*)subpixel_offset,
 		pointState.points2D,
 		feature_ptr,
diff --git a/submodules/diff-gaussian-rasterization/diff_gaussian_rasterization/__init__.py b/submodules/diff-gaussian-rasterization/diff_gaussian_rasterization/__init__.py
index 080302f..645927e 100644
--- a/submodules/diff-gaussian-rasterization/diff_gaussian_rasterization/__init__.py
+++ b/submodules/diff-gaussian-rasterization/diff_gaussian_rasterization/__init__.py
@@ -74,6 +74,8 @@ def forward(
             raster_settings.projmatrix,
             raster_settings.tanfovx,
             raster_settings.tanfovy,
+            raster_settings.cx,
+            raster_settings.cy,
             raster_settings.kernel_size,
             raster_settings.subpixel_offset,
             raster_settings.image_height,
@@ -125,6 +127,8 @@ def backward(ctx, grad_out_color, _):
                 raster_settings.projmatrix, 
                 raster_settings.tanfovx, 
                 raster_settings.tanfovy, 
+                raster_settings.cx,
+                raster_settings.cy,
                 raster_settings.kernel_size,
                 raster_settings.subpixel_offset,
                 grad_out_color, 
@@ -169,6 +173,8 @@ class GaussianRasterizationSettings(NamedTuple):
     image_width: int 
     tanfovx : float
     tanfovy : float
+    cx : float
+    cy : float
     kernel_size : float
     subpixel_offset: torch.Tensor
     bg : torch.Tensor
@@ -279,6 +285,8 @@ def integrate(self, points3D, means3D, means2D, opacities, shs = None, colors_pr
             raster_settings.projmatrix,
             raster_settings.tanfovx,
             raster_settings.tanfovy,
+            raster_settings.cx,
+            raster_settings.cy,
             raster_settings.kernel_size,
             raster_settings.subpixel_offset,
             raster_settings.image_height,
diff --git a/submodules/diff-gaussian-rasterization/rasterize_points.cu b/submodules/diff-gaussian-rasterization/rasterize_points.cu
index c675434..690c317 100644
--- a/submodules/diff-gaussian-rasterization/rasterize_points.cu
+++ b/submodules/diff-gaussian-rasterization/rasterize_points.cu
@@ -48,6 +48,7 @@ RasterizeGaussiansCUDA(
 	const torch::Tensor& projmatrix,
 	const float tan_fovx, 
 	const float tan_fovy,
+	const float cx, const float cy,
 	const float kernel_size,
 	const torch::Tensor& subpixel_offset,
     const int image_height,
@@ -111,6 +112,7 @@ RasterizeGaussiansCUDA(
 		campos.contiguous().data<float>(),
 		tan_fovx,
 		tan_fovy,
+		cx, cy,
 		kernel_size,
 		subpixel_offset.contiguous().data<float>(),
 		prefiltered,
@@ -136,6 +138,7 @@ std::tuple<torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Te
     const torch::Tensor& projmatrix,
 	const float tan_fovx,
 	const float tan_fovy,
+	const float cx, const float cy,
 	const float kernel_size,
 	const torch::Tensor& subpixel_offset,
     const torch::Tensor& dL_dout_color,
@@ -187,6 +190,7 @@ std::tuple<torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Te
 	  campos.contiguous().data<float>(),
 	  tan_fovx,
 	  tan_fovy,
+	  cx, cy,
 	  kernel_size,
 	  subpixel_offset.contiguous().data<float>(),
 	  radii.contiguous().data<int>(),
@@ -247,6 +251,7 @@ IntegrateGaussiansToPointsCUDA(
 	const torch::Tensor& projmatrix,
 	const float tan_fovx, 
 	const float tan_fovy,
+	const float cx, const float cy,
 	const float kernel_size,
 	const torch::Tensor& subpixel_offset,
     const int image_height,
@@ -330,6 +335,7 @@ IntegrateGaussiansToPointsCUDA(
 		campos.contiguous().data<float>(),
 		tan_fovx,
 		tan_fovy,
+		cx, cy,
 		kernel_size,
 		subpixel_offset.contiguous().data<float>(),
 		prefiltered,
diff --git a/submodules/diff-gaussian-rasterization/rasterize_points.h b/submodules/diff-gaussian-rasterization/rasterize_points.h
index 4bebb7b..bb62497 100644
--- a/submodules/diff-gaussian-rasterization/rasterize_points.h
+++ b/submodules/diff-gaussian-rasterization/rasterize_points.h
@@ -30,6 +30,7 @@ RasterizeGaussiansCUDA(
 	const torch::Tensor& projmatrix,
 	const float tan_fovx, 
 	const float tan_fovy,
+	const float cx, const float cy,
 	const float kernel_size,
 	const torch::Tensor& subpixel_offset,
     const int image_height,
@@ -55,6 +56,7 @@ std::tuple<torch::Tensor, torch::Tensor, torch::Tensor, torch::Tensor, torch::Te
     const torch::Tensor& projmatrix,
 	const float tan_fovx, 
 	const float tan_fovy,
+	const float cx, const float cy,
 	const float kernel_size,
 	const torch::Tensor& subpixel_offset,
     const torch::Tensor& dL_dout_color,
@@ -88,6 +90,7 @@ IntegrateGaussiansToPointsCUDA(
 	const torch::Tensor& projmatrix,
 	const float tan_fovx, 
 	const float tan_fovy,
+	const float cx, const float cy,
 	const float kernel_size,
 	const torch::Tensor& subpixel_offset,
     const int image_height,
diff --git a/train.py b/train.py
index 03c7630..8a260a1 100644
--- a/train.py
+++ b/train.py
@@ -121,7 +121,23 @@ def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoi
     ema_loss_for_log = 0.0
     progress_bar = tqdm(range(first_iter, opt.iterations), desc="Training progress")
     first_iter += 1
-    for iteration in range(first_iter, opt.iterations + 1):        
+    for iteration in range(first_iter, opt.iterations + 1):
+        if network_gui.conn == None:
+            network_gui.try_connect()
+        while network_gui.conn != None:
+            try:
+                net_image_bytes = None
+                custom_cam, do_training, pipe.convert_SHs_python, pipe.compute_cov3D_python, keep_alive, scaling_modifer = network_gui.receive()
+                if custom_cam != None:
+                    net_image = render(custom_cam, gaussians, pipe, background, scaling_modifer)["render"]
+                    net_image = net_image[:3, :, :]
+                    net_image_bytes = memoryview((torch.clamp(net_image, min=0, max=1.0) * 255).byte().permute(1, 2, 0).contiguous().cpu().numpy())
+                network_gui.send(net_image_bytes, dataset.source_path)
+                if do_training and ((iteration < int(opt.iterations)) or not keep_alive):
+                    break
+            except Exception as e:
+                network_gui.conn = None
+        
 
         iter_start.record()
 
@@ -267,6 +283,8 @@ def training(dataset, opt, pipe, testing_iterations, saving_iterations, checkpoi
                 if iteration < opt.iterations - 100:
                     # don't update in the end of training
                     gaussians.compute_3D_filter(cameras=trainCameras)
+                memory_usage = torch.cuda.memory_reserved(0)/1E9
+                print("torch.cuda.memory_reserved: ", memory_usage, "(GB)")
         
             # Optimizer step
             if iteration < opt.iterations:
@@ -370,7 +388,7 @@ def training_report(tb_writer, iteration, Ll1, loss, l1_loss, elapsed, testing_i
     torch.cuda.set_device(torch.device("cuda:0"))
     
     # # Start GUI server, configure and run training
-    # network_gui.init(args.ip, args.port)
+    network_gui.init(args.ip, args.port)
     torch.autograd.set_detect_anomaly(args.detect_anomaly)
     training(lp.extract(args), op.extract(args), pp.extract(args), args.test_iterations, args.save_iterations, args.checkpoint_iterations, args.start_checkpoint, args.debug_from)
 
diff --git a/utils/camera_utils.py b/utils/camera_utils.py
index 8674fc8..9da8aa6 100644
--- a/utils/camera_utils.py
+++ b/utils/camera_utils.py
@@ -50,6 +50,7 @@ def loadCam(args, id, cam_info, resolution_scale):
 
     return Camera(colmap_id=cam_info.uid, R=cam_info.R, T=cam_info.T, 
                   FoVx=cam_info.FovX, FoVy=cam_info.FovY, 
+                  focal_x=cam_info.focal_length_x, focal_y=cam_info.focal_length_y, cx=cam_info.cx, cy=cam_info.cy,
                   image=gt_image, gt_alpha_mask=loaded_mask,
                   image_name=cam_info.image_name, uid=id, data_device=args.data_device)
 
diff --git a/utils/depth_utils.py b/utils/depth_utils.py
index 1630972..f6455fd 100644
--- a/utils/depth_utils.py
+++ b/utils/depth_utils.py
@@ -8,9 +8,10 @@ def depths_to_points(view, depthmap):
     W, H = view.image_width, view.image_height
     fx = W / (2 * math.tan(view.FoVx / 2.))
     fy = H / (2 * math.tan(view.FoVy / 2.))
+    
     intrins = torch.tensor(
-        [[fx, 0., W/2.],
-        [0., fy, H/2.],
+        [[fx, 0., view.cx],
+        [0., fy, view.cy],
         [0., 0., 1.0]]
     ).float().cuda()
     grid_x, grid_y = torch.meshgrid(torch.arange(W, device='cuda').float() + 0.5, torch.arange(H, device='cuda').float() + 0.5, indexing='xy')
diff --git a/utils/graphics_utils.py b/utils/graphics_utils.py
index b4627d8..c961017 100644
--- a/utils/graphics_utils.py
+++ b/utils/graphics_utils.py
@@ -70,6 +70,41 @@ def getProjectionMatrix(znear, zfar, fovX, fovY):
     P[2, 3] = -(zfar * znear) / (zfar - znear)
     return P
 
+def getProjectionMatrixShift(znear, zfar, focal_x, focal_y, cx, cy, width, height, fovX, fovY):
+    tanHalfFovY = math.tan((fovY / 2))
+    tanHalfFovX = math.tan((fovX / 2))
+
+    # the origin at center of image plane
+    top = tanHalfFovY * znear
+    bottom = -top
+    right = tanHalfFovX * znear
+    left = -right
+
+    # shift the frame window due to the non-zero principle point offsets
+    offset_x = cx - (width/2)
+    offset_x = (offset_x/focal_x)*znear
+    offset_y = cy - (height/2)
+    offset_y = (offset_y/focal_y)*znear
+
+    top = top + offset_y
+    left = left + offset_x
+    right = right + offset_x
+    bottom = bottom + offset_y
+
+    P = torch.zeros(4, 4)
+
+    z_sign = 1.0
+
+    P[0, 0] = 2.0 * znear / (right - left)
+    P[1, 1] = 2.0 * znear / (top - bottom)
+    P[0, 2] = (right + left) / (right - left)
+    P[1, 2] = (top + bottom) / (top - bottom)
+    P[3, 2] = z_sign
+    P[2, 2] = z_sign * zfar / (zfar - znear)
+    P[2, 3] = -(zfar * znear) / (zfar - znear)
+    return P
+
+
 def fov2focal(fov, pixels):
     return pixels / (2 * math.tan(fov / 2))