Pulsar unified interface update

This commit addresses several braking changes to PyTorch3d camera conventions that made the unified system unusable at the moment (see facebookresearch#1352 and facebookresearch#1276). This commit fixes all these problems and tests using the test case from facebookresearch#1352. The results are equivalent for `FoVOrthographicCameras`, `OrthographicCameras`, `FoVPerspectiveCameras` and `PerspectiveCameras`.

We used to have a test case for the unified interface for Pulsar that compared the rendering results with some 'ground truth' images. This test seems to have been removed, thus leading to these breaking changes remaining undiscovered for a long time. Would be great if we could revive that test case!

pytorch3d/renderer/points/pulsar/unified.py

@@ -206,7 +206,7 @@ def _extract_intrinsics(  # noqa: C901
                         "The orthographic camera scale must be ((1.0, 1.0, 1.0),). "
                         f"{kwargs.get('scale_xyz', cameras.scale_xyz)[cloud_idx]}."
                     )
-                sensor_width = max_x - min_x
+                sensor_width = (max_x - min_x) * (self.renderer._renderer.width / self.renderer._renderer.height)
                 if not sensor_width > 0.0:
                     raise ValueError(
                         f"The orthographic camera must have positive size! Is: {sensor_width}."  # noqa: B950
@@ -220,24 +220,25 @@ def _extract_intrinsics(  # noqa: C901
                 focal_length_conf = kwargs.get("focal_length", cameras.focal_length)[
                     cloud_idx
                 ]
-                if (
-                    focal_length_conf.numel() == 2
-                    and focal_length_conf[0] * self.renderer._renderer.width
-                    - focal_length_conf[1] * self.renderer._renderer.height
-                    > 1e-5
-                ):
+                if torch.any(focal_length_conf <= 0.0):
+                    raise ValueError(f"Pulsar requires focal lengths > 0.0. Provided: {focal_length_conf}.")
+                if cameras.in_ndc():
+                    focal_length_conf *= self.renderer._renderer.height / 2.0
+                if (focal_length_conf.numel() == 2 and abs(focal_length_conf[0] - focal_length_conf[1]) > 1e-5):
                     raise ValueError(
                         "Pulsar only supports a single focal length! "
                         "Provided: %s." % (str(focal_length_conf))
                     )
                 if focal_length_conf.numel() == 2:
-                    sensor_width = 2.0 / focal_length_conf[0]
+                    focal_length_px = focal_length_conf[0]
                 else:
                     if focal_length_conf.numel() != 1:
                         raise ValueError(
                             "Focal length not parsable: %s." % (str(focal_length_conf))
                         )
-                    sensor_width = 2.0 / focal_length_conf
+                    focal_length_px = focal_length_conf
+                focal_length_px /= self.renderer._renderer.width / 2.0
+                sensor_width = 2.0 / focal_length_px
                 if "znear" not in kwargs.keys() or "zfar" not in kwargs.keys():
                     raise ValueError(
                         "pulsar needs znear and zfar values for "
@@ -248,16 +249,19 @@ def _extract_intrinsics(  # noqa: C901
                 zfar = kwargs["zfar"][cloud_idx]
                 principal_point_x = (
                     kwargs.get("principal_point", cameras.principal_point)[cloud_idx][0]
-                    * 0.5
-                    * self.renderer._renderer.width
                 )
                 principal_point_y = (
                     kwargs.get("principal_point", cameras.principal_point)[cloud_idx][1]
-                    * 0.5
-                    * self.renderer._renderer.height
                 )
+                if cameras.in_ndc():
+                    principal_point_x *= 0.5 * self.renderer._renderer.width * (self.renderer._renderer.height / self.renderer._renderer.width)
+                    principal_point_y *= -0.5 * self.renderer._renderer.height
+                else:
+                    principal_point_x = self.renderer._renderer.width / 2.0 - principal_point_x
+                    principal_point_y -= self.renderer._renderer.height / 2.0
         else:
             if not isinstance(cameras, PerspectiveCameras):
+                # This currently means FoVPerspectiveCameras.
                 # Create a virtual focal length that is closer than znear.
                 znear = kwargs.get("znear", cameras.znear)[cloud_idx]
                 zfar = kwargs.get("zfar", cameras.zfar)[cloud_idx]
@@ -266,7 +270,10 @@ def _extract_intrinsics(  # noqa: C901
                 afov = kwargs.get("fov", cameras.fov)[cloud_idx]
                 if kwargs.get("degrees", cameras.degrees):
                     afov *= math.pi / 180.0
-                sensor_width = math.tan(afov / 2.0) * 2.0 * focal_length
+                aspect_ratio = kwargs.get("aspect_ratio", cameras.aspect_ratio)[cloud_idx]
+                if aspect_ratio != 1.0:
+                    raise ValueError(f"Pulsar only supports aspect ration 1.0! Provided: {aspect_ratio}.")
+                sensor_width = math.tan(afov / 2.0) * 2.0 * focal_length * (self.renderer._renderer.width / self.renderer._renderer.height)
                 if not (
                     kwargs.get("aspect_ratio", cameras.aspect_ratio)[cloud_idx]
                     - self.renderer._renderer.width / self.renderer._renderer.height
@@ -286,10 +293,13 @@ def _extract_intrinsics(  # noqa: C901
                 focal_length_conf = kwargs.get("focal_length", cameras.focal_length)[
                     cloud_idx
                 ]
+                if torch.any(focal_length_conf <= 0.0):
+                    raise ValueError(f"Pulsar requires focal lengths > 0.0. Provided: {focal_length_conf}.")
+                if cameras.in_ndc():
+                    focal_length_conf *= self.renderer._renderer.height / 2.0
                 if (
                     focal_length_conf.numel() == 2
-                    and focal_length_conf[0] * self.renderer._renderer.width
-                    - focal_length_conf[1] * self.renderer._renderer.height
+                    and abs(focal_length_conf[0] - focal_length_conf[1])
                     > 1e-5
                 ):
                     raise ValueError(
@@ -312,6 +322,7 @@ def _extract_intrinsics(  # noqa: C901
                             "Focal length not parsable: %s." % (str(focal_length_conf))
                         )
                     focal_length_px = focal_length_conf
+                focal_length_px /= self.renderer._renderer.width / 2.0
                 focal_length = torch.tensor(
                     [
                         znear - 1e-6,
@@ -322,14 +333,16 @@ def _extract_intrinsics(  # noqa: C901
                 sensor_width = focal_length / focal_length_px * 2.0
                 principal_point_x = (
                     kwargs.get("principal_point", cameras.principal_point)[cloud_idx][0]
-                    * 0.5
-                    * self.renderer._renderer.width
                 )
                 principal_point_y = (
                     kwargs.get("principal_point", cameras.principal_point)[cloud_idx][1]
-                    * 0.5
-                    * self.renderer._renderer.height
                 )
+                if cameras.in_ndc():
+                    principal_point_x *= 0.5 * self.renderer._renderer.width * (self.renderer._renderer.height / self.renderer._renderer.width)
+                    principal_point_y *= -0.5 * self.renderer._renderer.height
+                else:
+                    principal_point_x = self.renderer._renderer.width / 2.0 - principal_point_x
+                    principal_point_y -= self.renderer._renderer.height / 2.0
         focal_length = _ensure_float_tensor(focal_length, device)
         sensor_width = _ensure_float_tensor(sensor_width, device)
         principal_point_x = _ensure_float_tensor(principal_point_x, device)
@@ -373,7 +386,7 @@ def _extract_extrinsics(
         return cam_pos, cam_rot
 
     def _get_vert_rad(
-        self, vert_pos, cam_pos, orthogonal_projection, focal_length, kwargs, cloud_idx
+        self, vert_pos, cam_pos, orthogonal_projection, focal_length, sensor_width, kwargs, cloud_idx
     ) -> torch.Tensor:
         """
         Get point radiuses.
@@ -403,12 +416,8 @@ def _get_vert_rad(
             )
         else:
             point_dists = torch.norm((vert_pos - cam_pos), p=2, dim=1, keepdim=False)
-            vert_rad = raster_rad / focal_length.to(vert_pos.device) * point_dists
-            if isinstance(self.rasterizer.cameras, PerspectiveCameras):
-                # NDC normalization happens through adjusted focal length.
-                pass
-            else:
-                vert_rad = vert_rad / 2.0  # NDC normalization.
+            vert_rad = raster_rad / focal_length.to(vert_pos.device) * point_dists * sensor_width
+            vert_rad = vert_rad / 2.0  # NDC normalization.
         return vert_rad
 
     # point_clouds is not typed to avoid a cyclic dependency.
@@ -503,6 +512,7 @@ def forward(self, point_clouds, **kwargs) -> torch.Tensor:
                 cam_pos,
                 orthogonal_projection,
                 focal_length,
+                sensor_width,
                 kwargs,
                 cloud_idx,
             )