Setting save_world_frame as False by default and Splat export panel in viewer

docs/extensions/blender_addon.md

@@ -6,7 +6,7 @@
 
 ## Overview
 
-This Blender add-on allows for compositing with a Nerfstudio render as a background layer by generating a camera path JSON file from the Blender camera path, as well as a way to import Nerfstudio JSON files as a Blender camera baked with the Nerfstudio camera path. This add-on also allows compositing multiple NeRF objects into a NeRF scene. This is achieved by importing a mesh or point-cloud representation of the NeRF scene from Nerfstudio into Blender and getting the camera coordinates relative to the transformations of the NeRF representation. Dynamic FOV from the Blender camera is supported and will match the Nerfstudio render. Perspective, equirectangular, VR180, and omnidirectional stereo (VR 360) cameras are supported.
+This Blender add-on allows for compositing with a Nerfstudio render as a background layer by generating a camera path JSON file from the Blender camera path, as well as a way to import Nerfstudio JSON files as a Blender camera baked with the Nerfstudio camera path. This add-on also allows compositing multiple NeRF objects into a NeRF scene. This is achieved by importing a mesh or point-cloud representation of the NeRF scene from Nerfstudio into Blender and getting the camera coordinates relative to the transformations of the NeRF representation. Dynamic FOV from the Blender camera is supported and will match the Nerfstudio render. Perspective, equirectangular, VR180, and omnidirectional stereo (VR 360) cameras are supported. This add-on also supports Gaussian Splatting scenes as well, however equirectangular and VR video rendering is not currently supported.
 
 <center>
  <img width="800" alt="image" src="https://user-images.githubusercontent.com/9502341/211442247-99d1ebc7-3ef9-46f7-9bcc-0e18553f19b7.PNG">
@@ -30,7 +30,7 @@ This Blender add-on allows for compositing with a Nerfstudio render as a backgro
 
 ## Scene Setup
 
-1. Export the mesh or point cloud representation of the NeRF from Nerfstudio, which will be used as reference for the actual NeRF in the Blender scene. Mesh export at a good quality is preferred, however, if the export is not clear or the NeRF is large, a detailed point cloud export will also work.
+1. Export the mesh or point cloud representation of the NeRF from Nerfstudio, which will be used as reference for the actual NeRF in the Blender scene. Mesh export at a good quality is preferred, however, if the export is not clear or the NeRF is large, a detailed point cloud export will also work. Keep the `save_world_frame` flag as False or in the viewer, de-select the "Save in world frame" checkbox to keep the correct coordinate system for the add-on.
 
 2. Import the mesh or point cloud representation of the NeRF into the scene. You may need to crop the mesh further. Since it is used as a reference and won't be visible in the final render, only the parts that the blender animation will interact with may be necessary to import.
 

nerfstudio/scripts/exporter.py

@@ -121,9 +121,9 @@ class ExportPointCloud(Exporter):
     """Number of rays to evaluate per batch. Decrease if you run out of memory."""
     std_ratio: float = 10.0
     """Threshold based on STD of the average distances across the point cloud to remove outliers."""
-    save_world_frame: bool = True
-    """If true, saves in the frame of the transform.json file, if false saves in the frame of the scaled 
-        dataparser transform"""
+    save_world_frame: bool = False
+    """If set, saves the point cloud in the same frame as the original dataset. Otherwise, uses the
+    scaled and reoriented coordinate space expected by the NeRF models."""
 
     def main(self) -> None:
         """Export point cloud."""
@@ -494,7 +494,7 @@ def main(self) -> None:
 
         model: GaussianSplattingModel = pipeline.model
 
-        filename = self.output_dir / "point_cloud.ply"
+        filename = self.output_dir / "splat.ply"
 
         map_to_tensors = {}
 

nerfstudio/viewer/export_panel.py

@@ -21,22 +21,33 @@
 from nerfstudio.data.scene_box import OrientedBox
 from nerfstudio.viewer.control_panel import ControlPanel
 from typing_extensions import Literal
+from nerfstudio.models.base_model import Model
+from nerfstudio.models.gaussian_splatting import GaussianSplattingModel
 
 
-def populate_export_tab(server: viser.ViserServer, control_panel: ControlPanel, config_path: Path) -> None:
-    crop_output = server.add_gui_checkbox("Use Crop", False)
+def populate_export_tab(
+    server: viser.ViserServer,
+    control_panel: ControlPanel,
+    config_path: Path,
+    viewer_model: Model,
+) -> None:
+    viewing_gsplat = isinstance(viewer_model, GaussianSplattingModel)
+    if not viewing_gsplat:
+        crop_output = server.add_gui_checkbox("Use Crop", False)
 
-    @crop_output.on_update
-    def _(_) -> None:
-        control_panel.crop_viewport = crop_output.value
+        @crop_output.on_update
+        def _(_) -> None:
+            control_panel.crop_viewport = crop_output.value
 
+    with server.add_gui_folder("Splat"):
+        populate_splat_tab(server, control_panel, config_path, viewing_gsplat)
     with server.add_gui_folder("Point Cloud"):
-        populate_point_cloud_tab(server, control_panel, config_path)
+        populate_point_cloud_tab(server, control_panel, config_path, viewing_gsplat)
     with server.add_gui_folder("Mesh"):
-        populate_mesh_tab(server, control_panel, config_path)
+        populate_mesh_tab(server, control_panel, config_path, viewing_gsplat)
 
 
-def show_command_modal(client: viser.ClientHandle, what: Literal["mesh", "point cloud"], command: str) -> None:
+def show_command_modal(client: viser.ClientHandle, what: Literal["mesh", "point cloud", "splat"], command: str) -> None:
     """Show a modal to each currently connected client.
 
     In the future, we should only show the modal to the client that pushes the
@@ -78,82 +89,123 @@ def populate_point_cloud_tab(
     server: viser.ViserServer,
     control_panel: ControlPanel,
     config_path: Path,
+    viewing_gsplat: bool,
 ) -> None:
-    server.add_gui_markdown("<small>Render depth, project to an oriented point cloud, and filter.</small> ")
-    num_points = server.add_gui_number("# Points", initial_value=1_000_000, min=1, max=None, step=1)
-    world_frame = server.add_gui_checkbox(
-        "Save in world frame",
-        True,
-        hint="Save the point cloud in the transforms.json frame, rather than scaled scene frame",
-    )
-    remove_outliers = server.add_gui_checkbox("Remove outliers", True)
-    normals = server.add_gui_dropdown(
-        "Normals",
-        # TODO: options here could depend on what's available to the model.
-        ("open3d", "model_output"),
-        initial_value="open3d",
-        hint="Normal map source.",
-    )
-    output_dir = server.add_gui_text("Output Directory", initial_value="exports/pcd/")
-    generate_command = server.add_gui_button("Generate Command", icon=viser.Icon.TERMINAL_2)
-
-    @generate_command.on_click
-    def _(event: viser.GuiEvent) -> None:
-        assert event.client is not None
-        command = " ".join(
-            [
-                "ns-export pointcloud",
-                f"--load-config {config_path}",
-                f"--output-dir {output_dir.value}",
-                f"--num-points {num_points.value}",
-                f"--remove-outliers {remove_outliers.value}",
-                f"--normal-method {normals.value}",
-                f"--use_bounding_box {control_panel.crop_viewport}",
-                f"--save-world-frame {world_frame.value}",
-                get_crop_string(control_panel.crop_obb),
-            ]
+    if not viewing_gsplat:
+        server.add_gui_markdown("<small>Render depth, project to an oriented point cloud, and filter</small> ")
+        num_points = server.add_gui_number("# Points", initial_value=1_000_000, min=1, max=None, step=1)
+        world_frame = server.add_gui_checkbox(
+            "Save in world frame",
+            False,
+            hint=(
+                "If checked, saves the point cloud in the same frame as the original dataset. Otherwise, uses the "
+                "scaled and reoriented coordinate space expected by the NeRF models."
+            ),
+        )
+        remove_outliers = server.add_gui_checkbox("Remove outliers", True)
+        normals = server.add_gui_dropdown(
+            "Normals",
+            # TODO: options here could depend on what's available to the model.
+            ("open3d", "model_output"),
+            initial_value="open3d",
+            hint="Normal map source.",
         )
-        show_command_modal(event.client, "point cloud", command)
+        output_dir = server.add_gui_text("Output Directory", initial_value="exports/pcd/")
+        generate_command = server.add_gui_button("Generate Command", icon=viser.Icon.TERMINAL_2)
+
+        @generate_command.on_click
+        def _(event: viser.GuiEvent) -> None:
+            assert event.client is not None
+            command = " ".join(
+                [
+                    "ns-export pointcloud",
+                    f"--load-config {config_path}",
+                    f"--output-dir {output_dir.value}",
+                    f"--num-points {num_points.value}",
+                    f"--remove-outliers {remove_outliers.value}",
+                    f"--normal-method {normals.value}",
+                    f"--use_bounding_box {control_panel.crop_viewport}",
+                    f"--save-world-frame {world_frame.value}",
+                    get_crop_string(control_panel.crop_obb),
+                ]
+            )
+            show_command_modal(event.client, "point cloud", command)
+
+    else:
+        server.add_gui_markdown("<small>Point cloud export is not currently supported with Gaussian Splatting</small>")
 
 
 def populate_mesh_tab(
     server: viser.ViserServer,
     control_panel: ControlPanel,
     config_path: Path,
+    viewing_gsplat: bool,
 ) -> None:
-    server.add_gui_markdown(
-        "<small>Render depth, project to an oriented point cloud, and run Poisson surface reconstruction.</small>"
-    )
-
-    normals = server.add_gui_dropdown(
-        "Normals",
-        ("open3d", "model_output"),
-        initial_value="open3d",
-        hint="Source for normal maps.",
-    )
-    num_faces = server.add_gui_number("# Faces", initial_value=50_000, min=1)
-    texture_resolution = server.add_gui_number("Texture Resolution", min=8, initial_value=2048)
-    output_directory = server.add_gui_text("Output Directory", initial_value="exports/mesh/")
-    num_points = server.add_gui_number("# Points", initial_value=1_000_000, min=1, max=None, step=1)
-    remove_outliers = server.add_gui_checkbox("Remove outliers", True)
-
-    generate_command = server.add_gui_button("Generate Command", icon=viser.Icon.TERMINAL_2)
-
-    @generate_command.on_click
-    def _(event: viser.GuiEvent) -> None:
-        assert event.client is not None
-        command = " ".join(
-            [
-                "ns-export poisson",
-                f"--load-config {config_path}",
-                f"--output-dir {output_directory.value}",
-                f"--target-num-faces {num_faces.value}",
-                f"--num-pixels-per-side {texture_resolution.value}",
-                f"--num-points {num_points.value}",
-                f"--remove-outliers {remove_outliers.value}",
-                f"--normal-method {normals.value}",
-                f"--use_bounding_box {control_panel.crop_viewport}",
-                get_crop_string(control_panel.crop_obb),
-            ]
+    if not viewing_gsplat:
+        server.add_gui_markdown(
+            "<small>Render depth, project to an oriented point cloud, and run Poisson surface reconstruction</small>"
         )
-        show_command_modal(event.client, "mesh", command)
+
+        normals = server.add_gui_dropdown(
+            "Normals",
+            ("open3d", "model_output"),
+            initial_value="open3d",
+            hint="Source for normal maps.",
+        )
+        num_faces = server.add_gui_number("# Faces", initial_value=50_000, min=1)
+        texture_resolution = server.add_gui_number("Texture Resolution", min=8, initial_value=2048)
+        output_directory = server.add_gui_text("Output Directory", initial_value="exports/mesh/")
+        num_points = server.add_gui_number("# Points", initial_value=1_000_000, min=1, max=None, step=1)
+        remove_outliers = server.add_gui_checkbox("Remove outliers", True)
+
+        generate_command = server.add_gui_button("Generate Command", icon=viser.Icon.TERMINAL_2)
+
+        @generate_command.on_click
+        def _(event: viser.GuiEvent) -> None:
+            assert event.client is not None
+            command = " ".join(
+                [
+                    "ns-export poisson",
+                    f"--load-config {config_path}",
+                    f"--output-dir {output_directory.value}",
+                    f"--target-num-faces {num_faces.value}",
+                    f"--num-pixels-per-side {texture_resolution.value}",
+                    f"--num-points {num_points.value}",
+                    f"--remove-outliers {remove_outliers.value}",
+                    f"--normal-method {normals.value}",
+                    f"--use_bounding_box {control_panel.crop_viewport}",
+                    get_crop_string(control_panel.crop_obb),
+                ]
+            )
+            show_command_modal(event.client, "mesh", command)
+
+    else:
+        server.add_gui_markdown("<small>Mesh export is not currently supported with Gaussian Splatting</small>")
+
+
+def populate_splat_tab(
+    server: viser.ViserServer,
+    control_panel: ControlPanel,
+    config_path: Path,
+    viewing_gsplat: bool,
+) -> None:
+    if viewing_gsplat:
+        server.add_gui_markdown("<small>Generate ply export of Gaussian Splat</small>")
+
+        output_directory = server.add_gui_text("Output Directory", initial_value="exports/splat/")
+        generate_command = server.add_gui_button("Generate Command", icon=viser.Icon.TERMINAL_2)
+
+        @generate_command.on_click
+        def _(event: viser.GuiEvent) -> None:
+            assert event.client is not None
+            command = " ".join(
+                [
+                    "ns-export gaussian-splat",
+                    f"--load-config {config_path}",
+                    f"--output-dir {output_directory.value}",
+                ]
+            )
+            show_command_modal(event.client, "splat", command)
+
+    else:
+        server.add_gui_markdown("<small>Splat export is only supported with Gaussian Splatting methods</small>")

nerfstudio/viewer/viewer.py

@@ -204,7 +204,7 @@ def __init__(
             )
 
         with tabs.add_tab("Export", viser.Icon.PACKAGE_EXPORT):
-            populate_export_tab(self.viser_server, self.control_panel, config_path)
+            populate_export_tab(self.viser_server, self.control_panel, config_path, self.pipeline.model)
 
         # Keep track of the pointers to generated GUI folders, because each generated folder holds a unique ID.
         viewer_gui_folders = dict()