Merge pull request #33 from catalystneuro/final-improvements

Change `FiberPhotometryResponseSeries` to hold data from both fibers

src/dombeck_lab_to_nwb/azcorra2023/azcorra2023_convert_all_sessions.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from pathlib import Path
 
+from neuroconv.tools.data_transfers import automatic_dandi_upload
 from neuroconv.utils import FolderPathType
 from nwbinspector.inspector_tools import save_report, format_messages
 from tqdm import tqdm
@@ -166,6 +167,7 @@ def convert_all_sessions(
     root_folder_path = Path("/Volumes/LaCie/CN_GCP/Dombeck/Azcorra2023/")
 
     # The folder containing the processed photometry data for each session.
+    # Use matlab_utils/convert_data6.m to convert the data6.mat files to .mat files expected by the converter.
     processed_photometry_folder_path = Path("/Volumes/LaCie/CN_GCP/Dombeck/tmp2/")
 
     # Mapping of the location names in the processed photometry data to the Allen Brain Atlas location names.
@@ -195,3 +197,8 @@ def convert_all_sessions(
         stub_test=stub_test,
         overwrite=overwrite,
     )
+
+    automatic_dandi_upload(
+        dandiset_id="001038",
+        nwb_folder_path=nwbfile_folder_path,
+    )

src/dombeck_lab_to_nwb/azcorra2023/azcorra2023_convert_session.py

@@ -113,56 +113,56 @@ def session_to_nwb(
 
     # Determine whether single or multi-fiber experiment and adjust conversion options accordingly
     fibers_metadata = extra_metadata["Ophys"]["FiberPhotometry"]["OpticalFibers"]
-    channel_id_to_fiber_photometry_series_name_mapping = dict()
-    channel_id_to_time_series_name_mapping = dict(A="Velocity")
+    trace_name_to_channel_id_mapping = dict(
+        FiberPhotometryResponseSeries=[],
+        FiberPhotometryResponseSeriesIsosbestic=[],
+    )
+    time_series_name_to_channel_id_mapping = dict(Velocity=["A"], Fluorescence=[])
     for fiber_metadata in fibers_metadata:
         fiber_name = fiber_metadata["name"]
         channel_name = fiber_metadata.pop("label")
-        channel_id_to_fiber_photometry_series_name_mapping.update(
-            {
-                channel_name: f"FiberPhotometryResponseSeriesGreen{fiber_name}",
-                f"{channel_name}405": f"FiberPhotometryResponseSeriesGreenIsosbestic{fiber_name}",
-            }
-        )
+        isosbestic_channel_name = f"{channel_name}405"
+
+        trace_name_to_channel_id_mapping["FiberPhotometryResponseSeries"].append(channel_name)
+        trace_name_to_channel_id_mapping["FiberPhotometryResponseSeriesIsosbestic"].append(isosbestic_channel_name)
+
         picoscope_channel_name = "C" if fiber_name == "Fiber1" else "B"
-        channel_id_to_time_series_name_mapping.update({picoscope_channel_name: f"Fluorescence{fiber_name}"})
+        time_series_name_to_channel_id_mapping["Fluorescence"].append(picoscope_channel_name)
 
     if binned_photometry_mat_file_path:
         conversion_options.update(
             dict(
                 FiberPhotometry=dict(
-                    channel_name_to_photometry_series_name_mapping=channel_id_to_fiber_photometry_series_name_mapping,
+                    trace_name_to_channel_id_mapping=trace_name_to_channel_id_mapping,
                     stub_test=stub_test,
                 ),
             ),
         )
 
     dff_channel_name_mapping = {
-        ch_name: "DfOverF" + series_name
-        for ch_name, series_name in channel_id_to_fiber_photometry_series_name_mapping.items()
+        f"DfOverF{series_name}": channel_names
+        for series_name, channel_names in trace_name_to_channel_id_mapping.items()
     }
 
     # Update conversion options
     conversion_options.update(
         dict(
             ProcessedFiberPhotometry=dict(
-                channel_name_to_photometry_series_name_mapping=dff_channel_name_mapping,
+                trace_name_to_channel_id_mapping=dff_channel_name_mapping,
                 stub_test=stub_test,
             ),
             PicoScopeTimeSeries=dict(
-                channel_id_to_time_series_name_mapping=channel_id_to_time_series_name_mapping,
+                time_series_name_to_channel_id_mapping=time_series_name_to_channel_id_mapping,
                 stub_test=stub_test,
             ),
             PicoScopeEvents=dict(stub_test=stub_test),
         )
     )
 
-    metadata = dict_deep_update(metadata, extra_metadata)
-
     # Run conversion
     converter.run_conversion(
         nwbfile_path=nwbfile_path,
-        metadata=metadata,
+        metadata=extra_metadata,
         conversion_options=conversion_options,
         overwrite=True,
     )

src/dombeck_lab_to_nwb/azcorra2023/interfaces/azcorra2023_fiberphotometryinterface.py

@@ -73,7 +73,7 @@ def add_to_nwbfile(
         self,
         nwbfile: NWBFile,
         metadata: dict,
-        channel_name_to_photometry_series_name_mapping: dict,
+        trace_name_to_channel_id_mapping: dict,
         stub_test: Optional[bool] = False,
     ) -> None:
         """
@@ -85,36 +85,46 @@ def add_to_nwbfile(
             The NWBFile to add the raw photometry data to.
         metadata : dict
             The metadata for the photometry data.
-        channel_name_to_photometry_series_name_mapping: dict
-            A dictionary that maps the channel names in the .mat file to the names of the photometry response series.
+        trace_name_to_channel_id_mapping: dict
+            A dictionary that maps the trace name to the channel ids. (e.g. {"FiberPhotometryResponseSeries": ["chRed", "chGreen"]})
         stub_test : bool, optional
             Whether to run the conversion as a stub test by writing 1-minute of data, by default False.
         """
 
-        channel_names = list(channel_name_to_photometry_series_name_mapping.keys())
-        assert all(
-            channel_name in self.column_names for channel_name in channel_names
-        ), f"Not all channel names are in {self.source_data['file_path']}."
+        for series_ind, (series_name, channel_names) in enumerate(trace_name_to_channel_id_mapping.items()):
+            assert all(
+                channel_name in self.column_names for channel_name in channel_names
+            ), f"Not all channel names are in {self.source_data['file_path']}."
 
-        for series_ind, (channel_name, series_name) in enumerate(
-            channel_name_to_photometry_series_name_mapping.items()
-        ):
             if series_name in nwbfile.acquisition:
-                raise ValueError(f"The fiber photometry series {series_name} already exists in the NWBfile.")
-
-            channel_index = self.column_names.index(channel_name)
-
-            data = self._photometry_data[channel_index]
-            if len(self.depth_ids) > 1:
-                data = data[self.depth_index]
-
+                raise ValueError(f"The fiber photometry series '{series_name}' already exists in the NWBfile.")
+
+            squeeze = False
+            if len(channel_names) == 1:
+                table_region = [series_ind]
+                squeeze = True
+            elif len(channel_names) == 2:
+                table_region_ind = series_ind * len(trace_name_to_channel_id_mapping.keys())
+                table_region = [table_region_ind, table_region_ind + 1]
+            else:
+                raise ValueError(f"Expected 1 or 2 channel names, found {len(channel_names)}.")
+
+            data_to_add = []
+            for channel_name in channel_names:
+                channel_index = self.column_names.index(channel_name)
+                data = self._photometry_data[channel_index]
+                if len(self.depth_ids) > 1:
+                    data = data[self.depth_index]
+                data_to_add.append(data if not stub_test else data[:6000])
+
+            fiber_data = np.column_stack(data_to_add)
             timestamps = self.get_timestamps(stub_test=stub_test)
             add_fiber_photometry_series(
                 nwbfile=nwbfile,
                 metadata=metadata,
-                data=data if not stub_test else data[:6000],
+                data=fiber_data if not squeeze else fiber_data.squeeze(axis=1),
                 timestamps=timestamps,
                 fiber_photometry_series_name=series_name,
-                table_region_ind=series_ind,
+                table_region=table_region,
                 parent_container="acquisition",
             )

src/dombeck_lab_to_nwb/azcorra2023/interfaces/azcorra2023_processedfiberphotometryinterface.py

@@ -149,7 +149,7 @@ def add_delta_f_over_f_traces(
         self,
         nwbfile: NWBFile,
         metadata: dict,
-        channel_name_to_photometry_series_name_mapping: dict,
+        trace_name_to_channel_id_mapping: dict,
         stub_test: bool = False,
     ):
         """
@@ -161,8 +161,8 @@ def add_delta_f_over_f_traces(
             The NWB file to which the data will be added.
         metadata : dict
             The metadata dictionary.
-        channel_name_to_photometry_series_name_mapping : dict
-            A dictionary mapping the channel names from the file to the photometry series names that are going to be added.
+        trace_name_to_channel_id_mapping : dict
+            A dictionary that maps the DF/F trace name to the channel ids. (e.g. {"DfOverFFiberPhotometryResponseSeries": ["chRed", "chGreen"]})
         stub_test : bool, optional
             Whether to run a stub test, by default False.
 
@@ -171,18 +171,32 @@ def add_delta_f_over_f_traces(
 
         timestamps = self.get_timestamps(stub_test=stub_test)
 
-        for series_ind, (channel_name, series_name) in enumerate(
-            channel_name_to_photometry_series_name_mapping.items()
-        ):
-            if channel_name not in self._processed_photometry_data:
-                print(f"Channel {channel_name} not found in the processed photometry data.")
-                continue
+        for series_ind, (series_name, channel_names) in enumerate(trace_name_to_channel_id_mapping.items()):
             if series_name in ophys_module.data_interfaces:
                 raise ValueError(f"The DF/F series {series_name} already exists in the NWBfile.")
 
+            data_to_add = []
+            for channel_name in channel_names:
+                if channel_name not in self._processed_photometry_data:
+                    print(f"Channel {channel_name} not found in the processed photometry data.")
+                    continue
+
+                data = self._processed_photometry_data[channel_name]
+                data_to_add.append(data if not stub_test else data[:6000])
+
+            squeeze = False
+            if len(channel_names) == 1:
+                table_region = [series_ind]
+                squeeze = True
+            elif len(channel_names) == 2:
+                table_region_ind = series_ind * len(trace_name_to_channel_id_mapping.keys())
+                table_region = [table_region_ind, table_region_ind + 1]
+            else:
+                raise ValueError(f"Expected 1 or 2 channel names, found {len(channel_names)}.")
+
             raw_series_name = series_name.replace("DfOverF", "")
-            data = self._processed_photometry_data[channel_name]
-            data_to_add = data if not stub_test else data[:6000]
+
+            fiber_data = np.column_stack(data_to_add)
             if raw_series_name in nwbfile.acquisition:
                 # Retrieve references to the raw photometry data
                 raw_response_series = nwbfile.acquisition[raw_series_name]
@@ -194,7 +208,7 @@ def add_delta_f_over_f_traces(
                 response_series = FiberPhotometryResponseSeries(
                     name=series_name,
                     description=description,
-                    data=data_to_add,
+                    data=fiber_data if not squeeze else fiber_data.squeeze(axis=1),
                     unit="n.a.",
                     rate=self._sampling_frequency,
                     starting_time=timestamps[0],
@@ -212,10 +226,10 @@ def add_delta_f_over_f_traces(
                 add_fiber_photometry_series(
                     nwbfile=nwbfile,
                     metadata=metadata,
-                    data=data_to_add,
+                    data=fiber_data if not squeeze else fiber_data.squeeze(axis=1),
                     timestamps=timestamps,
                     fiber_photometry_series_name=series_name,
-                    table_region_ind=series_ind,
+                    table_region=table_region,
                     parent_container="processing/ophys",
                 )
 
@@ -461,7 +475,7 @@ def add_to_nwbfile(
         self,
         nwbfile: NWBFile,
         metadata: dict,
-        channel_name_to_photometry_series_name_mapping: dict = None,
+        trace_name_to_channel_id_mapping: dict,
         stub_test: bool = False,
     ):
         """
@@ -473,8 +487,8 @@ def add_to_nwbfile(
             The NWB file to which the data will be added.
         metadata : dict
             The metadata dictionary.
-        channel_name_to_photometry_series_name_mapping : dict
-            A dictionary mapping the channel names from the file to the photometry series names that are going to be added.
+        trace_name_to_channel_id_mapping : dict
+            A dictionary that maps the DF/F trace name to the channel ids. (e.g. {"DfOverFFiberPhotometryResponseSeries": ["chRed", "chGreen"]})
         stub_test : bool, optional
             Whether to run a stub test, by default False.
 
@@ -492,6 +506,6 @@ def add_to_nwbfile(
         self.add_delta_f_over_f_traces(
             nwbfile=nwbfile,
             metadata=metadata,
-            channel_name_to_photometry_series_name_mapping=channel_name_to_photometry_series_name_mapping,
+            trace_name_to_channel_id_mapping=trace_name_to_channel_id_mapping,
             stub_test=stub_test,
         )

src/dombeck_lab_to_nwb/azcorra2023/interfaces/picoscope_timeseriesinterface.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from pathlib import Path
 
+import numpy as np
 from natsort import natsorted
 from neuroconv import BaseDataInterface
 from neuroconv.utils import FolderPathType
@@ -56,14 +57,9 @@ def get_metadata(self) -> dict:
         metadata["NWBFile"].update(session_start_time=session_start_time)
 
         metadata["PicoScopeTimeSeries"] = dict(
-            FluorescenceFiber1=dict(
-                name="FluorescenceFiber1",
-                description="The fluorescence traces from Fiber1 collected at 4000 Hz by Picoscope.",
-                unit="Volts",
-            ),
-            FluorescenceFiber2=dict(
-                name="FluorescenceFiber2",
-                description="The fluorescence traces from Fiber2 collected at 4000 Hz by Picoscope.",
+            Fluorescence=dict(
+                name="Fluorescence",
+                description="The fluorescence traces from one or two optical fibers during 405 nm and 470 nm illumination collected at 4000 Hz by Picoscope.",
                 unit="Volts",
             ),
             Velocity=dict(
@@ -86,7 +82,7 @@ def add_to_nwbfile(
         self,
         nwbfile: NWBFile,
         metadata: dict,
-        channel_id_to_time_series_name_mapping: dict,
+        time_series_name_to_channel_id_mapping: dict,
         stub_test: bool = False,
     ) -> None:
         """
@@ -98,8 +94,8 @@ def add_to_nwbfile(
             The NWBFile to which to add the Picoscope time series.
         metadata : dict
             The metadata for the Picoscope time series.
-        channel_id_to_time_series_name_mapping : dict
-            A mapping from the channel id to the time series name.
+        time_series_name_to_channel_id_mapping : dict
+            A mapping from the time series name to the channel id(s).
         stub_test : bool, optional
             Whether to run a stub test, by default False.
         """
@@ -109,16 +105,21 @@ def add_to_nwbfile(
         end_frame = 1000 if stub_test else None
 
         # Create TimeSeries for each data channel
-        for channel_id, time_series_name in channel_id_to_time_series_name_mapping.items():
+        for time_series_name, channel_names in time_series_name_to_channel_id_mapping.items():
             time_series_metadata = picoscope_time_series_metadata[time_series_name]
 
-            trace_extractor = self.get_trace_extractor_from_picoscope(channel_name=channel_id)
+            data_to_add = []
+            for channel_name in channel_names:
+                trace_extractor = self.get_trace_extractor_from_picoscope(channel_name=channel_name)
+                data = trace_extractor.get_traces(end_frame=end_frame)
+                data_to_add.append(data if not stub_test else data[:6000])
+
             sampling_frequency = trace_extractor.get_sampling_frequency()
-            data = trace_extractor.get_traces(end_frame=end_frame)
 
+            time_series = np.column_stack(data_to_add)
             picoscope_time_series = TimeSeries(
                 name=time_series_name,
-                data=data,
+                data=time_series if len(channel_names) > 1 else time_series.squeeze(axis=1),
                 rate=sampling_frequency,
                 description=time_series_metadata["description"],
                 unit=time_series_metadata["unit"],