Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add FacemapInterface #752

Draft
wants to merge 27 commits into
base: main
Choose a base branch
from
Draft
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
c558dee
add FacemapInterface, which currently only supports eye tracking.
bendichter Feb 18, 2024
275bf42
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Feb 18, 2024
16523f9
Update CHANGELOG.md
bendichter Feb 18, 2024
234ab4c
Merge branch 'main' into facemap
alessandratrapani Apr 5, 2024
78e8794
add motion svd
alessandratrapani Apr 5, 2024
07e21f9
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 5, 2024
2f25678
separate multivideo masks and ROI masks
alessandratrapani Apr 8, 2024
213ff8e
Merge branch 'main' into facemap
alessandratrapani Apr 8, 2024
056d787
separate multivideo and rois motion masks
alessandratrapani Apr 15, 2024
8ac9c47
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 15, 2024
ff03d31
Merge branch 'main' into facemap to update videodatainterface
alessandratrapani Apr 17, 2024
e413efa
set facempa test (not working)
alessandratrapani May 17, 2024
578eeb9
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 17, 2024
1682793
add get_metadata() function
alessandratrapani May 21, 2024
6f80805
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 21, 2024
52d5383
add tests for PupilTracking
alessandratrapani May 21, 2024
9c0aec3
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 21, 2024
d92ec25
motion svd metadata
alessandratrapani May 21, 2024
ecb5dde
test yey_tracking spatial series
alessandratrapani May 21, 2024
96651d9
Merge branch 'facemap' of https://github.com/catalystneuro/neuroconv …
alessandratrapani May 21, 2024
651eeb8
test pupil tracking time series
alessandratrapani May 21, 2024
b58ef5f
Merge branch 'main' into facemap
CodyCBakerPhD May 21, 2024
06302d3
add testing for MotionSVDMasks and MotionSVDSeries
alessandratrapani May 21, 2024
51b3a97
Merge branch 'facemap' of https://github.com/catalystneuro/neuroconv …
alessandratrapani May 21, 2024
79430bf
Merge branch 'main' into facemap
alessandratrapani Jun 10, 2024
f067458
set check_ragged to False to speed up add_row
alessandratrapani Jun 13, 2024
7b39363
Merge branch 'refs/heads/main' into facemap
alessandratrapani Jul 9, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
separate multivideo and rois motion masks
alessandratrapani committed Apr 15, 2024

Verified

This commit was signed with the committer’s verified signature. The key has expired.
Czaki Grzegorz Bokota
commit 056d78723d49272c8835899cae92ecd0e6a686f0
162 changes: 111 additions & 51 deletions src/neuroconv/datainterfaces/behavior/facemap/facemapdatainterface.py
Original file line number Diff line number Diff line change
@@ -5,10 +5,11 @@
from pynwb.base import TimeSeries
from pynwb.behavior import EyeTracking, PupilTracking, SpatialSeries
from pynwb.file import NWBFile

from ndx_facemap_motionsvd import MotionSVDMasks, MotionSVDSeries
from pynwb.core import DynamicTableRegion
from ..video.video_utils import get_video_timestamps
from ....basetemporalalignmentinterface import BaseTemporalAlignmentInterface
from ....tools import get_module, get_package
from ....tools import get_module
from ....utils import FilePathType


@@ -22,6 +23,7 @@
self,
mat_file_path: FilePathType,
video_file_path: FilePathType,
first_n_components: int = 500,
include_multivideo_SVD: bool = True,
verbose: bool = True,
):
@@ -34,10 +36,15 @@
Path to the .mat file.
video_file_path : string or Path
Path to the .avi file.
first_n_components : int, default: 500
Number of components to store.
include_multivideo_SVD : bool, default: True
Include multivideo motion SVD.
verbose : bool, default: True
Allows verbose.
"""
super().__init__(mat_file_path=mat_file_path, video_file_path=video_file_path, verbose=verbose)
self.first_n_components = first_n_components
self.include_multivideo_SVD = include_multivideo_SVD
self.original_timestamps = None
self.timestamps = None
@@ -83,95 +90,134 @@

behavior_module.add(pupil_tracking)

def add_motion_SVD(self, nwbfile: NWBFile):
def add_multivideo_motion_SVD(self, nwbfile: NWBFile):
"""
Add data motion SVD and motion mask for each ROI.
Add data motion SVD and motion mask for the whole video.

Parameters
----------
nwbfile : NWBFile
NWBFile to add motion SVD components data to.
"""
from ndx_facemap_motionsvd import MotionSVDMasks, MotionSVDSeries
from pynwb.core import DynamicTableRegion

# From documentation
# motSVD: cell array of motion SVDs [time x components] (in order: multivideo, ROI1, ROI2, ROI3)
# uMotMask: cell array of motion masks [pixels x components] (in order: multivideo, ROI1, ROI2, ROI3)
# motion masks of multivideo are reported as 2D-arrays npixels x components
# while ROIs motion masks are reported as 3D-arrays x_pixels x y_pixels x components

with h5py.File(self.source_data["mat_file_path"], "r") as file:

behavior_module = get_module(nwbfile=nwbfile, name="behavior", description="behavioral data")

timestamps = self.get_timestamps()

# Extract mask_coordinates
mask_coordinates = file[file[file["proc"]["ROI"][0][0]][0][0]]
y1 = int(np.round(mask_coordinates[0][0])-1) # correct matlab indexing
x1 = int(np.round(mask_coordinates[1][0])-1) # correct matlab indexing
y2 = y1 + int(np.round(mask_coordinates[2][0]))
x2 = x1 + int(np.round(mask_coordinates[3][0]))
mask_coordinates = [x1, y1, x2, y2]

# store multivideo motion mask and motion series
if self.include_multivideo_SVD:
# add multivideo mask
mask_ref = file["proc"]["uMotMask"][0][0]
motion_masks_table = MotionSVDMasks(
name=f"MotionSVDMasksMultivideo",
description=f"motion mask for multivideo",
)
motion_masks_table = MotionSVDMasks(
name=f"MotionSVDMasksMultivideo",
description=f"motion mask for multivideo",
mask_coordinates=mask_coordinates,
downsampling_factor=self._get_downsamplig_factor(),
processed_frame_dimension=self._get_processed_frame_dimension(),
)

multivideo_mask_ref = file["proc"]["wpix"][0][0]
multivideo_mask = file[multivideo_mask_ref]
multivideo_mask = multivideo_mask[:]
non_zero_multivideo_mask = np.where(multivideo_mask == 1)
y_indices, x_indices = non_zero_multivideo_mask
top = np.min(y_indices)
left = np.min(x_indices)
bottom = np.max(y_indices)
right = np.max(x_indices)
submask = multivideo_mask[top : bottom + 1, left : right + 1]
componendt_2d_shape = submask.shape

for component in file[mask_ref]:
componendt_2d = component.reshape(componendt_2d_shape)
motion_masks_table.add_row(image_mask=componendt_2d.T)
# add multivideo mask
mask_ref = file["proc"]["uMotMask"][0][0]
for c, component in enumerate(file[mask_ref]):
if c == self.first_n_components:
break
componendt_2d = component.reshape((y2-y1, x2-x1))
motion_masks_table.add_row(image_mask=componendt_2d.T)

motion_masks = DynamicTableRegion(
name="motion_masks",
data=list(range(len(file["proc"]["motSVD"][:]))),
description="all the multivideo motion mask",
table=motion_masks_table,
)

motion_masks = DynamicTableRegion(
name="motion_masks",
data=list(range(len(file["proc"]["motSVD"][:]))),
description="all the multivideo motion mask",
table=motion_masks_table,
)
series_ref = file["proc"]["motSVD"][0][0]
data = np.array(file[series_ref])
data = data[: self.first_n_components, :]

series_ref = file["proc"]["motSVD"][0][0]
data = np.array(file[series_ref])
motionsvd_series = MotionSVDSeries(
name=f"MotionSVDSeriesMultivideo",
description=f"SVD components for multivideo",
data=data.T,
motion_masks=motion_masks,
unit="unknown",
timestamps=timestamps,
)
behavior_module.add(motion_masks_table)
behavior_module.add(motionsvd_series)

motionsvd_series = MotionSVDSeries(
name=f"MotionSVDSeriesMultivideo",
description=f"SVD components for multivideo",
data=data.T,
motion_masks=motion_masks,
unit="unknown",
timestamps=timestamps,
)
behavior_module.add(motion_masks_table)
behavior_module.add(motionsvd_series)
return

def add_motion_SVD(self, nwbfile: NWBFile):
"""
Add data motion SVD and motion mask for each ROI.

Parameters
----------
nwbfile : NWBFile
NWBFile to add motion SVD components data to.
"""

# From documentation
# motSVD: cell array of motion SVDs [time x components] (in order: multivideo, ROI1, ROI2, ROI3)
# uMotMask: cell array of motion masks [pixels x components] (in order: multivideo, ROI1, ROI2, ROI3)
# ROIs motion masks are reported as 3D-arrays x_pixels x y_pixels x components

with h5py.File(self.source_data["mat_file_path"], "r") as file:

behavior_module = get_module(nwbfile=nwbfile, name="behavior", description="behavioral data")

timestamps = self.get_timestamps()
downsampling_factor=self._get_downsamplig_factor()
processed_frame_dimension=self._get_processed_frame_dimension()
# store ROIs motion mask and motion series
n = 1
for series_ref, mask_ref in zip(file["proc"]["motSVD"][1:][0], file["proc"]["uMotMask"][1:][0]):
for series_ref, mask_ref in zip(file["proc"]["motSVD"][1:], file["proc"]["uMotMask"][1:]):
series_ref = series_ref[0]
mask_ref = mask_ref[0]

# skipping the first ROI because it referes to "running" mask, from Facemap doc

Check failure on line 191 in src/neuroconv/datainterfaces/behavior/facemap/facemapdatainterface.py

GitHub Actions / Check for spelling errors

referes ==> refers, referees
mask_coordinates = file[file["proc"]["locROI"][n][0]]
y1 = int(np.round(mask_coordinates[0][0])-1) # correct matlab indexing
x1 = int(np.round(mask_coordinates[1][0])-1) # correct matlab indexing
y2 = y1 + int(np.round(mask_coordinates[2][0]))
x2 = x1 + int(np.round(mask_coordinates[3][0]))
mask_coordinates = [x1, y1, x2, y2]

motion_masks_table = MotionSVDMasks(
name=f"MotionSVDMasksROI{n}",
description=f"motion mask for ROI{n}",
mask_coordinates=mask_coordinates,
downsampling_factor=downsampling_factor,
processed_frame_dimension=processed_frame_dimension,
)
for component in file[mask_ref]:

for c, component in enumerate(file[mask_ref]):
if c == self.first_n_components:
break
motion_masks_table.add_row(image_mask=component.T)

motion_masks = DynamicTableRegion(
name="motion_masks",
data=list(range(len(file["proc"]["motSVD"][:]))),
data=list(range(self.first_n_components)),
description="all the ROIs motion mask",
table=motion_masks_table,
)

data = np.array(file[series_ref])
data = data[: self.first_n_components, :]

motionsvd_series = MotionSVDSeries(
name=f"MotionSVDSeriesROI{n}",
@@ -201,7 +247,19 @@

def set_aligned_timestamps(self, aligned_timestamps: np.ndarray) -> None:
self.timestamps = aligned_timestamps


def _get_downsamplig_factor(self) -> float:
with h5py.File(self.source_data["mat_file_path"], "r") as file:
downsamplig_factor = file["proc"]["sc"][0][0]
return downsamplig_factor

def _get_processed_frame_dimension(self) -> np.ndarray:
with h5py.File(self.source_data["mat_file_path"], "r") as file:
processed_frame_ref = file["proc"]["wpix"][0][0]
frame = file[processed_frame_ref]
return [frame.shape[1],frame.shape[0]]


def add_to_nwbfile(
self,
nwbfile: NWBFile,
@@ -226,3 +284,5 @@

self.add_pupil_data(nwbfile=nwbfile)
self.add_motion_SVD(nwbfile=nwbfile)
if self.add_multivideo_motion_SVD:
self.add_multivideo_motion_SVD(nwbfile=nwbfile)