Merge branch 'null-channel-testing'

src/flowkit/_models/sample.py

@@ -76,10 +76,10 @@ class Sample(object):
         - pathlib Path object to a CSV or TSV file
         - string of CSV text
 
-    :param null_channel_list: List of PnN labels for channels that were collected
-        but do not contain useful data. Note, this should only be used if there were
-        truly no fluorochromes used targeting those detectors and the channels
-        do not contribute to compensation.
+    :param null_channel_list: List of PnN labels for acquired channels that do not contain
+        useful data. Note, this should only be used if no fluorochromes were used to target
+        those detectors. Null channels do not contribute to compensation and should not be
+        included in a compensation matrix for this sample.
 
     :param ignore_offset_error: option to ignore data offset error (see above note), default is False
 
@@ -183,7 +183,12 @@ def __init__(
         except KeyError:
             self.version = None
 
-        self.null_channels = null_channel_list
+        # Ensure null channels is a list for checking later
+        if null_channel_list is None:
+            self.null_channels = []
+        else:
+            self.null_channels = null_channel_list
+
         self.event_count = flow_data.event_count
 
         # make a temp channels dict, self.channels will be a DataFrame built from it
@@ -229,7 +234,11 @@ def __init__(
             else:
                 channel_lin_log.append((0.0, 0.0))
 
-            if channel_label.lower()[:4] not in ['fsc-', 'ssc-', 'time']:
+            # Determine fluoro vs scatter vs time channels
+            # Null channels are excluded from any category.
+            if channel_label in self.null_channels:
+                pass
+            elif channel_label.lower()[:4] not in ['fsc-', 'ssc-', 'time']:
                 self.fluoro_indices.append(n - 1)
             elif channel_label.lower()[:4] in ['fsc-', 'ssc-']:
                 self.scatter_indices.append(n - 1)
@@ -468,7 +477,9 @@ def apply_compensation(self, compensation, comp_id='custom_spill'):
         Applies given compensation matrix to Sample events. If any
         transformation has been applied, it will be re-applied after
         compensation. Compensated events can be retrieved afterward
-        by calling `get_events` with `source='comp'`.
+        by calling `get_events` with `source='comp'`. Note, if the
+        sample specifies null channels then these must not be present
+        in the compensation matrix.
 
         :param compensation: Compensation matrix, which can be a:
 
@@ -485,17 +496,25 @@ def apply_compensation(self, compensation, comp_id='custom_spill'):
         :return: None
         """
         if isinstance(compensation, Matrix):
-            self.compensation = compensation
-            self._comp_events = self.compensation.apply(self)
+            tmp_matrix = compensation
         elif compensation is not None:
             detectors = [self.pnn_labels[i] for i in self.fluoro_indices]
             fluorochromes = [self.pns_labels[i] for i in self.fluoro_indices]
-            self.compensation = Matrix(comp_id, compensation, detectors, fluorochromes)
-            self._comp_events = self.compensation.apply(self)
+            tmp_matrix = Matrix(comp_id, compensation, detectors, fluorochromes)
         else:
-            # compensation must be None so clear any matrix and comp events
-            self.compensation = None
+            # compensation must be None, we'll clear any stored comp events later
+            tmp_matrix = None
+
+        if tmp_matrix is not None:
+            # We don't check null channels b/c Matrix.apply will catch them.
+            # tmp_matrix ensures we don't store comp events or compensation
+            # unless apply succeeds.
+            self._comp_events = tmp_matrix.apply(self)
+            self.compensation = tmp_matrix
+        else:
+            # compensation given was None, clear any matrix and comp events
             self._comp_events = None
+            self.compensation = None
 
         # Re-apply transform if set
         if self.transform is not None:
@@ -711,11 +730,11 @@ def _transform(self, transform, include_scatter=False):
     def apply_transform(self, transform, include_scatter=False):
         """
         Applies given transform to Sample events, and overwrites the `transform` attribute.
-        By default, only the fluorescent channels are transformed. For fully customized transformations
-        per channel, the `transform` can be specified as a dictionary mapping PnN labels to an instance
-        of the Transform subclass. If a dictionary of transforms is specified, the `include_scatter`
-        option is ignored and only the channels explicitly included in the transform dictionary will
-        be transformed.
+        By default, only the fluorescent channels are transformed (and excludes null channels).
+        For fully customized transformations per channel, the `transform` can be specified as a
+        dictionary mapping PnN labels to an instance of the Transform subclass. If a dictionary
+        of transforms is specified, the `include_scatter` option is ignored and only the channels
+        explicitly included in the transform dictionary will be transformed.
 
         :param transform: an instance of a Transform subclass or a dictionary where the keys correspond
             to the PnN labels and the value is an instance of a Transform subclass.

src/flowkit/_models/transforms/_matrix.py

@@ -1,9 +1,11 @@
 """
 Matrix class
 """
+from copy import copy
 import numpy as np
 import pandas as pd
 import flowutils
+from ...exceptions import FlowKitException
 
 
 class Matrix(object):
@@ -40,6 +42,9 @@ def __init__(
                 "uncompensated or compensated using the spill value from a Sample's metadata"
             )
 
+        # Copy detectors b/c it may be modified
+        detectors = copy(detectors)
+
         if isinstance(spill_data_or_file, np.ndarray):
             spill = spill_data_or_file
         else:
@@ -52,8 +57,16 @@ def __init__(
 
         self.id = matrix_id
         self.matrix = spill
-        # TODO: Should we use a different name other than 'fluorochromes'? They are typically antibodies or markers.
+
+        # Remove any null channels from detector list
+        if null_channels is not None:
+            for nc in null_channels:
+                if nc in detectors:
+                    detectors.remove(nc)
+
         self.detectors = detectors
+
+        # TODO: Should we use a different name other than 'fluorochromes'? They are typically antibodies or markers.
         # Note: fluorochromes attribute is required for compatibility with GatingML exports,
         #       as the GatingML 2.0 requires both the set of detectors and fluorochromes.
         if fluorochromes is None:
@@ -74,6 +87,12 @@ def apply(self, sample):
         :param sample: Sample instance with matching set of detectors
         :return: NumPy array of compensated events
         """
+        # Check that sample fluoro channels match the
+        # matrix detectors
+        sample_fluoro_labels = [sample.pnn_labels[i] for i in sample.fluoro_indices]
+        if not set(self.detectors).issubset(sample_fluoro_labels):
+            raise FlowKitException("Detectors must be a subset of the Sample's fluorochomes")
+
         indices = [
             sample.get_channel_index(d) for d in self.detectors
         ]
@@ -92,6 +111,12 @@ def inverse(self, sample):
         :param sample: Sample instance with matching set of detectors
         :return: NumPy array of compensated events
         """
+        # Check that sample fluoro channels match the
+        # matrix detectors
+        sample_fluoro_labels = [sample.pnn_labels[i] for i in sample.fluoro_indices]
+        if not set(self.detectors).issubset(sample_fluoro_labels):
+            raise FlowKitException("Detectors must be a subset of the Sample's fluorochomes")
+
         indices = [
             sample.get_channel_index(d) for d in self.detectors
         ]

tests/matrix_tests.py

@@ -62,6 +62,8 @@
     'CD4'
 ]
 
+csv_8c_comp_null_channel_file_path = 'data/8_color_data_set/den_comp_null_channel.csv'
+
 
 class MatrixTestCase(unittest.TestCase):
     """Tests related to compensation matrices and the Matrix class"""
@@ -117,3 +119,26 @@ def test_matrix_inverse():
         inv_data = matrix.inverse(sample)
 
         np.testing.assert_almost_equal(inv_data, data_raw, 10)
+
+    def test_null_channels(self):
+        # pretend FITC is a null channel
+        null_channels = ['TNFa FITC FLR-A']
+
+        comp_mat = fk.Matrix(
+            'my_spill',
+            csv_8c_comp_null_channel_file_path,
+            detectors_8c,
+            null_channels=null_channels
+        )
+
+        fcs_file_path = "data/8_color_data_set/fcs_files/101_DEN084Y5_15_E01_008_clean.fcs"
+
+        # test with a sample not using null channels and one using null channels
+        sample1 = fk.Sample(fcs_file_path, null_channel_list=None)
+        sample2 = fk.Sample(fcs_file_path, null_channel_list=null_channels)
+
+        comp_events1 = comp_mat.apply(sample1)
+        comp_events2 = comp_mat.apply(sample2)
+
+        self.assertIsInstance(comp_events1, np.ndarray)
+        self.assertIsInstance(comp_events2, np.ndarray)