Merge pull request #263 from NIEHS/dev

Dev

Smartscope.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: Smartscope
-Version: 0.9.2rc1
+Version: 0.9.3.dev0
 Summary: Smartscope module for automatic CryoEM grid screening
 Author: Jonathan Bouvette
 Author-email: [email protected]

Smartscope/__init__.py

@@ -7,7 +7,7 @@
 import sys
 
 version_file = Path(__file__).parents[1].resolve() / 'VERSION'
-__version__ = version_file.read_text()
+__version__ = version_file.read_text().strip()
 
 LOGLEVEL = os.getenv('LOGLEVEL') if os.getenv('LOGLEVEL') is not None else 'DEBUG'
 

Smartscope/core/config.py

@@ -1,37 +1,43 @@
 # from asyncio import protocols
 import os
-from typing import List
+from typing import List, Dict, Optional, Union
 from pathlib import Path
 import yaml
 
 import logging
 import importlib
 import sys
-from Smartscope.lib.Datatypes.base_plugin import Finder, Classifier, Selector
+from Smartscope.lib.Datatypes.base_plugin import BaseFeatureAnalyzer, Finder, Classifier, Selector
 from Smartscope.lib.Datatypes.base_protocol import BaseProtocol
 
 logger = logging.getLogger(__name__)
 
 
+
+
+def register_plugin(data):
+    if not 'pluginClass' in data.keys():
+        out_class = Finder
+        if 'Classifier' in data['targetClass']:
+            out_class = Classifier
+        if data['targetClass'] == ['Selector']:
+            out_class = Selector
+    else:
+        split = data['pluginClass'].split('.')
+        module = importlib.import_module('.'.join(split[:-1]))
+        out_class = getattr(module, split[-1])
+    return out_class
+
 def register_plugins(directories, factory):
     for directory in directories:
         for file in directory.glob('*.yaml'):
             logger.debug(f'Registering plugin {file}')
             with open(file) as f:
                 data = yaml.safe_load(f)
 
-            if not 'pluginClass' in data.keys():
-                out_class = Finder
-                if 'Classifier' in data['targetClass']:
-                    out_class = Classifier
-                if data['targetClass'] == ['Selector']:
-                    out_class = Selector
-            else:
-                split = data['pluginClass'].split('.')
-                module = importlib.import_module('.'.join(split[:-1]))
-                out_class = getattr(module, split[-1])
+            out_class = register_plugin(data)
 
-            factory[data['name']] = out_class.parse_obj(data)
+            factory[data['name']] = out_class.model_validate(data)
 
 def get_active_plugins_list(external_plugins_directory,external_plugins_list):
     with open(external_plugins_list,'r') as file:
@@ -43,21 +49,81 @@ def register_protocols(directories:List[Path], factory):
             logger.debug(f'Registering protocol {file}')
             with open(file) as f:
                 data = yaml.safe_load(f)
-            factory[data['name']] = BaseProtocol.parse_obj(data)
+            factory[data['name']] = BaseProtocol.model_validate(data)
 
 def register_external_plugins(external_plugins_list,plugins_factory,protocols_factory):   
     for path in external_plugins_list:
-        sys.path.insert(0, str(path))
         register_plugins([path/'smartscope_plugin'/'plugins'],plugins_factory)
         register_protocols([path/'smartscope_plugin'/'protocols'],protocols_factory)
-        sys.path.remove(str(path))
 
 def get_protocol_commands(external_plugins_list):
     from Smartscope.core.protocol_commands import protocolCommandsFactory
     for path in external_plugins_list:
-        sys.path.insert(0, str(path))
-        module = importlib.import_module('smartscope_plugin.protocol_commands')
+        module = importlib.import_module(path.name +'.smartscope_plugin.protocol_commands')
         protocol_commands= getattr(module,'protocolCommandsFactory')
-        sys.path.remove(str(path))
         protocolCommandsFactory.update(protocol_commands)
-    return protocolCommandsFactory
+    return protocolCommandsFactory
+
+class PluginDoesnNotExistError(Exception):
+
+    def __init__(self, plugin, plugins:Optional[Dict]= None) -> None:
+        if plugins is not None:
+            message = f'Plugin \'{plugin}\' does not exist. Available plugins are: {list(plugins.keys())}'
+        super().__init__(message)
+
+class PluginFactory:
+    _plugins_directory: Path
+    _external_plugins_directory: Optional[Path] = None
+    _external_plugins_list_file: Optional[Path] = None
+    _plugins_list: List[Path] = []
+    _plugins_data: Dict[str, Dict] = {}
+    _factory: Dict[str, BaseFeatureAnalyzer] = {}
+
+    def __init__(self, plugins_directory: Union[str,Path], external_plugins_list_file: Optional[Union[str,Path]]=None, external_plugins_directory: Optional[Union[str,Path]]=None) -> None:
+        self._plugins_directory = Path(plugins_directory)
+        if all([external_plugins_list_file is not None, external_plugins_directory is not None]):
+            self._external_plugins_list_file = Path(external_plugins_list_file)
+            self._external_plugins_directory = Path(external_plugins_directory)
+        # self.parse_plugins_directory()
+        # self.read_plugins_files()
+        # self.register_plugins()
+
+    def parse_plugins_directory(self) -> None:
+        self._plugins_list += list(self._plugins_directory.glob('*.yaml'))
+        if self._external_plugins_directory is not None:
+            pass
+
+    def read_plugins_files(self) -> None:
+        for file in self._plugins_list:
+            logger.debug(f'Reading plugin {file}')
+            with open(file) as f:
+                data = yaml.safe_load(f)
+            self._plugins_data[data['name']] = data
+
+    def register_plugin(self, name:str) -> None:
+        data = self._plugins_data[name]
+        out_class = register_plugin(data)
+        logger.debug(f'Registering plugin {name}')
+        self._factory[name] = out_class.model_validate(data)        
+
+    def register_plugins(self):
+        for name in self._plugins_data.keys():
+            self.register_plugin(name)
+
+    def get_plugin(self, name) -> BaseFeatureAnalyzer:
+        if (plugin := self._factory.get(name)) is not None:
+            logger.debug(f'Getting plugin {name}')
+            return plugin
+        if self._plugins_list.get(name) is not None:
+            logger.debug(f'Plugin {name} not registered, registering it now.')
+            return self._register_plugin(name)
+        raise PluginDoesnNotExistError(name, self._factory)
+
+    def get_plugins(self):
+        return self._factory
+
+    def reload_plugins(self):
+        self._plugins_list = []
+        self._plugins_data = {}
+        self._factory = {}
+        self.parse_plugins_directory(PLUGINS_DIRECTORY)

Smartscope/core/data_manipulations.py

@@ -0,0 +1,145 @@
+
+from typing import List, Optional, Dict
+import logging
+import random
+from copy import copy
+from Smartscope.lib.image.target import Target
+from smartscope_connector.Datatypes.querylist import QueryList
+from Smartscope.core.selector_sorter import SelectorSorter, SelectorValueParser, initialize_selector
+from Smartscope.core.settings.worker import PLUGINS_FACTORY
+import numpy as np
+
+from smartscope_connector import models
+
+logger = logging.getLogger(__name__)
+
+def create_target(target:models.target.Target, model:models.target.Target, finder:str, classifier:Optional[str]=None, start_number:int=0, **extra_fields):
+    target_dict = target.to_dict()
+    context = dict(number=start_number)
+    context['finders'] = [models.target_label.Finder.model_validate(target_dict | dict(method_name=finder))]
+    if classifier is not None:
+        context['classifiers'] = [models.target_label.Classifier(method_name=classifier,label=target.quality)]
+    data = target_dict | context | extra_fields
+    obj = model.model_validate(data)
+    return obj
+
+def add_targets(targets:List[Target], model:Target, finder:str, classifier:Optional[str]=None, start_number:int=0, **extra_fields):
+    output = []
+    for ind, target in enumerate(targets):
+        number = ind + start_number
+        output.append(create_target(target, model, finder, classifier, number, **extra_fields))
+    return QueryList(output)
+
+
+def get_target_methods(targets, method_type:['selectors','finders','classifiers']='selectors'):
+    def get_selector_methods_names(target):
+        items = getattr(target, method_type)
+        if isinstance(items, list):
+            return map(lambda x: x.method_name, items)
+        return map(lambda x: x.method_name ,list(items.all()))
+
+    return set().union(*map(get_selector_methods_names, targets))
+
+
+def randomized_choice(filtered_set: set, n: int):
+    choices = []
+    while n >= len(filtered_set):
+        choices += list(filtered_set)
+        n -= len(filtered_set)
+        logger.debug(f'More choices than length of filtered set, choosing one of each {choices}. {n} left to randomly choose from.')
+    for i in range(n):
+
+        choice = random.choice(list(filtered_set))
+        logger.debug(f'For {i}th choice, choosing {choice} from {filtered_set}.')
+        choices.append(choice)
+        filtered_set.remove(choice)
+    return choices
+
+def choose_get_index(lst, value):
+    indices = [i for i, x in enumerate(lst) if x == value]
+    if indices == []:
+        return None
+    choice = random.choice(indices)
+    del lst[choice]
+    return choice
+
+
+def filter_out_of_range(target):
+    return 0 if target.is_out_of_range() else 1
+
+
+def filter_targets(parent, targets):
+    classifiers = get_target_methods(targets, 'classifiers')
+    selectors = get_target_methods(targets, 'selectors')
+
+    ##Filter out of range targets
+    filtered = list(map(filter_out_of_range, targets))
+    logger.debug(f'Filtering {len(filtered)} targets.')
+
+    for classifier in classifiers:
+        for ind, target in enumerate(targets):
+            if filtered[ind] == 0:
+                continue
+            t_classifiers = target.classifiers
+            if not isinstance(t_classifiers, list):
+                t_classifiers = list(t_classifiers.all())
+            label = next(filter(lambda x: x.method_name == classifier, t_classifiers),None)
+            if label is None:
+                continue
+            if PLUGINS_FACTORY[classifier].classes[label.label].value <= 0:
+                filtered[ind] = 0
+                continue
+
+    filtered = np.array(filtered)
+    for selector in selectors:
+        sorter = initialize_selector(parent.grid_id, selector, targets)
+        filtered *= np.array(sorter.classes)
+    logger.debug(f'Filtered classes against classifiers {classifiers} and selectors {selectors}: {filtered}')
+
+    return filtered.tolist()
+
+def apply_filter(targets, filtered):
+    return [target for target, filt in zip(targets, filtered) if filt > 0]
+
+def select_random_areas(targets, filtered, n):
+    filtered_set = set(filtered)
+    if filtered_set == {0}:
+        return []
+    filtered_set.discard(0) 
+    logger.debug(f'Selecting from {len(filtered_set)} subsets.')
+    choices = randomized_choice(filtered_set, n)
+    logger.debug(f'Randomized choices: {choices}')
+    output = []
+    for choice in choices:
+        ind = choose_get_index(filtered, choice)
+        if ind is None:
+            break
+        output.append(targets[ind])
+    return output
+
+def select_n_areas(parent, n, is_bis=False):
+    additional_filters = dict()
+    if is_bis:
+        additional_filters['bis_type'] = 'center'
+    additional_filters['status__isnull'] = True
+    targets = list(parent.targets.filter(**additional_filters))
+    filtered= filter_targets(parent, targets)
+    if n <=0:
+        return apply_filter(targets, filtered)
+    return select_random_areas(targets, filtered, n)
+
+def set_or_update_refined_finder(instance, stage_x, stage_y, stage_z):
+    refined = next(filter(lambda x: x.method_name == 'Recentering',instance.finders), None)
+    if refined is None:
+        original_finder = instance.finders[0]
+        refined = models.target_label.Finder(method_name='Recentering',
+                                x= original_finder.x,
+                                y= original_finder.y,
+                                stage_x=stage_x,
+                                stage_y=stage_y,
+                                stage_z=stage_z,)
+        instance.finders.insert(0,refined)
+        return instance
+    index = instance.finders.index(refined)
+    instance.finders[index].set_stage_position(x=stage_x, y=stage_y, z=stage_z)
+    return instance

Smartscope/core/db_manipulations.py

@@ -132,11 +132,11 @@ def viewer_only(user):
     return False
 
 
-def group_holes_for_BIS(hole_models, max_radius=4, min_group_size=1, queue_all=False, iterations=500, score_weight=2):
+def group_holes_for_BIS(hole_models, max_radius=4, min_group_size=1, iterations=500, score_weight=2):
     if len(hole_models) == 0:
         return  hole_models
     logger.debug(
-        f'grouping params, max radius = {max_radius}, min group size = {min_group_size}, queue all = {queue_all}, max iterations = {iterations}, score_weight = {score_weight}')
+        f'grouping params, max radius = {max_radius}, min group size = {min_group_size}, max iterations = {iterations}, score_weight = {score_weight}')
     # Extract coordinated for the holes
     prefetch_related_objects(hole_models, 'finders')
     coords = []
@@ -198,10 +198,6 @@ def group_holes_for_BIS(hole_models, max_radius=4, min_group_size=1, queue_all=F
         center = hole_models[i]
         group_name = center.generate_bis_group_name()
 
-        if queue_all:
-            center.selected = True
-            center.status = 'queued'
-
         bis = g[g != i]
         for item in bis:
             i = hole_models[item]

Smartscope/core/frames.py

@@ -0,0 +1,36 @@
+import yaml
+import re
+import logging
+from Smartscope.core.models import AutoloaderGrid
+from Smartscope.core.settings.worker import SMARTSCOPE_CUSTOM_CONFIG
+
+logger = logging.getLogger(__name__)
+
+
+def get_frames_prefix(grid:AutoloaderGrid):
+    detector = grid.parent.detector_id
+    custom_paths = SMARTSCOPE_CUSTOM_CONFIG / 'custom_paths.yaml'
+    if not custom_paths.exists():
+        logger.debug(f'No custom paths file found at {custom_paths}')
+        return ''
+    file = yaml.safe_load(custom_paths.read_text())
+    key = f'detector_id_{detector.pk}'
+    paths = file.get(key, None)
+    if paths is None:
+        logger.debug(f'No key {key} file found at {custom_paths}')
+        return ''
+    return paths.get('frames_prefix', '')
+
+
+def parse_frames_prefix(prefix:str, grid:AutoloaderGrid):
+    pattern = r'.*(\{\{.*\}\})'
+    matches = re.findall(pattern, prefix)
+    for match in matches:
+        clean_match = match.replace('{{', '').replace('}}', '')
+        split = clean_match.split('.')
+        x = grid
+        for s in split:
+            x = getattr(x, s)
+        logger.debug(f'Parsed {match} to {x}')
+        prefix = prefix.replace(match,x)
+    return prefix

Smartscope/core/grid/diagnostics.py

@@ -14,7 +14,7 @@ def generate_diagnostic_figure(image:np.array, coords_set, outputpath:Path):
     for coords, color, perc_im_radius in coords_set:
         radius = int(image_color.shape[0]*(perc_im_radius/100))
         for coord in coords:
-            logger.info(f'{coord}, {type(coord)}')
+            # logger.info(f'{coord}, {type(coord)}')
             cv2.circle(image_color,coord,radius,color=color, thickness=cv2.FILLED)
     cv2.imwrite(str(outputpath), imutils.resize(image_color,512))
 

Smartscope/core/grid/grid_io.py

@@ -38,4 +38,10 @@ def create_dirs_docker(working_dir):
     def create_grid_directories(path: str) -> None:
         path = Path(path)
         for directory in [path, path / 'raw', path / 'pngs']:
-            directory.mkdir(exist_ok=True)
+            directory.mkdir(exist_ok=True)
+
+    @staticmethod
+    def create_grid_frames_directory(path, grid_dir):
+        directory = Path(path, grid_dir)
+        directory.mkdir(parents=True, exist_ok=True)
+        return grid_dir