Merge pull request #289 from NIEHS/stable

Stable

Docker/Dockerfile-smartscope

@@ -50,6 +50,7 @@ COPY --link ctffind /opt/
 COPY --link miniconda3/ /opt/miniconda3/
 COPY smartscope/ /opt/smartscope/
 COPY external_plugins/ /opt/external_plugins/
+COPY Smartscope-connector /opt/Smartscope-connector/
 
 
 # RUN wget docs.smartscope.org/downloads/Smartscope0.6.tar.gz --no-check-certificate && \
@@ -63,7 +64,8 @@ RUN	mkdir /opt/logs/ /mnt/fake_scope/ /mnt/fake_scope/raw/ /mnt/fake_scope/movie
 RUN chmod 777 -R /mnt/ && \
 	chmod 777 -R /opt/logs/ && \
 	chmod 777 /opt/shared/ && \
-	chmod 777 /opt/config/
+	chmod 777 /opt/config/ && \
+	chmod -R 777 /opt/Template_files
 	# chmod 777 /mnt/testfiles/
 
 USER smartscope_user

Docker/SmartScope/docker-compose.yml

@@ -5,7 +5,7 @@
 version: "3"
 services:
   smartscope:
-    image: ghcr.io/niehs/smartscope:latest
+    image: ghcr.io/niehs/smartscope:stable
     restart: always
     volumes:
       ######## FILL THIS ##########

Docker/SmartScope/smartscope.ps1

@@ -0,0 +1,163 @@
+# Define constants
+$cmd_file = "dockerCmd.txt"
+$dockerRepo = "ghcr.io/niehs/smartscope"
+$composeCmd = "docker compose"
+$dockerCmd = "docker"
+
+# Check if the command file exists and read commands
+if (Test-Path $cmd_file) {
+    $cmds = Get-Content $cmd_file
+    $dockerCmd = $cmds[0]
+    $composeCmd = $cmds[1]
+}
+
+# Define the help text function
+function Show-Help {
+    $OPTIONS = @"
+Usage: $($MyInvocation.MyCommand.Name) subcommand [command]
+
+Subcommand              DESCRIPTION
+==========              ===========
+start                   Start SmartScope
+stop                    Stop SmartScope
+restart                 Restart SmartScope
+setup                   Setup the SmartScope directories
+update [version]        Update SmartScope to the specified version. Choices: ['latest','stable'] (default: latest)
+run [command]           Run a SmartScope command in the SmartScope container
+exec [command]          Run any shell command in the SmartScope container
+python                  Runs an interactive ipython shell inside the SmartScope container
+"@
+    Write-Output $OPTIONS
+}
+
+# Define the stop function
+function Stop-SmartScope {
+    Write-Output "Stopping SmartScope"
+    & $composeCmd down
+}
+
+# Define the start function
+function Start-SmartScope {
+    $env:UID = [System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value
+    $env:GID = [System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value
+    Write-Output "Starting SmartScope"
+    & $composeCmd -f docker-compose.yml -f smartscope.yml up -d
+}
+
+# Define the prompt function
+function Prompt-YesNo ($question) {
+    while ($true) {
+        Write-Output "$question (y/n)"
+        $user_input = Read-Host
+
+        if ($user_input -eq "y") {
+            return $true
+        } elseif ($user_input -eq "n") {
+            return $false
+        } else {
+            Write-Output "Invalid choice. Please enter 'yes' or 'no'."
+        }
+    }    
+}
+
+# Define the check for updates function
+function Check-ForUpdates ($version) {
+    Write-Output "Checking for updates to version $version"
+    $online_sha = & $dockerCmd manifest inspect $dockerRepo:$version | Select-String -Pattern '"digest": "sha256:([^"]*)"' | ForEach-Object { $_.Matches.Groups[1].Value }
+    $local_sha = & $dockerCmd inspect --format "{{.ID}}" $dockerRepo:$version | ForEach-Object { $_.Split(':')[1] }
+    Write-Output "Online sha: $online_sha`nLocal sha: $local_sha"
+    if ($online_sha -eq $local_sha) {
+        return $false
+    } else {
+        return $true
+    }
+}
+
+# Process the argument
+$argument = if ($args.Count -gt 0) { $args[0] } else { Show-Help; exit 1 }
+
+Set-Location (Split-Path -Parent $MyInvocation.MyCommand.Path)
+
+switch ($argument) {
+    "start" {
+        if (Check-ForUpdates 'latest') {
+            Write-Output "A new release is available. Run 'smartscope.ps1 update latest' to update to the latest version."
+        }
+        if (Check-ForUpdates 'stable') {
+            Write-Output "A new beta version is available. Run 'smartscope.ps1 update stable' to update to the new beta version."
+        }
+        Start-SmartScope
+    }
+    "stop" {
+        Stop-SmartScope
+    }
+    "restart" {
+        Stop-SmartScope
+        Start-Sleep -Seconds 2
+        Start-SmartScope
+    }
+    "run" {
+        $cmd = "$composeCmd exec smartscope smartscope.py $($args[1..$args.Length - 1] -join ' ')"
+        Write-Output "Executing command inside the SmartScope container:`n$cmd"
+        iex $cmd
+    }
+    "help" { Show-Help }
+    "-h" { Show-Help }
+    "--help" { Show-Help }
+    "python" {
+        Write-Output "Running a python shell inside the SmartScope container"
+        & $composeCmd exec -it smartscope manage.py shell -i ipython
+    }
+    "exec" {
+        Write-Output "Executing shell command inside the SmartScope container:"
+        & $composeCmd exec -it smartscope $args[1..$args.Length - 1]
+    }
+    "setup" {
+        $version = if ($args.Count -gt 1) { $args[1] } else { 'latest' }
+        Write-Output "Setting up the latest version of SmartScope"
+        if ($version -eq 'latest') { $version = 'main' }
+        Write-Output "Setting up the SmartScope directories"
+        New-Item -ItemType Directory -Force -Path logs, shared/nginx, shared/auth, shared/smartscope, db, data, backups
+        foreach ($file in @("docker-compose.yml", "smartscope.yml", "smartscope.conf", "database.conf")) {
+            Write-Output "Pulling $file from $version"
+            if (Test-Path $file) {
+                Write-Output "$file already exists. Skipping."
+                continue
+            }
+            Invoke-WebRequest -Uri "https://raw.githubusercontent.com/NIEHS/SmartScope/$version/Docker/SmartScope/$file" -OutFile $file
+        }
+        Write-Output "Pulling initialdb.sql from $version"
+        Invoke-WebRequest -Uri "https://raw.githubusercontent.com/NIEHS/SmartScope/$version/Docker/SmartScope/shared/initialdb.sql" -OutFile "shared/initialdb.sql"
+    }
+    "update" {
+        $version = if ($args.Count -gt 1) { $args[1] } else { 'latest' }
+        Write-Output "Updating SmartScope to version: $version"
+        if (-not (Check-ForUpdates $version)) {
+            if (-not (Prompt-YesNo "You already have the docker image corresponding to version $version. Do you want to update this instance anyway?")) {
+                exit 0
+            }
+        }
+
+        $backupDir = "backups/$(Get-Date -Format 'yyyyMMdd')_config_pre_update"
+        if (Prompt-YesNo "Do you want to back up the database? (Highly recommended)") {
+            Write-Output "Backing up the database. This may take a few minutes."
+            & $composeCmd exec smartscope smartscope.py backup_db
+        }
+        Write-Output "Creating a backup of the configuration in $backupDir"
+        New-Item -ItemType Directory -Force -Path $backupDir
+        Copy-Item -Path docker-compose.yml, smartscope.yml, smartscope.conf, database.conf, smartscope.ps1 -Destination $backupDir
+        Write-Output "Pulling updated docker-compose.yml"
+        $repo_url = "https://raw.githubusercontent.com/NIEHS/SmartScope/$version/Docker/SmartScope"
+        Invoke-WebRequest -Uri "$repo_url/docker-compose.yml" -OutFile "docker-compose.yml"
+        Write-Output "Pulling docker image"
+        & $composeCmd pull smartscope
+        if (Prompt-YesNo "Files updated, do you want to restart SmartScope") {
+            & $MyInvocation.MyCommand.Path restart
+        }
+    }
+    default {
+        Write-Output "Unknown command error: $argument"
+        Show-Help
+        exit 1
+    }
+}

Smartscope/__init__.py

@@ -7,7 +7,7 @@
 import sys
 
 version_file = Path(__file__).parents[1].resolve() / 'VERSION'
-__version__ = version_file.read_text()
+__version__ = version_file.read_text().strip()
 
 LOGLEVEL = os.getenv('LOGLEVEL') if os.getenv('LOGLEVEL') is not None else 'DEBUG'
 

Smartscope/core/config.py

@@ -1,17 +1,27 @@
 # from asyncio import protocols
-import os
-from typing import List
+from typing import List, Dict, Optional, Union
 from pathlib import Path
 import yaml
 
 import logging
 import importlib
-import sys
-from Smartscope.lib.Datatypes.base_plugin import Finder, Classifier, Selector
+from Smartscope.lib.Datatypes.base_plugin import BaseFeatureAnalyzer, Finder, Classifier, Selector
 from Smartscope.lib.Datatypes.base_protocol import BaseProtocol
 
 logger = logging.getLogger(__name__)
 
+def register_plugin(data):
+    if not 'pluginClass' in data.keys():
+        out_class = Finder
+        if 'Classifier' in data['targetClass']:
+            out_class = Classifier
+        if data['targetClass'] == ['Selector']:
+            out_class = Selector
+    else:
+        split = data['pluginClass'].split('.')
+        module = importlib.import_module('.'.join(split[:-1]))
+        out_class = getattr(module, split[-1])
+    return out_class
 
 def register_plugins(directories, factory):
     for directory in directories:
@@ -20,20 +30,11 @@ def register_plugins(directories, factory):
             with open(file) as f:
                 data = yaml.safe_load(f)
 
-            if not 'pluginClass' in data.keys():
-                out_class = Finder
-                if 'Classifier' in data['targetClass']:
-                    out_class = Classifier
-                if data['targetClass'] == ['Selector']:
-                    out_class = Selector
-            else:
-                split = data['pluginClass'].split('.')
-                module = importlib.import_module('.'.join(split[:-1]))
-                out_class = getattr(module, split[-1])
+            out_class = register_plugin(data)
 
-            factory[data['name']] = out_class.parse_obj(data)
+            factory[data['name']] = out_class.model_validate(data)
 
-def get_active_plugins_list(external_plugins_directory,external_plugins_list):
+def get_active_plugins_list(external_plugins_directory,external_plugins_list) -> List[Path]:
     with open(external_plugins_list,'r') as file:
         return [external_plugins_directory / plugin.strip() for plugin in file.readlines()]
 
@@ -43,21 +44,95 @@ def register_protocols(directories:List[Path], factory):
             logger.debug(f'Registering protocol {file}')
             with open(file) as f:
                 data = yaml.safe_load(f)
-            factory[data['name']] = BaseProtocol.parse_obj(data)
+            factory[data['name']] = BaseProtocol.model_validate(data)
 
-def register_external_plugins(external_plugins_list,plugins_factory,protocols_factory):   
+def register_external_protocols(external_plugins_list,protocols_factory):   
     for path in external_plugins_list:
-        sys.path.insert(0, str(path))
-        register_plugins([path/'smartscope_plugin'/'plugins'],plugins_factory)
+        # register_plugins([path/'smartscope_plugin'/'plugins'],plugins_factory)
         register_protocols([path/'smartscope_plugin'/'protocols'],protocols_factory)
-        sys.path.remove(str(path))
 
 def get_protocol_commands(external_plugins_list):
     from Smartscope.core.protocol_commands import protocolCommandsFactory
     for path in external_plugins_list:
-        sys.path.insert(0, str(path))
-        module = importlib.import_module('smartscope_plugin.protocol_commands')
+        module = importlib.import_module(path.name +'.smartscope_plugin.protocol_commands')
         protocol_commands= getattr(module,'protocolCommandsFactory')
-        sys.path.remove(str(path))
         protocolCommandsFactory.update(protocol_commands)
-    return protocolCommandsFactory
+    return protocolCommandsFactory
+
+class PluginDoesnNotExistError(Exception):
+
+    def __init__(self, plugin, plugins:Optional[Dict]= None) -> None:
+        if plugins is not None:
+            plugins = '\n- '.join(sorted(list(plugins.keys())))
+            message = f"Plugin \'{plugin}\' does not exist. Available plugins are:\n- {plugins}"
+        super().__init__(message)
+
+class PluginFactory:
+    _plugins_directories: List[Path] = []
+    _external_plugins_directory: Optional[Path] = None
+    _external_plugins_list_file: Optional[Path] = None
+    _plugins_list: List[Path] = []
+    _plugins_data: Dict[str, Dict] = {}
+    _factory: Dict[str, BaseFeatureAnalyzer] = {}
+
+    def __init__(self, *plugins_directory: Union[str,Path], external_plugins_list_file: Optional[Union[str,Path]]=None, external_plugins_directory: Optional[Union[str,Path]]=None) -> None:
+        for directory in plugins_directory:
+            self._plugins_directories.append(Path(directory))
+        if all([external_plugins_list_file is not None, external_plugins_directory is not None]):
+            self._external_plugins_list_file = Path(external_plugins_list_file)
+            self._external_plugins_directory = Path(external_plugins_directory)
+
+    def parse_plugins_directories(self) -> None:
+        for directory in self._plugins_directories:
+            self._plugins_list += list(directory.glob('*.yaml'))
+        if self._external_plugins_directory is not None:
+            external_plugins = get_active_plugins_list(self._external_plugins_directory, self._external_plugins_list_file)
+            for plugin in external_plugins:
+                self._plugins_list += list((plugin/'smartscope_plugin'/'plugins').glob('*.yaml'))
+
+    def read_plugin_file(self, file) -> None:
+        logger.debug(f'Reading plugin {file}')
+        with open(file) as f:
+            data = yaml.safe_load(f)
+        self._plugins_data[data['name']] = data
+
+    def read_plugins_files(self) -> None:
+        for file in self._plugins_list:
+            self.read_plugin_file(file)
+
+
+    def register_plugin(self, name:str) -> None:
+        data = self._plugins_data[name]
+        out_class = register_plugin(data)
+        logger.debug(f'Registering plugin {name}')
+        self._factory[name] = out_class.model_validate(data)        
+
+    def register_plugins(self):
+        for name in self._plugins_data.keys():
+            self.register_plugin(name)
+
+    def get_plugin(self, name) -> BaseFeatureAnalyzer:
+        if self._factory == {}:
+            logger.debug('No plugins registered, loading plugins now.')
+            self.load_plugins()
+        if (plugin := self._factory.get(name)) is not None:
+            logger.debug(f'Getting plugin {name}')
+            return plugin
+        raise PluginDoesnNotExistError(name, self._factory)
+
+    def get_plugins(self):
+        if self._factory == {}:
+            logger.debug('No plugins registered, loading plugins now.')
+            self.load_plugins()
+        return self._factory
+
+    def load_plugins(self):
+        self._plugins_list = []
+        self._plugins_data = {}
+        self._factory = {}
+        self.parse_plugins_directories()
+        self.read_plugins_files()
+        self.register_plugins()
+
+    def reload_plugins(self):
+        self.load_plugins()