diff --git a/.gitignore b/.gitignore index 82ab4bdd..ef825953 100755 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,7 @@ docker-compose.yml.bak .config .python_history MyDocker/* - +.ipython +.mysql_history +.gitignore +.ipython/profile_default/history.sqlite diff --git a/.ipython/profile_default/startup/README b/.ipython/profile_default/startup/README new file mode 100644 index 00000000..61d47000 --- /dev/null +++ b/.ipython/profile_default/startup/README @@ -0,0 +1,11 @@ +This is the IPython startup directory + +.py and .ipy files in this directory will be run *prior* to any code or files specified +via the exec_lines or exec_files configurables whenever you load this profile. + +Files will be run in lexicographical order, so you can control the execution order of files +with a prefix, e.g.:: + + 00-first.py + 50-middle.py + 99-last.ipy diff --git a/Docker/Dockerfile-base b/Docker/Dockerfile-base index 14337e71..c1c923eb 100644 --- a/Docker/Dockerfile-base +++ b/Docker/Dockerfile-base @@ -1,50 +1,37 @@ -FROM nvidia/cuda:10.1-base-ubuntu18.04 - +FROM nvidia/cuda:12.1.0-base-ubuntu20.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y \ - python3-dev default-libmysqlclient-dev build-essential wget libglib2.0-0 ffmpeg libsm6 libxext6 curl mariadb-server && \ + python3-dev default-libmysqlclient-dev build-essential wget libglib2.0-0 ffmpeg libsm6 libxext6 curl mariadb-server git && \ apt-get clean && rm -rf /var/cache/apt/lists -# RUN service mariadb stop && service mariadb disable - RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 && \ rm Miniconda3-latest-Linux-x86_64.sh -RUN wget https://bio3d.colorado.edu/imod/AMD64-RHEL5/imod_4.11.15_RHEL7-64_CUDA10.1.sh && \ - yes | bash imod_4.11.15_RHEL7-64_CUDA10.1.sh -name IMOD && \ - rm imod_4.11.15_RHEL7-64_CUDA10.1.sh +RUN wget https://bio3d.colorado.edu/imod/AMD64-RHEL5/imod_4.11.24_RHEL7-64_CUDA10.1.sh && ls && \ + yes | bash imod_4.11.24_RHEL7-64_CUDA10.1.sh -name IMOD && \ + rm imod_4.11.24_RHEL7-64_CUDA10.1.sh ADD config/singularity/ctffind /usr/local/ ADD config/docker/requirements.txt . -# RUN mkdir /opt/logs/ /mnt/fake_scope/ /mnt/fake_scope/raw/ /opt/nginx-mount/ - - - -ENV PATH=$PATH:/opt/smartscope/Smartscope/bin:/opt/miniconda3/bin:$IMOD_DIR/bin +ENV PATH=$PATH:/opt/miniconda3/bin RUN conda update -y conda && \ - yes | conda install python=3.9 cudatoolkit=10.2 cudnn=7.6 && \ - yes | pip install numpy==1.21.0 && \ - yes | pip install torch==1.8.2 torchvision==0.9.2 torchaudio==0.8.2 --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu102 && \ - yes | pip install -r requirements.txt && \ - yes | pip install ipython && \ - yes | pip install sphinx-multiversion && \ + yes | conda install python=3.9 && \ + yes | conda install cudatoolkit=11.1 cudnn -c nvidia +RUN yes | conda install pytorch torchvision -c pytorch-lts -c nvidia && \ conda clean --all -# RUN wget docs.smartscope.org/downloads/Smartscope0.6.tar.gz --no-check-certificate &&\ -# tar -xvf Smartscope0.6.tar.gz -C /opt/ &&\ -# rm Smartscope0.6.tar.gz +RUN yes | pip install numpy==1.21.0 && \ + # yes | pip install torch==1.8.2 torchvision==0.9.2 torchaudio==0.8.2 --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu111 && \ + yes | pip install -r requirements.txt + # create a non-root user ARG USER_ID=9999 -RUN useradd -m --no-log-init --system --uid $USER_ID smartscope_user - -# RUN chmod 777 -R /mnt/ && \ -# chmod 777 /opt/logs/ && \ -# chmod 777 /opt/nginx-mount/ \ No newline at end of file +RUN useradd -m --no-log-init --system --uid $USER_ID smartscope_user \ No newline at end of file diff --git a/Docker/Dockerfile-smartscope b/Docker/Dockerfile-smartscope index 0ac1d618..37b72376 100644 --- a/Docker/Dockerfile-smartscope +++ b/Docker/Dockerfile-smartscope @@ -35,13 +35,15 @@ ENV ALLOWED_HOSTS=localhost \ MYSQL_HOST=db \ MYSQL_PORT=3306 \ MYSQL_USERNAME=root \ - MYSQL_ROOT_PASSWORD=pass \ + MYSQL_PASSWORD=pass \ DB_NAME=smartscope \ REDIS_HOST=cache \ REDIS_PORT=6379 \ USE_AWS=False \ FORCE_CPU=False +ENV PATH=$PATH:/opt/smartscope/Smartscope/bin:/usr/local/IMOD/bin + ADD . /opt/smartscope/ RUN wget docs.smartscope.org/downloads/Smartscope0.6.tar.gz --no-check-certificate && \ diff --git a/Docker/SmartScope/database.conf b/Docker/SmartScope/database.conf index 1df496f9..5be08b57 100644 --- a/Docker/SmartScope/database.conf +++ b/Docker/SmartScope/database.conf @@ -1,3 +1,6 @@ MYSQL_USER=root -MYSQL_ROOT_PASSWORD=pass -MYSQL_DATABASE=smartscope \ No newline at end of file +MYSQL_PASSWORD=pass +MYSQL_DATABASE=smartscope +MYSQL_HOST=db +MYSQL_PORT=3306 +MYSQL_SSL=False diff --git a/Docker/SmartScope/smartscope.conf b/Docker/SmartScope/smartscope.conf index 994294ef..7f604e84 100644 --- a/Docker/SmartScope/smartscope.conf +++ b/Docker/SmartScope/smartscope.conf @@ -3,12 +3,10 @@ ALLOWED_HOSTS=localhost AUTO_MIGRATION=true USE_SSL=false USE_LONGTERMSTORAGE=False +TIMEZONE=America/New_York ### EXPERT SETTINGS ### ### EVERYTHING BELOW SHOULD NOT NEED TO BE CHANGED### -#Other database settings -# MYSQL_HOST=db -# MYSQL_PORT=3306 # #Executables # IMOD_DIR=/usr/local/IMOD diff --git a/Smartscope/bin/smartscope.py b/Smartscope/bin/smartscope.py index ec12b455..5db14765 100755 --- a/Smartscope/bin/smartscope.py +++ b/Smartscope/bin/smartscope.py @@ -10,6 +10,7 @@ import logging from Smartscope.core.main_commands import main + logger = logging.getLogger(__name__) diff --git a/Smartscope/bin/smartscope.sh b/Smartscope/bin/smartscope.sh new file mode 100755 index 00000000..974e61cf --- /dev/null +++ b/Smartscope/bin/smartscope.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +smartscope.py $@ & \ No newline at end of file diff --git a/Smartscope/core/autoscreen.py b/Smartscope/core/autoscreen.py index 29e0e26d..83a9b645 100644 --- a/Smartscope/core/autoscreen.py +++ b/Smartscope/core/autoscreen.py @@ -2,17 +2,18 @@ import os import sys import time -import shlex from Smartscope.core.microscope_interfaces import FakeScopeInterface, TFSSerialemInterface, JEOLSerialemInterface from Smartscope.core.selectors import selector_wrapper -from Smartscope.core.models import ScreeningSession, HoleModel, SquareModel, Process, HighMagModel +from Smartscope.core.models import ScreeningSession, HoleModel, SquareModel, Process, HighMagModel, AutoloaderGrid from Smartscope.core.settings.worker import PROTOCOLS_FACTORY, PROTOCOL_COMMANDS_FACTORY from Smartscope.lib.image_manipulations import auto_contrast_sigma, fourier_crop, export_as_png from Smartscope.lib.montage import Montage,create_targets_from_center from Smartscope.core.finders import find_targets +from Smartscope.core.status import status, grid_status, FileSignals from Smartscope.core.protocols import get_or_set_protocol -from Smartscope.lib.Datatypes.microscope import Microscope,Detector,AtlasSettings +from Smartscope.lib.Datatypes.microscope import Microscope,Detector,AtlasSettings, MicroscopeInterface from Smartscope.lib.preprocessing_methods import processing_worker_wrapper +from Smartscope.core.preprocessing_pipelines import load_preprocessing_pipeline from Smartscope.lib.file_manipulations import get_file_and_process, create_grid_directories from Smartscope.lib.transformations import register_to_other_montage, register_targets_by_proximity from Smartscope.core.db_manipulations import update, select_n_areas, queue_atlas, add_targets, group_holes_for_BIS @@ -23,7 +24,6 @@ from django.utils import timezone import multiprocessing import logging -import subprocess import numpy as np from pathlib import Path @@ -32,18 +32,9 @@ def get_queue(grid): - """Query database to refresh the queue - - Args: - grid (AutoloaderGrid): AutoloadGrid object from Smartscope.server.models - - Returns: - (list): [squares,holes]; List of SquareModels that are status='queued' and List of HoleModels that are status='queued' - """ - squares = list(grid.squaremodel_set.filter(selected=True, status__in=['queued', 'started']).order_by('number')) - holes = list(grid.holemodel_set.filter(selected=True, square_id__status='completed').exclude(status='completed').order_by('square_id__completion_time', 'number')) - logger.debug(f'Pre-queue Holes: {holes}') - return squares, holes#[h for h in holes if not h.bisgroup_acquired] + square = grid.squaremodel_set.filter(selected=True, status__in=[status.QUEUED, status.STARTED]).order_by('number').first() + hole = grid.holemodel_set.filter(selected=True, square_id__status=status.COMPLETED).exclude(status=status.COMPLETED).order_by('square_id__completion_time', 'number').first() + return square, hole#[h for h in holes if not h.bisgroup_acquired] def resume_incomplete_processes(queue, grid, microscope_id): @@ -55,32 +46,32 @@ def resume_incomplete_processes(queue, grid, microscope_id): session (ScreeningSession): ScreeningSession object from Smartscope.server.models """ squares = grid.squaremodel_set.filter(selected=1).exclude( - status__in=['queued', 'started', 'completed']).order_by('number') + status__in=[status.QUEUED, status.STARTED, status.COMPLETED]).order_by('number') holes = grid.holemodel_set.filter(selected=1).exclude( - status__in=['queued', 'started', 'processed', 'completed']).order_by('square_id__number', 'number') + status__in=[status.QUEUED, status.STARTED, status.PROCESSED, status.COMPLETED]).order_by('square_id__number', 'number') for square in squares: logger.info(f'Square {square} was not fully processed') transaction.on_commit(lambda: queue.put([process_square_image, [square, grid, microscope_id], {}])) -def print_queue(squares, holes, session): - """Prints Queue to a file for displaying to the frontend +# def print_queue(squares, holes, session): +# """Prints Queue to a file for displaying to the frontend - Args: - squares (list): list of squares returned from the get_queue method - holes (list): list of holes returned from the get_queue method - session (ScreeningSession): ScreeningSession object from Smartscope.server.models - """ - string = ['------------------------------------------------------------\nCURRENT QUEUE:\n------------------------------------------------------------\nSquares:\n'] - for s in squares: - string.append(f"\t{s.number} -> {s.name}\n") - string.append(f'------------------------------------------------------------\nHoles: (total={len(holes)})\n') - for h in holes: - string.append(f"\t{h.number} -> {h.name}\n") - string.append('------------------------------------------------------------\n') - string = ''.join(string) - with open(os.path.join(session.directory, 'queue.txt'), 'w') as f: - f.write(string) +# Args: +# squares (list): list of squares returned from the get_queue method +# holes (list): list of holes returned from the get_queue method +# session (ScreeningSession): ScreeningSession object from Smartscope.server.models +# """ +# string = ['------------------------------------------------------------\nCURRENT QUEUE:\n------------------------------------------------------------\nSquares:\n'] +# for s in squares: +# string.append(f"\t{s.number} -> {s.name}\n") +# string.append(f'------------------------------------------------------------\nHoles: (total={len(holes)})\n') +# for h in holes: +# string.append(f"\t{h.number} -> {h.name}\n") +# string.append('------------------------------------------------------------\n') +# string = ''.join(string) +# with open(os.path.join(session.directory, 'queue.txt'), 'w') as f: +# f.write(string) def is_stop_file(sessionid: str) -> bool: @@ -96,7 +87,7 @@ def runAcquisition(scope,methods,params,instance): output = PROTOCOL_COMMANDS_FACTORY[method](scope,params,instance) return output -def run_grid(grid, session, processing_queue, scope): +def run_grid(grid:AutoloaderGrid, session:ScreeningSession, processing_queue:multiprocessing.JoinableQueue, scope:MicroscopeInterface): """Main logic for the SmartScope process Args: @@ -104,25 +95,25 @@ def run_grid(grid, session, processing_queue, scope): session (ScreeningSession): ScreeningSession object from Smartscope.server.models """ - if grid.status == 'complete': - logger.info(f'{grid.name} already complete') - return - if grid.status == 'aborting': - logger.info(f'Aborting {grid.name}') - return - session_id = session.pk microscope = session.microscope_id # Set the Websocket_update_decorator grid property update.grid = grid + if grid.status == grid_status.COMPLETED: + logger.info(f'{grid.name} already complete') + return + if grid.status == grid_status.ABORTING: + logger.info(f'Aborting {grid.name}') + update(grid, status=grid_status.COMPLETED) + return logger.info(f'Starting {grid.name}') grid = update(grid, refresh_from_db=True, last_update=None) - if grid.status is None: - grid = update(grid, status='started', start_time=timezone.now()) + if grid.status is grid_status.NULL: + grid = update(grid, status=grid_status.STARTED, start_time=timezone.now()) create_grid_directories(grid.directory) os.chdir(grid.directory) @@ -131,7 +122,8 @@ def run_grid(grid, session, processing_queue, scope): # ADD the new protocol loader protocol = get_or_set_protocol(grid) resume_incomplete_processes(processing_queue, grid, session.microscope_id) - subprocess.Popen(shlex.split(f'smartscope.py highmag_processing smartscopePipeline {grid.grid_id} 1')) + preprocessing = load_preprocessing_pipeline(Path('preprocessing.json')) + preprocessing.start(grid) is_stop_file(session_id) atlas = queue_atlas(grid) scope.loadGrid(grid.position) @@ -140,24 +132,23 @@ def run_grid(grid, session, processing_queue, scope): scope.reset_state() grid_type = grid.holeType grid_mesh = grid.meshMaterial - if atlas.status == 'queued' or atlas.status == 'started': - - atlas = update(atlas, status='started') + if atlas.status == status.QUEUED or atlas.status == status.STARTED: + atlas = update(atlas, status=status.STARTED) logger.info('Waiting on atlas file') runAcquisition(scope,protocol.atlas.acquisition,params,atlas) - atlas = update(atlas, status='acquired', completion_time=timezone.now()) - if atlas.status == 'acquired': + atlas = update(atlas, status=status.ACQUIRED, completion_time=timezone.now()) + if atlas.status == status.ACQUIRED: logger.info('Atlas acquired') montage = get_file_and_process(raw=atlas.raw, name=atlas.name, directory=microscope.scope_path) export_as_png(montage.image, montage.png) targets, finder_method, classifier_method, _ = find_targets(montage, protocol.atlas.targets.finders) squares = add_targets(grid, atlas, targets, SquareModel, finder_method, classifier_method) - atlas = update(atlas, status='processed', pixel_size=montage.pixel_size, + atlas = update(atlas, status=status.PROCESSED, pixel_size=montage.pixel_size, shape_x=montage.shape_x, shape_y=montage.shape_y, stage_z=montage.stage_z) - if atlas.status == 'processed': + if atlas.status == status.PROCESSED: selector_wrapper(protocol.atlas.targets.selectors, atlas, n_groups=5) select_n_areas(atlas, grid.params_id.squares_num) - atlas = update(atlas, status='completed') + atlas = update(atlas, status=status.COMPLETED) logger.info('Atlas analysis is complete') @@ -167,43 +158,43 @@ def run_grid(grid, session, processing_queue, scope): is_stop_file(session_id) grid = update(grid, refresh_from_db=True, last_update=None) params = grid.params_id - if grid.status == 'aborting': + if grid.status == grid_status.ABORTING: + preprocessing.stop(grid) break else: - squares, holes = get_queue(grid) - print_queue(squares, holes, session) - if len(holes) > 0 and (len(squares) == 0 or grid.collection_mode == 'screening'): + square, hole = get_queue(grid) + if hole is not None and (square is None or grid.collection_mode == 'screening'): is_done = False - hole = holes[0] - hole = update(hole, status='started') + hole = update(hole, status=status.STARTED) runAcquisition(scope,protocol.mediumMag.acquisition,params,hole) - hole = update(hole, status='acquired',completion_time=timezone.now()) + hole = update(hole, status=status.ACQUIRED,completion_time=timezone.now()) process_hole_image(hole, grid, microscope) scope.focusDrift(params.target_defocus_min, params.target_defocus_max, params.step_defocus, params.drift_crit) scope.reset_image_shift_values() - for hm in hole.targets.exclude(status__in=['acquired','completed']).order_by('hole_id__number'): - hm = update(hm, status='started') + for hm in hole.targets.exclude(status__in=[status.ACQUIRED,status.COMPLETED]).order_by('hole_id__number'): + hm = update(hm, status=status.STARTED) hm = runAcquisition(scope,protocol.highMag.acquisition,params,hm) - hm = update(hm, status='acquired', completion_time=timezone.now(), extra_fields=['is_x','is_y','offset','frames']) + hm = update(hm, status=status.ACQUIRED, completion_time=timezone.now(), extra_fields=['is_x','is_y','offset','frames']) if hm.hole_id.bis_type != 'center': - update(hm.hole_id, status='acquired', completion_time=timezone.now()) - update(hole, status='completed') + update(hm.hole_id, status=status.ACQUIRED, completion_time=timezone.now()) + update(hole, status=status.COMPLETED) + scope.reset_AFIS_image_shift(afis=params.afis) scope.refineZLP(params.zeroloss_delay) - elif len(squares) > 0: + scope.collectHardwareDark(params.hardwaredark_delay) + elif square is not None: is_done = False - square = squares[0] - if square.status == 'queued' or square.status == 'started': - square = update(square, status='started') + if square.status == status.QUEUED or square.status == status.STARTED: + square = update(square, status=status.STARTED) logger.info('Waiting on square file') runAcquisition(scope,protocol.square.acquisition,params,square) - square = update(square, status='acquired', completion_time=timezone.now()) + square = update(square, status=status.ACQUIRED, completion_time=timezone.now()) process_square_image(square, grid, microscope) elif is_done: microscope_id = session.microscope_id.pk if os.path.isfile(os.path.join(os.getenv('TEMPDIR'), f'.pause_{microscope_id}')): paused = os.path.join(os.getenv('TEMPDIR'), f'paused_{microscope_id}') open(paused, 'w').close() - update(grid, status='paused') + update(grid, status=grid_status.PAUSED) logger.info('SerialEM is paused') while os.path.isfile(paused): sys.stdout.flush() @@ -213,7 +204,7 @@ def run_grid(grid, session, processing_queue, scope): os.remove(next_file) running = False else: - update(grid, status='started') + update(grid, status=grid_status.STARTED) else: running = False else: @@ -223,7 +214,7 @@ def run_grid(grid, session, processing_queue, scope): is_done = True logger.debug(f'Running: {running}') else: - update(grid, status='complete') + update(grid, status=grid_status.COMPLETED) logger.info('Grid finished') return 'finished' @@ -244,23 +235,23 @@ def process_square_image(square, grid, microscope_id): params = grid.params_id is_bis = params.bis_max_distance > 0 montage = None - if square.status == 'acquired': + if square.status == status.ACQUIRED: montage = get_file_and_process(raw=square.raw, name=square.name, directory=microscope_id.scope_path) export_as_png(montage.image, montage.png) targets, finder_method, classifier_method, _ = find_targets(montage, protocol.finders) holes = add_targets(grid, square, targets, HoleModel, finder_method, classifier_method) - square = update(square, status='processed', shape_x=montage.shape_x, + square = update(square, status=status.PROCESSED, shape_x=montage.shape_x, shape_y=montage.shape_y, pixel_size=montage.pixel_size, refresh_from_db=True) transaction.on_commit(lambda: logger.debug('targets added')) - if square.status == 'processed': + if square.status == status.PROCESSED: if montage is None: montage = Montage(name=square.name) montage.load_or_process() selector_wrapper(protocol.selectors, square, n_groups=5, montage=montage) - square = update(square, status='selected') + square = update(square, status=status.TARGETS_SELECTED) transaction.on_commit(lambda: logger.debug('Selectors added')) - if square.status == 'selected': + if square.status == status.TARGETS_SELECTED: if is_bis: holes = list(HoleModel.display.filter(square_id=square.square_id)) holes = group_holes_for_BIS([h for h in holes if h.is_good() and not h.is_excluded()[0]], @@ -269,10 +260,10 @@ def process_square_image(square, grid, microscope_id): hole.save() logger.info(f'Picking holes on {square}') select_n_areas(square, grid.params_id.holes_per_square, is_bis=is_bis) - square = update(square, status='targets_picked') - if square.status == 'targets_picked': - square = update(square, status='completed', completion_time=timezone.now()) - if square.status == 'completed': + square = update(square, status=status.TARGETS_PICKED) + if square.status == status.TARGETS_PICKED: + square = update(square, status=status.COMPLETED, completion_time=timezone.now()) + if square.status == status.COMPLETED: logger.info(f'Square {square.name} analysis is complete') @@ -298,13 +289,15 @@ def process_hole_image(hole, grid, microscope_id): square_montage.load_or_process() image_coords = register_to_other_montage(np.array([x.coords for x in hole_group]),hole.coords, montage, square_montage) timer.report_timer('Initial registration to the higher mag image') + targets = [] + finder_method = 'Registration' + classifier_method=None if len(protocol.targets.finders) != 0: targets, finder_method, classifier_method, additional_outputs = find_targets(montage, protocol.targets.finders) generate_diagnostic_figure(montage.image,[([montage.center],(0,255,0), 1), ([t.coords for t in targets],(0,0,255),1)],Path(montage.directory / f'hole_recenter_it.png')) - else: + + if len(protocol.targets.finders) == 0 or targets == []: targets = create_targets_from_center(image_coords, montage) - finder_method = 'Registration' - classifier_method=None timer.report_timer('Identifying and registering targets') register = register_targets_by_proximity(image_coords,[target.coords for target in targets]) @@ -317,7 +310,7 @@ def process_hole_image(hole, grid, microscope_id): add_targets(grid,h,targets_to_register,HighMagModel,finder_method,classifier=classifier_method) timer.report_timer('Final registration and saving to db') update(hole, shape_x=montage.shape_x, - shape_y=montage.shape_y, pixel_size=montage.pixel_size, status='processed') + shape_y=montage.shape_y, pixel_size=montage.pixel_size, status=status.PROCESSED) def write_sessionLock(session, lockFile): @@ -335,7 +328,7 @@ def autoscreen(session_id): is_stop_file(session.session_id) if microscope.isLocked: logger.warning( - f'\nThe requested microscope is busy.\n\tLock file {microscope.lockFile} found\n\tSession id: {sessionLock} is currently running.\n\tIf you are sure that the microscope is not running, remove the lock file and restart.\nExiting.') + f'\nThe requested microscope is busy.\n\tLock file {microscope.lockFile} found\n\tSession id: {session} is currently running.\n\tIf you are sure that the microscope is not running, remove the lock file and restart.\nExiting.') sys.exit(0) write_sessionLock(session, microscope.lockFile) @@ -370,6 +363,9 @@ def autoscreen(session_id): except Exception as e: logger.exception(e) status = 'error' + if grid in locals(): + update.grid = grid + update(grid, status=grid_status.ERROR) except KeyboardInterrupt: logger.info('Stopping Smartscope.py autoscreen') status = 'killed' diff --git a/Smartscope/core/cache.py b/Smartscope/core/cache.py index 840fb206..aaedd3b5 100644 --- a/Smartscope/core/cache.py +++ b/Smartscope/core/cache.py @@ -6,10 +6,8 @@ logger = logging.getLogger(__name__) - -def save_multishot_from_cache(multishot_id, directory): - multishot = cache.get(multishot_id) - logger.debug(f'Getting multishot from cache at id {multishot_id}:\n{multishot}') - with open(Path(directory,'multishot.json'),'w') as f: - f.write(multishot) - +def save_json_from_cache(item_id, directory, file_root_name): + cached_item = cache.get(item_id) + logger.debug(f'Getting {file_root_name} from cache at id {item_id}:\n{cached_item}') + with open(Path(directory,f'{file_root_name}.json'),'w') as f: + f.write(cached_item) \ No newline at end of file diff --git a/Smartscope/core/db_manipulations.py b/Smartscope/core/db_manipulations.py index c561ee9d..1bb553fa 100755 --- a/Smartscope/core/db_manipulations.py +++ b/Smartscope/core/db_manipulations.py @@ -1,22 +1,24 @@ from typing import Any, Callable, List, Union -from Smartscope.core.models import * -from scipy.spatial.distance import cdist +from datetime import timedelta import numpy as np import random import logging +from scipy.spatial.distance import cdist + from django.db.models.query import prefetch_related_objects from django.db import transaction -from datetime import timedelta -from Smartscope.server.api.serializers import update_to_fullmeta, SvgSerializer +from django.contrib.contenttypes.models import ContentType + from channels.layers import get_channel_layer from asgiref.sync import async_to_sync -from django.contrib.contenttypes.models import ContentType +from Smartscope.core.models import * +from Smartscope.lib.multishot import load_multishot_from_file +from Smartscope.server.api.serializers import update_to_fullmeta, SvgSerializer logger = logging.getLogger(__name__) - class Websocket_update_decorator: def __init__(self, f: Callable[[Any], List[Any]] = None, grid: Union[AutoloaderGrid, None] = None): @@ -161,22 +163,34 @@ def set_or_update_refined_finder(object_id, stage_x, stage_y, stage_z): new.save() -def get_hole_count(grid, hole_list=None): +def get_hole_count(grid:AutoloaderGrid, hole_list=None): if hole_list is not None: queued = len(hole_list) else: queued = HoleModel.display.filter(grid_id=grid.grid_id,status='queued').count() + queued_exposures = queued + if grid.params_id.multishot_per_hole: + mutlishot_file = Path(grid.directory,'multishot.json') + multishot = load_multishot_from_file(mutlishot_file) + if multishot is not None: + queued_exposures = queued*multishot.n_shots completed = HighMagModel.objects.filter(grid_id=grid.grid_id) num_completed = completed.count() holes_per_hour = 0 last_hour = 0 + elapsed = 0 + remaining = 0 if grid.start_time is not None: - holes_per_hour = round(num_completed / (grid.time_spent.total_seconds() / 3600), 1) + elapsed = grid.time_spent + holes_per_hour = round(num_completed / (elapsed.total_seconds() / 3600), 1) last_hour_date_time = grid.end_time - timedelta(hours=1) last_hour = completed.filter(completion_time__gte=last_hour_date_time).count() + remaining = timedelta(hours=queued_exposures/last_hour) + logger.debug(f'{num_completed} completed holes, {queued} queued holes, {holes_per_hour} holes per hour, {last_hour} holes in the last hour') - return dict(completed=num_completed, queued=queued, perhour=holes_per_hour, lasthour=last_hour) + + return dict(completed=num_completed, queued=queued, queued_exposures=queued_exposures, perhour=holes_per_hour, lasthour=last_hour, elapsed=str(elapsed).split('.', 2)[0], remaining=str(remaining).split('.', 2)[0]) def viewer_only(user): @@ -188,6 +202,8 @@ def viewer_only(user): def group_holes_for_BIS(hole_models, max_radius=4, min_group_size=1, queue_all=False, iterations=500, score_weight=2): + if len(hole_models) == 0: + return logger.debug( f'grouping params, max radius = {max_radius}, min group size = {min_group_size}, queue all = {queue_all}, max iterations = {iterations}, score_weight = {score_weight}') # Extract coordinated for the holes diff --git a/Smartscope/core/finders.py b/Smartscope/core/finders.py index 6f4f1e81..ea3401f8 100644 --- a/Smartscope/core/finders.py +++ b/Smartscope/core/finders.py @@ -20,3 +20,5 @@ def find_targets(montage: Montage, methods: list): logger.debug(f"{method} was successful: {success}, Is Classifier: {method.target_class is TargetClass.CLASSIFIER}") return targets, method.name, method.name if method.target_class is TargetClass.CLASSIFIER else None, additional_outputs + + return [], '', None, dict() diff --git a/Smartscope/core/microscope_interfaces.py b/Smartscope/core/microscope_interfaces.py index 20664f78..d0ee771a 100755 --- a/Smartscope/core/microscope_interfaces.py +++ b/Smartscope/core/microscope_interfaces.py @@ -162,7 +162,6 @@ def tiltTo(self,tiltAngle): sem.TiltTo(tiltAngle) self.state.tiltAngle = tiltAngle - def medium_mag_hole(self, file=''): sem.AllowFileOverwrite(1) self.acquire_medium_mag() @@ -202,9 +201,13 @@ def setup(self, saveframes, framesName=None): sem.KeepCameraSetChanges('P') sem.SetLowDoseMode(1) - def refineZLP(self, zerolossDelay): + def refineZLP(self, zerolossDelay:float): if self.detector.energyFilter and zerolossDelay > 0: sem.RefineZLP(zerolossDelay * 60) + + def collectHardwareDark(self, harwareDarkDelay:int): + if harwareDarkDelay > 0: + sem.UpdateHWDarkRef(harwareDarkDelay) def disconnect(self, close_valves=True): @@ -244,9 +247,16 @@ def loadGrid(self, position): def reset_image_shift(self): return sem.ResetImageShift() + + def reset_image_shift_values(self): + self.state.reset_image_shift_values() + self.state.preAFISimageShiftX, self.state.preAFISimageShiftY = sem.ReportImageShift()[:2] + + def reset_AFIS_image_shift(self, afis:bool=False): + sem.SetImageShift(self.state.preAFISimageShiftX, self.state.preAFISimageShiftY, 1, int(afis)) - def image_shift_by_microns(self,isX,isY,tiltAngle): - sem.ImageShiftByMicrons(isX - self.state.imageShiftX, isY - self.state.imageShiftY, 0) + def image_shift_by_microns(self,isX,isY,tiltAngle, afis:bool=False): + sem.ImageShiftByMicrons(isX - self.state.imageShiftX, isY - self.state.imageShiftY, 1, int(afis)) self.state.imageShiftX = isX self.state.imageShiftY = isY sem.SetDefocus(self.state.currentDefocus - isY * math.sin(math.radians(tiltAngle))) @@ -345,7 +355,7 @@ def square(self, file=''): def align(): pass - def image_shift_by_microns(self, isX, isY, tiltAngle): + def image_shift_by_microns(self, isX, isY, tiltAngle, afis=False): return super().image_shift_by_microns(isX, isY, tiltAngle) def reset_image_shift(self): diff --git a/Smartscope/core/models/models_actions.py b/Smartscope/core/models/models_actions.py index 436de853..a6e286d1 100755 --- a/Smartscope/core/models/models_actions.py +++ b/Smartscope/core/models/models_actions.py @@ -14,9 +14,12 @@ def targets_methods(instance): if (output:=cache.get(cache_key)) is not None: logger.debug(f'Loading targets_method for {instance} from cache.') return output + default_output = dict(finders=[], classifiers=[], selectors=[]) + if not hasattr(instance,'targets'): + return default_output targets = instance.targets.values_list('pk', flat=True) if len(targets) == 0: - return dict(finders=[], classifiers=[], selectors=[]) + return default_output contenttype = ContentType.objects.get_for_model(instance.targets.first()) finders = list(Finder.objects.filter(content_type=contenttype, object_id__in=targets).values_list('method_name', flat=True).distinct()) diff --git a/Smartscope/core/models/session.py b/Smartscope/core/models/session.py index 7b545eff..a6319776 100644 --- a/Smartscope/core/models/session.py +++ b/Smartscope/core/models/session.py @@ -270,6 +270,7 @@ class GridCollectionParams(BaseModel): holes_per_square = models.IntegerField(default=3) # If -1 means all bis_max_distance = models.FloatField(default=3) # 0 means not BIS min_bis_group_size = models.IntegerField(default=1) + afis = models.BooleanField(default=False, verbose_name='AFIS') target_defocus_min = models.FloatField(default=-2) target_defocus_max = models.FloatField(default=-2) step_defocus = models.FloatField(default=0) # 0 deactivates step defocus @@ -280,6 +281,7 @@ class GridCollectionParams(BaseModel): offset_targeting = models.BooleanField(default=True) offset_distance = models.FloatField(default=-1) zeroloss_delay = models.IntegerField(default=-1) + hardwaredark_delay = models.IntegerField(default=-1,verbose_name='Hardware Dark Delay') multishot_per_hole = models.BooleanField(default=False) class Meta(BaseModel.Meta): @@ -502,7 +504,7 @@ def high_mag(self): @ property def end_time(self): try: - hole = self.highmagmodel_set.filter(status='completed').order_by('-completion_time').first() + hole = self.highmagmodel_set.order_by('-completion_time').first() if hole is None: raise @@ -601,7 +603,7 @@ def targets(self): return self.squaremodel_set.all() # @cached_model_property(key_prefix='svg', extra_suffix_from_function=['method'], timeout=3600) - def toSVG(self, display_type, method): + def svg(self, display_type, method): targets = list(SquareModel.display.filter(atlas_id=self.atlas_id)) return drawAtlas(self,targets , display_type, method) @@ -782,7 +784,7 @@ def targets(self): # @cached_model_property(key_prefix='svg', extra_suffix_from_function=['method'], timeout=3600) - def toSVG(self, display_type, method): + def svg(self, display_type, method): holes = list(HoleModel.display.filter(square_id=self.square_id)) sq = drawSquare(self, holes, display_type, method) @@ -883,7 +885,7 @@ def id(self): return self.hole_id # @cached_model_property(key_prefix='svg', extra_suffix_from_function=['method'], timeout=3600) - def toSVG(self, display_type, method): + def svg(self, display_type, method): holes = list(self.targets) if self.shape_x is None: # There was an error in previous version where shape wasn't set. set_shape_values(self) @@ -981,9 +983,8 @@ def parent(self): def set_parent(self, parent): self.hole_id = parent # endaliases - - @property - def SVG(self): + + def svg(self, *args, **kwargs): return drawHighMag(self) @property diff --git a/Smartscope/core/preprocessing_pipelines.py b/Smartscope/core/preprocessing_pipelines.py index 381fb7d6..674799b1 100755 --- a/Smartscope/core/preprocessing_pipelines.py +++ b/Smartscope/core/preprocessing_pipelines.py @@ -1,18 +1,27 @@ from abc import ABC, abstractmethod from functools import partial -import signal import time -from typing import List +from typing import List, Mapping, Union, Optional, Dict, Any +import os +import multiprocessing +import logging +import psutil +import subprocess as sub +import shlex +from pathlib import Path +from pydantic import BaseModel, Field, validator + + from Smartscope.core.db_manipulations import websocket_update from Smartscope.core.models import AutoloaderGrid -from pathlib import Path +from Smartscope.lib.Datatypes.models import generate_unique_id from Smartscope.lib.preprocessing_methods import get_CTFFIN4_data, process_hm_from_average, process_hm_from_frames, processing_worker_wrapper from Smartscope.core.models.models_actions import update_fields -import os -import sys -import multiprocessing -import logging +from Smartscope.core.settings.worker import DEFAULT_PREPROCESSING_PIPELINE +from Smartscope.lib.logger import add_log_handlers + from django.db import transaction +from django import forms logger = logging.getLogger(__name__) @@ -20,22 +29,42 @@ class PreprocessingPipeline(ABC): + name: str + verbose_name: str + description:str + cmdkwargs_handler: Any + pipeline_form: forms.Form + def __init__(self, grid: AutoloaderGrid): self.grid = grid self.directory = grid.directory + @classmethod + def form(cls,data:Union[Mapping,None]=None): + return cls.pipeline_form(data=data) + + @classmethod + def pipeline_data(cls,data:Dict): + return PreprocessingPipelineCmd(pipeline=cls.name,kwargs=cls.cmdkwargs_handler.parse_obj(data)) + @abstractmethod def start(self): pass - @abstractmethod def list_incomplete_processes(self): - pass + self.incomplete_processes = list(self.grid.highmagmodel_set.filter(status='acquired').order_by('completion_time')) @abstractmethod def stop(self): pass + def is_stop_file(self): + stopfile = Path('preprocessing.stop') + if stopfile.is_file(): + stopfile.unlink() + return True + return False + def update_processes(self): to_update = [] for instance in self.incomplete_processes: @@ -45,35 +74,105 @@ def update_processes(self): @abstractmethod def check_for_update(self, instance): pass + + +class SmartScopePreprocessingCmdKwargs(BaseModel): + n_processes:int = 1 + frames_directory:Union[Path,None] = None + @validator('frames_directory') + def is_frame_directory_empty(cls,v): + logger.debug(f'{v}, {type(v)}') + if v == '' or v == Path('.'): + return None + return v + +class PreprocessingPipelineCmd(BaseModel): + pipeline:str + cache_id:str = Field(default_factory=generate_unique_id) + process_pid: Optional[int] + kwargs: Any + + def is_running(self): + if self.process_pid is None: + return False + try: + process = psutil.Process(self.process_pid) + if process.is_running(): + return True + else: + return False + except psutil.NoSuchProcess: + return False + + def stop(self, grid:AutoloaderGrid): + stop_file = Path(grid.directory,'preprocessing.stop') + stop_file.touch() + while self.is_running(): + try: + process = psutil.Process(self.process_pid) + except psutil.NoSuchProcess: + break + logger.debug(f'Will check again if process as been killed. This may take a while for all actions to complete.') + time.sleep(2) + logger.info('Preprocessing has been killed gracefully.') + + def start(self, grid:AutoloaderGrid): + proc = sub.call(shlex.split(f'/opt/smartscope/Smartscope/bin/smartscope.sh highmag_processing {grid.grid_id}')) + time.sleep(3) + + +class SmartScopePreprocessingPipelineForm(forms.Form): + n_processes = forms.IntegerField(initial=1, min_value=1, max_value=4, help_text='Number of parallel processes to use for preprocessing.') + frames_directory = forms.CharField(help_text='Locations to look for the frames file other. Will look in the default smartscope/movies location by default.') + + def __init__(self, *args,**kwargs): + super().__init__(*args, **kwargs) + for visible in self.visible_fields(): + visible.field.widget.attrs['class'] = 'form-control' + visible.field.required = False class SmartscopePreprocessingPipeline(PreprocessingPipeline): + verbose_name = 'SmartScope Preprocessing Pipeline' + name = 'smartscopePipeline' + description = 'Default CPU-based Processing pipeline using IMOD alignframe and CTFFIND4.' to_process_queue = multiprocessing.JoinableQueue() processed_queue = multiprocessing.Queue() child_process = [] to_update = [] incomplete_processes = [] + cmdkwargs_handler = SmartScopePreprocessingCmdKwargs + pipeline_form= SmartScopePreprocessingPipelineForm - def __init__(self, grid: AutoloaderGrid, frames_directory=None): + def __init__(self, grid: AutoloaderGrid, cmd_data:Dict): super().__init__(grid=grid) self.microscope = self.grid.session_id.microscope_id self.detector = self.grid.session_id.detector_id + self.cmd_data = self.cmdkwargs_handler.parse_obj(cmd_data) + logger.debug(self.cmd_data) self.frames_directory = [Path(self.detector.frames_directory)] + if self.cmd_data.frames_directory is not None: + self.frames_directory.append(self.cmd_data.frames_directory) - if frames_directory is not None: - self.frames_directory.append(frames_directory) + def clear_queue(self): + while True: + try: + self.to_process_queue.get_nowait() + self.to_process_queue.task_done() + except multiprocessing.queues.Empty: + break - def start(self, n_processes: int = 1): - os.chdir(self.grid.directory) + def start(self): session = self.grid.session_id - for n in range(int(n_processes)): + logger.info(f'Starting the preprocessing with {self.cmd_data.n_processes}') + for n in range(int(self.cmd_data.n_processes)): proc = multiprocessing.Process(target=processing_worker_wrapper, args=( session.directory, self.to_process_queue,), kwargs={'output_queue': self.processed_queue}) proc.start() self.child_process.append(proc) self.list_incomplete_processes() - while True: + while not self.is_stop_file(): self.queue_incomplete_processes() self.to_process_queue.join() self.check_for_update() @@ -81,10 +180,10 @@ def start(self, n_processes: int = 1): self.list_incomplete_processes() self.grid.refresh_from_db() if self.grid.status == 'complete' and len(self.incomplete_processes) == 0: - break + return def list_incomplete_processes(self): - self.incomplete_processes = list(self.grid.highmagmodel_set.filter(status='acquired').order_by('completion_time')[:20]) + self.incomplete_processes = list(self.grid.highmagmodel_set.filter(status__in=['acquired','skipped']).order_by('status','completion_time')[:5*self.cmd_data.n_processes]) def queue_incomplete_processes(self): from_average = partial(process_hm_from_average, scope_path_directory=self.microscope.scope_path, @@ -100,20 +199,30 @@ def queue_incomplete_processes(self): def stop(self): for proc in self.child_process: self.to_process_queue.put('exit') + for proc in self.child_process: proc.join() logger.debug('Process joined') def check_for_update(self): while self.processed_queue.qsize() > 0: movie = self.processed_queue.get() + data = dict() if not movie.check_metadata(): + data['status'] = 'skipped' + instance = [obj for obj in self.incomplete_processes if obj.name == movie.name][0] + if instance.status != 'skipped': + self.to_update.append(update_fields(instance, data)) continue logger.debug(f'Updating {movie.name}') try: data = get_CTFFIN4_data(movie.ctf) except Exception as err: logger.exception(err) - logger.info(f'An error occured while getting CTF data from {movie.name}. Will try again on the next cycle.') + logger.info(f'An error occured while getting CTF data from {movie.name}. Will try again later.') + data['status'] = 'skipped' + instance = [obj for obj in self.incomplete_processes if obj.name == movie.name][0] + if instance.status != 'skipped': + self.to_update.append(update_fields(instance, data)) continue data['status'] = 'completed' movie.read_data() @@ -128,8 +237,9 @@ def check_for_update(self): def update_processes(self): logger.info(f'Updating {len(self.to_update)} items in the database') if len(self.to_update) == 0: - logger.info(f'No items to update, waiting 30 seconds before checking again.') - return time.sleep(30) + sleep_time = 10 + logger.info(f'No items to update, waiting {sleep_time} seconds before checking again.') + return time.sleep(sleep_time) with transaction.atomic(): for obj in self.to_update: print(obj.shape_x) @@ -141,12 +251,37 @@ def update_processes(self): PREPROCESSING_PIPELINE_FACTORY = dict(smartscopePipeline=SmartscopePreprocessingPipeline) +def load_preprocessing_pipeline(file:Path): + if file.exists(): + return PreprocessingPipelineCmd.parse_file(file) + logger.info(f'Preprocessing file {file} does not exist. Loading default pipeline.') + for default in DEFAULT_PREPROCESSING_PIPELINE: + if default.exists(): + return PreprocessingPipelineCmd.parse_file(default) + logger.info(f'Default preprocessing pipeline not found.') + return None + -def highmag_processing(pipeline: PreprocessingPipeline, grid_id: str, n_processes: int = 1) -> None: +def highmag_processing(grid_id: str, *args, **kwargs) -> None: try: grid = AutoloaderGrid.objects.get(grid_id=grid_id) - pipeline = PREPROCESSING_PIPELINE_FACTORY[pipeline](grid) - pipeline.start(n_processes) + os.chdir(grid.directory) + # logging.getLogger('Smartscope').handlers.pop() + # logger.debug(f'Log handlers:{logger.handlers}') + add_log_handlers(directory=grid.session_id.directory, name='proc.out') + logger.debug(f'Log handlers:{logger.handlers}') + preprocess_file = Path('preprocessing.json') + cmd_data = load_preprocessing_pipeline(preprocess_file) + if cmd_data is None: + logger.info('Trying to load preprocessing parameters from command line arguments.') + cmd_data = PreprocessingPipelineCmd.parse_obj(**kwargs) + if cmd_data.is_running(): + logger.info(f'Processings with PID:{cmd_data.process_pid} seem to already be running, please kill the other one before continuing.') + return + cmd_data.process_pid=os.getpid() + preprocess_file.write_text(cmd_data.json()) + pipeline = PREPROCESSING_PIPELINE_FACTORY[cmd_data.pipeline](grid, cmd_data.kwargs) + pipeline.start() except Exception as e: logger.exception(e) diff --git a/Smartscope/core/protocol_commands.py b/Smartscope/core/protocol_commands.py index 6ee9eecc..e9770b97 100644 --- a/Smartscope/core/protocol_commands.py +++ b/Smartscope/core/protocol_commands.py @@ -83,7 +83,7 @@ def highMag(scope, params,instance): offset = add_IS_offset(grid_type.hole_size, grid_mesh.name, offset_in_um=params.offset_distance) isX, isY = stage_x - finder.stage_x + offset, (stage_y - finder.stage_y) * cos(radians(params.tilt_angle)) logger.debug(f'The tilt angle is {params.tilt_angle}, Y axis image-shift corrected from {stage_y - finder.stage_y:.2f} to {isY:.2f}') - scope.image_shift_by_microns(isX,isY,params.tilt_angle) + scope.image_shift_by_microns(isX,isY,params.tilt_angle, afis=params.afis) frames = scope.highmag(file=instance.raw, frames=params.save_frames, earlyReturn=any([params.force_process_from_average, params.save_frames is False])) instance.is_x=isX instance.is_y=isY diff --git a/Smartscope/core/settings/server_docker.py b/Smartscope/core/settings/server_docker.py index 0375be8b..a3e79907 100755 --- a/Smartscope/core/settings/server_docker.py +++ b/Smartscope/core/settings/server_docker.py @@ -26,8 +26,8 @@ # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ SECRET_KEY = os.getenv('SECRET_KEY') -DEBUG = os.getenv('DEBUG') -DEPLOY = True +DEBUG = eval(os.getenv('DEBUG')) +DEPLOY = os.getenv('DEPLOY', True) ALLOWED_HOSTS = os.getenv('ALLOWED_HOSTS').split(',') @@ -41,14 +41,10 @@ USE_AWS = eval(os.getenv('USE_AWS')) USE_MICROSCOPE = eval(os.getenv('USE_MICROSCOPE')) -# WORKER_HOSTNAME = os.getenv('WORKER_HOSTNAME') - if USE_LONGTERMSTORAGE: AUTOSCREENSTORAGE = os.getenv('AUTOSCREENSTORAGE') - AUTOSCREENINGSTORAGE_URL = '/autoscreeningstorage/' else: AUTOSCREENSTORAGE = None - AUTOSCREENINGSTORAGE_URL = None # if DEPLOY is False: # autoscreening = FileSystemStorage(location=AUTOSCREENDIR, base_url=AUTOSCREENING_URL) @@ -127,7 +123,7 @@ 'default': { 'ENGINE': 'django.db.backends.mysql', 'NAME': os.getenv('MYSQL_DATABASE'), - 'PASSWORD': os.getenv('MYSQL_ROOT_PASSWORD'), + 'PASSWORD': os.getenv('MYSQL_PASSWORD'), 'CONN_MAX_AGE': 0, } @@ -138,10 +134,16 @@ } else: DATABASES['default']['USER'] = os.getenv('MYSQL_USER') - DATABASES['default']['PASSWORD'] = os.getenv('MYSQL_ROOT_PASSWORD') + DATABASES['default']['PASSWORD'] = os.getenv('MYSQL_PASSWORD') DATABASES['default']['HOST'] = os.getenv('MYSQL_HOST') DATABASES['default']['PORT'] = os.getenv('MYSQL_PORT') + ssl = eval(os.getenv('MYSQL_SSL', 'False')) + if ssl: + DATABASES['default']['OPTIONS'] = { + 'ssl': ssl, + } + # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -183,7 +185,7 @@ LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'EST' +TIME_ZONE = os.getenv('TIMEZONE', 'America/New_York') USE_I18N = True diff --git a/Smartscope/core/settings/worker.py b/Smartscope/core/settings/worker.py index e98a9a8a..0e8c4cd9 100644 --- a/Smartscope/core/settings/worker.py +++ b/Smartscope/core/settings/worker.py @@ -24,4 +24,6 @@ PLUGINS_FACTORY['CTF Viewer'] = CTFFitViewer() ##Register available protocol commands -PROTOCOL_COMMANDS_FACTORY = get_protocol_commands(EXTERNAL_PLUGINS_LIST) \ No newline at end of file +PROTOCOL_COMMANDS_FACTORY = get_protocol_commands(EXTERNAL_PLUGINS_LIST) + +DEFAULT_PREPROCESSING_PIPELINE = [ SMARTSCOPE_CUSTOM_CONFIG / 'default_preprocessing.json', SMARTSCOPE_DEFAULT_CONFIG / 'default_preprocessing.json' ] \ No newline at end of file diff --git a/Smartscope/core/status.py b/Smartscope/core/status.py new file mode 100644 index 00000000..1a98dea6 --- /dev/null +++ b/Smartscope/core/status.py @@ -0,0 +1,90 @@ +import os +import logging +from dataclasses import dataclass +from pathlib import Path + + +logger = logging.getLogger(__name__) + +class status: + """ + ENUMS don't play well with Django. This is an attempt at creating something equivalent that works + """ + + NULL=None + STARTED='started' + QUEUED='queued' + ACQUIRED='acquired' + PROCESSED='processed' + TARGETS_PICKED='targets_picked' + TARGETS_SELECTED='selected' + ERROR='error' + SKIPPED='skipped' + COMPLETED='completed' + +class grid_status: + """ + ENUMS don't play well with Django. This is an attempt at creating something equivalent that works + """ + + NULL=None + STARTED='started' + ERROR='error' + SKIPPED='skipped' + ABORTING='aborting' + PAUSED='paused' + COMPLETED='complete' + + + +class FileSignal: + + def __init__(self, path:os.PathLike): + if not isinstance(path,Path): + self._path = Path(path) + return + self._path = path + + @property + def exists(self): + return self._path.exists + + def create(self): + return self._path.touch() + + def remove(self): + return self._path.unlink() + + + # @property + # def is_paused(self): + # return self.paused_file.exists() + + + + # @property + # def is_stop_file(self): + # if not self.stop_file.exists(): + # return False + # logger.debug(f'Stop file {self.stop_file} found.') + # self.stop_file.unlink() + # raise KeyboardInterrupt() + +@dataclass +class FileSignals: + + microscope_id:str + session_id:str + grid_id:str + + @property + def paused_file(self) -> FileSignal: + return FileSignal(Path(os.getenv('TEMPDIR'), f'paused_{self.microscope_id}')) + + @property + def stop_file(self) -> FileSignal: + return Path(os.getenv('TEMPDIR'), f'{self.session_id}.stop') + + @property + def session_lock(self) -> FileSignal: + return Path() \ No newline at end of file diff --git a/Smartscope/core/svg_plots.py b/Smartscope/core/svg_plots.py index 813d75e4..e6c124ea 100644 --- a/Smartscope/core/svg_plots.py +++ b/Smartscope/core/svg_plots.py @@ -84,10 +84,17 @@ def asSvg(self, outputFile=None): returnString = outputFile is None if returnString: outputFile = StringIO() - startStr = ''' - + + myDrawging: d = myDrawging(atlas.shape_y, atlas.shape_x, id='square-svg', displayInline=False) d.append(draw.Image(0, 0, d.width, d.height, path=atlas.png, embed= not atlas.is_aws)) @@ -174,10 +181,10 @@ def drawAtlas(atlas, targets, display_type, method): d.append(text) d.append(add_scale_bar(atlas.pixel_size, d.width, d.height)) d.append(add_legend(set(labels_list), d.width, d.height, atlas.pixel_size)) - return d.asSvg() + return d -def drawSquare(square, targets, display_type, method): +def drawSquare(square, targets, display_type, method) -> myDrawging: d = myDrawging(square.shape_y, square.shape_x, id='square-svg', displayInline=False) d.append(draw.Image(0, 0, d.width, d.height, path=square.png, embed= not square.is_aws)) @@ -221,10 +228,10 @@ def drawSquare(square, targets, display_type, method): d.append(text) d.append(add_scale_bar(square.pixel_size, d.width, d.height, id_type='square')) d.append(add_legend(set(labels_list), d.width, d.height, square.pixel_size)) - return d.asSvg() + return d -def drawMediumMag(hole, targets, display_type, method, **kwargs): +def drawMediumMag(hole, targets, display_type, method, **kwargs) -> myDrawging: d = myDrawging(hole.shape_y, hole.shape_x, id='hole-svg', displayInline=False) d.append(draw.Image(0, 0, d.width, d.height, path=hole.png, embed= not hole.is_aws)) @@ -254,13 +261,11 @@ def drawMediumMag(hole, targets, display_type, method, **kwargs): d.append(text) d.append(add_scale_bar(hole.pixel_size, d.width, d.height, id_type='hole')) d.append(add_legend(set(labels_list), d.width, d.height, hole.pixel_size)) - return d.asSvg() + return d -def drawHighMag(highmag): +def drawHighMag(highmag) -> myDrawging: d = myDrawging(highmag.shape_y, highmag.shape_x, id=f'{highmag.name}-svg', displayInline=False) d.append(draw.Image(0, 0, d.width, d.height, path=highmag.png, embed= not highmag.is_aws)) - d.append(add_scale_bar(highmag.pixel_size, d.width, d.height, id_type=highmag.name)) - - return d.asSvg \ No newline at end of file + return d \ No newline at end of file diff --git a/Smartscope/core/test_commands.py b/Smartscope/core/test_commands.py index 5229b2ae..f7b1f01d 100755 --- a/Smartscope/core/test_commands.py +++ b/Smartscope/core/test_commands.py @@ -148,99 +148,3 @@ def list_plugins(): def list_protocols(): from Smartscope.core.settings.worker import PROTOCOLS_FACTORY [print(f"{'#'*60}\n{name}:\n\n\t{plugin}\n{'#'*60}\n") for name,plugin in PROTOCOLS_FACTORY.items()] - - -def acquire_fast_atlas_test(microscope_id): - import serialem as sem - from Smartscope.lib.microscope_methods import FastAtlas,make_serpent_pattern_in_mask - - microscope = Microscope.objects.get(pk=microscope_id) - with TFSSerialemInterface(ip=microscope.serialem_IP, - port=microscope.serialem_PORT, - directory=microscope.windows_path, - scope_path=microscope.scope_path, - energyfilter=False, - loader_size=microscope.loader_size, - frames_directory='X:/testing/') as scope: - scope.set_atlas_optics(62,100,5) - sem.Search() - sem.EarlyReturnNextShot(-1) - X, Y, _, _, pixel_size, _ = sem.ImageProperties('A') - logger.info(f'Setting atlas for {X}, {Y}, {pixel_size}') - atlas = FastAtlas(atlas_imsize_x=X,atlas_imsize_y=Y,pixel_size_in_angst=pixel_size*10) - atlas.generate_tile_mask(atlas_radius_in_um=990) - atlas.make_stage_pattern(make_serpent_pattern_in_mask) - sem.SetContinuous('R',1) - sem.UseContinuousFrames(1) - sem.StartFrameWaitTimer(-1) - start_time = time.time() - time_distances = [] - sem.MoveStageTo(*atlas.stage_movements[0][0]) - - for ind, m in enumerate(atlas.stage_movements): - start, end = m - distance = abs(start[1]-end[1]) - exposure_time = 0.011 * distance + 4 - logger.info(f'Setting exposure time to {exposure_time:.1f}') - sem.SetExposure('R', exposure_time) - sem.MoveStageTo(*start) - start_movement = time.time() - logger.info(f'Starting movement {ind} at {start}, moving to {end}') - sem.Echo(f'Movemement {ind}') - sem.MoveStageTo(*end) - movement_elapsed = time.time() - start_movement - distance = abs(start[1]-end[1]) - logger.info(f'Movement {ind} took {movement_elapsed:.1f} for a distance of {distance}') - time_distances.append([movement_elapsed,distance]) - - sem.SetContinuous('R',0) - elapsed = time.time() - start_time - logger.info(f'Finished acquisition in {elapsed:.0f} seconds.') - -def acquire_fast_atlas_test2(microscope_id,detector_id): - from Smartscope.core.models import Microscope, Detector - import Smartscope.lib.Datatypes.microscope as micModels - import serialem as sem - from Smartscope.lib.microscope_methods import FastAtlas,make_serpent_pattern_in_mask - - microscope = Microscope.objects.get(pk=microscope_id) - detector = Detector.objects.get(pk=detector_id) - with TFSSerialemInterface(microscope = micModels.Microscope.from_orm(microscope), - detector= micModels.Detector.from_orm(detector), - atlasSettings= micModels.AtlasSettings.from_orm(detector)) as scope: - # scope.set_atlas_optics(62,100,5) - sem.SetColumnOrGunValve(1) - sem.Search() - X, Y, _, _, pixel_size, _ = sem.ImageProperties('A') - logger.info(f'Setting atlas for {X}, {Y}, {pixel_size}') - atlas = FastAtlas(atlas_imsize_x=X,atlas_imsize_y=Y,pixel_size_in_angst=pixel_size*10) - atlas.generate_tile_mask(atlas_radius_in_um=990) - atlas.make_stage_pattern(make_serpent_pattern_in_mask) - sem.SetContinuous('R',1) - sem.UseContinuousFrames(1) - sem.StartFrameWaitTimer(-1) - start_time = time.time() - time_distances = [] - sem.MoveStageTo(*atlas.stage_movements[0][0]) - - for ind, m in enumerate(atlas.stage_movements): - start, end = m - distance = abs(start[1]-end[1]) - Path(scope.microscope.scopePath,'raw','atlas',str(ind)).mkdir(parents=True,exist_ok=True) - sem.SetFolderForFrames(f'{scope.microscope.directory}raw/atlas/{ind}') - sem.MoveStageTo(*start) - sem.Record() - start_movement = time.time() - logger.info(f'Starting movement {ind} at {start}, moving to {end}') - sem.Echo(f'Movemement {ind}') - stage_movement = end - start - sem.MoveStageWithSpeed(*stage_movement,0.5) - sem.StopContinuous() - movement_elapsed = time.time() - start_movement - distance = abs(start[1]-end[1]) - logger.info(f'Movement {ind} took {movement_elapsed:.1f} for a distance of {distance}') - time_distances.append([movement_elapsed,distance]) - - sem.SetContinuous('R',0) - elapsed = time.time() - start_time - logger.info(f'Finished acquisition in {elapsed:.0f} seconds.') diff --git a/Smartscope/lib/Datatypes/microscope.py b/Smartscope/lib/Datatypes/microscope.py index e28acd99..725b5843 100755 --- a/Smartscope/lib/Datatypes/microscope.py +++ b/Smartscope/lib/Datatypes/microscope.py @@ -16,6 +16,8 @@ class MicroscopeState: stageY: float = 0 stageZ: float = 0 tiltAngle: float = None + preAFISimageShiftX: float = 0 + preAFISimageShiftY: float = 0 def setStage(self,stageX,stageY,stageZ): self.stageX = stageX @@ -105,7 +107,7 @@ def report_stage(self): return 0,0,0 @abstractmethod - def eucentricHeight(self, tiltTo=10, increments=-5) -> float: + def eucentricHeight(self, tiltTo:int=10, increments:int=-5) -> float: pass @abstractmethod @@ -174,7 +176,7 @@ def load_hole_ref(self): pass @abstractmethod - def image_shift_by_microns(self,isX,isY,tiltAngle): + def image_shift_by_microns(self,isX,isY,tiltAngle, afis:bool=False): pass @abstractmethod @@ -203,3 +205,11 @@ def loadGrid(self, position): def refineZLP(self, zerolossDelay): pass + + def collectHardwareDark(self, harwareDarkDelay:int): + pass + + def reset_AFIS_image_shift(self, afis:bool=False): + pass + + diff --git a/Smartscope/lib/Finders/AIFinder/detectors/detect_squares.py b/Smartscope/lib/Finders/AIFinder/detectors/detect_squares.py index 7ca38f43..45d229f6 100755 --- a/Smartscope/lib/Finders/AIFinder/detectors/detect_squares.py +++ b/Smartscope/lib/Finders/AIFinder/detectors/detect_squares.py @@ -104,7 +104,7 @@ def detect(atlas, device = '0', imgsz = 2048, thresh = 0.2, iou = 0.3, weights = cfg.MODEL.WEIGHTS = weights cfg.INPUT.MIN_SIZE_TEST = 2048 cfg.INPUT.MAX_SIZE_TEST = 2048 - cfg.TEST.DETECTIONS_PER_IMAGE = 400 + cfg.TEST.DETECTIONS_PER_IMAGE = 500 if device == 'cpu': cfg.MODEL.DEVICE='cpu' diff --git a/Smartscope/lib/Finders/AIFinder/wrapper.py b/Smartscope/lib/Finders/AIFinder/wrapper.py index f9304dee..32575177 100755 --- a/Smartscope/lib/Finders/AIFinder/wrapper.py +++ b/Smartscope/lib/Finders/AIFinder/wrapper.py @@ -1,5 +1,7 @@ import os import sys +import cv2 +import numpy as np sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'detectors')) from .detectors.detect_squares import detect @@ -7,8 +9,9 @@ from ..basic_finders import find_square_center import logging import torch -# logger = logging.getLogger('processing') -# logger = logging.getLogger('autoscreen') + +from Smartscope.lib.image_manipulations import fourier_crop +from Smartscope.lib.montage import Montage logger = logging.getLogger(__name__) WEIGHT_DIR = os.path.join(os.getenv("TEMPLATE_FILES"), 'weights') @@ -30,18 +33,31 @@ def find_squares(montage, **kwargs): return (squares, labels), success, dict() -def find_holes(montage, **kwargs): +def find_holes(montage:Montage, **kwargs): logger.info('Running AI hole detection') # centroid = find_square_center(montage.image) kwargs['weights_circle'] = os.path.join(WEIGHT_DIR, kwargs['weights_circle']) if not IS_CUDA: kwargs['device'] = 'cpu' - holes, _ = detect_holes(montage.image, **kwargs) + image = montage.image + binning=1 + if montage.pixel_size < 100: + logger.info(f'Resizing image') + binning = (150/montage.pixel_size) + image = fourier_crop(image, height=montage.shape_x/binning) + pad_x = int((montage.shape_x - image.shape[0]) //2) + pad_y = int((montage.shape_y - image.shape[1]) //2) + image = cv2.copyMakeBorder(image,pad_x,pad_x,pad_y,pad_y,cv2.BORDER_CONSTANT,value=0) + logger.debug(f'Resized shape: {image.shape}, original: {montage.image.shape}') + holes, _ = detect_holes(image, **kwargs) + logger.info(f'AI hole detection found {len(holes)} holes') success = True if len(holes) < 10: success = False - # holes = [i.numpy() for i in holes] - logger.info(f'AI hole detection found {len(holes)} holes') + logger.debug(f'{holes[0]},{type(holes[0])}') + + holes = [(np.array(hole)-np.array(list(montage.center)*2))*binning + np.array(list(montage.center)*2) for hole in holes] + logger.debug(f'{holes[0]},{type(holes[0])}') return holes, success, dict() diff --git a/Smartscope/lib/generic_position.py b/Smartscope/lib/generic_position.py index 0b1de10c..758b09c4 100755 --- a/Smartscope/lib/generic_position.py +++ b/Smartscope/lib/generic_position.py @@ -13,7 +13,7 @@ def parse_mdoc(mdocFile: str, movie: bool = False) -> pd.DataFrame: Opens an mdoc file and returns a dataframe with the different values from the file and the pixel size """ pattern = re.compile(r'(\w*)\s=\s([\-\w\.\s\\:]+)\n') - + metadata= None with open(mdocFile, 'r') as file: mdocFile = file.read().split('\n\n') diff --git a/Smartscope/lib/microscope_methods.py b/Smartscope/lib/microscope_methods.py deleted file mode 100644 index 2d50ff34..00000000 --- a/Smartscope/lib/microscope_methods.py +++ /dev/null @@ -1,133 +0,0 @@ -import numpy as np -import matplotlib.pyplot as plt -from typing import List, Union -from dataclasses import dataclass -from matplotlib.patches import Circle, Rectangle -from pathlib import Path -import logging -import time - -logger = logging.getLogger(__name__) - -@dataclass -class FastAtlas: - _max_radius_in_um:int = 990 - atlas_imsize_x:int = 5700 - atlas_imsize_y:int = 4096 - pixel_size_in_angst:float = 650 - overlap_fraction:float = 0.05 - _lattice_mask = None - - @property - def pixel_size_um(self): - return self.pixel_size_in_angst/10_000 - - @property - def imsize_x_um(self): - return self.atlas_imsize_x * self.pixel_size_um - - @property - def imsize_y_um(self): - return self.atlas_imsize_y * self.pixel_size_um - - @property - def imsize(self): - return np.array([self.imsize_x_um,self.imsize_y_um]) - - @property - def lattice_mask(self): - if self._lattice_mask is not None: - return self._lattice_mask - raise ValueError('Lattice mask was not set. Please use generate_tile_mask method') - - @property - def lattice_mask_center(self): - return np.array(self.lattice_mask.shape)//2 - - - def generate_tile_mask(self, atlas_radius_in_um:int = 600) -> np.ndarray: - padded_max_axis = max([int(self._max_radius_in_um*2//self.imsize_x_um*(1-self.overlap_fraction)), int(self._max_radius_in_um*2//self.imsize_y_um*(1-self.overlap_fraction))]) + 2 - lattice_mask = np.zeros((padded_max_axis,padded_max_axis)) - center = np.array(lattice_mask.shape)//2 - - for x,xval in enumerate(lattice_mask): - for y,yval in enumerate(xval): - coord = (np.array([x,y])-center) * self.imsize - dist = np.sqrt(np.sum(np.power(coord,2))) - if dist > atlas_radius_in_um: - continue - lattice_mask[x,y] = 1 - self._lattice_mask = lattice_mask - return self.lattice_mask - - def make_stage_pattern(self, pattern_method: callable): - self.movements = pattern_method(self.lattice_mask) - self.stage_movements = (np.array(self.movements)-self.lattice_mask_center) * self.imsize*(1-self.overlap_fraction) - - def plot_movements_on_stage(self, export_to:Union[Path,None]=None): - fig = plt.figure(figsize=(10,10)) - ax = fig.add_subplot(111) - limits = Circle([0,0],self.atlas_radius_in_um,fill=False) - ax.add_patch(limits) - ax.set_ylim(-self._max_radius_in_um,self._max_radius_in_um) - ax.set_xlim(-self._max_radius_in_um,self._max_radius_in_um) - for i, mov in enumerate(self.movements): - a = (np.array(mov)-self.lattice_mask_center) * self.imsize*(1-self.overlap_fraction) - ax.text(a[0,0],a[0,1], i+1) - ax.plot(a[:,0],a[:,1], marker='o') - if export_to is not None: - fig.savefig(str(export_to),dpi=100,pad_inches=0, bbox_inches='tight') - - -def make_spiral_pattern_in_mask(mask:np.ndarray) -> List: - start = np.array(mask.shape) // 2 - ind = 1 - movements = [] - while True: - if ind+1 >= max(mask.shape): - break - - for i in [np.array([1,0]),np.array([0,1])]: - axis = np.where(i == 1)[0] - temp_ind = ind - if ind % 2 == 0: - temp_ind *=-1 - - end=start+(i*temp_ind) - - if start[0] == end[0]: - order= np.sort(np.array([start[1],end[1]])) - mov_slice = (start[0],slice(order[0],order[1],None)) - else: - order= np.sort(np.array([start[0],end[0]])) - mov_slice = (slice(order[0],order[1],None), start[1]) - - if (mask[mov_slice] != 0).any(): - indexes = np.where(mask[mov_slice] == 1)[0] - new_start, new_end = start.copy(), end.copy() - new_start[axis] = start[axis]+indexes[0] if temp_ind > 0 else start[axis]-indexes[0] - new_end[axis] = new_start[axis] + len(indexes) -1 if temp_ind > 0 else new_start[axis] - len(indexes) +1 - movements.append([new_start,new_end]) - start=end - ind += 1 - return movements - -def make_serpent_pattern_in_mask(mask:np.ndarray) -> List: - movement_direction = 1 - movements =[] - for ind,line in enumerate(mask): - if (line != 0).any(): - indexes = np.where(line == 1)[0] - start = [ind,indexes[0] if movement_direction == 1 else indexes[-1]] - end = [ind, indexes[-1] if movement_direction == 1 else indexes[0]] - movements.append([start,end]) - movement_direction *= -1 - return movements - - - - - - - - diff --git a/Smartscope/lib/multishot.py b/Smartscope/lib/multishot.py index f9edd417..ba275981 100644 --- a/Smartscope/lib/multishot.py +++ b/Smartscope/lib/multishot.py @@ -33,14 +33,17 @@ def hole_mask(self): return cv2.circle(mask,self.center,radius,1,cv2.FILLED) -class RecordParams: - def __init__(self,detector_size_x:int,detector_size_y:int,pixel_size:float,beam_size:int,hole_size:float): - self.detector_size = np.array([detector_size_x,detector_size_y]) - self.pixel_size = pixel_size - self.beam_size = beam_size - self.hole_size = hole_size - # self.to_microns() - +class RecordParams(BaseModel): + detector_size_x:int + detector_size_y:int + pixel_size:float + beam_size:int + hole_size:float + + @property + def detector_size(self): + return np.array([self.detector_size_x,self.detector_size_y]) + @property def pixel_size_um(self): return self.pixel_size / 10_000 @@ -74,6 +77,7 @@ class MultiShot(BaseModel): in_hole: float coverage: float display: Optional[str] + params: Optional[RecordParams] cache_id:str = Field(default_factory=generate_unique_id) class Config: @@ -93,10 +97,11 @@ def convert_shots_to_pixel(self, pixel_size_in_um:float) -> ArrayLike: def set_display(self,record_params: RecordParams): stream = display_multishot_matplotlib(self.shots,record_params.hole_size,record_params.beam_size_um,record_params.detector_size_um) self.display=base64.b64encode(stream.getvalue()).decode() + self.params = record_params -def make_beam_and_fov_masks(shots,beam_size,fov_size,box:MaskBox): - radius = int(beam_size/2//box.pix_size) +def make_beam_and_fov_masks(shots,beam_size,fov_size,box:MaskBox, beam_padding=1.1): + radius = int(beam_size*beam_padding/2//box.pix_size) half_fov = np.int8(fov_size/box.pix_size//2) beam_masks=[] fov_masks=[] @@ -143,7 +148,7 @@ def check_fraction_in_hole(fov_masks,box): return sum_in_hole/init_sum, sum_in_hole, init_sum -def set_shots_per_hole(number_of_shots:int,hole_size:float,beam_size,image_size:np.ndarray,radius_step:int=0.05,starting_angle:float=0, consider_aspect=True, min_efficiency=0.85): +def set_shots_per_hole(number_of_shots:int,hole_size:float,beam_size:float,image_size:np.ndarray,radius_step:int=0.02,starting_angle:float=0, consider_aspect=True, min_efficiency=0.85): hole_area = np.pi*(hole_size/2)**2 min_allowed_coverage = np.prod(image_size)/hole_area angle_between_shots = 2*np.pi / number_of_shots @@ -155,7 +160,8 @@ def set_shots_per_hole(number_of_shots:int,hole_size:float,beam_size,image_size: aspect_step = 0.1 steps=20 - start_radius = (hole_size/2 - np.sqrt(np.sum(image_size**2))/2) if number_of_shots != 1 else 0 + # start_radius = (hole_size/2 - np.sqrt(np.sum(image_size**2))/2) if number_of_shots != 1 else 0 + start_radius = beam_size/2000 if number_of_shots != 1 else 0 max_radius = hole_size/2 box = MaskBox(hole_size=hole_size) best_shots = None diff --git a/Smartscope/lib/preprocessing_methods.py b/Smartscope/lib/preprocessing_methods.py index a4894dbe..25e52c84 100755 --- a/Smartscope/lib/preprocessing_methods.py +++ b/Smartscope/lib/preprocessing_methods.py @@ -75,6 +75,9 @@ def process_hm_from_frames(name, frames_file_name, frames_directories: list, sph logger.info(f'Mdoc file not found {mdoc}. Skipping.') return movie movie.metadata = parse_mdoc(mdocFile=mdoc, movie=True) + if movie.metadata is None: + logger.info(f'Mdoc file not found {mdoc}. Skipping.') + return movie time.sleep(10) if not movie.shifts.exists() or not movie.ctf.exists(): try: @@ -119,6 +122,8 @@ def processing_worker_wrapper(logdir, queue, output_queue=None): item = queue.get() logger.info(f'Got item {item} from queue') if item == 'exit': + queue.task_done() + logger.info('Breaking processing worker loop.') break if item is not None: logger.debug(f'Running {item[0]} {item[1]} {item[2]} from queue') diff --git a/Smartscope/server/api/migrations/0012_gridcollectionparams_hardwaredark_delay_and_more.py b/Smartscope/server/api/migrations/0012_gridcollectionparams_hardwaredark_delay_and_more.py new file mode 100644 index 00000000..07f16eed --- /dev/null +++ b/Smartscope/server/api/migrations/0012_gridcollectionparams_hardwaredark_delay_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.0.5 on 2023-05-13 01:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('API', '0011_alter_screeningsession_version'), + ] + + operations = [ + migrations.AddField( + model_name='gridcollectionparams', + name='hardwaredark_delay', + field=models.IntegerField(default=-1), + ), + migrations.AlterField( + model_name='detector', + name='frames_directory', + field=models.CharField(default='/mnt/scope/movies/', help_text='Location of the frames directory from SmartScope that point to the same location as frames_windows_directory.', max_length=200), + ), + migrations.AlterField( + model_name='detector', + name='frames_windows_directory', + field=models.CharField(default='movies', help_text='Location of the frames from the perspective of SerialEM. This values will use the SetDirectory command.', max_length=200), + ), + ] diff --git a/Smartscope/server/api/migrations/0013_gridcollectionparams_afis.py b/Smartscope/server/api/migrations/0013_gridcollectionparams_afis.py new file mode 100644 index 00000000..cbe02b9f --- /dev/null +++ b/Smartscope/server/api/migrations/0013_gridcollectionparams_afis.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0.5 on 2023-05-16 14:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('API', '0012_gridcollectionparams_hardwaredark_delay_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='gridcollectionparams', + name='afis', + field=models.BooleanField(default=False), + ), + ] diff --git a/Smartscope/server/api/serializers.py b/Smartscope/server/api/serializers.py index f1efbd28..4713862f 100755 --- a/Smartscope/server/api/serializers.py +++ b/Smartscope/server/api/serializers.py @@ -263,7 +263,7 @@ def to_representation(self, instance): 'display_type': self.display_type, 'method': self.method, 'element': models_to_serializers[self.instance.__class__.__name__]['element'], - 'svg': self.instance.toSVG(display_type=self.display_type, method=self.method,), + 'svg': self.instance.svg(display_type=self.display_type, method=self.method,).asSvg(), 'fullmeta': self.load_meta() } diff --git a/Smartscope/server/api/templates/holecard.html b/Smartscope/server/api/templates/holecard.html index 9519953f..fa660bae 100755 --- a/Smartscope/server/api/templates/holecard.html +++ b/Smartscope/server/api/templates/holecard.html @@ -1,5 +1,4 @@ {% load rest_framework %} -{% for hole in holes %}
@@ -20,10 +19,12 @@
Hole {{hole.hole_id.number}}-{{hole.number}}
+
+ {% include "autoscreenViewer/download_images_dropdown.html" with instance=hole %}
- +
@@ -31,7 +32,7 @@
Hole {{hole.hole_id.number}}-{{hole.number}}
- {{hole.SVG | safe }} + {{svg | safe }}
@@ -54,5 +55,4 @@
Hole {{hole.hole_id.number}}-{{hole.number}}
- -{% endfor %} \ No newline at end of file + \ No newline at end of file diff --git a/Smartscope/server/api/templates/holecards.html b/Smartscope/server/api/templates/holecards.html new file mode 100755 index 00000000..76400fba --- /dev/null +++ b/Smartscope/server/api/templates/holecards.html @@ -0,0 +1,4 @@ +{% load rest_framework %} +{% for card in cards %} +{{card}} +{% endfor %} \ No newline at end of file diff --git a/Smartscope/server/api/templates/mapcard.html b/Smartscope/server/api/templates/mapcard.html index e9a8d5eb..0a6b692a 100755 --- a/Smartscope/server/api/templates/mapcard.html +++ b/Smartscope/server/api/templates/mapcard.html @@ -11,32 +11,7 @@

{{instance.alias_name}}

data-bs-toggle="tooltip" data-bs-placement="top" title="Toggle legend"> -
- -
- -
-
+ {% include "autoscreenViewer/download_images_dropdown.html" with instance=instance %}
+
+ +
+
\ No newline at end of file diff --git a/Smartscope/server/frontend/templates/autoscreenViewer/run_session.html b/Smartscope/server/frontend/templates/autoscreenViewer/run_session.html index 40e06727..8ee04e3e 100755 --- a/Smartscope/server/frontend/templates/autoscreenViewer/run_session.html +++ b/Smartscope/server/frontend/templates/autoscreenViewer/run_session.html @@ -58,19 +58,19 @@

SerialEM is paused

-
+
Main Output
-
@@ -78,7 +78,7 @@
Main Output
Processing
-
diff --git a/Smartscope/server/frontend/templates/forms/formFieldsBase.html b/Smartscope/server/frontend/templates/forms/formFieldsBase.html index bf7a91a3..cebaa336 100644 --- a/Smartscope/server/frontend/templates/forms/formFieldsBase.html +++ b/Smartscope/server/frontend/templates/forms/formFieldsBase.html @@ -1,5 +1,5 @@ -
+
{% for field in form %}
diff --git a/Smartscope/server/frontend/templates/smartscopeSetup/preprocessing/preprocessing_pipeline.html b/Smartscope/server/frontend/templates/smartscopeSetup/preprocessing/preprocessing_pipeline.html new file mode 100644 index 00000000..5f063ea5 --- /dev/null +++ b/Smartscope/server/frontend/templates/smartscopeSetup/preprocessing/preprocessing_pipeline.html @@ -0,0 +1,31 @@ +{% extends "general/right_sidebar.html" %} +{% block title %}Preprocessing Pipeline{% endblock %} + +{% block sidebar_content %} +
+ + {% include "forms/formFieldsBase.html" with form=form row=false %} + + +
+ +
+ {% if grid %} + {% include "./preprocessing_pipeline_form.html" with form=pipeline_form description=description pipeline=pipeline grid_id=grid.grid_id %} + {% endif %} +
+ +{% if grid %} +{% include "./preprocessing_pipeline_process.html" with data=pipeline_data grid_id=grid.grid_id %} +{% endif %} + + +{% endblock %} \ No newline at end of file diff --git a/Smartscope/server/frontend/templates/smartscopeSetup/preprocessing/preprocessing_pipeline_form.html b/Smartscope/server/frontend/templates/smartscopeSetup/preprocessing/preprocessing_pipeline_form.html new file mode 100644 index 00000000..c063fc45 --- /dev/null +++ b/Smartscope/server/frontend/templates/smartscopeSetup/preprocessing/preprocessing_pipeline_form.html @@ -0,0 +1,30 @@ +
+
+ Description: {{description}} +
+ {% csrf_token %} + + {% include "forms/formFieldsBase.html" with form=form row=false %} +
+ + +
+
+
+ + \ No newline at end of file diff --git a/Smartscope/server/frontend/templates/smartscopeSetup/preprocessing/preprocessing_pipeline_process.html b/Smartscope/server/frontend/templates/smartscopeSetup/preprocessing/preprocessing_pipeline_process.html new file mode 100644 index 00000000..547633f4 --- /dev/null +++ b/Smartscope/server/frontend/templates/smartscopeSetup/preprocessing/preprocessing_pipeline_process.html @@ -0,0 +1,19 @@ +
+
+ Status: {% if data.is_running %}Running{% else %}Stopped{% endif %} + PID: {{data.process_pid}} + {% if data.is_running %} + + {% else %} + + {% endif %} + +
+ It may take a few minutes to complete current actions. + +
+
\ No newline at end of file diff --git a/Smartscope/server/frontend/templates/smartscopeSetup/run_setup.html b/Smartscope/server/frontend/templates/smartscopeSetup/run_setup.html index dc328786..45bb5388 100755 --- a/Smartscope/server/frontend/templates/smartscopeSetup/run_setup.html +++ b/Smartscope/server/frontend/templates/smartscopeSetup/run_setup.html @@ -11,6 +11,14 @@

COLLECTION PARAMETERS

{% include "forms/formFieldsBase.html" with form=form_params row=True %} +

PREPROCESSING PIPELINE

+ + {% include "forms/formFieldsBase.html" with form=form_preprocess row=True id='formPreprocess' %}

AUTOLOADER

diff --git a/Smartscope/server/frontend/urls.py b/Smartscope/server/frontend/urls.py index 4df6efc0..4790e799 100755 --- a/Smartscope/server/frontend/urls.py +++ b/Smartscope/server/frontend/urls.py @@ -15,7 +15,14 @@ path('multishot/', views.MultiShotView.as_view(),name='setMultishot'), path('protocol/',views.ProtocolView.as_view(), name='protocol'), path('microscopes/status/', views.MicroscopeStatus.as_view(), name='microscopeStatus'), - + path('preprocessing/',views.PreprocessingPipeline.as_view(),name='preprocessingPipeline'), + path('preprocessing/getpipeline/',views.PreprocessingPipeline().get_pipeline,name='getPreprocessingPipeline'), + path('preprocessing/',views.PreprocessingPipeline().get_grid_pipeline,name='getGridPreprocessingPipeline'), + path('preprocessing/setpipeline//',views.PreprocessingPipeline().set_pipeline,kwargs={'grid_id': ''},name='setPreprocessingPipeline'), + path('preprocessing/setpipeline//',views.PreprocessingPipeline().set_pipeline,name='setPreprocessingPipeline'), + path('preprocessing//start',views.PreprocessingPipeline().start,name='startPreprocessingPipeline'), + path('preprocessing//stop',views.PreprocessingPipeline().stop,name='stopPreprocessingPipeline'), + path('collectionstats//',views.CollectionStatsView.as_view(),name='collectionStats'), ] if settings.USE_MICROSCOPE: urlpatterns += [path('run/', RedirectView.as_view(url='setup/'), name='run'), diff --git a/Smartscope/server/frontend/views.py b/Smartscope/server/frontend/views.py index 385e682b..6f0da71d 100755 --- a/Smartscope/server/frontend/views.py +++ b/Smartscope/server/frontend/views.py @@ -4,23 +4,27 @@ import psutil from datetime import datetime import logging +import plotly.graph_objs as go + from django.shortcuts import render from django.contrib.auth import logout, authenticate, login from django.contrib.auth.mixins import LoginRequiredMixin from django.conf import settings from django.http import JsonResponse, HttpResponse +from django.template.response import TemplateResponse from django.views.generic import TemplateView from django.shortcuts import redirect from django.utils.timezone import now from .forms import * -from Smartscope.core.db_manipulations import viewer_only +from Smartscope.core.db_manipulations import viewer_only, get_hole_count from Smartscope.core.protocols import get_or_set_protocol from Smartscope.lib.file_manipulations import create_grid_directories from Smartscope.lib.multishot import RecordParams,set_shots_per_hole, load_multishot_from_file -from Smartscope.core.cache import save_multishot_from_cache +from Smartscope.core.cache import save_json_from_cache from Smartscope.core.protocols import load_protocol, set_protocol +from Smartscope.core.preprocessing_pipelines import PREPROCESSING_PIPELINE_FACTORY, load_preprocessing_pipeline logger =logging.getLogger(__name__) @@ -59,9 +63,11 @@ def get_context_data(self, **kwargs): if not 'form_general' in kwargs.keys(): form_general = ScreeningSessionForm() form_params = GridCollectionParamsForm() + form_preprocess = PreprocessingPipelineIDForm() else: form_general = kwargs['form_general'] form_params = kwargs['form_params'] + form_preprocess = kwargs['form_preprocess'] grids = [] form = AutoloaderGridForm(prefix=1) @@ -69,7 +75,7 @@ def get_context_data(self, **kwargs): grids.append(form) - context = dict(form_general=form_general, form_params=form_params, grids=grids) + context = dict(form_general=form_general, form_params=form_params,form_preprocess=form_preprocess, grids=grids) sessions = ScreeningSession.objects.all().order_by('-date')[:10] context['sessions'] = sessions print(context['grids']) @@ -79,11 +85,12 @@ def post(self, request, **kwargs): is_viewer_only = viewer_only(self.request.user) form_general = ScreeningSessionForm(request.POST) form_params = GridCollectionParamsForm(request.POST) + form_preprocess = PreprocessingPipelineIDForm(request.POST) if not is_viewer_only: num_grids = set([k.split('-')[0] for k in request.POST.keys() if k.split('-')[0].isnumeric()]) - if form_general.is_valid() and form_params.is_valid(): + if form_general.is_valid() and form_params.is_valid() and form_preprocess.is_valid(): session, created = ScreeningSession.objects.get_or_create(**form_general.cleaned_data, date=datetime.today().strftime('%Y%m%d')) if created: @@ -91,6 +98,7 @@ def post(self, request, **kwargs): # multishot = form_params.cleaned_data.pop('multishot_per_hole') multishot_per_hole_id = form_params.cleaned_data.pop('multishot_per_hole_id') + preprocessing_pipeline_id = form_preprocess.cleaned_data.pop('preprocessing_pipeline_id',False) params, created = GridCollectionParams.objects.get_or_create(**form_params.cleaned_data) if created: logger.debug(f'{params} newly created') @@ -111,7 +119,9 @@ def post(self, request, **kwargs): logger.debug(f'Setting protocol {protocol} for {grid}') get_or_set_protocol(grid,protocol) if params.multishot_per_hole: - save_multishot_from_cache(multishot_per_hole_id, grid.directory) + save_json_from_cache(multishot_per_hole_id, grid.directory,'multishot') + if preprocessing_pipeline_id != '': + save_json_from_cache(preprocessing_pipeline_id, grid.directory,'preprocessing') return redirect(f'../session/{session.session_id}') @@ -373,4 +383,121 @@ def get_context_data(self,*args, **kwargs): def get(self,request, *args, **kwargs): context = self.get_context_data(*args, **kwargs) - return render(request,self.template_name, context) \ No newline at end of file + return render(request,self.template_name, context) + +class PreprocessingPipeline(TemplateView): + template_name= "smartscopeSetup/preprocessing/preprocessing_pipeline.html" + + def get(self,request, *args, **kwargs): + context = self.get_context_data(**kwargs) + return render(request,self.template_name, context) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['form'] = SelectPeprocessingPipilelineForm() + return context + + def get_grid_context_data(self,grid_id): + context = dict() + grid = AutoloaderGrid.objects.get(pk=grid_id) + pipeline_data = load_preprocessing_pipeline(Path(grid.directory, 'preprocessing.json')) + pipeline = PREPROCESSING_PIPELINE_FACTORY[pipeline_data.pipeline] + context['grid'] = grid + context['form'] = SelectPeprocessingPipilelineForm(data={'pipeline':pipeline_data.pipeline}) + context['pipeline_form'] = pipeline.form(data=pipeline_data.kwargs) + context['pipeline'] = pipeline_data.pipeline + context['description'] = pipeline.description + context['pipeline_data'] = pipeline_data + return context + + def get_grid_pipeline(self, request, *args ,grid_id, **kwargs): + context = self.get_grid_context_data(grid_id) + return render(request,self.template_name, context) + + + def get_pipeline(self, request, *args, **kwargs,): + try: + context = {} + pipeline = request.GET.get('pipeline',None) + # form = SelectPeprocessingPipilelineForm(request.POST) + # is_valid = form.is_valid() + # if not is_valid: + # return HttpResponse('Form invalid') + # if is_valid: + # logger.debug(form.cleaned_data) + # data = form.cleaned_data + pipeline_obj = PREPROCESSING_PIPELINE_FACTORY[pipeline] + logger.debug(pipeline) + context['pipeline'] = pipeline + context['description'] = pipeline_obj.description + context['form'] = pipeline_obj.form() + # context['grid_id'] = None + return TemplateResponse(request=request,template="smartscopeSetup/preprocessing/preprocessing_pipeline_form.html",context=context) + except Exception as err: + logger.exception(err) + + def set_pipeline(self, request,pipeline, *args, grid_id=None, **kwargs): + try: + logger.debug(request.POST) + pipeline_obj = PREPROCESSING_PIPELINE_FACTORY[pipeline] + form = pipeline_obj.form(request.POST) + logger.debug(f'Updating pipeline for {grid_id}') + if form.is_valid(): + pipeline_data = pipeline_obj.pipeline_data(form.cleaned_data) + if grid_id is None or not grid_id: + cache.set(pipeline_data.cache_id,pipeline_data.json(exclude={'cache_id'}),timeout=30*60) + logger.debug(pipeline_data) + return TemplateResponse(request=request, + template='forms/formFieldsBase.html', + context=dict(form=PreprocessingPipelineIDForm(data=dict(preprocessing_pipeline_id=pipeline_data.cache_id)), + row=True, + id='formPreprocess')) + grid = AutoloaderGrid.objects.get(pk=grid_id) + Path(grid.directory,'preprocessing.json').write_text(pipeline_data.json(exclude={'cache_id'})) + logger.info('Updated pipeline for existing grid') + return self.get_grid_pipeline(request, grid_id=grid_id) + except Exception as err: + logger.exception(err) + + def start(self, request, grid_id, *args, **kwargs): + context = self.get_grid_context_data(grid_id) + context['pipeline_data'].start(context['grid']) + return self.get_grid_pipeline(request,grid_id=grid_id) + + def stop(self, request, grid_id, *args, **kwargs): + context = self.get_grid_context_data(grid_id) + context['pipeline_data'].stop(context['grid']) + return self.get_grid_pipeline(request,grid_id=grid_id) + +class CollectionStatsView(TemplateView): + template_name = "autoscreenViewer/collection_stats.html" + + def ctfGraph(self,grid_id): + ### NEED TO MOVE THE GRAPHING LOGIC OUTSIDE OF HERE + data = list(HighMagModel.objects.filter(status='completed', grid_id=grid_id, ctffit__lte=15).values_list('ctffit', flat=True)) # replace with your own data source + hist = go.Histogram(x=data, nbinsx=30) + layout = go.Layout( + title='CTF fit distribution', + xaxis=dict( + title='CTF fit resolution (Angstrom)' + ), + yaxis=dict( + title='Number of exposures' + ), + showlegend=False, + ) + fig = go.Figure(data=[hist],layout=layout,) + graph = fig.to_html(full_html=False) + return graph + + + def get_context_data(self, grid_id, **kwargs): + context = super().get_context_data(**kwargs) + grid= AutoloaderGrid.objects.get(pk=grid_id) + context.update(get_hole_count(grid)) + context['graph'] = self.ctfGraph(grid_id) + return context + + def get(self,request, grid_id, *args, **kwargs): + context = self.get_context_data(grid_id, **kwargs) + return render(request,self.template_name, context) \ No newline at end of file diff --git a/Smartscope/server/main/templates/base.html b/Smartscope/server/main/templates/base.html index e043fc42..b2594df1 100755 --- a/Smartscope/server/main/templates/base.html +++ b/Smartscope/server/main/templates/base.html @@ -11,6 +11,7 @@ + diff --git a/Smartscope/server/main/urls.py b/Smartscope/server/main/urls.py index 4f429b9e..60e825cc 100755 --- a/Smartscope/server/main/urls.py +++ b/Smartscope/server/main/urls.py @@ -43,10 +43,10 @@ ] if settings.DEPLOY is False: - urlpatterns.append(re_path(r'^autoscreening/(?P.*)$', serve, {'document_root': settings.AUTOSCREENDIR})) - urlpatterns.append(re_path(r'^autoscreeningstorage/(?P.*)$', serve, {'document_root': settings.AUTOSCREENSTORAGE})) + # urlpatterns.append(re_path(r'^autoscreening/(?P.*)$', serve, {'document_root': settings.AUTOSCREENDIR})) + # urlpatterns.append(re_path(r'^autoscreeningstorage/(?P.*)$', serve, {'document_root': settings.AUTOSCREENSTORAGE})) urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) - urlpatterns.append(re_path(r'^docs/(?P.*)$', serve, {'document_root': f'{settings.STATIC_ROOT}/docs-build/html'})) + # urlpatterns.append(re_path(r'^docs/(?P.*)$', serve, {'document_root': f'{settings.STATIC_ROOT}/docs-build/html'})) # Api urls diff --git a/VERSION b/VERSION index b63ba696..f374f666 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.9 +0.9.1 diff --git a/config/docker/requirements.txt b/config/docker/requirements.txt index 5f4f8fb4..e8a3cc08 100755 --- a/config/docker/requirements.txt +++ b/config/docker/requirements.txt @@ -3,7 +3,8 @@ # torch==1.8.2 # torchvision==0.9.2 # torchaudio==0.8.2 ---find-links https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.8/index.html +--find-links https://dl.fbaipublicfiles.com/detectron2/wheels/cu111/torch1.8/index.html +cliYAML @ git+https://github.com/JoQCcoz/cliYAML.git@main detectron2==0.6 absl-py==0.13.0 aioredis==1.3.1 @@ -96,7 +97,8 @@ parso pathspec==0.8.1 pexpect pickleshare -Pillow==8.3.1 +Pillow +plotly portalocker==2.3.0 prompt-toolkit protobuf==3.17.3 diff --git a/config/docker/templates_SSL/default.conf b/config/docker/templates_SSL/default.conf index e0419258..d6483dbc 100755 --- a/config/docker/templates_SSL/default.conf +++ b/config/docker/templates_SSL/default.conf @@ -7,7 +7,9 @@ server { listen 80; listen [::]:80; #Change next line with your own domain - server_name _; + server_name _; + server_tokens off; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; return 301 https://$server_name$request_uri; } server { @@ -16,16 +18,22 @@ server { #Change next line with your own domain server_name _; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + + server_tokens off; + location / { proxy_pass http://smartscope:48001; proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; proxy_redirect off; } location /websocket/ { proxy_pass http://smartscope:48001; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -45,4 +53,4 @@ server { ssl_prefer_server_ciphers on; ssl_session_cache shared:SSL:5m; ssl_protocols TLSv1.2; -} \ No newline at end of file +} diff --git a/config/smartscope/default_preprocessing.json b/config/smartscope/default_preprocessing.json new file mode 100644 index 00000000..75739a3d --- /dev/null +++ b/config/smartscope/default_preprocessing.json @@ -0,0 +1,7 @@ +{ +"pipeline": "smartscopePipeline", +"kwargs": { + "n_processes": 1, + "frames_directory": null + } +} \ No newline at end of file diff --git a/static/reports.js b/static/reports.js index 3ce455ab..ca693da8 100755 --- a/static/reports.js +++ b/static/reports.js @@ -314,6 +314,27 @@ function hideSVGlabel(el, parentid) { element.setAttribute('visibility', final); } } +// function downloadPNG(element_id) { +// // Get the SVG element +// const svg = document.getElementById(element_id); + +// // Create a canvas element with the same dimensions as the SVG +// const canvas = document.createElement("canvas"); +// canvas.width = svg.clientWidth; +// canvas.height = svg.clientHeight; + +// // Use canvg to rasterize the SVG onto the canvas +// canvg(canvas, svg.outerHTML); + +// // Create a data URL from the canvas +// const dataURL = canvas.toDataURL("image/png"); + +// // Create a download link with the data URL and click it to start the download +// const link = document.createElement("a"); +// link.download = "image.png"; +// link.href = dataURL; +// link.click(); +// } function openMenu(el, menu) { menu.classList.remove('show') @@ -631,7 +652,7 @@ async function reportMain() { zoomedContentCln = document.getElementById('zoomedContent').cloneNode(true) fullmeta = await loadMeta() populateReportHead() - renderCounts() + // renderCounts() if (fullmeta.status != null) { console.log(fullmeta.atlas[Object.keys(fullmeta.atlas)[0]].status) if (fullmeta.atlas[Object.keys(fullmeta.atlas)[0]].status == 'completed') { diff --git a/static/run_session.js b/static/run_session.js index f37ecbc6..14ad5148 100755 --- a/static/run_session.js +++ b/static/run_session.js @@ -9,8 +9,8 @@ async function loadlogs() { if (data.reload === true) { location.reload(); } - queue = document.getElementById('queue') - queue.innerHTML = data.queue + // queue = document.getElementById('queue') + // queue.innerHTML = data.queue out = document.getElementById('out') out.innerHTML = data.out proc = document.getElementById('proc') diff --git a/static/setup_session.js b/static/setup_session.js index ba284ff2..06261249 100755 --- a/static/setup_session.js +++ b/static/setup_session.js @@ -36,12 +36,12 @@ function fillFromPrevious(prev_num) { const curr_num = prev_num + 1 for (item of ['holeType', 'meshSize', "meshMaterial", 'protocol']) { - var prevVal = $(`[name='${prev_num}-${item}'`).val() + var prevVal = $(`[name='${prev_num}-${item}']`).val() - $(`[name='${curr_num}-${item}'`).val(prevVal) + $(`[name='${curr_num}-${item}']`).val(prevVal) } - let prevName = $(`[name='${prev_num}-name'`).val() - $(`[name='${curr_num}-position'`).val(parseInt($(`[name='${prev_num}-position'`).val(), 10) + 1) + let prevName = $(`[name='${prev_num}-name']`).val() + $(`[name='${curr_num}-position']`).val(parseInt($(`[name='${prev_num}-position']`).val(), 10) + 1) var prevNameSplit = prevName.split('_') var popped = prevNameSplit.pop() @@ -51,7 +51,7 @@ function fillFromPrevious(prev_num) { popped = [popped, 1] } newName = prevNameSplit.concat(popped).join('_') - $(`[name='${curr_num}-name'`).val(newName) + $(`[name='${curr_num}-name']`).val(newName) }