Merge pull request #243 from NIEHS/stable

0.9.2

.gitignore

@@ -5,7 +5,7 @@ Untitled.ipynb
 cryolo_models/
 Tomo.egg-info
 .DS_Store
-Smartscope.egg-info/
+# Smartscope.egg-info/
 .vscode/
 conf.env
 Smartscope/Finders/AIFinder/detectors/weights/
@@ -30,8 +30,15 @@ docker-compose.yml.bak
 .nv
 .config
 .python_history
-MyDocker/*
+Smartscope/core/__pycache__
+Smartscope/mydatase
 .ipython
 .mysql_history
 .gitignore
 .ipython/profile_default/history.sqlite
+mydatabase
+SerialEM-python
+serialem.egg-info/
+SmartScope-docs
+data
+Smartscope/.local.env

Docker/Dockerfile-base

@@ -6,29 +6,30 @@ RUN apt-get update && apt-get install -y \
 	python3-dev default-libmysqlclient-dev build-essential wget libglib2.0-0 ffmpeg libsm6 libxext6 curl mariadb-server git && \
 	apt-get clean && rm -rf /var/cache/apt/lists
 
-RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
-	bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 && \
-	rm Miniconda3-latest-Linux-x86_64.sh
+# RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+# 	bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 && \
+# 	rm Miniconda3-latest-Linux-x86_64.sh
 
 RUN wget https://bio3d.colorado.edu/imod/AMD64-RHEL5/imod_4.11.24_RHEL7-64_CUDA10.1.sh && ls && \
 	yes | bash imod_4.11.24_RHEL7-64_CUDA10.1.sh -name IMOD && \
 	rm imod_4.11.24_RHEL7-64_CUDA10.1.sh
 
 ADD config/singularity/ctffind /usr/local/
 
-ADD config/docker/requirements.txt .
-
 ENV PATH=$PATH:/opt/miniconda3/bin
 
-RUN conda update -y conda && \
-	yes | conda install python=3.9 && \
-	yes | conda install cudatoolkit=11.1 cudnn -c nvidia 
-RUN	yes | conda install pytorch torchvision -c pytorch-lts -c nvidia && \
-	conda clean --all
+# RUN conda update -y conda && \
+# 	yes | conda install python=3.9 && \
+# 	yes | conda install cudatoolkit=11.1 cudnn -c nvidia \
+# 	yes | conda install pytorch torchvision -c pytorch-lts -c nvidia && \
+# 	conda clean --all
+
+# ADD config/docker/requirements.txt .
 
-RUN	yes | pip install numpy==1.21.0 && \
-	# yes | pip install torch==1.8.2 torchvision==0.9.2 torchaudio==0.8.2 --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu111 && \
-	yes | pip install -r requirements.txt
+# RUN	yes | pip install numpy==1.21.0 && \
+# 	# yes | pip install torch==1.8.2 torchvision==0.9.2 torchaudio==0.8.2 --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu111 && \
+# 	yes | pip install -r requirements.txt && \
+# 	pip cache purge
 
 
 # create a non-root user

Docker/Dockerfile-smartscope

@@ -1,4 +1,4 @@
-FROM smartscope-base
+FROM smartscope-base-test
 
 ARG VERSION='0.8-rc.2'
 ARG BRANCH='stable'
@@ -11,7 +11,7 @@ LABEL org.opencontainers.image.licenses=BSD-3-Clause
 #General environment variables
 ENV VERSION=$VERSION \
 	IMOD_DIR=/usr/local/IMOD \	
-	CTFFIND=/usr/local/ctffind \
+	CTFFIND=/opt/ctffind \
 	APP=/opt/smartscope/ \
 	AUTOSCREENDIR=/mnt/data/ \
 	TEMPDIR=/tmp/ \
@@ -22,7 +22,7 @@ ENV VERSION=$VERSION \
 ENV	ALLOWED_HOSTS=localhost \
 	CONFIG=/opt/config/ \
 	USE_SSL=false \
-	EXTERNAL_PLUGINS_DIRECTORY=/opt/smartscope/external_plugins/ \
+	EXTERNAL_PLUGINS_DIRECTORY=/opt/external_plugins/ \
 	AUTO_MIGRATION=true \
 	DJANGO_SETTINGS_MODULE=Smartscope.core.settings.server_docker \
 	USE_STORAGE=True \
@@ -31,6 +31,7 @@ ENV	ALLOWED_HOSTS=localhost \
 	DEFAULT_UMASK=002 \
 	LOGLEVEL=INFO \
 	DEBUG=False \
+	DEPLOY=True \
 	TEST_FILES=/mnt/testfiles/ \
 	MYSQL_HOST=db \
 	MYSQL_PORT=3306 \
@@ -44,21 +45,26 @@ ENV	ALLOWED_HOSTS=localhost \
 
 ENV PATH=$PATH:/opt/smartscope/Smartscope/bin:/usr/local/IMOD/bin
 
-ADD . /opt/smartscope/
+COPY --link Template_files/ /opt/Template_files/
+COPY --link ctffind /opt/
+COPY --link miniconda3/ /opt/miniconda3/
+COPY smartscope/ /opt/smartscope/
+COPY external_plugins/ /opt/external_plugins/
 
-RUN wget docs.smartscope.org/downloads/Smartscope0.6.tar.gz --no-check-certificate && \
-	tar -xvf Smartscope0.6.tar.gz -C /opt/ && \
-	rm Smartscope0.6.tar.gz
 
-RUN	mkdir /opt/logs/ /mnt/fake_scope/ /mnt/fake_scope/raw/ /opt/shared/ /opt/config/ /mnt/testfiles
-RUN pip install -e /opt/smartscope/ --no-dependencies && \
-	pip install -e /opt/smartscope/SerialEM-python --no-dependencies
+# RUN wget docs.smartscope.org/downloads/Smartscope0.6.tar.gz --no-check-certificate && \
+# 	tar -xvf Smartscope0.6.tar.gz -C /opt/ && \
+# 	rm Smartscope0.6.tar.gz
+
+RUN	mkdir /opt/logs/ /mnt/fake_scope/ /mnt/fake_scope/raw/ /mnt/fake_scope/movies/ /opt/shared/ /opt/config/ /mnt/testfiles
+# RUN pip install -e /opt/smartscope/ --no-dependencies && \
+# 	pip install -e /opt/smartscope/SerialEM-python --no-dependencies
 
 RUN chmod 777 -R /mnt/ && \
 	chmod 777 -R /opt/logs/ && \
 	chmod 777 /opt/shared/ && \
-	chmod 777 /opt/config/ && \
-	chmod 777 /mnt/testfiles/
+	chmod 777 /opt/config/
+	# chmod 777 /mnt/testfiles/
 
 USER smartscope_user
 

Docker/SmartScope/database.conf

@@ -6,6 +6,6 @@ MYSQL_PORT=3306
 MYSQL_SSL=False
 
 ## This is used for creating the user and the database. 
-## Can be replaced with MYSQL_ROOT_USER and MYSQL_ROOT_PASSWORD
+
 MYSQL_ROOT_USER=root
 MYSQL_ROOT_PASSWORD=pass

Docker/SmartScope/docker-compose.yml

@@ -6,7 +6,6 @@ version: "3"
 services:
   smartscope:
     image: ghcr.io/niehs/smartscope:latest
-    container_name: smartscope
     restart: always
     volumes:
       ######## FILL THIS ##########
@@ -16,10 +15,13 @@ services:
       # - /path/to/where/serialem/will/write/files/:/mnt/your_scope_name_here/
       # Example:
       # - /mnt/gatan_Raid_X/smartscope/:/mnt/arctica/
-      ###### OTHER DEFAULT VOLUMES - NO NEED TO CHANGE ####### 
+      ###### OTHER DEFAULT VOLUMES - NO NEED TO CHANGE #######
+      - ./shared/auth/:/opt/auth/:rw
       - ./shared/nginx/:/opt/shared/:rw
       - ./shared/smartscope/:/opt/config/:rw
       - ./testfiles/:/mnt/testfiles/:rw
+      - ./logs/:/opt/logs/
+      - ./backups/:/mnt/backups/
     env_file:
       - smartscope.conf
       - database.conf
@@ -30,13 +32,11 @@ services:
       - smartscopenet
   db:
     image: mariadb:10.5
-    container_name: smartscope-db
     restart: always
     env_file:
       - database.conf
     volumes:
       - ./db/:/var/lib/mysql/
-      # - ./shared/initialdb.sql:/docker-entrypoint-initdb.d/initialdb.sql
     networks:
       - smartscopenet
   cache:
@@ -48,8 +48,6 @@ services:
   nginx:
     image: nginx
     restart: always
-    ports:
-      - 48000:80
     volumes:
       - ./shared/nginx/:/opt/shared/:ro #Doesn't need to be changed
     networks:

Docker/SmartScope/smartscope.conf

@@ -3,6 +3,7 @@ ALLOWED_HOSTS=localhost
 AUTO_MIGRATION=true
 USE_SSL=false
 USE_LONGTERMSTORAGE=False
+USE_CUSTOM_PATHS=False
 TIMEZONE=America/New_York
 
 ### EXPERT SETTINGS ###
@@ -33,6 +34,7 @@ TIMEZONE=America/New_York
 # LOGLEVEL=INFO
 # DEBUG=False
 # TEST_FILES=/mnt/testfiles/
+# CORS_ORIGIN_ALLOW_ALL=False
 
 # #Cache
 # REDIS_HOST=cache

Docker/SmartScope/smartscope.sh

@@ -1,5 +1,15 @@
 #! /bin/bash
 
+readonly cmd_file="dockerCmd.txt"
+readonly dockerRepo="ghcr.io/niehs/smartscope"
+composeCmd="docker compose"
+dockerCmd="docker"
+
+if [ -e "$cmd_file" ]; then
+    read -r dockerCmd < "$cmd_file"
+    read -r composeCmd < "$cmd_file"
+fi
+
 helpText () {
     OPTIONS="
 Usage: $(basename $0) \e[3msubcommand [command]\e[0m
@@ -8,12 +18,60 @@ Subcommand              DESCRIPTION
 ==========              ===========
 start                   Start SmartScope
 stop                    Stop smartscope
+restart                 Restart smartscope
+setup                   Setup the smartscope directories
+update \e[3mversion\e[0m                 Update smartscope to the specified version. Choices: ['latest','stable'] (default: latest)
 run \e[3mcommand\e[0m             Run a smartscope command in the smartscope container
+exec \e[3mcommand\e[0m            Run any shell command in the smartscope container
 python                  Runs an interactive ipython shell inside the smartscope container
 "
     echo -e "$OPTIONS"
 }
 
+stop () {
+    echo "Stopping smartscope"
+    $composeCmd down
+}
+
+start () {
+    export UID=$(id -u)
+    export GID=$(id -g)
+    echo "Starting smartscope"
+    $composeCmd -f docker-compose.yml -f smartscope.yml up -d
+}
+
+promptYesNo () {
+    local question=$1
+    while true; do
+        echo "$question (y/n)"
+        read user_input
+
+        if [[ "$user_input" == "y" ]]; then
+            return 0  # Return true
+        elif [[ "$user_input" == "n" ]]; then
+            return 1  # Return false
+        else
+            echo "Invalid choice. Please enter 'yes' or 'no'."
+        fi
+    done    
+}
+
+checkForUpdates () {
+    local version=$1
+    echo "Checking for updates to version $version"
+    online_sha=$($dockerCmd manifest inspect $dockerRepo:$version | sed -n '/"config": {/,/"digest": "sha256:\([^"]*\)"/s/.*"digest": "sha256:\([^"]*\)".*/\1/p')
+    local_sha=$($dockerCmd inspect --format "{{.ID}}" $dockerRepo:$version | cut -d ':' -f 2)
+    echo "Online sha: $online_sha
+Local sha: $local_sha"
+    if [[ "$online_sha" == "$local_sha" ]]; then
+        # echo "No updates available"
+        return 1
+    else
+        # echo "Updates available"
+        return 0
+    fi
+}
+
 [ -n "$1" ] && argument=$1 || {
     helpText
     exit 1
@@ -22,25 +80,81 @@ cd $(dirname "$(readlink -f "$0")")
 
 case $argument in
     start)
-        export UID=$(id -u)
-        export GID=$(id -g)
-        echo "Starting smartscope"
-        docker compose -f docker-compose.yml -f smartscope.yml up -d ;;
+        if checkForUpdates 'latest'; then
+            echo "A new release is available. Run 'smartscope.sh update latest' to update to the latest version."
+        fi        
+        if checkForUpdates 'stable'; then
+            echo "A new beta version is available. Run 'smartscope.sh update stable' to update to the new beta version."
+        fi 
+        start;;
     stop)
-        echo "Stopping smartscope"
-        docker compose down ;;
+        stop;;
+    restart)
+        stop
+        sleep 2
+        start;;
     run)
-        cmd="docker exec smartscope smartscope.py ${@:2}"
+        cmd="$composeCmd exec smartscope smartscope.py ${@:2}"
         echo -e "Executing command inside the smartscope container:
     \e[3m$cmd\e[0m"
         exec $cmd;;
     help|-h|--help)
         helpText;;
     python)
         echo "Running a python shell inside the smartscope container"
-        docker exec -it smartscope manage.py shell -i ipython ;;
+        $composeCmd exec -it smartscope manage.py shell -i ipython ;;
+    exec)
+        echo "Executing shell command inside the smartscope container:"
+        $composeCmd exec -it smartscope ${@:2};;
+    setup)
+        version=${2:-latest}
+        echo "Setting up the latest version of smartscope"
+        if [ "$version" == 'latest' ]; then
+            version='main'
+        fi
+        echo "Setting up the smartscope directories"
+        mkdir -p logs shared/nginx shared/auth shared/smartscope db data backups
+        for file in docker-compose.yml smartscope.yml smartscope.conf database.conf; do
+            echo "Pulling $file from $version"
+            if [ -e "$file" ]; then
+                echo "$file already exists. Skipping."
+                continue
+            fi
+            wget https://raw.githubusercontent.com/NIEHS/SmartScope/$version/Docker/SmartScope/$file
+        done
+        echo "Pulling initialdb.sql from $version"
+        wget https://raw.githubusercontent.com/NIEHS/SmartScope/$version/Docker/SmartScope/shared/initialdb.sql -O shared/initialdb.sql 
+        ;;
+    update)
+        version=${2:-latest}
+        echo "Updating smartscope to version: $version"
+        if ! checkForUpdates $version; then
+            if ! promptYesNo "You already have the docker image correspoding to version $version. Do you want to update this instance anyway?"; then
+                exit 0
+            fi
+        fi 
+
+        backupDir="backups/$(date +%Y%m%d)_config_pre_update"
+        if promptYesNo "Do you want to back up the database? (Highly recommended)"; then
+            echo "Backing up the database. This may take a few minutes."
+            $composeCmd exec smartscope smartscope.py backup_db
+        fi
+        echo "Creating a backup of the configuration in $backupDir"
+        mkdir $backupDir
+        cp docker-compose.yml smartscope.yml smartscope.conf database.conf smartscope.sh $backupDir
+        echo "Pulling updated docker-compose.yml"
+        repo_url="https://raw.githubusercontent.com/NIEHS/SmartScope/$version/Docker/SmartScope"
+        wget $repo_url/docker-compose.yml -O docker-compose.yml
+        echo "Pulling docker image"
+        $composeCmd pull smartscope
+        if promptYesNo  "Files updated, do you want to restart smartscope"; then
+            exec $0 restart
+        fi
+        ;;
+
     *)
         echo Unkown command error: $argument
         helpText
         exit 1;;
 esac
+

Docker/SmartScope/smartscope.yml

@@ -13,3 +13,9 @@ services:
       # - /path/to/where/serialem/will/write/files/:/mnt/your_scope_name_here/
       # Example:
       # - /mnt/gatan_Raid_X/smartscope/:/mnt/arctica/
+  nginx:
+    ports:
+      #### BROADCASTED PORT ####
+      # Default port is 48000 but this can be changed by any port of your choosing.
+      # Note: You need sudo permissions to use port 80
+      - 48000:80

Docker/build.sh

@@ -7,4 +7,4 @@ docker build \
     --label org.opencontainers.image.documentation=https://docs.smartscope.org/docs/$VERSION/index.html \
     --label org.opencontainers.image.version=$VERSION \
     --build-arg VERSION=$VERSION \
-    -t smartscope:$VERSION -t ghcr.io/niehs/smartscope:$VERSION -f Docker/Dockerfile-smartscope .
+    -t smartscope:$VERSION -t ghcr.io/niehs/smartscope:$VERSION -f smartscope/Docker/Dockerfile-smartscope .