Merge pull request #554 from NREL/new-solver-setup

New solver setup

.gitattributes

@@ -1 +1,2 @@
 #pattern  filter=crypt diff=crypt
+julia_src/xpress/licenseserver/xpauth.xpr filter=crypt diff=crypt merge=crypt

.github/disabled_cbc_workflows/push_test_cbc.yml

@@ -26,4 +26,4 @@ jobs:
       - name: Check django logs
         run: docker logs django
       - name: test
-        run: docker-compose exec -T celery python manage.py test reo.tests.test_custom_rates reo.tests.test_demand_ratchet -v 2 --failfast --no-input
+        run: docker-compose exec -T celery python manage.py test reoptjl.test.test_job_endpoint -v 2 --failfast --no-input

.github/scripts/decrypt.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-transcrypt/transcrypt --flush-credentials -y || true
-transcrypt/transcrypt -c aes-256-cbc -p "$TRANSCRYPT_PASSWORD" -y
+# transcrypt/transcrypt --flush-credentials -y || true
+# transcrypt/transcrypt -c aes-256-cbc -p "$TRANSCRYPT_PASSWORD" -y
 
 

.github/workflows/pull_request_tests.yml

@@ -18,21 +18,23 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2      
-      - name: Decrypt
-        env:
-          TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }}
-        run: ./.github/scripts/decrypt.sh 
+      # - name: Decrypt
+      #   env:
+      #     TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }}
+      #   run: ./.github/scripts/decrypt.sh 
       - name: Make keys.py
         env:
           NREL_DEV_API_KEY: ${{ secrets.NREL_DEV_API_KEY }}
         run: ./.github/scripts/make_keys.py.sh 
       - name: Build containers
-        run: docker-compose up -d
+        run: docker-compose -f docker-compose.highs.yml up -d
       - name: Check running containers
         run: docker ps -a
       - name: Wait for julia_api
         uses: jakejarvis/wait-action@master
         with:
           time: '150s'
-      - name: test
-        run: docker exec reopt_api_celery_1 python manage.py test -v 2 --failfast --no-input
+      - name: test v3 validator
+        run: docker exec reopt_api_celery_1 python manage.py test reoptjl.test.test_validator -v 2 --failfast --no-input          
+      - name: test v3 http.jl endpoints
+        run: docker exec reopt_api_celery_1 python manage.py test reoptjl.test.test_http_endpoints -v 2 --failfast --no-input

.github/workflows/push_tests.yml

@@ -14,21 +14,23 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Decrypt
-        env:
-          TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }}
-        run: ./.github/scripts/decrypt.sh 
+      # - name: Decrypt
+      #   env:
+      #     TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }}
+      #   run: ./.github/scripts/decrypt.sh 
       - name: Make keys.py
         env:
           NREL_DEV_API_KEY: ${{ secrets.NREL_DEV_API_KEY }}
         run: ./.github/scripts/make_keys.py.sh 
       - name: Build containers
-        run: docker-compose up -d
+        run: docker-compose -f docker-compose.highs.yml up -d
       - name: Check running containers
         run: docker ps -a
       - name: Wait for julia_api
         uses: jakejarvis/wait-action@master
         with:
           time: '150s'
-      - name: test
-        run: docker exec reopt_api_celery_1 python manage.py test -v 2 --failfast --no-input
+      - name: test v3 validator
+        run: docker exec reopt_api_celery_1 python manage.py test reoptjl.test.test_validator -v 2 --failfast --no-input          
+      - name: test v3 http.jl endpoints
+        run: docker exec reopt_api_celery_1 python manage.py test reoptjl.test.test_http_endpoints -v 2 --failfast --no-input

.gitignore

@@ -115,3 +115,8 @@ docs/formulation/*aux
 docs/formulation/*log
 
 /.werf_secret_key
+
+# Customized solver setup and docker files
+julia_src/xpress/licenseserver
+julia_src/Dockerfile.xpress
+docker-compose.xpress.yml

.helm/templates/julia-deployment.yaml

@@ -34,7 +34,7 @@ spec:
       containers:
         - name: {{ .Chart.Name }}-julia
           image: {{ index .Values.werf.image "julia-api" }}
-          args: ["julia", "--project=/opt/julia_src", "http.jl"]
+          args: ["julia", "--project=/opt/julia_src/xpress", "http.jl"]
           ports:
             - containerPort: 8081
           envFrom:

docker-compose.highs.yml

@@ -0,0 +1,73 @@
+version: "2.1"
+
+services:
+
+  redis:
+    image: redis
+    command: redis-server
+    expose:
+      - 6379
+
+  db:
+    image: postgres
+    restart: always
+    environment:
+      - POSTGRES_USER=reopt
+      - POSTGRES_PASSWORD=reopt
+      - POSTGRES_DB=reopt
+    expose:
+      - 5432
+
+  celery:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: base-api-image
+    command:  >
+      "celery -A reopt_api worker -l info"
+    environment:
+      - APP_ENV=local
+      - SQL_HOST=db
+      - SQL_PORT=5432
+      - REDIS_HOST=redis
+    volumes:
+      - .:/opt/reopt
+    depends_on:
+      - db
+      - redis
+      - julia
+
+  django:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: base-api-image
+    command:  >
+      "python manage.py migrate
+      && /opt/reopt/bin/wait-for-it.bash -t 0 julia:8081 -- python manage.py runserver 0.0.0.0:8000"
+    environment:
+      - APP_ENV=local
+      - SQL_HOST=db
+      - SQL_PORT=5432
+      - REDIS_HOST=redis
+    depends_on:
+      - db
+      - redis
+      - julia
+    ports:
+      - 8000:8000
+    volumes:
+      - .:/opt/reopt
+
+  julia:
+    container_name: julia_api
+    build:
+      context: julia_src/
+      dockerfile: Dockerfile.highs
+    environment:
+      - SOLVER=HiGHS      
+    command: julia --project=/opt/julia_src/highs highs/http.jl
+    ports:
+      - "8081:8081"
+    volumes:
+      - ./julia_src:/opt/julia_src

docker-compose.nginx.yml

@@ -78,6 +78,7 @@ services:
     container_name: julia-nginx
     build:
       context: julia_src/
+      dockerfile: Dockerfile.highs
     command: bash ./run_julia_servers.sh 8
     environment:
       - JULIA_NUM_THREADS=2

julia_src/Dockerfile

@@ -5,10 +5,6 @@ ARG NREL_ROOT_CERT_URL_ROOT=""
 RUN set -x && if [ -n "$NREL_ROOT_CERT_URL_ROOT" ]; then curl -fsSLk -o /usr/local/share/ca-certificates/nrel_root.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_root.pem" && curl -fsSLk -o /usr/local/share/ca-certificates/nrel_xca1.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_xca1.pem" &&  update-ca-certificates; fi
 ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 
-# Install gfortran for /ghpghx
-RUN apt-get update && apt-get install -y \
-    gfortran
-
 # Install Julia packages
 ENV JULIA_NUM_THREADS=2
 WORKDIR /opt/julia_src/

julia_src/Dockerfile.highs

@@ -0,0 +1,17 @@
+FROM julia:1.8.5
+
+# Install NREL root certs for machines running on NREL's network.
+ARG NREL_ROOT_CERT_URL_ROOT=""
+RUN set -x && if [ -n "$NREL_ROOT_CERT_URL_ROOT" ]; then curl -fsSLk -o /usr/local/share/ca-certificates/nrel_root.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_root.pem" && curl -fsSLk -o /usr/local/share/ca-certificates/nrel_xca1.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_xca1.pem" &&  update-ca-certificates; fi
+ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+
+# Install Julia packages
+ENV JULIA_NUM_THREADS=2
+
+WORKDIR /opt/julia_src
+COPY . .
+RUN julia --project=/opt/julia_src/highs -e 'import Pkg; Pkg.instantiate();'
+RUN julia --project=/opt/julia_src/highs highs/precompile.jl
+EXPOSE 8081
+
+CMD ["bash"]

julia_src/TODO.txt

@@ -0,0 +1,3 @@
+Make separate functions which all/some http.jl uses for any solver
+Added HiGHS and Cbc to the same Julia env setup with http.jl, in julia_src/highs env
+Add input to Settings input for choice of solver between HiGHS and Cbc