diff --git a/.env.sample b/.env.sample index e9e826020..197ad0921 100644 --- a/.env.sample +++ b/.env.sample @@ -6,6 +6,7 @@ ##### automatically invalidate them and your course _will_ break. ##### + ##### ##### ##### Core Settings @@ -20,37 +21,10 @@ NAME=cs310 ## created inside it. ORG=CS310-2017Jan -## GitHub org identifier for the test organization (you probably do not want to change this) -ORGTEST=classytest - -## Course name for the test instance (you probably do not want to change this) -NAMETEST=classytest - ## The external name used for the Classy service (used by GitHub WebHooks) ## Must start with https:// and should not have a trailing slash PUBLICHOSTNAME=https://classy.cs.ubc.ca -## Set the logging verbosity: TRACE (default), INFO, WARN, ERROR, TEST, NONE -LOG_LEVEL=INFO - -##### -##### -##### Host config for portal/backend; no trailing slash -##### https://localhost is usually used for testing -##### -##### - - -## URL (no trailing slash) for Classy backend; different than HOSTNAME as this is the -## internal name (e.g., as Classy is addressed to other local services) -BACKEND_URL=https://localhost -BACKEND_PORT=3000 - -## Full path to fullchain.pem (Can be self-signed for localhost testing) -SSL_CERT_PATH=/DEVPATH/classy/packages/portal/backend/ssl/fullchain.pem -## Full path to privkey.pem (Can be self-signed for localhost testing) -SSL_KEY_PATH=/DEVPATH/classy/packages/portal/backend/ssl/privkey.pem - ##### ##### @@ -58,45 +32,25 @@ SSL_KEY_PATH=/DEVPATH/classy/packages/portal/backend/ssl/privkey.pem ##### ##### -## For testing, you can spin up a basic mongo instance (w/o authentication) using: -## `docker run -p 27017:27017 mongo` -## Specify the DB_URL as below to connect: -# DB_URL=mongodb://localhost:27017 - - ## To spin up a mongo instance with authentication, specify a username and password below. ## Notes: ## - you must specify the username and password twice (once for the MONGO_INITDB_ROOT_* and once in the DB_URL) ## - the username/password will only be applied on the **FIRST** launch of the db service (otherwise they have no effect) -## - when deploying with Docker Compose, replace _localhost_ with the value of CONTAINER_NAME_DATABASE (set below). +## - when deploying with Docker Compose, replace `localhost` with `db`. ## - the DB_URL must be URI encoded if it contains special characters +## For local testing, you can spin up a basic mongo instance (w/o authentication) using: `docker run -p 27017:27017 mongo` +## and setting DB_URL=mongodb://localhost:27017 MONGO_INITDB_ROOT_USERNAME=mongoadmin MONGO_INITDB_ROOT_PASSWORD=strongpasswd DB_URL=mongodb://mongoadmin:strongpasswd@localhost:27017/?authMechanism=DEFAULT + ##### ##### ##### GitHub Configuration ##### ##### -## GitHub API host (no trailing slash). This is because the API host is often different than the web host. -## For public github it will be: https://api.github.com -## For hosted github it will be: https://https://api.github.ugrad.cs.ubc.ca (or possibly https://github.ugrad.cs.ubc.ca/api/v3) -GH_API=https://api.github.com - -## GitHub Web root (no trailing slash) -## For public GitHub it will be https://github.com -GH_HOST=https://github.com - -## The name of the GitHub bot account the students will call -## You must have access to this account because it needs to be -## added to both the admin and staff teams so it can admin and -## comment on repos. Do not include the @ in the username. -## The bot needs to be added to your org with admin privileges -## e.g., for public GitHub here: https://github.com/orgs/ORGNAME/people -GH_BOT_USERNAME=ubcbot - ## A GitHub token so the bot can use the GitHub API without going ## through authentication. It is important that this token be well ## protected as without it you can lose programmatic access to student @@ -104,10 +58,8 @@ GH_BOT_USERNAME=ubcbot ## GH_BOT_TOKEN=token d4951x.... ## (yes the word token is required) ## If you want to use ubcbot, contact Reid Holmes for a token. - GH_BOT_TOKEN=token d4951x... - ## Before you can authenticate against GitHub you will need to create ## two OAuth applications on the org; e.g., for public GitHub you can ## do this here: https://github.com/organizations/ORGNAME/settings/applications @@ -119,22 +71,53 @@ GH_BOT_TOKEN=token d4951x... ## ## The Client ID and Client Secret for the OAuth profile (testing or prod) ## you intend to use should be included below. These _must_ be protected. - GH_CLIENT_ID=f42b49hut... GH_CLIENT_SECRET=1337secretTokenCharsHere... +## GitHub API host (no trailing slash). This is because the API host is often different than the web host. +## For public github it will be: https://api.github.com +## For hosted github it will be: https://https://api.github.ugrad.cs.ubc.ca (or possibly https://github.ugrad.cs.ubc.ca/api/v3) +GH_API=https://api.github.com + +## GitHub Web root (no trailing slash) +## For public GitHub it will be https://github.com +GH_HOST=https://github.com + +## The name of the GitHub bot account the students will call +## You must have access to this account because it needs to be +## added to both the admin and staff teams so it can admin and +## comment on repos. Do not include the @ in the username. +## The bot needs to be added to your org with admin privileges +## e.g., for public GitHub here: https://github.com/orgs/ORGNAME/people +GH_BOT_USERNAME=ubcbot + -##### ##### ##### ##### AutoTest Settings ##### ##### -##### + +## The uid for the (non-root) user that should run the containers (if following deploy instructions, should be the uid +## for the classy user). Also used by the AutoTest service to configure permissions on directories shared between autotest +## and the grading container. +UID=993 + +## The group id for the docker group on the host. Use `cut -d: -f3 < <(getent group docker)` to get the id. +## Used by containers that need to access the docker socket. +GID=989 + +## GitHub token with permission to clone the repository containing the Dockerfile for the grading container +GH_DOCKER_TOKEN=asb865... + +## Include a hostname to IP address mapping for outgoing requests from grading containers. +## This mapping is required since the grading container will not be able to make DNS requests. +## Format hostname:IP +HOSTS_ALLOW=classy.cs.ubc.ca:142.103.6.191 ## When using docker-compose, an entry is added to the hosts file for each ## dependent service. Thus, we just need to specify the service name in the URL. -AUTOTEST_URL=http://localhost +AUTOTEST_URL=http://autotest ## AutoTest instance port. AUTOTEST_PORT=11333 @@ -148,52 +131,49 @@ AUTOTEST_POSTBACK=false ## Where the AutoTest service should store persistent data (e.g. grade container execution logs) ## This path is on the HOST machine (and is the mount point for PERSIST_DIR inside the grade container) -HOST_DIR=./data/runs +HOST_DIR=/var/opt/classy/runs ## Where the AutoTest service should store persistent data (e.g. grade container execution logs) ## This path is INSIDE the container (and is bound to HOST_DIR on the host machine) -PERSIST_DIR=/DEVPATH/classy/packages/autotest/test/data +PERSIST_DIR=/output -## The uid for the (non-root) user that should run the containers (if following deploy instructions, should be the uid -## for the classy user). Also used by the AutoTest service to configure permissions on directories shared between autotest -## and the grading container. -UID=993 -## [SDMM/310 Only] Port that the geo-location service should listen on -## MUST BE SET TO 11316 (this is baked into the service's dockerfile) -GEO_PORT=11316 +##### +##### +##### Portal Settings +##### +##### -## [SDMM/310 Only] Port that the reference UI service should listen on. -## MUST BE SET TO 11315 (this is baked into the service's dockerfile) -UI_PORT=11315 +## URL (no trailing slash) for Classy backend; different than HOSTNAME as this is the +## internal name (e.g., as Classy is addressed to other local services) +## https://localhost is usually used for testing +BACKEND_URL=https://portal +BACKEND_PORT=3000 ##### -##### Deployment Only +##### +##### Miscellaneous Settings +##### ##### -# The Docker daemon socket. Specify a value here if not using the default unix:///var/run/docker.sock. -DOCKER_HOST_URL=tcp://$hostname:2376 - -## The name docker-compose will prefix to every container -COMPOSE_PROJECT_NAME=classy +## Full path to fullchain.pem (Can be self-signed for localhost testing) +SSL_CERT_PATH=/etc/ssl/fullchain.pem +## Full path to privkey.pem (Can be self-signed for localhost testing) +SSL_KEY_PATH=/etc/ssl/privkey.pem -## The location of the SSL certificate and private key. +## The location of the SSL certificate and private key on the host (if deployed) HOST_SSL_CERT_PATH=/opt/classy/ssl/fullchain.pem HOST_SSL_KEY_PATH=/opt/classy/ssl/privkey.pem -## GitHub token with read access to clone repositories in the org for the particular course offering -COURSE_GH_ORG_TOKEN=asb865... +## The name docker-compose will prefix to every container +COMPOSE_PROJECT_NAME=classy -## GitHub token with permission to clone the repository containing the Dockerfile for the grading container -GH_DOCKER_TOKEN=asb865... +## GitHub org identifier for the test organization (you probably do not want to change this) +ORGTEST=classytest -## [310/SDMM Only] A single hosts entry used to resolve the hostname of the server running geolocation. -## This is required since the grading container will not be able to make DNS requests. -## Format hostname:IP -HOSTS_ALLOW=classy.cs.ubc.ca:142.103.6.191 +## Course name for the test instance (you probably do not want to change this) +NAMETEST=classytest -## Specifies the mode under which the reference implementation should operate. -## Currently supports values 'd1' and 'd2'. -## Affects the reference UI and the grading container. -PLATFORM=d1 +## Set the logging verbosity: TRACE (default), INFO, WARN, ERROR, TEST, NONE +LOG_LEVEL=INFO diff --git a/docker-compose.yml b/docker-compose.yml index 8b2e55ef2..2ba318451 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ # - Services specified here can be extended (and additional services can be added) by creating additional # docker-compose.yml files. See https://docs.docker.com/compose/extends/#example-use-case. -# NOTE: Do not change the service names. They are used to refer to the service throughout the codebase in http requests. +# NOTE: Do not change the container names. They are used to refer to the service throughout the codebase in http requests. version: "3.5" @@ -25,19 +25,20 @@ services: build: context: ./ dockerfile: ./packages/autotest/Dockerfile + container_name: autotest depends_on: - db env_file: .env expose: - ${AUTOTEST_PORT} restart: always - user: "${UID}" + user: "${UID}:${GID}" volumes: - "${HOST_DIR}:${PERSIST_DIR}" - - "${HOST_SSL_CERT_PATH}:${SSL_CERT_PATH}" - - "${HOST_SSL_KEY_PATH}:${SSL_KEY_PATH}" + - "/var/run/docker.sock:/var/run/docker.sock" db: command: --quiet + container_name: db environment: - MONGO_INITDB_ROOT_USERNAME - MONGO_INITDB_ROOT_PASSWORD @@ -55,15 +56,13 @@ services: - GH_BOT_EMAIL context: ./ dockerfile: ./packages/portal/Dockerfile + container_name: portal depends_on: - db - autotest env_file: .env expose: - ${BACKEND_PORT} - # Hack for SDMM since the github webhook hits the port directly - ports: - - 5000:${BACKEND_PORT} restart: always user: "${UID}" volumes: @@ -78,6 +77,7 @@ services: - BACKEND_PORT context: ./ dockerfile: ./packages/proxy/Dockerfile + container_name: proxy depends_on: - portal ports: diff --git a/docs/deploy.md b/docs/deploy.md index fc47d467e..c8d474988 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -32,95 +32,61 @@ The following software should be installed on the host before attempting to depl **NOTE:** Make sure you logout and back in to see the new user and group. 2. Use `certbot` to get SSL certificates for the host from Let's Encrypt: - - ```bash - sudo certbot certonly --standalone -d $(hostname) - ``` - - Then create the following pre-renewal and deploy hooks: - ```bash - cat <<- EOF > /etc/letsencrypt/renewal-hooks/pre/stop-classy.sh - #!/bin/sh - set -e - - # Stop Classy so that port 80 and 443 can be used by certbot - cd /opt/classy - /usr/local/bin/docker-compose stop || true - EOF - - cat <<- EOF > /etc/letsencrypt/renewal-hooks/deploy/copy-certs.sh - #!/bin/sh - set -e - - # Copies the latest certificates to Classy - mkdir -p /opt/classy/ssl - \cp -Hf /etc/letsencrypt/live/$(hostname)/* /opt/classy/ssl/ - chown -R --reference=/opt/classy /opt/classy/ssl - chmod -R 0050 /opt/classy/ssl - EOF - - cat <<- EOF > /etc/letsencrypt/renewal-hooks/post/start-classy.sh - #!/bin/sh - set -e - - # Restart classy - cd /opt/classy - /usr/local/bin/docker-compose up --detach - EOF - - chmod +x /etc/letsencrypt/renewal-hooks/pre/stop-classy.sh - chmod +x /etc/letsencrypt/renewal-hooks/deploy/copy-certs.sh - chmod +x /etc/letsencrypt/renewal-hooks/post/start-classy.sh - ``` - **SECURITY WARNING:** Possession of the certificate is equivalent to having root access on the host since the Docker - daemon will be configured to accept TCP connections from clients presenting that certificate. - - **NOTE:** This Let's Encrypt certificate is used for all services requiring a certificate on the host. This includes - the Classy service and the Docker daemon. - -3. Configure the Docker daemon to allow HTTPS (TCP+TLS) access: - - ```bash - cat <<- EOF > /etc/docker/daemon.json - { - "tls": true, - "tlscert": "/etc/letsencrypt/live/$(hostname)/fullchain.pem", - "tlskey": "/etc/letsencrypt/live/$(hostname)/privkey.pem", - "hosts": ["unix:///var/run/docker.sock", "tcp://$(hostname):2376"] - } - EOF - systemctl restart docker - ``` - - To verify the configuration, check that following two versions of the `docker version` command complete successfully: - ```bash - # (1) Using the default socket (only accessible to root) - docker version - - # (2) Using TCP (accessible to ANY user with the appropriate certificate) - docker --host=tcp://$(hostname):2376 \ - --tlsverify=1 \ - --tlscacert=/etc/letsencrypt/live/$(hostname)/fullchain.pem \ - --tlscert=/etc/letsencrypt/live/$(hostname)/fullchain.pem \ - --tlskey=/etc/letsencrypt/live/$(hostname)/privkey.pem \ - version - ``` - - (For reference) Check that the docker client inside a container can access the daemon on host (Note: this basically - runs the same command as (2) above except by using env vars and mounting the certs into the default search location--see - https://docs.docker.com/engine/security/https/#secure-by-default): - ```bash - docker run --rm \ - --env DOCKER_HOST=tcp://$(hostname):2376 \ - --env DOCKER_TLS_VERIFY=1 \ - --volume $(readlink -f /etc/letsencrypt/live/$(hostname)/fullchain.pem):/root/.docker/ca.pem \ - --volume $(readlink -f /etc/letsencrypt/live/$(hostname)/fullchain.pem):/root/.docker/cert.pem \ - --volume $(readlink -f /etc/letsencrypt/live/$(hostname)/privkey.pem):/root/.docker/key.pem \ - docker version - ``` - -4. Add the firewall rules to block unwanted traffic (if using autotest). + 1. Create a certbot deploy hook that will run when new certificates are obtained: + ```bash + cat <<- EOF > /etc/letsencrypt/renewal-hooks/deploy/copy-certs.sh + #!/bin/sh + set -e + + # Copies the latest certificates to Classy + mkdir -p /opt/classy/ssl + \cp -Hf /etc/letsencrypt/live/$(hostname)/* /opt/classy/ssl/ + chown -R --reference=/opt/classy /opt/classy/ssl + chmod -R 0050 /opt/classy/ssl + EOF + + chmod +x /etc/letsencrypt/renewal-hooks/deploy/copy-certs.sh + ``` + + 2. Get the initial certificates: + ```bash + sudo certbot certonly --standalone -d $(hostname) --agree-tos -m user@example.com --no-eff-email -n + ``` + + Confirm that there are certificates in /etc/letsencrypt/live/$(hostname) (e.g. *.pem files). The deploy hook should + have copied the certificate files to /opt/classy/ssl. If not, manually run + `sudo /etc/letsencrypt/renewal-hooks/deploy/copy-certs.sh` (this only needs to be done once). + + 3. Configure pre- and post-renewal hooks to automatically start and stop Classy when it is time to renew: + ```bash + cat <<- EOF > /etc/letsencrypt/renewal-hooks/pre/stop-classy.sh + #!/bin/sh + set -e + + # Stop Classy so that port 80 and 443 can be used by certbot + cd /opt/classy + /usr/local/bin/docker-compose stop || true + EOF + + cat <<- EOF > /etc/letsencrypt/renewal-hooks/post/start-classy.sh + #!/bin/sh + set -e + + # Restart classy + cd /opt/classy + /usr/local/bin/docker-compose up --detach + EOF + + chmod +x /etc/letsencrypt/renewal-hooks/pre/stop-classy.sh + chmod +x /etc/letsencrypt/renewal-hooks/post/start-classy.sh + ``` + + Classy needs to be stopped so that port 80 isn't bound when certbot attempts to renew the certificates. It + would need to be restarted in all cases to mount the new certificates. Note: the deploy hook should also run on + successfully renewal copy the latest version of the certificates to `/opt/classy/ssl` before restarting Classy. + +3. Add the firewall rules to block unwanted traffic (if using autotest). ```bash # These two rules will block all traffic coming FROM the subnet (i.e. grading container) @@ -148,7 +114,7 @@ The following software should be installed on the host before attempting to depl added to grading container. - These rules **must always be applied** (i.e. they should persist across reboots). -5. Test the system. To ensure the firewall rules are working as expected we can run some simple commands from a container +4. Test the system. To ensure the firewall rules are working as expected we can run some simple commands from a container connected to the subnet. ```bash @@ -176,25 +142,26 @@ The following software should be installed on the host before attempting to depl ```bash git clone https://github.com/ubccpsc/classy.git ~/classy - sudo cp -r ~/classy /opt && rm -rf ~/classy - sudo chown classy:classy /opt/classy - sudo chmod g+rwx,o-rwx /opt/classy + sudo -i # stay as root for remainder of this Classy Configuration section + cp -r ~/classy /opt && rm -rf ~/classy + cp /opt/classy/.env.sample /opt/classy/.env + chown -Rh classy:classy /opt/classy + find /opt/classy -type d -exec chmod 770 {} \; + find /opt/classy -type f -exec chmod 660 {} \; - # Set GRADER_HOST_DIR to /var/opt/classy/runs - # Set database storage to /var/opt/classy/db - sudo mkdir /var/opt/classy - sudo mkdir /var/opt/classy/backups # for database backups - sudo mkdir /var/opt/classy/db # for database storage - sudo mkdir /var/opt/classy/runs # for grading container output - sudo chown -R classy:classy /var/opt/classy - sudo chmod -R g+rwx,o-rwx /var/opt/classy + mkdir /var/opt/classy + mkdir /var/opt/classy/backups # for database backups + mkdir /var/opt/classy/db # for database storage + mkdir /var/opt/classy/runs # for grading container output + chown -Rh classy:classy /var/opt/classy + find /var/opt/classy -type d -exec chmod 770 {} \; + find /var/opt/classy -type f -exec chmod 660 {} \; ``` 2. Configure the `.env` (more instructions inside this file) ```bash cd /opt/classy - cp .env.sample .env nano .env ``` @@ -225,7 +192,7 @@ The following software should be installed on the host before attempting to depl Note: you can also use the additional options for [mongodump](https://docs.mongodb.com/manual/reference/program/mongodump/) and [mongorestore](https://docs.mongodb.com/manual/reference/program/mongorestore/) described in the docs. -5. Archive old execution. AutoTest stores the output of each run on disk and, depending on the size of the output, can cause space issues. +5. Archive old executions. AutoTest stores the output of each run on disk and, depending on the size of the output, can cause space issues. You can apply the following cron job (as root) that will archive (and then remove) runs more than a week old. Adapt as needed: this will run every Wednesday at 0300 and archive runs older than 7 days (based on last modified time); all runs are stored together in a single compressed tarball called `runs-TIMESTAMP.tar.gz` under `/cs/portal-backup/cs310.ugrad.cs.ubc.ca/classy`. diff --git a/packages/autotest/src/server/RouteHandler.ts b/packages/autotest/src/server/RouteHandler.ts index ef24f0337..7288214f0 100644 --- a/packages/autotest/src/server/RouteHandler.ts +++ b/packages/autotest/src/server/RouteHandler.ts @@ -1,7 +1,6 @@ import * as Docker from "dockerode"; import * as fs from "fs-extra"; import * as restify from "restify"; -import {URL} from "url"; import Config, {ConfigKey} from "../../../common/Config"; import Log from "../../../common/Log"; import {CommitTarget} from "../../../common/types/ContainerTypes"; @@ -23,21 +22,8 @@ export default class RouteHandler { // Running tests; don't need to connect to the Docker daemon this.docker = null; } else { - const dockerHost = Config.getInstance().getProp(ConfigKey.dockerHost) || ""; - if (dockerHost.startsWith("https") || dockerHost.startsWith("http") || dockerHost.startsWith("tcp")) { - const dockerUrl = new URL(dockerHost); - RouteHandler.docker = new Docker({ - host: dockerUrl.hostname, - port: dockerUrl.port, - ca: fs.readFileSync("/etc/ssl/certs/ca-certificates.crt"), - cert: fs.readFileSync(Config.getInstance().getProp(ConfigKey.sslCertPath)), - key: fs.readFileSync(Config.getInstance().getProp(ConfigKey.sslKeyPath)), - version: "v1.30" - }); - } else { - Log.info("RouteHandler::getDocker() - Defaulting to Docker socket."); - RouteHandler.docker = new Docker(); - } + // Connect to the Docker socket using defaults + RouteHandler.docker = new Docker(); } } diff --git a/packages/common/Config.ts b/packages/common/Config.ts index f458659de..5c312ff6b 100644 --- a/packages/common/Config.ts +++ b/packages/common/Config.ts @@ -54,7 +54,6 @@ export enum ConfigKey { hostDir = "hostDir", dockerUid = "dockerUid", hostsAllow = "hostsAllow", - dockerHost = "dockerHost", timeout = "timeout", botName = "botName", postback = "postback", @@ -90,7 +89,6 @@ export default class Config { persistDir: process.env.PERSIST_DIR, dockerUid: process.env.UID, hostsAllow: process.env.HOSTS_ALLOW, - dockerHost: process.env.DOCKER_HOST_URL, timeout: Number(process.env.GRADER_TIMEOUT), botName: process.env.GH_BOT_USERNAME,