From 9728d8722d564b6077e6cddf6ebd1c0e191a79a4 Mon Sep 17 00:00:00 2001 From: Ryan McGuire Date: Thu, 26 Jan 2023 06:12:40 -0500 Subject: [PATCH] CLI script (#20) * cli script * `d.rymcg.tech create` script * CLI script README * `make readme` * `make readme` * `make readme` * d.rymcg.tech BASH completions * cli README * cli tab completion * README --- DIGITALOCEAN.md | 8 +- MAKEFILE_OPS.md | 5 + Makefile | 18 ++ README.md | 181 +++++++++++-- _scripts/Makefile.projects | 1 + _scripts/Makefile.projects-custom-build | 1 + ...efile.projects-custom-build-custom-install | 1 + _scripts/Makefile.projects-custom-install | 1 + _scripts/Makefile.projects-external | 11 + _scripts/Makefile.projects-no-open | 1 + _scripts/Makefile.readme | 4 + _scripts/create | 62 +++++ _scripts/d.rymcg.tech | 237 ++++++++++++++++++ _scripts/funcs.sh | 7 +- _scripts/user/README.md | 3 + _scripts/user/d.rymcg.tech | 1 + _templates/README.md | 2 + _templates/bare/.env-dist | 20 ++ _templates/bare/.gitignore | 6 + _templates/bare/Makefile | 13 + _templates/bare/README.md | 62 +++++ _templates/bare/docker-compose.yaml | 95 +++++++ _templates/bare/whoami/Dockerfile | 10 + 23 files changed, 718 insertions(+), 32 deletions(-) create mode 100644 _scripts/Makefile.projects-external create mode 100644 _scripts/Makefile.readme create mode 100755 _scripts/create create mode 100755 _scripts/d.rymcg.tech create mode 100644 _scripts/user/README.md create mode 120000 _scripts/user/d.rymcg.tech create mode 100644 _templates/README.md create mode 100644 _templates/bare/.env-dist create mode 100644 _templates/bare/.gitignore create mode 100644 _templates/bare/Makefile create mode 100644 _templates/bare/README.md create mode 100644 _templates/bare/docker-compose.yaml create mode 100644 _templates/bare/whoami/Dockerfile diff --git a/DIGITALOCEAN.md b/DIGITALOCEAN.md index 27ab1d74..05a16e33 100644 --- a/DIGITALOCEAN.md +++ b/DIGITALOCEAN.md @@ -167,10 +167,10 @@ Setup the docker context to tunnel through your ssh connection (this lets your workstation docker client control the remote docker server): ``` -# From your workstation (replace ssh.d.example.com with your own docker server): -DOCKER_SERVER=ssh.d.example.com -docker context create ${DOCKER_SERVER} --docker "host=ssh://${DOCKER_SERVER}" -docker context use ${DOCKER_SERVER} +# From your workstation (replace d.example.com with your own docker server): +DOMAIN=d.example.com +docker context create ${DOMAIN} --docker "host=ssh://ssh.${DOMAIN}" +docker context use ${DOMAIN} ``` List all of your docker contexts (your current context is denoted with an diff --git a/MAKEFILE_OPS.md b/MAKEFILE_OPS.md index d1fc58c1..57563fb9 100644 --- a/MAKEFILE_OPS.md +++ b/MAKEFILE_OPS.md @@ -541,6 +541,7 @@ same: $ cd ~/git/vendor/enigmacurry/d.rymcg.tech/whoami $ make help Makefile help for /home/ryan/git/vendor/enigmacurry/d.rymcg.tech/whoami: +make readme - Open the README.md file in your web browser make install - (re)builds images and (re)starts services (only if changed) make uninstall - Remove service containers, leaving the volumes intact make reinstall - Remove service containers, and re-install (volumes left intact). @@ -731,3 +732,7 @@ $ make clean instance=foo `make clean-all` will remove all of the environment files and saved passwords.json for all docker contexts. + +### make readme + +`make readme` will open the project README in your web browser. diff --git a/Makefile b/Makefile index 202b04c2..9b758dba 100644 --- a/Makefile +++ b/Makefile @@ -91,3 +91,21 @@ userns-remap-off: .PHONY: userns-remap-check # Check current Docker User Namespace Remap setting userns-remap-check: @${BIN}/userns-remap check + +.PHONY: readme # Open the README.md in your web browser +readme: + xdg-open "https://github.com/EnigmaCurry/d.rymcg.tech/tree/master#readme" + +.PHONY: install-cli +install-cli: + @echo "## Add this to the bottom of your ~/.bashrc or ~/.profile ::" + @echo "" + @echo "## d.rymcg.tech" + @echo "export PATH=\"$(realpath ${ROOT_DIR})/_scripts/user:\$${PATH}\"" + @echo "## optional TAB completion:" + @echo "eval \$$(d.rymcg.tech completion bash)" + @echo "complete -F __d.rymcg.tech_completions d.rymcg.tech" + @echo "## If you make an alias to the d.rymcg.tech (eg. 'dry')," + @echo "## then you can make completion support for the alias too:" + @echo "#complete -F __d.rymcg.tech_completions dry" + @echo "" diff --git a/README.md b/README.md index 28717b31..0b92b5a2 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@ # d.rymcg.tech -This is a collection of docker-compose projects consisting of +This is a collection of Docker Compose projects consisting of [Traefik](https://doc.traefik.io/traefik/) as a TLS HTTP/TCP reverse -proxy and other various applications and services behind this proxy. -Each project is in its own sub-directory containing its own -`docker-compose.yaml` and `.env` file (as well as `.env-dist` sample -file). This structure allows you to pick and choose which services you -wish to enable. You may also integrate your own external -docker-compose projects into this framework. +proxy and other various self-hosted applications and services behind +this proxy. Each project is in its own sub-directory containing its +own `docker-compose.yaml` and `.env-dist` sample config file. This +structure allows you to pick and choose which services you wish to +enable. You may also integrate your own external Docker Compose +projects into this framework. -Each project also has a `Makefile` to simplify configuration, -installation, and maintainance tasks. The setup for any sub-project is -as easy as running: +Each project has a `Makefile` to simplify configuration, installation, +and maintainance tasks. The setup for any sub-project is as easy as +running: * `make config` and interactively answering some questions to generate the `.env` file automatically. @@ -19,8 +19,8 @@ as easy as running: * `make open` to automatically open your web browser to the newly deployed application URL. -Under the covers, setup is pure `docker-compose`, with *all* -configuration derived from the `.env` file. +Under the covers, setup is pure `docker compose`, with *all* +configuration derived from your customized `.env` file. # Contents @@ -74,7 +74,10 @@ state is managed as part of the container/volume lifecycle. [Install Docker Server](https://docs.docker.com/engine/install/#server) on your own -public internet server or cloud host. +public internet server or cloud host. You may also install to a +private server behind a firewall (but in this case be sure to setup +the Traefik ACME DNS Challenge, because the default TLS challenge +requires an open port 443 public to the internet). See [SECURITY.md](SECURITY.md) for a list of security concerns when choosing a hosting provider. @@ -323,6 +326,12 @@ Host ssh.d.example.com `*.d.example.com` or an explicit `A` record having been created for this hostname.) +Note: if you use a workstation that goes to sleep, or loses network +connectivity, you may find that your shared+multiplexed SSH +connections will sometimes become zombies and stop communication. Get +used to running `killall ssh` before trying to restablish the +connection. + ### Set remote Docker context On your local workstation, create a new [Docker @@ -372,9 +381,14 @@ Use the same command again to switch to any other context.) ``` git clone https://github.com/EnigmaCurry/d.rymcg.tech.git \ ${HOME}/git/vendor/enigmacurry/d.rymcg.tech + cd ${HOME}/git/vendor/enigmacurry/d.rymcg.tech ``` +You may clone to any path you like, but the path suggested above is a +vendor neutral way of organizing third party repositories, with the +intention of making the same path work on all machines. + ## Main configuration Run the configuration wizard, and answer the questions: @@ -467,15 +481,18 @@ Bespoke things: ## Command line interaction -As alluded to earlier, this project offers two ways to control Docker: +As alluded to earlier, this project offers multiple ways to control +Docker: 1. Editing `.env` files by hand, and running `docker compose` commands yourself. 2. Running `make` targets that edit the `.env` files automatically and runs `docker compose` for you (this is the author's preferred method). + 3. Running the `d.rymcg.tech` CLI script, which runs the `make` + targets from any working directory. -Both of these methods are compatible, and they both get you to the +All of these methods are compatible, and they will all get you to the same place. The Makefiles offer a more streamlined approach with a configuration wizard and sensible defaults. Most of the sub-project README files reflect the `make` command style for config. Editing the @@ -518,7 +535,7 @@ directory you are in. default values from `.env-dist` (and based upon your `ROOT_DOMAIN` specified earlier). You can accept the suggested default values, or use the backspace key and edit the value, to fill in your own - answers. + answers. * The suffix of the .env filename, `_default`, refers to the [instance](#creating-multiple-instances-of-a-service) of the service (each instance has a different name, with `_default` being @@ -561,6 +578,106 @@ the `.env` files too). For a more in depth guide on using the Makefiles, see [MAKEFILE_OPS.md](MAKEFILE_OPS.md) +### Using the `d.rymcg.tech` CLI script (optional) + +By default, both `make` and `docker compose` expect you to change your +working directory to use them (however, you *can* work around this +using `make -C` and `docker compose -f`). There is a third option to +use the eponymous [`d.rymcg.tech` script](_scripts/d.rymcg.tech) +included in this repository. In addition to letting you run any +project `make` target from any working directory, this script also +offers a convenient way to create [external +projects](#integrating-external-projects) from a skeleton template. + +To install the script, you need to add it to your `PATH` shell +variable, and optionally enable the BASH completion support: + +``` +## Add this at the bottom of your ~/.bashrc config: + +## d.rymcg.tech +export PATH="${HOME}/git/vendor/enigmacurry/d.rymcg.tech/_scripts/user:${PATH}" +## optional TAB completion: +eval $(d.rymcg.tech completion bash) +complete -F __d.rymcg.tech_completions d.rymcg.tech + +## You might want to use a more convenient alias, (eg. 'dry'), +#alias dry="d.rymcg.tech" +## You can make completion support work for the alias too: +#complete -F __d.rymcg.tech_completions dry +``` + +Once installed, run `d.rymcg.tech` to see the command help text. + +``` +## Main d.rymcg.tech sub-commands: +cd Enter a sub-shell and go to the ROOT_DIR directory +create Create a new external project +make Run a make command for the given d.rymcg.tech project name + +## Documentation sub-commands: +help Show this help screen +list List available d.rymcg.tech projects + (not including external projects) +readme [PROJECT] Open the README.md for the given project name +readme Open the main d.rymcg.tech README.md in your browser +readme raspberry_pi Open the RASPBERRY_PI.md documentation +readme makefile_ops Open the MAKEFILE_OPS.md documentation +readme security Open the SECURITY.md documentation +readme digitalocean Open the DIGITALOCEAN.md documentation +readme license Open the LICENSE.txt software license +``` + +You can use this script to run the make targets for any of the bundled +projects, usable from any working directory: + + * `d.rymcg.tech list` (retrieve list of all available projects) + * `d.rymcg.tech make -- status` (view status of all installed + projects) + * `d.rymcg.tech make traefik config` (run the Traefik `make config` target) + * `d.rymcg.tech make traefik install` (run the Traefik `make install` target) + * `d.rymcg.tech make whoami logs` (run the whoami `make logs` target) + * `d.rymcg.tech make piwigo logs SERVICE=db` (you can also add any + variable assignments, just like with `make`) + +`d.rymcg.tech make [PROJECT_NAME] ...` is a simple wrapper for `make +-C ~/git/vendor/enigmacurry/d.rymcg.tech/${PROJECT_NAME} ...` (the +script will detect the correct path that you cloned to) so that you +can run all of the same things as outlined in +[MAKEFILE_OPS.md](MAKEFILE_OPS.md), but from any directory. The +special project placeholder value `-` (any number of consecutive +dashes) indicates to use the [root Makefile](Makefile) rather than any +particular project Makefile. + +You can get into the root d.rymcg.tech directory quickly, from +anywhere: + +``` +## This enters a subshell and changes the working directory to the d.rymcg.tech root: +## (You can also specify a second argument to specify the sub-directory.) +d.rymcg.tech cd +``` + +Press `Ctrl-D` to exit the sub-shell and jump back to wherever you +came from. + +From any working directory, you can create a new, barebones, [external +project](#integrating-external-projects): + +``` +# This creates a new project directory in your current working directory: +# It will ask you to enter the name of the project and choose the template. +# Optional 2nd and 3rd args will skip the asking: PROJECT_NAME TEMPLATE_NAME +d.rymcg.tech create +``` + +Open the README for any project in your web browser (omit the second +arg to open the root README): + +``` +d.rymcg.tech readme traefik +``` + ## Creating multiple instances of a service By default, each project supports deploying a single instance per @@ -764,17 +881,29 @@ Enter the name of the backup file, and all of the `.env` and You can integrate your own docker-compose projects that exist in external git repositories, and have them use the d.rymcg.tech -framework: +framework. + +The easiest method of creating an external project, is by setting up +the [`d.rymcg.tech` +script](https://github.com/EnigmaCurry/d.rymcg.tech/tree/cli-script#using-the-drymcgtech-cli-script-optional), +then run: + +``` +## Run this from any directory: +d.rymcg.tech create +``` + +To do this same thing manually, here are the steps: - * Clone d.rymcg.tech and set it up (Install Traefik, and whoami, make - sure that works first). * Create a new project directory, or clone your existing project, to - any other directory. (It does not need to be a sub-directory of + any directory. (It does not need to be a sub-directory of `d.rymcg.tech`, but it can be). * In your own project repository directory, create the files for - `docker-compose.yaml`, `.env-dist`, and `Makefile`. As an example, - you can use any of the d.rymcg.tech sub-projects, like - [whoami](whoami). + `docker-compose.yaml`, `Makefile`, `.env-dist`, `.gitignore`and + `README.md`. As an example, you can use any of the d.rymcg.tech + sub-projects, like [whoami](whoami), or take a look at the [bare + template](_templates/bare) that `d.rymcg.tech create` uses (the + template requires the use of `envsubst`). Create the `Makefile` in your own separate repository so that it includes the main d.rymcg.tech `Makefile.projects` file from @@ -785,7 +914,7 @@ elsewhere: # ROOT_DIR can be a relative or absolute path to the d.rymcg.tech directory: ROOT_DIR = ${HOME}/git/vendor/enigmacurry/d.rymcg.tech -include ${ROOT_DIR}/_scripts/Makefile.projects +include ${ROOT_DIR}/_scripts/Makefile.projects-external .PHONY: config-hook # Configure .env file config-hook: @@ -795,7 +924,9 @@ config-hook: A minimal `Makefile`, like the one above, should include a `config-hook` target that reconfigures your `.env` file based upon the -example variables given in `.env-dist`. +example variables given in `.env-dist`. This is what the user will +have to answer qusetions for when running `make config` for your +project. Now in your own project directory, you can use all the regular `make` commands that d.rymcg.tech provides: diff --git a/_scripts/Makefile.projects b/_scripts/Makefile.projects index d516c395..f28bb750 100644 --- a/_scripts/Makefile.projects +++ b/_scripts/Makefile.projects @@ -7,4 +7,5 @@ include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.open include ${ROOT_DIR}/_scripts/Makefile.reconfigure +include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-custom-build b/_scripts/Makefile.projects-custom-build index 54403eaa..c58350a7 100644 --- a/_scripts/Makefile.projects-custom-build +++ b/_scripts/Makefile.projects-custom-build @@ -7,6 +7,7 @@ include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.open include ${ROOT_DIR}/_scripts/Makefile.reconfigure +include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-custom-build-custom-install b/_scripts/Makefile.projects-custom-build-custom-install index 2f8aca90..c287a372 100644 --- a/_scripts/Makefile.projects-custom-build-custom-install +++ b/_scripts/Makefile.projects-custom-build-custom-install @@ -6,6 +6,7 @@ include ${ROOT_DIR}/_scripts/Makefile.clean include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.open include ${ROOT_DIR}/_scripts/Makefile.reconfigure +include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-custom-install b/_scripts/Makefile.projects-custom-install index de97a1a3..e7084a75 100644 --- a/_scripts/Makefile.projects-custom-install +++ b/_scripts/Makefile.projects-custom-install @@ -6,4 +6,5 @@ include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.clean include ${ROOT_DIR}/_scripts/Makefile.open include ${ROOT_DIR}/_scripts/Makefile.reconfigure +include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-external b/_scripts/Makefile.projects-external new file mode 100644 index 00000000..f28bb750 --- /dev/null +++ b/_scripts/Makefile.projects-external @@ -0,0 +1,11 @@ +include ${ROOT_DIR}/_scripts/Makefile.globals +include ${ROOT_DIR}/_scripts/Makefile.help +include ${ROOT_DIR}/_scripts/Makefile.build +include ${ROOT_DIR}/_scripts/Makefile.install +include ${ROOT_DIR}/_scripts/Makefile.clean +include ${ROOT_DIR}/_scripts/Makefile.override +include ${ROOT_DIR}/_scripts/Makefile.lifecycle +include ${ROOT_DIR}/_scripts/Makefile.open +include ${ROOT_DIR}/_scripts/Makefile.reconfigure +include ${ROOT_DIR}/_scripts/Makefile.readme +include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.projects-no-open b/_scripts/Makefile.projects-no-open index a43577b1..781dd06b 100644 --- a/_scripts/Makefile.projects-no-open +++ b/_scripts/Makefile.projects-no-open @@ -6,4 +6,5 @@ include ${ROOT_DIR}/_scripts/Makefile.lifecycle include ${ROOT_DIR}/_scripts/Makefile.override include ${ROOT_DIR}/_scripts/Makefile.clean include ${ROOT_DIR}/_scripts/Makefile.reconfigure +include ${ROOT_DIR}/_scripts/Makefile.readme include ${ROOT_DIR}/.env_$(shell ${BIN}/docker_context) diff --git a/_scripts/Makefile.readme b/_scripts/Makefile.readme new file mode 100644 index 00000000..c242b19f --- /dev/null +++ b/_scripts/Makefile.readme @@ -0,0 +1,4 @@ +.PHONY: readme # Open the README.md in your web browser +readme: +# @URL="https://github.com/EnigmaCurry/d.rymcg.tech/tree/master/$$(pwd | grep -Po \"$$(realpath ${ROOT_DIR})\K.*\")"; set -x; xdg-open "$${URL}" + @PROJECT_DIR=$$(realpath $$(pwd) | grep -Po "$$(realpath ${ROOT_DIR})\K.*"); URL="https://github.com/EnigmaCurry/d.rymcg.tech/tree/master$${PROJECT_DIR}#readme"; set -x; xdg-open "$${URL}" diff --git a/_scripts/create b/_scripts/create new file mode 100755 index 00000000..b8cdd458 --- /dev/null +++ b/_scripts/create @@ -0,0 +1,62 @@ +#!/bin/bash + +## BIN is the _scripts directory inside of d.rymcg.tech +BIN=$(dirname $(realpath ${BASH_SOURCE})) +## ROOT_DIR is the root path of the d.rymcg.tech project +ROOT_DIR=$(dirname ${BIN}) + +source ${BIN}/funcs.sh +set -eo pipefail + +echo "ROOT_DIR=${ROOT_DIR}" + +if [[ $# != 3 ]]; then + fault "Usage: create PROJECT_NAME TEMPLATE_NAME OUTPUT_DIR" +fi + +PROJECT_NAME="$1" +TEMPLATE_NAME="$2" +OUTPUT_DIR="$3" + +if [[ -e "${OUTPUT_DIR}" ]]; then + if [[ -d "${OUTPUT_DIR}" ]]; then + fault "The directory already exists: ${OUTPUT_DIR}" + fi + fault "There is an existing file with the same name: ${OUTPUT_DIR}" +fi + +mkdir -p "${OUTPUT_DIR}" + +TEMPLATE_DIR="${ROOT_DIR}/_templates/${TEMPLATE_NAME}" +TEMPLATE_FILES=("docker-compose.yaml" "Makefile" ".env-dist" ".gitignore" "README.md") + +if [[ ! -d "${TEMPLATE_DIR}" ]]; then + fault "The template directory is missing: ${TEMPLATE_DIR}" +fi +for template in ${TEMPLATE_FILES[@]}; do + if [[ ! -f "${TEMPLATE_DIR}/${template}" ]]; then + fault "The template is missing ${template} : ${TEMPLATE_DIR}" + fi +done + +TEMPLATE_VARS='${CREATE_TEMPLATE_PROJECT_NAME},${CREATE_TEMPLATE_PROJECT_URL_NAME},${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE},${CREATE_TEMPLATE_ROOT_DIR}' + +export CREATE_TEMPLATE_ROOT_DIR="${ROOT_DIR}" +export CREATE_TEMPLATE_PROJECT_NAME="$(echo ${PROJECT_NAME} | tr '[:upper:]' '[:lower:]' | tr ' ' '_' | tr '-' '_')" +export CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE="$(echo ${CREATE_TEMPLATE_PROJECT_NAME} | tr '[:lower:]' '[:upper:]')" +export CREATE_TEMPLATE_PROJECT_URL_NAME="$(echo ${CREATE_TEMPLATE_PROJECT_NAME} | tr '_' '-')" + +## TODO: use ytt to template YAML instead of envsubst +## HOWEVER, this is blocked on https://github.com/carvel-dev/ytt/issues/63 + +for filename in ${TEMPLATE_FILES[@]}; do + echo "# Rendering ${OUTPUT_DIR}/${filename}" + cat "${TEMPLATE_DIR}/${filename}" | envsubst "${TEMPLATE_VARS}" > "${OUTPUT_DIR}/${filename}" +done + +echo "# Copying other files to ${OUTPUT_DIR} ..." +cp -v "${TEMPLATE_DIR}/README.md" "${OUTPUT_DIR}/README.md" +find ${TEMPLATE_DIR} -maxdepth 1 -type d -printf "%P\n" | xargs -iXX cp -av "${TEMPLATE_DIR}/XX" "${OUTPUT_DIR}/XX" + +echo "" +echo "# Finished creating new project directory: ${OUTPUT_DIR}" diff --git a/_scripts/d.rymcg.tech b/_scripts/d.rymcg.tech new file mode 100755 index 00000000..abebcd87 --- /dev/null +++ b/_scripts/d.rymcg.tech @@ -0,0 +1,237 @@ +#!/bin/bash + +## The eponymous CLI script for the project d.rymcg.tech + +## This script should be symlinked to a directory on your PATH. You +## may add the `user` subdirectory to your path, which contains a +## symlink prepared for you. DO NOT add the whole _scripts directory +## to your PATH, but only the _scripts/user subdirectory! DO NOT move +## this script, only make symlinks to it! +## +## For example (choose one or the other): +## +## # Option 1 - This assumes ~/bin is already in your PATH: +## ln -s ~/git/vendor/enigmacurry/d.rymcg.tech/_scripts/d.rymcg.tech ~/bin +## +## # Option 2 - You would add this to the botom of your ~/.bashrc or ~/.profile +## export PATH=~/git/vendor/enigmacurry/d.rymcg.tech/_scripts/user:${PATH} + +## BIN is the _scripts directory inside of d.rymcg.tech +BIN=$(dirname $(realpath ${BASH_SOURCE})) +## ROOT_DIR is the root path of the d.rymcg.tech project +ROOT_DIR=$(dirname ${BIN}) + +source ${BIN}/funcs.sh + +__help() { + echo "Found ROOT_DIR=${ROOT_DIR}" + echo "" + echo "## Main d.rymcg.tech sub-commands:" + ( + echo -e "cd\tEnter a sub-shell and go to the ROOT_DIR directory" + echo -e "create\tCreate a new external project" + echo -e "make\tRun a make command for the given d.rymcg.tech project name" + ) | expand -t 15 + echo "" + echo "## Documentation sub-commands:" + ( + echo -e "help\tShow this help screen" + echo -e "list\tList available d.rymcg.tech projects" + echo -e "\t(not including external projects)" + echo -e "readme [PROJECT]\tOpen the README.md for the given project name" + echo -e "readme\tOpen the main d.rymcg.tech README.md in your browser" + echo -e "readme raspberry_pi\tOpen the RASPBERRY_PI.md documentation" + echo -e "readme makefile_ops\tOpen the MAKEFILE_OPS.md documentation" + echo -e "readme security\tOpen the SECURITY.md documentation" + echo -e "readme digitalocean\tOpen the DIGITALOCEAN.md documentation" + echo -e "readme license\tOpen the LICENSE.txt software license" + ) | expand -t 22 + echo "" +} + +__create() { + echo "This script will create a new docker-compose project and integrate d.rymcg.tech Makefiles" + if [[ $# -gt 0 ]]; then + PROJECT_NAME="$1" + else + ask_no_blank "Enter a name for the new docker-compose project" PROJECT_NAME + echo "" + fi + if [[ $# -gt 1 ]]; then + TEMPLATE_NAME="$2" + else + echo "Available templates:" + find "${ROOT_DIR}/_templates" -maxdepth 1 -type d -printf "%P\n" | column + echo "" + ask_no_blank "Enter the template name to use" TEMPLATE_NAME bare + echo "" + fi + ${BIN}/create "${PROJECT_NAME}" "${TEMPLATE_NAME}" "./${PROJECT_NAME}" +} + +__change-directory() { + echo "Entering sub-shell. Press Ctrl-D to pop back to the parent shell." + if [[ $# -gt 0 ]]; then + /bin/bash --rcfile <(echo "cd ${ROOT_DIR}/$1") + else + /bin/bash --rcfile <(echo "cd ${ROOT_DIR}") + fi + echo "Exited sub-shell." +} + +__make() { + if [[ $# -gt 0 ]]; then + PROJECT_NAME="$1"; shift + pattern="^-+$" + if [[ "${PROJECT_NAME}" =~ $pattern ]]; then + PROJECT_DIR=${ROOT_DIR} + else + PROJECT_DIR=${ROOT_DIR}/${PROJECT_NAME} + fi + test -d "${PROJECT_DIR}" || fault "Project directory does not exist: ${PROJECT_DIR}" + make -C "${PROJECT_DIR}" "$@" + else + __list_projects + error "Missing project name argument. Choose one from the above." + error "To invoke the root Makefile use '-' as the name" + fi + DIR_NAME="$1"; shift +} + +__list_projects() { + ( + if [[ "$1" == "--raw" ]]; then + find "${ROOT_DIR}" -maxdepth 1 -type d -printf "%P\n" | grep -v "^_" | grep -v "^\." | sort -u | xargs -iXX /bin/bash -c "test -f ${ROOT_DIR}/XX/Makefile && echo XX" + else + echo "List of available d.rymcg.tech projects (not including external projects):" + echo "" + set +e + find "${ROOT_DIR}" -maxdepth 1 -type d -printf "%P\n" | grep -v "^_" | grep -v "^\." | sort -u | xargs -iXX /bin/bash -c "test -f ${ROOT_DIR}/XX/Makefile && echo XX" | column + echo "" + fi + ) +} + +__readme() { + declare -A DOCS_LIST=([README]= [DIGITALOCEAN]= [MAKEFILE_OPS]= [RASPBERRY_PI]= [SECURITY]= [LICENSE]=) + if [[ $# -gt 0 ]]; then + NAME="${1}"; shift + NAME_UPPERCASE="$(echo "${NAME}" | tr '[:lower:]' '[:upper:]')" + if [[ "${NAME_UPPERCASE}" == "LICENSE" ]]; then + (set +x; xdg-open "https://github.com/EnigmaCurry/d.rymcg.tech/blob/master/LICENSE.txt") + elif [[ -v DOCS_LIST["${NAME_UPPERCASE}"] ]]; then + (set +x; xdg-open "https://github.com/EnigmaCurry/d.rymcg.tech/blob/master/${NAME_UPPERCASE}.md#readme") + else + __make "${NAME}" readme "$@" + fi + else + __make -- readme + fi +} + +__info() { + if [[ "$1" == "ROOT_DIR" ]]; then + echo "${ROOT_DIR}" + fi +} + +__d.rymcg.tech_completions() { + ### BASH completion + ## dev links: + ### https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion-Builtins.html#Programmable-Completion-Builtins + ### https://iridakos.com/programming/2018/03/01/bash-programmable-completion-tutorial + ### https://github.com/git/git/blob/master/contrib/completion/git-completion.bash + ## COMP_WORDS: an array of all the words typed + ## COMP_CWORD: an index of the COMP_WORDS array pointing to the word the current cursor is at + ## COMP_LINE: the current command line + ## COMPREPLY is an array variable used to store the output completions + ROOT_PROJECT_PLACEHOLDER="^-+$" + if [[ ${COMP_CWORD} == 1 ]]; then + ## Complete the main command: + ## d.rymcg.tech ... + COMMANDS="help create cd make list readme" + COMPREPLY=($(compgen -W "${COMMANDS}" "${COMP_WORDS[1]}")) + elif [[ ${COMP_CWORD} == 2 ]]; then + ALL_PROJECTS=$(d.rymcg.tech list --raw) + ## Dispatch completion for the sub-command's first argument: + ## d.rymcg.tech make ... + case ${COMP_WORDS[1]} in + create) + COMPREPLY=("#" "Type the project name");; + cd) + COMPREPLY=($(compgen -W "${ALL_PROJECTS}" "${COMP_WORDS[2]}"));; + readme) + COMPREPLY=($(compgen -W "README DIGITALOCEAN SECURITY RASPBERRY_PI MAKEFILE_OPS LICENSE readme digitalocean security raspberry_pi makefile_ops license ${ALL_PROJECTS}" "${COMP_WORDS[2]}"));; + make) + COMPREPLY=($(compgen -W "-- ${ALL_PROJECTS}" "${COMP_WORDS[2]}"));; + esac + elif [[ ${COMP_CWORD} == 3 ]]; then + ## Dispatch completion for the sub-commands second argument: + ## d.rymcg.tech make traefik ... + if [[ "${COMP_WORDS[1]}" == "make" ]]; then + ROOT_DIR=$(d.rymcg.tech info ROOT_DIR) + PROJECT=${COMP_WORDS[2]} + PROJECT_DIR="${ROOT_DIR}/${PROJECT}" + if [[ "${PROJECT}" =~ ${ROOT_PROJECT_PLACEHOLDER} ]]; then + PROJECT_DIR="${ROOT_DIR}" + fi + ## Thanks Chris Down https://unix.stackexchange.com/a/230050 + MAKE_TARGETS="$(make -C "${PROJECT_DIR}" -qp 2>/dev/null | awk -F':' '/^[a-zA-Z0-9][^$#\/\t=]*:([^=]|$)/ {split($1,A,/ /);for(i in A)print A[i]}' | sort -u)" + COMPREPLY=($(compgen -W "${MAKE_TARGETS}" "${COMP_WORDS[3]}")) + elif [[ "${COMP_WORDS[1]}" == "create" ]]; then + COMPREPLY=("bare") + fi + fi +} + +__completion() { + # Print the completion script to be evaluated + if [[ $# -lt 1 ]]; then + echo "## To enable BASH shell completion support for d.rymcg.tech," + echo "## add the following lines into your ~/.bashrc ::" + echo "" + echo "eval \$(d.rymcg.tech completion bash)" + echo "complete -F __d.rymcg.tech_completions d.rymcg.tech" + echo "" + echo "## If your script is named something different," + echo "## rename that last argument from d.rymcg.tech to whatever you called it." + else + if [[ "${1}" == "bash" ]]; then + declare -f __d.rymcg.tech_completions + else + fault "Sorry, this script only support BASH shell completion" + fi + fi +} + +main() { + set -eo pipefail + if [[ $# -gt 0 ]]; then + test -f "${ROOT_DIR}/Makefile" || fault "d.rymcg.tech ROOT_DIR directory not found: ${ROOT_DIR}" + COMMAND=$1; shift + case ${COMMAND} in + help) + __help "$@";; + create) + __create "$@";; + cd) + __change-directory "$@";; + make) + __make "$@";; + list) + __list_projects "$@";; + readme) + __readme "$@";; + completion) + __completion "$@";; + info) + __info "$@";; + *) + fault "Invalid command" + esac + else + __help + fi +} + +main "$@" diff --git a/_scripts/funcs.sh b/_scripts/funcs.sh index 426511db..4371f99b 100644 --- a/_scripts/funcs.sh +++ b/_scripts/funcs.sh @@ -2,13 +2,14 @@ BIN=$(dirname ${BASH_SOURCE}) -fault(){ test -n "$1" && echo $1; echo "Exiting." >/dev/stderr ; exit 1; } +error(){ echo "Error: $@" >/dev/stderr; } +fault(){ test -n "$1" && error $1; echo "Exiting." >/dev/stderr; exit 1; } check_var(){ __missing=false __vars="$@" for __var in ${__vars}; do if [[ -z "${!__var}" ]]; then - echo "${__var} variable is missing." >/dev/stderr + error "${__var} variable is missing." __missing=true fi done @@ -112,7 +113,7 @@ docker_exec() { ytt() { set -e - docker build -t localhost/ytt -f- . >/dev/null <<'EOF' + docker image inspect localhost/ytt >/dev/null || docker build -t localhost/ytt -f- . >/dev/null <<'EOF' FROM debian:stable-slim as ytt ARG YTT_VERSION=v0.43.0 RUN apt-get update && apt-get install -y wget && wget "https://github.com/vmware-tanzu/carvel-ytt/releases/download/${YTT_VERSION}/ytt-linux-$(dpkg --print-architecture)" -O ytt && install ytt /usr/local/bin/ytt diff --git a/_scripts/user/README.md b/_scripts/user/README.md new file mode 100644 index 00000000..cb796591 --- /dev/null +++ b/_scripts/user/README.md @@ -0,0 +1,3 @@ +This directory exists so that you can add it to your PATH. It contains +symlinks to a limited set of scripts that you will then be able to run +from any directory. diff --git a/_scripts/user/d.rymcg.tech b/_scripts/user/d.rymcg.tech new file mode 120000 index 00000000..8996f4b1 --- /dev/null +++ b/_scripts/user/d.rymcg.tech @@ -0,0 +1 @@ +../d.rymcg.tech \ No newline at end of file diff --git a/_templates/README.md b/_templates/README.md new file mode 100644 index 00000000..9eaae2ce --- /dev/null +++ b/_templates/README.md @@ -0,0 +1,2 @@ +This directory contains project templates to be used with the +`d.rymcg.tech create` script. diff --git a/_templates/bare/.env-dist b/_templates/bare/.env-dist new file mode 100644 index 00000000..fe0c3faf --- /dev/null +++ b/_templates/bare/.env-dist @@ -0,0 +1,20 @@ +#The domain name for the main service: +${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_TRAEFIK_HOST=${CREATE_TEMPLATE_PROJECT_NAME}.example.com + +#The custom name shown in all HTTP responses +${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_NAME= + +#Filter access by IP address source range (CIDR): +##Disallow all access: +#${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_IP_SOURCERANGE="0.0.0.0/32" +##Allow all access: +${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_IP_SOURCERANGE="0.0.0.0/0" + +##You can customize the UID and GID that the container runs as, this +##is passed to the Dockerfile as a build ARG. +${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_UID=54321 +${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_GID=54321 + +##If deploying multiple instances, this is the identifier for each: +##`make config` will set this to `default` if not specified +${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE= diff --git a/_templates/bare/.gitignore b/_templates/bare/.gitignore new file mode 100644 index 00000000..cd71200c --- /dev/null +++ b/_templates/bare/.gitignore @@ -0,0 +1,6 @@ +.env +.env_* +*.tgz +*.tgz.gpg +passwords.json +docker-compose.override*.yaml diff --git a/_templates/bare/Makefile b/_templates/bare/Makefile new file mode 100644 index 00000000..63fdf118 --- /dev/null +++ b/_templates/bare/Makefile @@ -0,0 +1,13 @@ +## ROOT_DIR is the full path to the git clone of d.rymcg.tech: +ROOT_DIR = ${CREATE_TEMPLATE_ROOT_DIR} +## This Makefile inherits abilities from d.rymcg.tech Makefiles +include ${ROOT_DIR}/_scripts/Makefile.projects-external +include ${ROOT_DIR}/_scripts/Makefile.instance + +.PHONY: config-hook +config-hook: + ## config-hook is for the custom configuration the user must enter when running `make config`: + # reconfigure_ask will ask the user to enter the value for a variable, with the given prompt, with the given default value, and save it to the env file: + @${BIN}/reconfigure_ask ${ENV_FILE} ${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_TRAEFIK_HOST "Enter the ${CREATE_TEMPLATE_PROJECT_NAME} domain name" ${CREATE_TEMPLATE_PROJECT_URL_NAME}${INSTANCE_URL_SUFFIX}.${ROOT_DOMAIN} + # reconfigure will directly set a value in the env file + @${BIN}/reconfigure ${ENV_FILE} ${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE=$${instance:-default} diff --git a/_templates/bare/README.md b/_templates/bare/README.md new file mode 100644 index 00000000..ed2fb463 --- /dev/null +++ b/_templates/bare/README.md @@ -0,0 +1,62 @@ +# Example docker compose project + +This example wraps [traefik/whoami](https://github.com/traefik/whoami) +as an example of a containerized service, built from Dockerfile. + +## Setup + +This example project integrates with +[d.rymcg.tech](https://github.com/EnigmaCurry/d.rymcg.tech#readme). +Before proceeding, you must first clone and setup `d.rymcg.tech` on +your workstation. + +This project is an example of a so-called +["external"](https://github.com/enigmacurry/d.rymcg.tech#integrating-external-projects) +project to `d.rymcg.tech`, as it does not live in the same source tree +as `d.rymcg.tech`, but makes a link to inherit its Makefiles and to +gain its superpowers. + +## Configure + +Once +[d.rymcg.tech](https://github.com/EnigmaCurry/d.rymcg.tech#readme) has +been installed, you can come back to this directory. + +Run: + +``` +make config +``` + +This will create the `.env_{DOCKER_CONTEXT}` configuration file for +your service. + +## Install + +Run: + +``` +make install +``` + +## Open in your web browser + +Run: + +``` +make open +``` + +## Customizing + +The [whoami](whoami) sub-directory contains the source for the image +that this service builds. It is just an example, and can be replaced +with your own Dockerfile for your own service. If you change the name +of the directory, be sure to update the `docker-compose.yaml` +`build.context` as well. + +If you don't need to build an image, but instead you want to pull an +image from a docker registry, remove the `build` directive and replace +with `image: your_image_name`, and then you can delete the +[whoami](whoami) directory (no Dockerfile is needed if you are just +pulling a prebuilt image). diff --git a/_templates/bare/docker-compose.yaml b/_templates/bare/docker-compose.yaml new file mode 100644 index 00000000..ad3c4df8 --- /dev/null +++ b/_templates/bare/docker-compose.yaml @@ -0,0 +1,95 @@ +version: "3.9" +services: + ## The name of your service: + ${CREATE_TEMPLATE_PROJECT_NAME}: + ## Build the docker image from the local Dockerfile: + build: + context: whoami + args: + UID: ${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_UID} + GID: ${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_GID} + USERNAME: ${CREATE_TEMPLATE_PROJECT_NAME} + ## The docker image can be pulled instead of built if specified: + #image: "traefik/whoami" + + ## user is optional, it overrides the USER of the Dockerfile + user: ${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_UID}:${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_GID} + ## Setting a restart policy is important, especially in case of system reboot: + restart: unless-stopped + labels: + ## These labels configure Traefik to proxy for this service: + ## Traefik ignores all containers by default, you must explicitly set traefik.enable=true: + - "traefik.enable=true" + + ## ${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_TRAEFIK_HOST is the domain name of the service. + ## ${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE is the name of the instance if running more than one ('default' by default). + ## The service, router, and middle ware are all namespaced with the ${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE name. + + ## This is the router rule that matches the domain name with the service: + - "traefik.http.routers.${CREATE_TEMPLATE_PROJECT_NAME}-${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE:-default}.rule=Host(`${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_TRAEFIK_HOST}`)" + ## loadbalancer.server.port is only needed if the Dockerfile for the image did not use EXPOSE: + - "traefik.http.services.${CREATE_TEMPLATE_PROJECT_NAME}-${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE:-default}.loadbalancer.server.port=8000" + ## This exposes the service on the Traefik entrypoint named websecure (on port :443) + - "traefik.http.routers.${CREATE_TEMPLATE_PROJECT_NAME}-${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE:-default}.entrypoints=websecure" + ## This creates a middleware to filter traffic by client source IP address range: + - "traefik.http.middlewares.${CREATE_TEMPLATE_PROJECT_NAME}-${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE:-default}-whitelist.ipwhitelist.sourcerange=${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_IP_SOURCERANGE}" + ## This enables the middleware to operate on this service: + - "traefik.http.routers.${CREATE_TEMPLATE_PROJECT_NAME}-${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE:-default}.middlewares=${CREATE_TEMPLATE_PROJECT_NAME}-${${CREATE_TEMPLATE_PROJECT_NAME_UPPERCASE}_INSTANCE:-default}-whitelist" + + ## Security options + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + sysctls: + - net.ipv4.ip_unprivileged_port_start=1024 + cap_add: + ## These are all of the available Linux system capabilities that containers can access + ## https://man.archlinux.org/man/capabilities.7 + ## + ## These are the capabilities that Docker would add by default, + ## except not, because we use cap_drop: ALL + ## https://github.com/moby/moby/blob/2b9de2e24a9877843c1998366ac97959c12b45aa/oci/caps/defaults.go#L6-L19 + ## You can try commenting these out and test if you app works + ## without them: + - CHOWN + - DAC_OVERRIDE + - FSETID + - FOWNER + - MKNOD + - NET_RAW + - SETGID + - SETUID + - SETFCAP + - SETPCAP + - NET_BIND_SERVICE + - SYS_CHROOT + - KILL + - AUDIT_WRITE + ## The rest of these are PRIVILEGED capabilities: + ## Unless you know you need one of these, you should delete + ## them, or leave them commented out, they are for reference only! + ## - AUDIT_CONTROL + ## - AUDIT_READ + ## - BLOCK_SUSPEND + ## - DAC_READ_SEARCH + ## - IPC_LOCK + ## - IPC_OWNER + ## - LEASE + ## - LINUX_IMMUTABLE + ## - MAC_ADMIN + ## - MAC_OVERRIDE + ## - NET_ADMIN + ## - NET_BROADCAST + ## - SYS_ADMIN + ## - SYS_BOOT + ## - SYSLOG + ## - SYS_MODULE + ## - SYS_NICE + ## - SYS_PACCT + ## - SYS_PTRACE + ## - SYS_RAWIO + ## - SYS_RESOURCE + ## - SYS_TIME + ## - SYS_TTY_CONFIG + ## - WAKE_ALARM diff --git a/_templates/bare/whoami/Dockerfile b/_templates/bare/whoami/Dockerfile new file mode 100644 index 00000000..f47c2884 --- /dev/null +++ b/_templates/bare/whoami/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine:3 +ARG UID +ARG GID +ARG USERNAME +RUN addgroup -g ${GID} ${USERNAME} && \ + adduser -D -u ${UID} -G ${USERNAME} -h /home/${USERNAME} ${USERNAME} +COPY --from=traefik/whoami:latest /whoami /usr/local/bin/whoami + +EXPOSE 8000 +CMD /usr/local/bin/whoami --port 8000