From ff655035ca600cf8760368e85d8ad6053c74e9ee Mon Sep 17 00:00:00 2001 From: Tracey Jaquith Date: Sat, 30 Mar 2024 02:36:55 -0700 Subject: [PATCH 1/3] mac HinD basically working; just a nomad run cpu constraint remains --- README.md | 4 ++++ bin/bootstrap.sh | 16 ++++++++++--- install.sh | 60 ++++++++++++++++++++++++++++++++---------------- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index d2ef964..d47f6d7 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,10 @@ net.core.wmem_max=134217728' |sudo tee /etc/sysctl.d/90-tcp-memory.conf # Miscellaneous - client IP addresses will be in request header 'X-Forwarded-For' (per `caddy`) +- pop inside the HinD container: +``` +sudo podman exec -it hind zsh +``` - get list of `consul` services: ``` wget -qO- 'localhost:8500/v1/catalog/services?tags=1' | jq . diff --git a/bin/bootstrap.sh b/bin/bootstrap.sh index 64f8b09..1b6518d 100755 --- a/bin/bootstrap.sh +++ b/bin/bootstrap.sh @@ -43,9 +43,19 @@ else pkill -SIGQUIT nomad sleep 5 - consul keygen |tr -d '^\n' | podman secret create HIND_C - - nomad operator gossip keyring generate |tr -d '^\n' | podman secret create HIND_N - - grep -F 'Secret ID' /tmp/bootstrap |cut -f2- -d= |tr -d ' ' | podman secret create NOMAD_TOKEN - + + if [ "$HOST_UNAME" = Darwin ]; then + apt-get install -yqq fuse-overlayfs + echo; echo + echo -n 'echo -n ' + grep -F 'Secret ID' /tmp/bootstrap |cut -f2- -d= |tr -d ' \n' + echo ' | podman secret create NOMAD_TOKEN -' + echo; echo + else + consul keygen |tr -d '^\n' | podman secret create HIND_C - + nomad operator gossip keyring generate |tr -d '^\n' | podman secret create HIND_N - + grep -F 'Secret ID' /tmp/bootstrap |cut -f2- -d= |tr -d ' ' | podman secret create NOMAD_TOKEN - + fi rm -f /tmp/* diff --git a/install.sh b/install.sh index 4117e93..87aa714 100755 --- a/install.sh +++ b/install.sh @@ -21,6 +21,24 @@ export FQDN=$(hostname -f) podman -v > /dev/null || echo 'please install the podman package first' podman -v > /dev/null || exit 1 +if [ "$HOST_UNAME" = Darwin ]; then + ARGS_INIT='' + ARGS_RUN='-p 8000:80 -p 4000:443 --secret NOMAD_TOKEN,type=env' + # previously had also added above: '-v /sys/fs/cgroup:/sys/fs/cgroup:rw' + + PV=$HOME/pv + # export FQDN=http://localhost + export FQDN=http://$FQDN +else + # In rare case this is a symlink, ensure we mount the proper source. + # NOTE: we map in /var/lib/containers here so `podman secret create` inside the `podman run` + # container will effect us, the outside/VM. + VLC=$(realpath /var/lib/containers 2>/dev/null || echo /var/lib/containers) + SOCK=$(podman info |grep -F podman.sock |rev |cut -f1 -d ' ' |rev) + ARGS_INIT="--net=host -v ${VLC}:/var/lib/containers" + ARGS_RUN="--net=host --cgroupns=host -v /opt/nomad/data/alloc:/opt/nomad/data/alloc -v $SOCK:$SOCK" + PV=/pv +fi ( # clear any prior run (likely fail?) @@ -40,21 +58,16 @@ podman -v > /dev/null || exit 1 # bootstrap the general image to a customized image for your cluster, leveraging podman secrets IMG=ghcr.io/internetarchive/hind:main - # In rare case this is a symlink, ensure we mount the proper source. - # NOTE: we map in /var/lib/containers here so `podman secret create` inside the `podman run` - # container will effect us, the outside/VM. - VLC=$(realpath /var/lib/containers 2>/dev/null || echo /var/lib/containers) - set -x # We need to shared these 2 directories "inside" the running `hind` container, and "outside" on # the VM itself. We want to persist HTTPS cert files, and any `data/alloc` directories setup # on the "inside" (eg: `nomad run`) need to be available to nomad jobs running on the outside/VM. - mkdir -p -m777 /pv/CERTS + mkdir -p -m777 $PV/CERTS mkdir -p -m777 /opt/nomad/data/alloc podman pull $QUIET $IMG > $OUT - podman run --net=host --privileged --cgroupns=host \ - -v ${VLC}:/var/lib/containers \ + podman run --privileged --cgroupns=host \ + $ARGS_INIT \ -e FQDN -e HOST_UNAME \ --name hind-init $QUIET "$@" $IMG podman commit $QUIET hind-init localhost/hind > $OUT 2>&1 @@ -62,23 +75,29 @@ podman -v > /dev/null || exit 1 ) +if [ "$HOST_UNAME" = Darwin ]; then + echo VEhJUy1HRVRTLVJFUExBQ0VELUlULURPRVMtUklMTFk= |tr -d '\n' | podman secret create HIND_C - + echo VEhJUy1HRVRTLVJFUExBQ0VELUlULURPRVMtUklMTFk= |tr -d '\n' | podman secret create HIND_N - + + set +x + echo ' + +COPY/PASTE THE NOMAD_TOKEN secret create ABOVE NOW + +' + read cont +fi + + # Now run the new docker image in the background. # NOTE: we switch `-v /var/lib/containers` to volume mounting the `podman.sock`, since we want HinD # container to `podman run` nomad jobs on the outside/VM, not inside itself ( - SOCK=$(podman info |grep -F podman.sock |rev |cut -f1 -d ' ' |rev) - if [ "$HOST_UNAME" = Darwin ]; then - ARGS='-p 6000:4646 -p 8000:80 -p 4000:443 -v /sys/fs/cgroup:/sys/fs/cgroup:rw' - else - ARGS='--net=host' - fi set -x - podman run --privileged --cgroupns=host \ - $ARGS \ - -v $SOCK:$SOCK \ - -v /opt/nomad/data/alloc:/opt/nomad/data/alloc \ - -v /pv:/pv \ + podman run --privileged \ + $ARGS_RUN \ + -v $PV:/pv \ --secret HIND_C,type=env --secret HIND_N,type=env \ --restart=always --name hind -d $QUIET "$@" localhost/hind > $OUT 2>&1 ) @@ -93,6 +112,7 @@ SUCCESS! exit 0 fi +set +x echo ' Congratulations! @@ -106,7 +126,7 @@ anywhere you have downloaded a `nomad` binary): ' if [ $HOST_UNAME = Darwin ]; then - echo "export NOMAD_ADDR=http://$FQDN:6000" + echo "export NOMAD_ADDR=$FQDN:8000" else echo "export NOMAD_ADDR=https://$FQDN" fi From ee44179a0a24637a0be01f3af90fa0f9a3d29204 Mon Sep 17 00:00:00 2001 From: Tracey Jaquith Date: Sat, 30 Mar 2024 02:56:14 -0700 Subject: [PATCH 2/3] bootstrap fix. mac setup simplify --- Dockerfile | 4 ++++ install.sh | 10 +++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index fa1a909..81f004b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,10 @@ ENV REVERSE_PROXY "" ENV ON_DEMAND_TLS_ASK "" ENV HOST_UNAME Linux +# replaced at runtime: +ENV HIND_N "VEhJUy1HRVRTLVJFUExBQ0VELUlULURPRVMtUklMTFk=" +ENV HIND_C "VEhJUy1HRVRTLVJFUExBQ0VELUlULURPRVMtUklMTFk=" + ENV DEBIAN_FRONTEND noninteractive ENV TZ Etc/UTC ENV TERM xterm diff --git a/install.sh b/install.sh index 87aa714..45db21f 100755 --- a/install.sh +++ b/install.sh @@ -35,8 +35,8 @@ else # container will effect us, the outside/VM. VLC=$(realpath /var/lib/containers 2>/dev/null || echo /var/lib/containers) SOCK=$(podman info |grep -F podman.sock |rev |cut -f1 -d ' ' |rev) - ARGS_INIT="--net=host -v ${VLC}:/var/lib/containers" - ARGS_RUN="--net=host --cgroupns=host -v /opt/nomad/data/alloc:/opt/nomad/data/alloc -v $SOCK:$SOCK" + ARGS_INIT="--net=host --cgroupns=host -v ${VLC}:/var/lib/containers" + ARGS_RUN="--net=host --cgroupns=host -v /opt/nomad/data/alloc:/opt/nomad/data/alloc -v $SOCK:$SOCK --secret HIND_C,type=env --secret HIND_N,type=env" PV=/pv fi @@ -66,7 +66,7 @@ fi mkdir -p -m777 /opt/nomad/data/alloc podman pull $QUIET $IMG > $OUT - podman run --privileged --cgroupns=host \ + podman run --privileged \ $ARGS_INIT \ -e FQDN -e HOST_UNAME \ --name hind-init $QUIET "$@" $IMG @@ -76,9 +76,6 @@ fi if [ "$HOST_UNAME" = Darwin ]; then - echo VEhJUy1HRVRTLVJFUExBQ0VELUlULURPRVMtUklMTFk= |tr -d '\n' | podman secret create HIND_C - - echo VEhJUy1HRVRTLVJFUExBQ0VELUlULURPRVMtUklMTFk= |tr -d '\n' | podman secret create HIND_N - - set +x echo ' @@ -98,7 +95,6 @@ fi podman run --privileged \ $ARGS_RUN \ -v $PV:/pv \ - --secret HIND_C,type=env --secret HIND_N,type=env \ --restart=always --name hind -d $QUIET "$@" localhost/hind > $OUT 2>&1 ) From b5ad54e8ee0d662ce6b82c6bd04b88459a05d292 Mon Sep 17 00:00:00 2001 From: Tracey Jaquith Date: Sat, 30 Mar 2024 03:00:57 -0700 Subject: [PATCH 3/3] finesse/style --- install.sh | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/install.sh b/install.sh index 45db21f..2961e36 100755 --- a/install.sh +++ b/install.sh @@ -22,22 +22,22 @@ podman -v > /dev/null || echo 'please install the podman package first' podman -v > /dev/null || exit 1 if [ "$HOST_UNAME" = Darwin ]; then + export FQDN=http://$FQDN + PV=$HOME/pv + ARGS_INIT='' ARGS_RUN='-p 8000:80 -p 4000:443 --secret NOMAD_TOKEN,type=env' # previously had also added above: '-v /sys/fs/cgroup:/sys/fs/cgroup:rw' - - PV=$HOME/pv - # export FQDN=http://localhost - export FQDN=http://$FQDN else # In rare case this is a symlink, ensure we mount the proper source. # NOTE: we map in /var/lib/containers here so `podman secret create` inside the `podman run` # container will effect us, the outside/VM. VLC=$(realpath /var/lib/containers 2>/dev/null || echo /var/lib/containers) SOCK=$(podman info |grep -F podman.sock |rev |cut -f1 -d ' ' |rev) + PV=/pv + ARGS_INIT="--net=host --cgroupns=host -v ${VLC}:/var/lib/containers" ARGS_RUN="--net=host --cgroupns=host -v /opt/nomad/data/alloc:/opt/nomad/data/alloc -v $SOCK:$SOCK --secret HIND_C,type=env --secret HIND_N,type=env" - PV=/pv fi ( @@ -66,10 +66,7 @@ fi mkdir -p -m777 /opt/nomad/data/alloc podman pull $QUIET $IMG > $OUT - podman run --privileged \ - $ARGS_INIT \ - -e FQDN -e HOST_UNAME \ - --name hind-init $QUIET "$@" $IMG + podman run --privileged $ARGS_INIT -e FQDN -e HOST_UNAME --name hind-init $QUIET "$@" $IMG podman commit $QUIET hind-init localhost/hind > $OUT 2>&1 podman rm -v hind-init > $OUT 2>&1 ) @@ -90,12 +87,9 @@ fi # NOTE: we switch `-v /var/lib/containers` to volume mounting the `podman.sock`, since we want HinD # container to `podman run` nomad jobs on the outside/VM, not inside itself ( - set -x - podman run --privileged \ - $ARGS_RUN \ - -v $PV:/pv \ - --restart=always --name hind -d $QUIET "$@" localhost/hind > $OUT 2>&1 + podman run --privileged $ARGS_RUN -v $PV:/pv --restart=always --name hind -d $QUIET "$@" localhost/hind \ + > $OUT 2>&1 )