diff --git a/chart/templates/_fileserver.tpl b/chart/templates/_fileserver.tpl index 693679f..d188aab 100644 --- a/chart/templates/_fileserver.tpl +++ b/chart/templates/_fileserver.tpl @@ -140,6 +140,20 @@ mountPath: {{ printf "%s/%s/%s" $globalValues.spacefxDirectories.base $volumeNam {{- end }} {{- end }} +{{- define "spacefx.fileserver.clientapp.volumemount.mountpath" }} +{{- $serviceValues := .serviceValues }} +{{- $globalValues := .globalValues }} +{{- $volumeName := .volumeName }} +{{- $shareName := printf "%s-%s" $serviceValues.appName $volumeName }} +{{- $mountPath := printf "%s/%s/%s" $globalValues.spacefxDirectories.base $volumeName $serviceValues.appName }} +{{- if and (eq $serviceValues.appName "hostsvc-link") (eq $volumeName "allxfer") }} +{{ printf "%s/%s" $globalValues.spacefxDirectories.base $volumeName }} +{{- else }} +{{ printf "%s/%s/%s" $globalValues.spacefxDirectories.base $volumeName $serviceValues.appName }} +{{- end }} +{{- end }} + + {{- define "spacefx.fileserver.clientapp.volume" }} {{- $serviceValues := .serviceValues }} {{- $volumeName := .volumeName }} diff --git a/chart/templates/_initcontainers.tpl b/chart/templates/_initcontainers.tpl new file mode 100644 index 0000000..955561e --- /dev/null +++ b/chart/templates/_initcontainers.tpl @@ -0,0 +1,25 @@ +{{- define "spacefx.initcontainers.setperms" }} +{{- $serviceValues := .serviceValues }} +{{- $globalValues := .globalValues }} +{{- $mountPaths := "" }} +initContainers: + - name: init-permissions + image: docker.io/rancher/mirrored-library-busybox:1.36.1 + volumeMounts: +{{- range $volumeKey, $volumeName := $globalValues.xferVolumes }} +{{- $fileServerVolumeMount := printf "%s" (include "spacefx.fileserver.clientapp.volumemount" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | nindent 2 | trim) }} +{{- $fileServerVolumeMountPath := printf "%s" (include "spacefx.fileserver.clientapp.volumemount.mountpath" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | trim) }} +{{- $mountPaths = printf "%s %s" $mountPaths $fileServerVolumeMountPath }} +{{- printf "- %s" $fileServerVolumeMount | nindent 5 }} +{{- end }} +{{- if $serviceValues.xferVolumes }} +{{- range $volumeKey, $volumeName := $serviceValues.xferVolumes }} +{{- $fileServerVolumeMount := printf "%s" (include "spacefx.fileserver.clientapp.volumemount" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | nindent 2 | trim) }} +{{- $fileServerVolumeMountPath := printf "%s" (include "spacefx.fileserver.clientapp.volumemount.mountpath" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | trim) }} +{{- $mountPaths = printf "%s %s" $mountPaths $fileServerVolumeMountPath }} +{{- printf "- %s" $fileServerVolumeMount | nindent 5 }} +{{- end }} +{{- end }} + command: ["sh", "-c", "chown -R {{ $serviceValues.runAsUserId }}:{{ $serviceValues.runAsUserId }} {{ $mountPaths }}"] +{{- end }} + diff --git a/chart/templates/_service.tpl b/chart/templates/_service.tpl index c973024..95abe89 100644 --- a/chart/templates/_service.tpl +++ b/chart/templates/_service.tpl @@ -44,9 +44,18 @@ spec: {{- if eq $serviceValues.appName "platform-mts" }} dnsPolicy: ClusterFirstWithHostNet hostNetwork: true + {{- end }} + {{- if and ($globalValues.security.forceNonRoot) (ne $serviceValues.appName "hostsvc-link") }} +{{- include "spacefx.initcontainers.setperms" (dict "globalValues" $globalValues "serviceValues" $serviceValues) | indent 6 }} {{- end }} containers: - name: {{ $serviceValues.appName | quote }} + {{- if and ($globalValues.security.forceNonRoot) (ne $serviceValues.appName "hostsvc-link") }} + securityContext: + runAsUser: {{ $serviceValues.runAsUserId }} + runAsGroup: {{ $serviceValues.runAsUserId }} + runAsNonRoot: true + {{- end }} ports: - name: app-port containerPort: 50051 @@ -70,12 +79,6 @@ spec: timeoutSeconds: {{ $globalValues.probes.startup.timeoutSeconds }} {{- end }} {{- end }} - {{- if $serviceValues.securityContext }} - securityContext: - {{- range $index, $securityContext := $serviceValues.securityContext }} - {{ $securityContext.name }}: {{ $securityContext.value }} - {{- end }} - {{- end }} {{- if $serviceValues.debugShim }} image: "{{ $serviceValues.repository }}:latest" imagePullPolicy: "Never" diff --git a/chart/values.yaml b/chart/values.yaml index 6fc66b6..9fcae0c 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -113,6 +113,7 @@ global: security: payloadAppNetworkRestrictionsEnabled: true topicRestrictionEnabled: true + forceNonRoot: true subcharts: dapr: enabled: false @@ -134,6 +135,7 @@ services: workingDirectory: /workspaces/exampleapp repository: kaniko-project/executor tag: v1.20.1-slim + runAsUserId: 701 registry: appName: coresvc-registry containerCommand: @@ -163,6 +165,7 @@ services: cpu: limit: 1000m request: 10m + runAsUserId: 702 switchboard: appName: coresvc-switchboard serviceNamespace: coresvc @@ -192,6 +195,7 @@ services: enabled: true hasBase: false enabled: false + runAsUserId: 703 platform: mts: appConfig: @@ -215,6 +219,7 @@ services: hasBase: false enabled: false workingDir: /workspaces/platform-mts + runAsUserId: 704 deployment: appName: platform-deployment appHealthChecks: true @@ -269,6 +274,7 @@ services: hasBase: false enabled: false workingDir: /workspace/platform-deployment + runAsUserId: 705 vth: appConfig: - name: enableRoutingToMTS @@ -291,6 +297,7 @@ services: hasBase: false enabled: false workingDir: /workspaces/vth + runAsUserId: 706 host: sensor: appConfig: @@ -315,6 +322,7 @@ services: enabled: true hasBase: false enabled: false + runAsUserId: 707 workingDir: /workspaces/hostsvc-sensor link: appConfig: @@ -371,6 +379,7 @@ services: enabled: true hasBase: false enabled: false + runAsUserId: 708 workingDir: /workspaces/hostsvc-logging position: appName: hostsvc-position @@ -390,6 +399,7 @@ services: enabled: true hasBase: false enabled: false + runAsUserId: 709 workingDir: /workspaces/hostsvc-position payloadapp: payloadappTemplate: diff --git a/scripts/deploy_spacefx.sh b/scripts/deploy_spacefx.sh index 7286e25..4e96725 100755 --- a/scripts/deploy_spacefx.sh +++ b/scripts/deploy_spacefx.sh @@ -86,11 +86,15 @@ function deploy_spacefx_service_group(){ info_log "Scanning '${service_group}' spacefx services for deploying..." run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group} | to_entries[] | select(.value.${enabled_filter} == true) | .key' -r" spacefx_services + run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.global.security.forceNonRoot' -r" spacefx_forceNonRoot + run_a_script "kubectl --kubeconfig ${KUBECONFIG} get deployments -A -o json" services_deployed_cache --disable_log for service in $spacefx_services; do run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group}.${service}.appName' -r" spacefx_service_appName run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group}.${service}.serviceNamespace' -r" spacefx_service_serviceNamespace + run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group}.${service}.runAsUserId' -r" spacefx_service_userid + run_a_script "jq -r '.items[] | select(.metadata.name == \"${spacefx_service_appName}\" and (.metadata.namespace == \"${spacefx_service_serviceNamespace}\")) | true' <<< \${services_deployed_cache}" service_deployed @@ -99,6 +103,78 @@ function deploy_spacefx_service_group(){ continue fi + # Create users and groups if the service needs one + if [[ "${spacefx_forceNonRoot}" == "true" ]] && [[ "${spacefx_service_userid}" != "null" ]]; then + info_log "...checking if group '${spacefx_service_appName}' (GID: '${spacefx_service_userid}') exists..." + + # This will return the group_name for the groupID. i.e. "702" + run_a_script "getent group ${spacefx_service_userid}" preexisting_groupid_by_id --ignore_error + preexisting_groupid_by_id="${preexisting_groupid_by_id%%:*}" + + # This will check if a group exists and gets its ID + run_a_script "getent group ${spacefx_service_appName}" preexisting_groupid_by_name --ignore_error + preexisting_groupid_by_name="${preexisting_groupid_by_name%%:*}" + + + if [[ -n "${preexisting_groupid_by_name}" ]] && [[ "${preexisting_groupid_by_id}" == "${preexisting_groupid_by_name}" ]]; then + info_log "...group '${spacefx_service_appName}' (GID: '${preexisting_groupid_by_id}') already exists. Nothing to do" + else + if [[ -n "${preexisting_groupid_by_id}" ]]; then + info "...GID '${spacefx_service_userid}' already in use, but isn't assigned to '${spacefx_service_appName}'. Attempting to delete..." + run_a_script "getent group ${spacefx_service_userid}" group_to_del + group_to_del="${group_to_del%%:*}" + + run_a_script "groupdel -f ${group_to_del}" + info "...successfully deleted previous group '${group_to_del}' (GID: '${username_to_del}')" + fi + + if [[ -n "${preexisting_groupid_by_name}" ]]; then + info "...Group '${spacefx_service_appName}' already in use, but isn't assigned to '${spacefx_service_userid}'. Attempting to delete..." + run_a_script "groupdel -f ${spacefx_service_appName}" + info "...successfully deleted previous group '${spacefx_service_appName}'" + fi + + info_log "...creating group '${spacefx_service_appName}' with GID '${spacefx_service_userid}'..." + run_a_script "groupadd -r -g ${spacefx_service_userid} ${spacefx_service_appName}" --no_log + info_log "...successfully created group '${spacefx_service_appName}' (GID: '${spacefx_service_userid}')." + fi + + + info_log "...checking if user '${spacefx_service_appName}' (UID: '${spacefx_service_userid}') exists..." + + # This will return a user id if the userid exists. i.e. "701" + run_a_script "id -u ${spacefx_service_userid}" preexisting_userid --ignore_error + + # This will return the user id for the username. i.e. "702" + run_a_script "id -u ${spacefx_service_appName}" preexisting_userid_for_username --ignore_error + + if [[ -n "${preexisting_userid_for_username}" ]] && [[ "${preexisting_userid}" == "${preexisting_userid_for_username}" ]]; then + info_log "...user '${spacefx_service_appName}' (UID: '${spacefx_service_userid}') already exists. Nothing to do" + else + if [[ -n "${preexisting_userid}" ]]; then + info "...UID '${spacefx_service_userid}' already in use, but isn't assigned to '${spacefx_service_appName}'. Attempting to delete..." + run_a_script "getent passwd ${spacefx_service_userid}" username_to_del + username_to_del="${username_to_del%%:*}" + run_a_script "userdel -f ${username_to_del}" + info "...successfully deleted previous user '${username_to_del}' (UID: '${username_to_del}')" + fi + + if [[ -n "${preexisting_userid_for_username}" ]]; then + info "...Username '${spacefx_service_appName}' already in use, but isn't assigned to '${spacefx_service_userid}'. Attempting to delete..." + run_a_script "userdel -f ${spacefx_service_appName}" + info "...successfully deleted previous user '${spacefx_service_appName}' (UID: '${preexisting_userid_for_username}')" + fi + + info_log "...creating user '${spacefx_service_appName}' with UID '${spacefx_service_userid}'..." + run_a_script "useradd -r -u ${spacefx_service_userid} -g ${spacefx_service_appName} -d /nonexistent -s /usr/sbin/nologin ${spacefx_service_appName}" --no_log + info_log "...successfully created user '${spacefx_service_appName}' (UID: '${spacefx_service_userid}')." + fi + + + fi + + + info_log "...adding '${service}'..." deploy_group_cmd="${deploy_group_cmd} --set services.${service_group}.${service}.enabled=true \ --set services.${service_group}.${service}.provisionVolumeClaims=true \ @@ -203,6 +279,15 @@ function deploy_apps_to_deployment_service(){ info_log "...successfully copied chart to '${SPACEFX_DIR}/xfer/platform-deployment/tmp/chart/${SPACEFX_VERSION}'" + run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.global.security.forceNonRoot' -r" spacefx_forceNonRoot + + if [[ "${spacefx_forceNonRoot}" == "true" ]]; then + info_log "Updating permissions for '${SPACEFX_DIR}/xfer/platform-deployment' to user 'platform-deployment'..." + run_a_script "chown -R platform-deployment:platform-deployment ${SPACEFX_DIR}/xfer/platform-deployment" + run_a_script "chmod -R u+rwx ${SPACEFX_DIR}/xfer/platform-deployment" + info_log "Permissions successfully updated" + fi + info_log "FINISHED: ${FUNCNAME[0]}" } diff --git a/scripts/stage_spacefx.sh b/scripts/stage_spacefx.sh index 9c72c27..4a8068f 100755 --- a/scripts/stage_spacefx.sh +++ b/scripts/stage_spacefx.sh @@ -239,6 +239,10 @@ function toggle_security_restrictions(){ run_a_script "yq eval '.global.security.topicRestrictionEnabled = false' -i \"${SPACEFX_DIR}/chart/values.yaml\"" info_log "'DEV_ENVIRONMENT' = true. Allowing Links to Platform-Deployment..." run_a_script "yq eval '(.services.host.link.appConfig[] | select(.name == \"allowLinksToDeploymentSvc\") .value) = true' -i \"${SPACEFX_DIR}/chart/values.yaml\"" + + info_log "'DEV_ENVIRONMENT' = true. Enabling run as root..." + run_a_script "yq eval '.global.security.forceNonRoot = false' -i \"${SPACEFX_DIR}/chart/values.yaml\"" + info_log "...security restrictions configured for development" else info_log "'DEV_ENVIRONMENT' = false. Enabling Network Restrictions..." @@ -251,6 +255,9 @@ function toggle_security_restrictions(){ run_a_script "yq eval '(.services.host.link.appConfig[] | select(.name == \"allowLinksToDeploymentSvc\") .value) = false' -i \"${SPACEFX_DIR}/chart/values.yaml\"" info_log "...Network and Topic restrictions successfully enabled." + info_log "'DEV_ENVIRONMENT' = true. Enabling run as root..." + run_a_script "yq eval '.global.security.forceNonRoot = true' -i \"${SPACEFX_DIR}/chart/values.yaml\"" + info_log "...security restrictions configured for production" fi diff --git a/tests/dev_cluster.sh b/tests/dev_cluster.sh index 5080281..55f1cf9 100755 --- a/tests/dev_cluster.sh +++ b/tests/dev_cluster.sh @@ -7,6 +7,7 @@ # # "bash ./tests/dev_cluster.sh" set -e +MAX_WAIT_SECS=300 SCRIPT_NAME=$(basename "$0") WORKING_DIR="$(git rev-parse --show-toplevel)" @@ -22,6 +23,57 @@ ARTIFACT_PATH=${WORKING_DIR}/output/spacefx-dev/devcontainer-feature-spacefx-dev echo "Microsoft Azure Orbital Space SDK - Development Cluster Test" +############################################################ +# Given a namespace, this function will wait for all pods to enter a running state +############################################################ +function wait_for_namespace_to_provision(){ + + local namespace="" + + while [[ "$#" -gt 0 ]]; do + case $1 in + --namespace) + shift + namespace=$1 + ;; + esac + shift + done + + if [[ -z $namespace ]]; then + echo "--namespace not provided to wait_for_namespace_to_provision function" + exit 1 + fi + + echo "Waiting for namespace '${namespace}' to fully provision (max $MAX_WAIT_SECS seconds)..." + + # This returns any pods that are not completed nor succeeded + k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length') + + start_time=$(date +%s) + + while [[ $k3s_deployments_not_ready != "0" ]]; do + k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length') + + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + + if [[ $elapsed_time -ge $MAX_WAIT_SECS ]]; then + echo "Timed out waiting for deployment to complete. Check logs for more information" + exit 1 + fi + + kubectl get pods -A + kubectl get deployments -A + + echo "Found incomplete deployments. Rechecking in 5 seconds" + sleep 5 + done + + echo "Namespace '${namespace}' is provisioned" + +} + if [[ -d "/var/spacedev" ]]; then echo "Preexisting /var/spacedev found. Resetting enviornment with big_red_button.sh" /var/spacedev/scripts/big_red_button.sh @@ -94,6 +146,9 @@ if [[ ! -f "${KUBECONFIG}" ]]; then echo "KUBECONFIG '${KUBECONFIG}' not found. Cluster did not initialize." exit 1 fi + + + kubectl --kubeconfig ${KUBECONFIG} get deployment/coresvc-registry -n coresvc kubectl --kubeconfig ${KUBECONFIG} get deployment/coresvc-switchboard -n coresvc @@ -107,6 +162,9 @@ kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-deployment -n platfor kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-mts -n platformsvc kubectl --kubeconfig ${KUBECONFIG} get deployment/vth -n platformsvc +wait_for_namespace_to_provision --namespace coresvc +wait_for_namespace_to_provision --namespace hostsvc +wait_for_namespace_to_provision --namespace platformsvc echo "" echo "" diff --git a/tests/prod_cluster.sh b/tests/prod_cluster.sh index 8688f67..e3396cd 100755 --- a/tests/prod_cluster.sh +++ b/tests/prod_cluster.sh @@ -9,9 +9,61 @@ set -e SCRIPT_NAME=$(basename "$0") WORKING_DIR="$(git rev-parse --show-toplevel)" - +MAX_WAIT_SECS=300 echo "Microsoft Azure Orbital Space SDK - Production Cluster Test" + +############################################################ +# Given a namespace, this function will wait for all pods to enter a running state +############################################################ +function wait_for_namespace_to_provision(){ + + local namespace="" + + while [[ "$#" -gt 0 ]]; do + case $1 in + --namespace) + shift + namespace=$1 + ;; + esac + shift + done + + if [[ -z $namespace ]]; then + echo "--namespace not provided to wait_for_namespace_to_provision function" + exit 1 + fi + + echo "Waiting for namespace '${namespace}' to fully provision (max $MAX_WAIT_SECS seconds)..." + + # This returns any pods that are not completed nor succeeded + k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length') + + start_time=$(date +%s) + + while [[ $k3s_deployments_not_ready != "0" ]]; do + k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length') + + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + + if [[ $elapsed_time -ge $MAX_WAIT_SECS ]]; then + echo "Timed out waiting for deployment to complete. Check logs for more information" + exit 1 + fi + + kubectl get pods -A + kubectl get deployments -A + + echo "Found incomplete deployments. Rechecking in 5 seconds" + sleep 5 + done + + echo "Namespace '${namespace}' is provisioned" + +} + if [[ -d "/var/spacedev" ]]; then echo "Resetting enviornment with big_red_button.sh" /var/spacedev/scripts/big_red_button.sh @@ -46,6 +98,12 @@ kubectl --kubeconfig ${KUBECONFIG} get deployment/hostsvc-position -n hostsvc kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-deployment -n platformsvc kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-mts -n platformsvc + +wait_for_namespace_to_provision --namespace coresvc +wait_for_namespace_to_provision --namespace hostsvc +wait_for_namespace_to_provision --namespace platformsvc + + echo "" echo "" echo "" diff --git a/tests/prod_cluster_with_smb.sh b/tests/prod_cluster_with_smb.sh index ee86e3b..9d4d619 100755 --- a/tests/prod_cluster_with_smb.sh +++ b/tests/prod_cluster_with_smb.sh @@ -9,9 +9,61 @@ set -e SCRIPT_NAME=$(basename "$0") WORKING_DIR="$(git rev-parse --show-toplevel)" - +MAX_WAIT_SECS=300 echo "Microsoft Azure Orbital Space SDK - Production Cluster Test" + +############################################################ +# Given a namespace, this function will wait for all pods to enter a running state +############################################################ +function wait_for_namespace_to_provision(){ + + local namespace="" + + while [[ "$#" -gt 0 ]]; do + case $1 in + --namespace) + shift + namespace=$1 + ;; + esac + shift + done + + if [[ -z $namespace ]]; then + echo "--namespace not provided to wait_for_namespace_to_provision function" + exit 1 + fi + + echo "Waiting for namespace '${namespace}' to fully provision (max $MAX_WAIT_SECS seconds)..." + + # This returns any pods that are not completed nor succeeded + k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length') + + start_time=$(date +%s) + + while [[ $k3s_deployments_not_ready != "0" ]]; do + k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length') + + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + + if [[ $elapsed_time -ge $MAX_WAIT_SECS ]]; then + echo "Timed out waiting for deployment to complete. Check logs for more information" + exit 1 + fi + + kubectl get pods -A + kubectl get deployments -A + + echo "Found incomplete deployments. Rechecking in 5 seconds" + sleep 5 + done + + echo "Namespace '${namespace}' is provisioned" + +} + if [[ -d "/var/spacedev" ]]; then echo "Resetting enviornment with big_red_button.sh" /var/spacedev/scripts/big_red_button.sh @@ -47,6 +99,10 @@ kubectl --kubeconfig ${KUBECONFIG} get deployment/hostsvc-position -n hostsvc kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-deployment -n platformsvc kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-mts -n platformsvc +wait_for_namespace_to_provision --namespace coresvc +wait_for_namespace_to_provision --namespace hostsvc +wait_for_namespace_to_provision --namespace platformsvc + echo "" echo "" echo ""