Skip to content

Commit

Permalink
Add option to install everything in global venvs
Browse files Browse the repository at this point in the history
Since we are python3 only for openstack we create a single python3
virtualenv to install all the packages into. This gives us the benefits
of installing into a virtualenv while still ensuring coinstallability.
This is a major change and will likely break many things.

There are several reasons for this. The change that started this effort
was pip stopped uninstalling packages which used distutils to generate
their package installation. Many distro packages do this which meant
that pip installed packages and distro packages could not coexist in the
global install space. More recently git has made pip installing repos as
root more difficult due to file ownership concerns.

Currently the switch to the global venv is optional, but if we go down
this path we should very quickly remove the old global installation
method as it has only caused us problems.

Major hurdles we have to get over are convincing rootwrap to trust
binaries in the virtualenvs (so you'll notice we update rootwrap
configs).

Some distros still have issues, keep them using the old setup for now.

Depends-On: https://review.opendev.org/c/openstack/grenade/+/880266
Co-Authored-By: Dr. Jens Harbott <[email protected]>
Change-Id: If9bc7ba45522189d03f19b86cb681bb150ee2f25
  • Loading branch information
cboylan and osfrickler committed Aug 2, 2023
1 parent 9dba099 commit a40f9cb
Show file tree
Hide file tree
Showing 12 changed files with 92 additions and 6 deletions.
8 changes: 8 additions & 0 deletions .zuul.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -700,6 +700,9 @@
# TODO(kopecmartin) n-v until the following is resolved:
# https://bugs.launchpad.net/neutron/+bug/1979047
voting: false
vars:
devstack_localrc:
GLOBAL_VENV: false

- job:
name: devstack-platform-debian-bullseye
Expand All @@ -709,6 +712,9 @@
timeout: 9000
vars:
configure_swap_size: 4096
devstack_localrc:
# TODO(frickler): drop this once wheel build is fixed
MYSQL_GATHER_PERFORMANCE: false

- job:
name: devstack-platform-rocky-blue-onyx
Expand All @@ -718,6 +724,8 @@
timeout: 9000
vars:
configure_swap_size: 4096
devstack_localrc:
GLOBAL_VENV: false

- job:
name: devstack-platform-ubuntu-focal
Expand Down
1 change: 1 addition & 0 deletions files/apache-horizon.template
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,5 @@
CustomLog /var/log/%APACHE_NAME%/horizon_access.log combined
</VirtualHost>

%WSGIPYTHONHOME%
WSGISocketPrefix /var/run/%APACHE_NAME%
5 changes: 5 additions & 0 deletions functions-common
Original file line number Diff line number Diff line change
Expand Up @@ -1522,6 +1522,7 @@ function write_user_unit_file {
mkdir -p $SYSTEMD_DIR

iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
iniset -sudo $unitfile "Service" "User" "$user"
iniset -sudo $unitfile "Service" "ExecStart" "$command"
iniset -sudo $unitfile "Service" "KillMode" "process"
Expand Down Expand Up @@ -1549,6 +1550,7 @@ function write_uwsgi_user_unit_file {
mkdir -p $SYSTEMD_DIR

iniset -sudo $unitfile "Unit" "Description" "Devstack $service"
iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\""
iniset -sudo $unitfile "Service" "SyslogIdentifier" "$service"
iniset -sudo $unitfile "Service" "User" "$user"
iniset -sudo $unitfile "Service" "ExecStart" "$command"
Expand Down Expand Up @@ -1614,6 +1616,9 @@ function _run_under_systemd {
fi
local env_vars="$5"
if [[ "$command" =~ "uwsgi" ]] ; then
if [[ "$GLOBAL_VENV" == "True" ]] ; then
cmd="$cmd --venv $DEVSTACK_VENV"
fi
write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
else
write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars"
Expand Down
34 changes: 31 additions & 3 deletions inc/python
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,23 @@ function join_extras {
# Python Functions
# ================

# Setup the global devstack virtualenvs and the associated environment
# updates.
function setup_devstack_virtualenv {
# We run devstack out of a global virtualenv.
if [[ ! -d $DEVSTACK_VENV ]] ; then
# Using system site packages to enable nova to use libguestfs.
# This package is currently installed via the distro and not
# available on pypi.
python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV
pip_install -U pip
fi
if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then
export PATH="$DEVSTACK_VENV/bin:$PATH"
export PYTHON="$DEVSTACK_VENV/bin/python3"
fi
}

# Get the path to the pip command.
# get_pip_command
function get_pip_command {
Expand Down Expand Up @@ -60,8 +77,11 @@ function get_python_exec_prefix {
fi
$xtrace

local PYTHON_PATH=/usr/local/bin
echo $PYTHON_PATH
if [[ "$GLOBAL_VENV" == "True" ]] ; then
echo "$DEVSTACK_VENV/bin"
else
echo "/usr/local/bin"
fi
}

# Wrapper for ``pip install`` that only installs versions of libraries
Expand Down Expand Up @@ -166,6 +186,14 @@ function pip_install {
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
local sudo_pip="env"
elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then
# We have to check that the DEVSTACK_VENV exists because early
# devstack boostrapping needs to operate in a system context
# too bootstrap pip. Once pip is bootstrapped we create the
# global venv and can start to use it.
local cmd_pip=$DEVSTACK_VENV/bin/pip
local sudo_pip="env"
echo "Using python $PYTHON3_VERSION to install $package_dir"
else
local cmd_pip="python$PYTHON3_VERSION -m pip"
# See
Expand Down Expand Up @@ -439,7 +467,7 @@ function setup_package {

pip_install $flags "$project_dir$extras"
# ensure that further actions can do things like setup.py sdist
if [[ "$flags" == "-e" ]]; then
if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then
safe_chown -R $STACK_USER $1/*.egg-info
fi
}
Expand Down
5 changes: 5 additions & 0 deletions inc/rootwrap
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ function configure_rootwrap {
sudo install -o root -g root -m 644 $rootwrap_conf_src_dir/rootwrap.conf /etc/${project}/rootwrap.conf
sudo sed -e "s:^filters_path=.*$:filters_path=/etc/${project}/rootwrap.d:" -i /etc/${project}/rootwrap.conf

# Rely on $PATH set by devstack to determine what is safe to execute
# by rootwrap rather than use explicit whitelist of paths in
# rootwrap.conf
sudo sed -e 's/^exec_dirs=.*/#&/' -i /etc/${project}/rootwrap.conf

# Set up the rootwrap sudoers
local tempfile
tempfile=$(mktemp)
Expand Down
3 changes: 3 additions & 0 deletions lib/glance
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ USE_CINDER_FOR_GLANCE=$(trueorfalse False USE_CINDER_FOR_GLANCE)
# from CINDER_ENABLED_BACKENDS
GLANCE_CINDER_DEFAULT_BACKEND=${GLANCE_CINDER_DEFAULT_BACKEND:-lvmdriver-1}
GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/local/etc/glance
if [[ "$GLOBAL_VENV" == "True" ]] ; then
GLANCE_STORE_ROOTWRAP_BASE_DIR=${DEVSTACK_VENV}/etc/glance
fi
# When Cinder is used as a glance store, you can optionally configure cinder to
# optimize bootable volume creation by allowing volumes to be cloned directly
# in the backend instead of transferring data via Glance. To use this feature,
Expand Down
6 changes: 6 additions & 0 deletions lib/horizon
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,11 @@ function configure_horizon {
local horizon_conf
horizon_conf=$(apache_site_config_for horizon)

local wsgi_venv_config=""
if [[ "$GLOBAL_VENV" == "True" ]] ; then
wsgi_venv_config="WSGIPythonHome $DEVSTACK_VENV"
fi

# Configure apache to run horizon
# Set up the django horizon application to serve via apache/wsgi
sudo sh -c "sed -e \"
Expand All @@ -124,6 +129,7 @@ function configure_horizon {
s,%APACHE_NAME%,$APACHE_NAME,g;
s,%DEST%,$DEST,g;
s,%WEBROOT%,$HORIZON_APACHE_ROOT,g;
s,%WSGIPYTHONHOME%,$wsgi_venv_config,g;
\" $FILES/apache-horizon.template >$horizon_conf"

if is_ubuntu; then
Expand Down
7 changes: 5 additions & 2 deletions lib/tls
Original file line number Diff line number Diff line change
Expand Up @@ -364,8 +364,11 @@ function deploy_int_CA {
function fix_system_ca_bundle_path {
if is_service_enabled tls-proxy; then
local capath
capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')

if [[ "$GLOBAL_VENV" == "True" ]] ; then
capath=$($DEVSTACK_VENV/bin/python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
else
capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
fi
if [[ ! $capath == "" && ! $capath =~ ^/etc/.* && ! -L $capath ]]; then
if is_fedora; then
sudo rm -f $capath
Expand Down
12 changes: 12 additions & 0 deletions stack.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env bash


# ``stack.sh`` is an opinionated OpenStack developer installation. It
# installs and configures various combinations of **Cinder**, **Glance**,
# **Horizon**, **Keystone**, **Nova**, **Neutron**, and **Swift**
Expand Down Expand Up @@ -824,6 +825,17 @@ fi
source $TOP_DIR/tools/fixup_stuff.sh
fixup_all

if [[ "$GLOBAL_VENV" == "True" ]] ; then
# TODO(frickler): find a better solution for this
sudo ln -sf /opt/stack/data/venv/bin/privsep-helper /usr/local/bin
sudo ln -sf /opt/stack/data/venv/bin/cinder-rtstool /usr/local/bin
sudo ln -sf /opt/stack/data/venv/bin/openstack /usr/local/bin
sudo ln -sf /opt/stack/data/venv/bin/tox /usr/local/bin
sudo ln -sf /opt/stack/data/venv/bin/nova-manage /usr/local/bin

setup_devstack_virtualenv
fi

# Install subunit for the subunit output stream
pip_install -U os-testr

Expand Down
8 changes: 8 additions & 0 deletions stackrc
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,14 @@ IDENTITY_API_VERSION=3
# each services ${SERVICE}_ENFORCE_SCOPE variables
ENFORCE_SCOPE=$(trueorfalse False ENFORCE_SCOPE)

# Devstack supports the use of a global virtualenv. These variables enable
# and disable this functionality as well as set the path to the virtualenv.
# Note that the DATA_DIR is selected because grenade testing uses a shared
# DATA_DIR but different DEST dirs and we don't want two sets of venvs,
# instead we want one global set.
GLOBAL_VENV=$(trueorfalse True GLOBAL_VENV)
DEVSTACK_VENV=${DEVSTACK_VENV:-$DATA_DIR/venv}

# Enable use of Python virtual environments. Individual project use of
# venvs are controlled by the PROJECT_VENV array; every project with
# an entry in the array will be installed into the named venv.
Expand Down
2 changes: 2 additions & 0 deletions tools/install_prereqs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ if [[ -n "$SYSLOG" && "$SYSLOG" != "False" ]]; then
fi
fi

# TODO(clarkb) remove these once we are switched to global venv by default
export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null || which python3 2>/dev/null)

# Mark end of run
# ---------------
Expand Down
7 changes: 6 additions & 1 deletion tools/memory_tracker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@

set -o errexit

PYTHON=${PYTHON:-python3}
# TODO(frickler): make this use stackrc variables
if [ -x /opt/stack/data/venv/bin/python ]; then
PYTHON=/opt/stack/data/venv/bin/python
else
PYTHON=${PYTHON:-python3}
fi

# time to sleep between checks
SLEEP_TIME=20
Expand Down

0 comments on commit a40f9cb

Please sign in to comment.