diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..b7efcb4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "test/bats"] + path = test/bats + url = https://github.com/bats-core/bats-core.git +[submodule "test/test_helper/bats-support"] + path = test/test_helper/bats-support + url = https://github.com/bats-core/bats-support.git +[submodule "test/test_helper/bats-assert"] + path = test/test_helper/bats-assert + url = https://github.com/bats-core/bats-assert.git diff --git a/ReadMe.md b/ReadMe.md index e0bf756..793177c 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -2,7 +2,7 @@ # Docker real-time guide for `PREEMPT_RT` -Author: [Tobit Flatscher](https://github.com/2b-t) (August 2021 - April 2022) +Author: [Tobit Flatscher](https://github.com/2b-t) (August 2021 - July 2022) @@ -35,28 +35,30 @@ The manual set-up of `PREEMPT_RT` takes quite a while (see [`doc/realtime_basics ### 1.1 Installing `PREEMPT_RT` -The installation procedure either by compilation from source or from an existing [Debian package](https://packages.debian.org/buster/linux-image-rt-amd64) is lined out in [`doc/realtime_basics/PreemptRt.md`](./doc/realtime_basics/PreemptRt.md). The same procedure can also be performed with the provided scripts [`install_debian_preemptrt.sh`](./install_debian_preemptrt) and [`compile_kernel_preemptrt.sh`](./compile_kernel_preemptrt.sh). +The installation procedure either by compilation from source or from an existing [Debian package](https://packages.debian.org/buster/linux-image-rt-amd64) is lined out in [`doc/realtime_basics/PreemptRt.md`](./doc/realtime_basics/PreemptRt.md). The same procedure can also be performed with the provided scripts [`src/install_debian_preemptrt`](./src/install_debian_preemptrt) and [`src/compile_kernel_preemptrt`](./src/compile_kernel_preemptrt). -[`install_debian_preemptrt.sh`](./install_debian_preemptrt) checks online if there are already precompiled `PREEMPT_RT` packages available and lets you select a suiting version graphically, while [`compile_kernel_preemptrt.sh`](./compile_kernel_preemptrt.sh) compiles the kernel from scratch from you and installs it. Before using the scripts be sure to make them executable on your system with `$ sudo chmod +x install_debian_preemptrt.sh`. +[`src/install_debian_preemptrt`](./src/install_debian_preemptrt) checks online if there are already precompiled `PREEMPT_RT` packages available and lets you select a suiting version graphically, while [`src/compile_kernel_preemptrt`](./src/compile_kernel_preemptrt) compiles the kernel from scratch from you and installs it. Before using the scripts be sure to make them executable on your system with `$ sudo chmod +x install_debian_preemptrt`. #### 1.1.1 Installation from pre-compiled Debian package (recommended) -Start of by launching [`install_debian_preemptrt.sh`](./install_debian_preemptrt) and follow the installation instructions +Start of by launching [`src/install_debian_preemptrt`](./src/install_debian_preemptrt) and follow the installation instructions ```shell -$ ./install_debian_preemptrt.sh +$ cd src +$ ./install_debian_preemptrt ``` Afterwards you can reboot your system (be sure to select the correct kernel!) and should already be ready to go. You can check the kernel version with `$ uname -r` to verify that you are using the correct kernel and the installation was indeed successful. #### 1.1.2 Compilation of the kernel -If the installation above fails or for some good reason you have to compile the kernel yourself you can use the [`compile_kernel_preemptrt.sh`](./compile_kernel_preemptrt.sh) script. +If the installation above fails or for some good reason you have to compile the kernel yourself you can use the [`src/compile_kernel_preemptrt`](./src/compile_kernel_preemptrt) script. You can launch it in two different ways: ```shell -$ ./compile_kernel_preemptrt.sh +$ cd src +$ ./compile_kernel_preemptrt ``` will install the required dependencies and then open a dialog which lets you browse the possible versions and available options manually, reducing the copy and paste operations. @@ -64,7 +66,8 @@ will install the required dependencies and then open a dialog which lets you bro If you supply a correct real-time patch version from the list of available ones as an input argument, launching the command with superuser privileges it will download all files, patch the kernel, create a Debian package if no official one is available and install it automatically. ```shell -$ sudo ./compile_kernel_preemptrt.sh 5.10.78-rt55 +$ cd src +$ sudo ./compile_kernel_preemptrt 5.10.78-rt55 ``` This might be helpful for deploying a new kernel automatically on a remote system. The possible version numbers can be found at [here](https://mirrors.edge.kernel.org/pub/linux/kernel/projects/rt/). diff --git a/compile_kernel_preemptrt.sh b/compile_kernel_preemptrt.sh deleted file mode 100755 index 3ed1e7b..0000000 --- a/compile_kernel_preemptrt.sh +++ /dev/null @@ -1,254 +0,0 @@ -#!/bin/bash -# Script for compiling a kernel with the PREEMPT_RT real-time patch with a simple graphical interface -# Tobit Flatscher - github.com/2b-t (2022) -# -# Usage: -# - '$ ./patch_kernel_preemprt.sh' and go through with a graphical user interface -# - Select a PREEMPT_RT version at https://mirrors.edge.kernel.org/pub/linux/kernel/projects/rt/" -# and compile with '$ sudo ./patch_kernel_preemprt.sh 5.10.78-rt55' - - -# Install required dependencies -function install_dependencies { - echo "Installing dependencies..." - sudo apt-get install -y grep curl sed - sudo apt-get install -y build-essential bc ca-certificates gnupg2 libssl-dev lsb-release libelf-dev bison flex dwarves zstd libncurses-dev dpkg-dev - echo "Dependencies installed successfully!" -} - -# Get major versions by crawling website -function get_preemptrt_major_versions { - echo $(curl -Ls https://mirrors.edge.kernel.org/pub/linux/kernel/projects/rt | grep -o -P '(?<=href\=\")(\d\.\d*)+(?=/\">)') -} - -# Select the desired major version with a user dialog -function select_preemptrt_major_version { - local PREEMPTRT_MAJOR_VERSIONS=$(get_preemptrt_major_versions) - local DIALOG_PREEMPTRT_MAJOR_VERSIONS="" - for VER in ${PREEMPTRT_MAJOR_VERSIONS} - do - DIALOG_PREEMPTRT_MAJOR_VERSIONS="${DIALOG_PREEMPTRT_MAJOR_VERSIONS} ${VER} ${VER}" - done - CURRENT_KERNEL_VERSION=$(uname -r | sed 's/\.[^\.]*//2g') - echo $(dialog --no-tags --stdout --default-item ${CURRENT_KERNEL_VERSION} --menu "Select a major kernel version:" 30 40 10 ${DIALOG_PREEMPTRT_MAJOR_VERSIONS}) -} - -# Get the full versions by crawling website -function get_preemptrt_full_versions { - echo $(curl -Ls https://mirrors.edge.kernel.org/pub/linux/kernel/projects/rt/${PREEMPT_RT_VER} | grep -o -P '(?<=href\=\"patch-).*(?=.patch.gz\">)') -} - -# Select the desired major version with a user dialog -function select_preemptrt_full_version { - local PREEMPTRT_FULL_VERSIONS=$(get_preemptrt_full_versions) - local DIALOG_PREEMPTRT_FULL_VERSIONS="" - for VER in ${PREEMPTRT_FULL_VERSIONS} - do - DIALOG_PREEMPTRT_FULL_VERSIONS="${DIALOG_PREEMPTRT_FULL_VERSIONS} ${VER} ${VER}" - done - echo $(dialog --no-tags --stdout --menu "Select the desired version of PREEMPT_RT:" 30 40 10 ${DIALOG_PREEMPTRT_FULL_VERSIONS}) -} - -# Reconstruct the corresponding kernel major version -function reconstruct_kernel_major_version { - local KERNEL_MAJOR_VERSION=$(echo "${PREEMPT_RT_VER}" | grep -o -P '^\s*(\d)+') - echo "$(curl -Ls https://www.kernel.org/pub/linux/kernel | grep -o -P "(?<=href\=\")(v${KERNEL_MAJOR_VERSION}.*)(?=/\">)")" -} - -# Download and extract the vanilla kernel -function download_and_extract_kernel { - echo "Downloading kernel '${KERNEL_VER_FULL}'..." - curl -SLO --fail https://www.kernel.org/pub/linux/kernel/${KERNEL_VER}/linux-${KERNEL_VER_FULL}.tar.xz - curl -SLO --fail https://www.kernel.org/pub/linux/kernel/${KERNEL_VER}/linux-${KERNEL_VER_FULL}.tar.sign - xz -d linux-${KERNEL_VER_FULL}.tar.xz -} - -# Download and extract the PREEMPT_RT patch -function download_and_extract_preemptrt { - echo "Downloading PREEMPT_RT patch '${PREEMPT_RT_VER_FULL}'..." - curl -SLO --fail https://mirrors.edge.kernel.org/pub/linux/kernel/projects/rt/${PREEMPT_RT_VER}/patch-${PREEMPT_RT_VER_FULL}.patch.xz - curl -SLO --fail https://mirrors.edge.kernel.org/pub/linux/kernel/projects/rt/${PREEMPT_RT_VER}/patch-${PREEMPT_RT_VER_FULL}.patch.sign - xz -d patch-${PREEMPT_RT_VER_FULL}.patch.xz -} - -# Sign the kernel and the patch -function sign_kernel_and_preemptrt { - echo "Signing keys..." - # Catch non-zero exit code despite "set -e", see https://stackoverflow.com/a/57189853 - if gpg2 --verify linux-${KERNEL_VER_FULL}.tar.sign; then - : - else - local KERNEL_KEY=$(gpg2 --verify linux-${KERNEL_VER_FULL}.tar.sign 2>&1 | grep -o -P '(?<=RSA key )(.*)') - gpg2 --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys ${KERNEL_KEY} - gpg2 --verify linux-${KERNEL_VER_FULL}.tar.sign - fi - - if gpg2 --verify patch-${PREEMPT_RT_VER_FULL}.patch.sign; then - : - else - local PREEMPT_KEY=$(gpg2 --verify patch-${PREEMPT_RT_VER_FULL}.patch.sign 2>&1 | grep -o -P '(?<=RSA key )(.*)') - gpg2 --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys ${PREEMPT_KEY} - gpg2 --verify patch-${PREEMPT_RT_VER_FULL}.patch.sign - fi -} - -# Apply the patch to the kernel -function apply_patch_to_kernel { - echo "Applying PREEMPT_RT patch '${PREEMPT_RT_VER_FULL}' to kernel '${KERNEL_VER_FULL}'..." - tar xf linux-${KERNEL_VER_FULL}.tar - cd linux-${KERNEL_VER_FULL} - patch -p1 < ../patch-${PREEMPT_RT_VER_FULL}.patch -} - -# Generate a new kernel configuration -function generate_new_kernel_configuration { - # Only option set in the configuration is full PREEMPT_RT, rest is left on default - echo "Generating kernel configuration" - echo "4" | make oldconfig -} - -# Function for finding a certain setting, replacing it or adding it if not existing -# @param $1 The name of the setting -# @param $2 its desired value -function find_and_replace_in_config { - local CONFIG_FILE=".config" - grep -E "$1=" $CONFIG_FILE && sed -i "s/$1=.*/$1=$2/" $CONFIG_FILE || echo $1"="$2 >> ${CONFIG_FILE} -} - -# Function for commenting out certain settings in the configuration files -# @param $1 The name of the setting to be commented out -function comment_out_in_config { - local CONFIG_FILE=".config" - sed -E "/$1/ s/^#*/#/" -i ${CONFIG_FILE} -} - -# Unsign the kernel configuration -function unsign_kernel_configuration { - echo "Forcing unsigned kernel..." - find_and_replace_in_config "CONFIG_SYSTEM_TRUSTED_KEYS" '""' - find_and_replace_in_config "CONFIG_SYSTEM_REVOCATION_KEYS" '""' -} - -# Select the installation modality -function select_installation_mode { - echo $(dialog --stdout --default-item "Debian package" --menu "Select the desired installation mode:" 30 40 10 "Debian" "Debian package" "Classic" "Install directly") -} - -# Generate a debian package for easier installation -function generate_preemptrt_kernel_debian_package { - echo "Generating Debian package..." - sudo make -j$(nproc) deb-pkg -} - -# Function for deciding whether to install the kernel now or install it later -function select_install_now { - dialog --stdout --title "Install Debian package" --yesno "Want to install the Debian package now" 7 60 - echo $? -} - -# Install PREEMPT_RT from the Debian package -function install_preemptrt_kernel_debian_package { - echo "Installing Debian package..." - sudo dpkg -i ../linux-image-${PREEMPT_RT_VER_FULL}_${PREEMPT_RT_VER_FULL}-1_$(dpkg --print-architecture).deb -} - -# Install the kernel directly without creating a Debian package first -function install_preemptrt_kernel_directly { - echo "Installing in the classic way..." - sudo make -j$(nproc) - sudo make modules_install -j$(nproc) - sudo make install -j$(nproc) -} - -# Install and let the user decide the kernel etc. -function install_kernel_interactive { - { - sudo apt-get install -y dialog - install_dependencies - } || { - echo "Warning: Could not install dependencies. Installation might fail!" - } - - # Get patch version via dialog - local PREEMPT_RT_VER=$(select_preemptrt_major_version) # e.g. 5.10 - local PREEMPT_RT_VER_FULL=$(select_preemptrt_full_version) # e.g. 5.10.78-rt55 - local KERNEL_VER=$(reconstruct_kernel_major_version) # e.g. v5.x - local KERNEL_VER_FULL=$(echo "${PREEMPT_RT_VER_FULL}" | sed 's/-rt.*//g') # e.g. 5.10.78 - - # Download and extract the files for kernel and patch - download_and_extract_kernel - download_and_extract_preemptrt - sign_kernel_and_preemptrt - - # Apply patch to kernel - apply_patch_to_kernel - generate_new_kernel_configuration - unsign_kernel_configuration - - # Choose between Debian package and installing directly - local INSTALLATION_MODE=$(select_installation_mode) - if [ "${INSTALLATION_MODE}" == "Debian" ] - then - generate_preemptrt_kernel_debian_package - INSTALL_NOW=$(select_install_now) - if [ "${INSTALL_NOW}" -eq 0 ] - then - install_preemptrt_kernel_debian_package - echo "Done: Installation with Debian package completed!" - else - echo "Done: Debian package generated!" - fi - else - install_preemptrt_kernel_directly - echo "Done: Classic installation complete!" - fi -} - -# Installs the kernel as Debian package from a given command-line argument without interactive choices -function install_kernel_noninteractive { - { - install_dependencies - } || { - echo "Warning: Could not install dependencies. Installation might fail!" - } - - local PREEMPT_RT_VER_FULL=$1 # e.g. 5.10.78-rt55 - local KERNEL_VER_FULL=$(echo "${PREEMPT_RT_VER_FULL}" | sed 's/-rt.*//g') # e.g. 5.10.78 - local PREEMPT_RT_VER=$(echo "${KERNEL_VER_FULL}" | sed 's/\.[^\.]*//2g') # e.g. 5.10 - local KERNEL_VER="v"$(echo "${PREEMPT_RT_VER}" | sed -n 's/\(.*\)[.]\(.*\)/\1.x/p') # e.g. v5.x - - download_and_extract_kernel - download_and_extract_preemptrt - sign_kernel_and_preemptrt - apply_patch_to_kernel - generate_new_kernel_configuration - unsign_kernel_configuration - generate_preemptrt_kernel_debian_package - install_preemptrt_kernel_debian_package - echo "Done: Installation with Debian package completed!" -} - - -# Install the kernel either in an interactive or non-interactive way depending if an input argument is given -function main { - set -e # Exit immediately in case of failure - if [ "$#" -eq 0 ] - then - install_kernel_interactive - elif [ "$#" -eq 1 ] && [ "${EUID}" -eq 0 ] - then - install_kernel_noninteractive $@ - else - echo "Error: Could not run installation script!" - echo "Either launch it as: " - echo " - '$ ./patch_kernel_preemprt.sh'" - echo " - '$ sudo ./patch_kernel_preemprt.sh 5.10.78-rt55'" - echo "For the available PREEMPT_RT versions see: " - echo "see https://mirrors.edge.kernel.org/pub/linux/kernel/projects/rt/" - exit 1 - fi - exit 0 -} - -main $@ - diff --git a/doc/realtime_basics/PreemptRt.md b/doc/realtime_basics/PreemptRt.md index db78d2f..c616fd6 100644 --- a/doc/realtime_basics/PreemptRt.md +++ b/doc/realtime_basics/PreemptRt.md @@ -14,11 +14,11 @@ The `PREEMPT_RT` kernels are known for causing a headache with the **official Nv The installation of `PREEMPT_RT` can be performed either by installing the patch from an existing **Debian package** or by re-compiling the kernel yourself. While the first option should be preferred for having a more detailed control over the kernel set-up latter might still be desired or might be your only options in rare cases of lacking compatibility. -In the main folder of this I have provided two **scripts `install_debian_preemptrt.sh` and `compile_kernel_preemptrt.sh`** which is able to install either from an existing Debian package or by recompiling the kernel automatically. This means it should not be necessary to perform this steps manually. Nonetheless I will leave this here as a reference! +In the main folder of this I have provided two **scripts `src/install_debian_preemptrt` and `src/compile_kernel_preemptrt`** which is able to install either from an existing Debian package or by recompiling the kernel automatically. This means it should not be necessary to perform this steps manually. Nonetheless I will leave this here as a reference! #### 1.2.1 Installation from Debian package (recommended) -The installation from a Debian package is way **simpler** than the recompilation listed below but is at the same time **less flexible** as it is only available for a limited number of kernels and kernel configurations. You might not be able to make the Debian package work with your particular system and might have to re-compile the kernel anyways. Nonetheless it is **highly advised** that you follow these simple steps before turning to a full re-compilation of the kernel. The script that already performs the steps for you is `install_debian_preemptrt.sh`. +The installation from a Debian package is way **simpler** than the recompilation listed below but is at the same time **less flexible** as it is only available for a limited number of kernels and kernel configurations. You might not be able to make the Debian package work with your particular system and might have to re-compile the kernel anyways. Nonetheless it is **highly advised** that you follow these simple steps before turning to a full re-compilation of the kernel. The script that already performs the steps for you is `src/install_debian_preemptrt`. Have a look at the search results resulting from [this query on package.debian.org](https://packages.debian.org/search?keywords=linux-image-rt-amd64) (potentially changing the architecture!) and see if you can find a kernel close to yours, e.g. [this one](https://packages.debian.org/bullseye/linux-image-rt-amd64). If you can find one click on the architecture `amd64` under `Download linux-image-rt-amd64` on the bottom and select a geographically suiting mirror and save the image in a location of your choice. @@ -33,7 +33,7 @@ Jump to section 2.1.3 and then try to reboot. In case it does not work you will #### 1.2.2 Re-compilation of the kernel -The re-compilation of the kernel is described in the [official Ubuntu installation guide](https://help.ubuntu.com/lts/installation-guide/amd64/install.en.pdf#page=98) as well as on the [Franka Emika installation guide](https://frankaemika.github.io/docs/installation_linux.html#setting-up-the-real-time-kernel) page but [might depend on the precise version](https://stackoverflow.com/a/51709420). In case you are running into issues you might have to consider [this](https://askubuntu.com/a/1338150) and [this](https://askubuntu.com/a/1329625) post. As already mentioned the script `compile_kernel_preemptrt.sh` does the steps that are listed in this section. Therefore it should not be necessary for you to do the following steps manually! +The re-compilation of the kernel is described in the [official Ubuntu installation guide](https://help.ubuntu.com/lts/installation-guide/amd64/install.en.pdf#page=98) as well as on the [Franka Emika installation guide](https://frankaemika.github.io/docs/installation_linux.html#setting-up-the-real-time-kernel) page but [might depend on the precise version](https://stackoverflow.com/a/51709420). In case you are running into issues you might have to consider [this](https://askubuntu.com/a/1338150) and [this](https://askubuntu.com/a/1329625) post. As already mentioned the script `src/compile_kernel_preemptrt` does the steps that are listed in this section. Therefore it should not be necessary for you to do the following steps manually! Start by installing the Debian packages required for the re-compilation. For a Debian-based Linux distribution this can be done conveniently with: diff --git a/docker/Dockerfile b/docker/Dockerfile index 072c392..80db712 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,6 +17,11 @@ RUN apt-get update && apt-get install -y \ python3 \ python3-distutils \ gnuplot-qt +# For unit-testing bash +RUN apt-get update && apt-get install -y \ + bats \ + dialog \ + xdotool # Fetch most recent version of rt-tests and mklatencyplot RUN cd ${WS_DIR} \ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f0cdb9d..1d32782 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -7,4 +7,6 @@ services: tty: true privileged: true network_mode: host - + volumes: + - ../src:/benchmark/src + - ../test:/benchmark/test diff --git a/install_debian_preemptrt.sh b/install_debian_preemptrt.sh deleted file mode 100755 index 056a869..0000000 --- a/install_debian_preemptrt.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# This script searches https://packages.debian.org/ for kernels with the PREEMPT_RT patch and allows the user to choose a desired version interactively -# This should be preferred over the re-compilation of the Linux kernel -# Tobit Flatscher - github.com/2b-t (2022) - - -# Install the kernel from the available pre-compiled packages online -function main { - set -e # Exit immediately in case of failure - sudo apt-get install -y dialog - sudo apt-get install -y coreutils curl grep dpkg-dev - - # Select Debian version - POSSIBLE_DEBIAN_VERSIONS=$(cat /etc/debian_version | tr / " ") - local DIALOG_POSSIBLE_DEBIAN_VERSIONS="" - for VER in ${POSSIBLE_DEBIAN_VERSIONS} - do - PREEMPTRT_FILE=$(curl -Ls https://packages.debian.org/${VER}/$(dpkg --print-architecture)/linux-image-rt-$(dpkg --print-architecture)/download | grep -o -P '(?<=