From 510b797da9a9282bda98fa10fccf1e76030f5949 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Thu, 30 Nov 2023 15:05:14 -0500 Subject: [PATCH 001/147] conf: move templates Upstream OE now enforces a requirement that the configuration templates be stored in a path like: ${layer}/conf/templates/${name}. Move the meta-nilrt conf templates to a compliant location. Signed-off-by: Alex Stewart --- conf/{ => templates/default}/bblayers.conf.sample | 0 conf/{ => templates/default}/local.conf.sample | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename conf/{ => templates/default}/bblayers.conf.sample (100%) rename conf/{ => templates/default}/local.conf.sample (100%) diff --git a/conf/bblayers.conf.sample b/conf/templates/default/bblayers.conf.sample similarity index 100% rename from conf/bblayers.conf.sample rename to conf/templates/default/bblayers.conf.sample diff --git a/conf/local.conf.sample b/conf/templates/default/local.conf.sample similarity index 100% rename from conf/local.conf.sample rename to conf/templates/default/local.conf.sample From 6c5a000cbe6a312a50a39558f1a6d95dd7a51cf6 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 29 Nov 2023 17:59:29 -0600 Subject: [PATCH 002/147] kernel-performance-tests: Add container load test Add a cyclictest ptest with container start/stop in background. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 30d75a96dcdaf1abb4092427b72565eb246418dc) Signed-off-by: Alex Stewart --- .../run-container-load | 14 ++++++++++++++ .../kernel-performance-tests-files/run-ptest | 1 + .../test_kernel_cyclictest_container_load.sh | 15 +++++++++++++++ .../kernel-tests/kernel-performance-tests.bb | 6 +++++- 4 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 recipes-kernel/kernel-tests/kernel-performance-tests-files/run-container-load create mode 100644 recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_container_load.sh diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-container-load b/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-container-load new file mode 100644 index 000000000..05b435450 --- /dev/null +++ b/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-container-load @@ -0,0 +1,14 @@ +#!/bin/bash + +function setup_container_load() +{ + docker pull hello-world +} + +function start_container_load() +{ + while true; do + docker run hello-world > /dev/null + sleep 1 + done +} diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-ptest b/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-ptest index 0b17a3bdb..4ee102807 100644 --- a/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-ptest +++ b/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-ptest @@ -4,4 +4,5 @@ ./test_kernel_cyclictest_hackbench.sh ./test_kernel_cyclictest_fio.sh ./test_kernel_cyclictest_iperf.sh +./test_kernel_cyclictest_container_load.sh exit 0 diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_container_load.sh b/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_container_load.sh new file mode 100644 index 000000000..9b2883104 --- /dev/null +++ b/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_container_load.sh @@ -0,0 +1,15 @@ +#!/bin/bash +source ./run-cyclictest +source ./run-container-load + +# start background container load +setup_container_load +start_container_load & + +# measure the system latency under container load +run_cyclictest "container" + +# clean-up +kill $(jobs -p) 2>/dev/null + +exit $CYCLICTEST_RESULT diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests.bb b/recipes-kernel/kernel-tests/kernel-performance-tests.bb index 5c5858249..b06816ac7 100644 --- a/recipes-kernel/kernel-tests/kernel-performance-tests.bb +++ b/recipes-kernel/kernel-tests/kernel-performance-tests.bb @@ -11,7 +11,7 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}-files:" S = "${WORKDIR}" DEPENDS = "virtual/kernel" -RDEPENDS:${PN}-ptest += "bash rt-tests fio iperf3 python3 python3-pip" +RDEPENDS:${PN}-ptest += "bash rt-tests fio iperf3 python3 python3-pip docker" RDEPENDS:${PN}-ptest:append:x64 = " fw-printenv" RDEPENDS:${PN}-ptest:append:armv7a = " u-boot-fw-utils" @@ -27,6 +27,8 @@ SRC_URI += "\ file://test_kernel_cyclictest_hackbench.sh \ file://test_kernel_cyclictest_fio.sh \ file://test_kernel_cyclictest_iperf.sh \ + file://run-container-load \ + file://test_kernel_cyclictest_container_load.sh \ " do_install_ptest:append() { @@ -39,6 +41,8 @@ do_install_ptest:append() { install -m 0755 ${S}/test_kernel_cyclictest_hackbench.sh ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_fio.sh ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_iperf.sh ${D}${PTEST_PATH} + install -m 0755 ${S}/run-container-load ${D}${PTEST_PATH} + install -m 0755 ${S}/test_kernel_cyclictest_container_load.sh ${D}${PTEST_PATH} } pkg_postinst_ontarget:${PN}-ptest:append() { From 8cd2c623ff04c6f51014afe5a89d3e6bc809d487 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 1 Dec 2023 14:28:18 -0500 Subject: [PATCH 003/147] layer.conf: assert nanbield compatibility Signed-off-by: Alex Stewart --- conf/layer.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/layer.conf b/conf/layer.conf index ff6fb4a6a..dfc47a16a 100644 --- a/conf/layer.conf +++ b/conf/layer.conf @@ -9,4 +9,4 @@ BBFILE_COLLECTIONS += "meta-nilrt" BBFILE_PATTERN_meta-nilrt = "^${LAYERDIR}/" BBFILE_PRIORITY_meta-nilrt = "25" -LAYERSERIES_COMPAT_meta-nilrt = "kirkstone" +LAYERSERIES_COMPAT_meta-nilrt = "nanbield" From 0b1fe79ebddc6723384539e414bf0b4bbae06ba5 Mon Sep 17 00:00:00 2001 From: Mike Petersen Date: Fri, 15 Sep 2023 13:30:46 -0500 Subject: [PATCH 004/147] linux-nilrt-nohz: Downgrade to 5.15 kernel Some NI driver teams have noticed increased jitter on the 6.1 NO_HZ kernel, which has not yet been addressed. Since this issue was not present in the 5.15 kernel, we can downgrade this recipe to 5.15 to avoid performance regressions in the 2023Q4 release. Signed-off-by: Mike Petersen (cherry picked from commit e805904f94f98e9c37f573a7908a99aff4786b58) Rebased-by: Alex Stewart Rebased to apply the same change to the kirkstone master branch. Signed-off-by: Alex Stewart (cherry picked from commit afb05b05939c4459eb10fdd51d0ba8148cea217d) Signed-off-by: Alex Stewart --- recipes-kernel/linux/linux-nilrt-nohz_git.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/linux/linux-nilrt-nohz_git.bb b/recipes-kernel/linux/linux-nilrt-nohz_git.bb index 6e639fcc2..e51a80897 100644 --- a/recipes-kernel/linux/linux-nilrt-nohz_git.bb +++ b/recipes-kernel/linux/linux-nilrt-nohz_git.bb @@ -1,6 +1,6 @@ DESCRIPTION = "NILRT linux kernel full dynamic ticks (NO_HZ_FULL) build" NI_RELEASE_VERSION = "master" -LINUX_VERSION = "6.1" +LINUX_VERSION = "5.15" LINUX_KERNEL_TYPE = "nohz" require linux-nilrt-alternate.inc From 15cd89f8f0b658d7ae40be49c4aeac83b129ec80 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 5 Dec 2023 12:41:32 -0500 Subject: [PATCH 005/147] ni-rtfeatures: fixup spelling in initscript Signed-off-by: Alex Stewart (cherry picked from commit b279f07a0472e3256a0b34362be2bc9947f55ba0) Signed-off-by: Alex Stewart --- recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd b/recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd index 3ea8f4d60..735310d56 100755 --- a/recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd +++ b/recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd @@ -56,7 +56,7 @@ log_reset_source () { log INFO "reset_source=${reason} # RT watchdog timer expired" ;; *) - log ERROR "Unkown reset_source string: \"$reason\"" + log ERROR "Unknown reset_source string: \"$reason\"" ;; esac } From e7a3d3ea26efabb0a7fc34feaf0efa198c396081 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 5 Dec 2023 12:51:12 -0500 Subject: [PATCH 006/147] ni-rtfeatures: handle poweron reset_source When the reset_source register is `0x00` (no flags) - as in cases where the system is totally powered off and then on - the ni-rtfeatures driver writes "poweron" into the reset_source sysfs entry. The ni-rtfeatures initscript does not handle this string, and warns that the sysfs is malformed. Handle this case in the initscript, and provide some useful logging output. Signed-off-by: Alex Stewart (cherry picked from commit b42be87dfbd2a855d3c01c8ae409c41c7aac85a2) Signed-off-by: Alex Stewart --- recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd | 3 +++ recipes-ni/ni-rtfeatures/ni-rtfeatures.bb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd b/recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd index 735310d56..a35b22955 100755 --- a/recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd +++ b/recipes-ni/ni-rtfeatures/files/ni-rtfeatures.initd @@ -46,6 +46,9 @@ log_reset_source () { ironclad) log INFO "reset_source=${reason} # Ironclad watchdog timer expired" ;; + poweron) + log INFO "reset_source=${reason} # System powered-on; not reset." + ;; processor) log INFO "reset_source=${reason} # Reset from MAX or command line" ;; diff --git a/recipes-ni/ni-rtfeatures/ni-rtfeatures.bb b/recipes-ni/ni-rtfeatures/ni-rtfeatures.bb index 1e674c12e..ee030a53b 100644 --- a/recipes-ni/ni-rtfeatures/ni-rtfeatures.bb +++ b/recipes-ni/ni-rtfeatures/ni-rtfeatures.bb @@ -8,7 +8,7 @@ LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda DEPENDS += "update-rc.d-native" -PV = "2.0" +PV = "2.1" SRC_URI += "\ From d553da29b18fff2ebd9143a8c8ea9c23f5b182d2 Mon Sep 17 00:00:00 2001 From: Dylan Turner Date: Tue, 5 Dec 2023 10:50:46 -0600 Subject: [PATCH 007/147] k-cont-perf-tests: print run-cyclictest output run-cyclictest is run from a container, and although the output is logged, it is never printed, meaning the tests do not print the PASS/FAIL messages required of ptests Print the logs to have this output. Signed-off-by: Dylan Turner (cherry picked from commit faa26152949e9dc3b454c210e9943c5e3a72d59a) Signed-off-by: Alex Stewart --- .../cyclictest-container/call_run_ct.sh | 2 +- .../test_kernel_cyclictest_fio_containerized.sh | 4 +++- .../test_kernel_cyclictest_hackbench_containerized.sh | 4 +++- .../test_kernel_cyclictest_idle_containerized.sh | 4 +++- .../test_kernel_cyclictest_iperf_containerized.sh | 4 +++- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/cyclictest-container/call_run_ct.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/cyclictest-container/call_run_ct.sh index 7e2f3c223..cd897943c 100755 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/cyclictest-container/call_run_ct.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/cyclictest-container/call_run_ct.sh @@ -1,7 +1,7 @@ #!/bin/sh cd /ptests source ./run-cyclictest -TEST_LOG=${LOG_DIR}/run_cyclictest-`date +'%Y_%m_%d-%H_%M_%S'`-${1}.log +TEST_LOG=${LOG_DIR}/run_cyclictest-${1}.log run_cyclictest "${1}" > ${TEST_LOG} 2>&1 echo "${CYCLICTEST_RESULT}" diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh index 1e01d1888..1cd101e3c 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh @@ -37,9 +37,11 @@ RESULT=$(docker run --privileged --network=host \ bash call_run_ct.sh "fio_containerized" \ | tr -d '\r' | tr -d '\n') +# Make sure we print the PASS/FAIL message +cat ${LOG_DIR}/run_cyclictest-fio_containerized.log + # Clean up the background container docker exec ${LOAD_CONT} bash -c "killall -INT fio > /dev/null 2>&1" -echo "Test Result: ${RESULT}" exit ${RESULT} diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh index b1aaa06d6..25e3ccc27 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh @@ -37,10 +37,12 @@ RESULT=$(docker run --privileged --network=host \ bash call_run_ct.sh "hackbench_containerized" \ | tr -d '\r' | tr -d '\n') +# Make sure we print the PASS/FAIL message +cat ${LOG_DIR}/run_cyclictest-hackbench_containerized.log + # Clean up the background container docker exec ${LOAD_CONT} \ bash -c "killall -INT hackbench > /dev/null 2>&1" -echo "Test Result: ${RESULT}" exit ${RESULT} diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh index 0c2a87352..9cf74f401 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh @@ -24,6 +24,8 @@ RESULT=$(docker run --privileged --network=host \ bash call_run_ct.sh "idle_containerized" \ | tr -d '\r' | tr -d '\n') -echo "Test result: ${RESULT}" +# Make sure we print the PASS/FAIL message +cat ${LOG_DIR}/run_cyclictest-idle_containerized.log + exit ${RESULT} diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh index 8b26264af..18f8c6f9d 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh @@ -54,10 +54,12 @@ RESULT=$(docker run --privileged --network=host \ bash call_run_ct.sh "iperf_containerized" \ | tr -d '\r' | tr -d '\n') +# Make sure we print the PASS/FAIL message +cat ${LOG_DIR}/run_cyclictest-iperf_containerized.log + # Clean up the background container docker exec ${LOAD_CONT} \ bash -c "killall -INT iperf3 > /dev/null 2>&1" -echo "Test result: ${RESULT}" exit ${RESULT} From f0d5fe632489e37b57a036d8b6f07c32cacc22d0 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 11 Dec 2023 17:10:48 -0500 Subject: [PATCH 008/147] ptest-runner: add a user/group for ptest As of OE-core commit d58cf08421e0f89840820370dd3c5b6e153efcbb, ptest-runner now creates a user and group called `ptest`. Since NILRT uses static uid/gids, we have to reserve a slot for this account. Signed-off-by: Alex Stewart --- files/group | 1 + files/passwd | 1 + 2 files changed, 2 insertions(+) diff --git a/files/group b/files/group index 4b09a0a59..40b4358cf 100644 --- a/files/group +++ b/files/group @@ -6,6 +6,7 @@ ni:x:500: openvpn:x:499: niwscerts:x:498: # free space +ptest:x:401: ossec:x:400: mysql:x:399: datetime:x:398: diff --git a/files/passwd b/files/passwd index 6cf8186f8..1e2f796a1 100644 --- a/files/passwd +++ b/files/passwd @@ -6,6 +6,7 @@ webserv:x:501:::: lvuser:x:500:::: openvpn:x:499:::: # free space +ptest:x:401:::: ossec:x:400:::: mysql:x:399:::: polkitd:x:398:::: From 6cabd057c17c8e6d8875c63bf88ec78683dd2a34 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 11 Dec 2023 17:14:07 -0500 Subject: [PATCH 009/147] mousepad: upgrade bbappend to 0.6.% Currency upgrade for meta-oe nanbield. Signed-off-by: Alex Stewart --- .../mousepad/{mousepad_0.5.%.bbappend => mousepad_0.6.%.bbappend} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename recipes-apps/mousepad/{mousepad_0.5.%.bbappend => mousepad_0.6.%.bbappend} (100%) diff --git a/recipes-apps/mousepad/mousepad_0.5.%.bbappend b/recipes-apps/mousepad/mousepad_0.6.%.bbappend similarity index 100% rename from recipes-apps/mousepad/mousepad_0.5.%.bbappend rename to recipes-apps/mousepad/mousepad_0.6.%.bbappend From 3ec34370b0b40a93a43e8126537253a962c689b0 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 11 Dec 2023 17:15:49 -0500 Subject: [PATCH 010/147] procps: make bbappend apply to all versions The changes applied by the procps.bbappend are generic enough that they should apply to all versions of procps. Instead of reversioning the bbappend on each OE rebase, just make the append version match generic. Signed-off-by: Alex Stewart --- .../procps/{procps_3.%.bbappend => procps_%.bbappend} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename recipes-extended/procps/{procps_3.%.bbappend => procps_%.bbappend} (100%) diff --git a/recipes-extended/procps/procps_3.%.bbappend b/recipes-extended/procps/procps_%.bbappend similarity index 100% rename from recipes-extended/procps/procps_3.%.bbappend rename to recipes-extended/procps/procps_%.bbappend From f540e351c5bda2304fa21e49d464df78a0753bcf Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 12 Dec 2023 15:49:39 -0500 Subject: [PATCH 011/147] gnupg: remove bbappend The meta-nilrt gnupg bbappend only ignores a CVE, which has been fixed in master. Since we are now based on upstream master refs, this ignore-rule is no longer needed. Remove it and drop the empty bbappend. Signed-off-by: Alex Stewart --- recipes-support/gnupg/gnupg_2.3.%.bbappend | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 recipes-support/gnupg/gnupg_2.3.%.bbappend diff --git a/recipes-support/gnupg/gnupg_2.3.%.bbappend b/recipes-support/gnupg/gnupg_2.3.%.bbappend deleted file mode 100644 index 3ffb2c584..000000000 --- a/recipes-support/gnupg/gnupg_2.3.%.bbappend +++ /dev/null @@ -1,5 +0,0 @@ -# libksba has already been upgraded to version 1.6.3 in openembedded-core. -# However, this CVE is still getting flagged as other configurations call -# out gnupg 2.3.0 (inclusive) - 2.4.0 (exclusive). Upstream has already -# bumped gnupg in master so let's just ignore the CVE here. -CVE_CHECK_IGNORE:append = " CVE-2022-3515" From aa156cffb38b219bd6a9cf5919820e282bfc01dc Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 12 Dec 2023 15:54:16 -0500 Subject: [PATCH 012/147] syslog-ng: upgrade bbappend to 4.% Currency upgrade for meta-oe nanbield. Signed-off-by: Alex Stewart --- .../syslog-ng/{syslog-ng_3.%.bbappend => syslog-ng_4.%.bbappend} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename recipes-support/syslog-ng/{syslog-ng_3.%.bbappend => syslog-ng_4.%.bbappend} (100%) diff --git a/recipes-support/syslog-ng/syslog-ng_3.%.bbappend b/recipes-support/syslog-ng/syslog-ng_4.%.bbappend similarity index 100% rename from recipes-support/syslog-ng/syslog-ng_3.%.bbappend rename to recipes-support/syslog-ng/syslog-ng_4.%.bbappend From e1e26ad64ea300a9151f336c994e66949ac7008f Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 12 Dec 2023 15:56:34 -0500 Subject: [PATCH 013/147] onboard: remove recipe Upstream meta-OE has dropped the onboard virtual keyboard recipe, so meta-nilrt's bbappends have nothing to apply to. Until we can find a new solution or resurrect the onboard recipe, remove it from meta-nilrt. Signed-off-by: Alex Stewart --- .../packagegroups/packagefeed-ni-extra.bb | 1 - .../packagegroup-ni-graphical.bb | 1 - ...001-add-xfce-to-autostart-onlyshowin.patch | 10 --- .../onboard/files/01-gnome-accessibility | 3 - recipes-support/onboard/files/NI.colors | 62 ------------------- recipes-support/onboard/files/NI.theme | 16 ----- .../onboard/files/onboard-defaults.conf | 13 ---- recipes-support/onboard/onboard_%.bbappend | 38 ------------ 8 files changed, 144 deletions(-) delete mode 100644 recipes-support/onboard/files/0001-add-xfce-to-autostart-onlyshowin.patch delete mode 100644 recipes-support/onboard/files/01-gnome-accessibility delete mode 100644 recipes-support/onboard/files/NI.colors delete mode 100644 recipes-support/onboard/files/NI.theme delete mode 100644 recipes-support/onboard/files/onboard-defaults.conf delete mode 100644 recipes-support/onboard/onboard_%.bbappend diff --git a/recipes-core/packagegroups/packagefeed-ni-extra.bb b/recipes-core/packagegroups/packagefeed-ni-extra.bb index a4d283423..445243f35 100644 --- a/recipes-core/packagegroups/packagefeed-ni-extra.bb +++ b/recipes-core/packagegroups/packagefeed-ni-extra.bb @@ -26,7 +26,6 @@ RDEPENDS:${PN}:append:x64 = "\ networkmanager \ numlockx \ openbox \ - onboard \ sessreg \ setxkbmap \ sysconfig-settings-ui \ diff --git a/recipes-core/packagegroups/packagegroup-ni-graphical.bb b/recipes-core/packagegroups/packagegroup-ni-graphical.bb index 2601f3d9f..5ad251275 100644 --- a/recipes-core/packagegroups/packagegroup-ni-graphical.bb +++ b/recipes-core/packagegroups/packagegroup-ni-graphical.bb @@ -6,6 +6,5 @@ PACKAGE_ARCH = "${MACHINE_ARCH}" inherit packagegroup RDEPENDS:${PN} = "\ - onboard \ packagegroup-ni-xfce \ " diff --git a/recipes-support/onboard/files/0001-add-xfce-to-autostart-onlyshowin.patch b/recipes-support/onboard/files/0001-add-xfce-to-autostart-onlyshowin.patch deleted file mode 100644 index 28f4dff62..000000000 --- a/recipes-support/onboard/files/0001-add-xfce-to-autostart-onlyshowin.patch +++ /dev/null @@ -1,10 +0,0 @@ -diff --git a/data/onboard-autostart.desktop.in b/data/onboard-autostart.desktop.in -index 8fb55ac..cf022a0 100644 ---- a/data/onboard-autostart.desktop.in -+++ b/data/onboard-autostart.desktop.in -@@ -9,4 +9,4 @@ NoDisplay=true - X-Ubuntu-Gettext-Domain=onboard - AutostartCondition=GSettings org.gnome.desktop.a11y.applications screen-keyboard-enabled - X-GNOME-AutoRestart=true --OnlyShowIn=GNOME;Unity;MATE; -+OnlyShowIn=GNOME;Unity;MATE;XFCE; diff --git a/recipes-support/onboard/files/01-gnome-accessibility b/recipes-support/onboard/files/01-gnome-accessibility deleted file mode 100644 index a92f5648d..000000000 --- a/recipes-support/onboard/files/01-gnome-accessibility +++ /dev/null @@ -1,3 +0,0 @@ -# We need accessibility on in order to use Onboard's auto-show. -[org/gnome/desktop/interface] -toolkit-accessibility=true diff --git a/recipes-support/onboard/files/NI.colors b/recipes-support/onboard/files/NI.colors deleted file mode 100644 index 7f8217b92..000000000 --- a/recipes-support/onboard/files/NI.colors +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - - icon0 - - - - icon1, icon2 - - - - - - - icon3, - RCTL, LCTL, RALT, LALT, LWIN, CAPS, - LFSH, RTSH, NMLK, - MENU, RWIN, BKSP, TAB, RTRN, - KPDL, KPEN, KPSU, KPDV, KPAD, KPMU, - LEFT, RGHT, UP, DOWN, INS, DELE, HOME, END, PGUP, PGDN, - F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11, F12, - Prnt, Pause, ESC, Scroll, - secondaryclick, middleclick, doubleclick, dragclick, hoverclick, - hide, showclick, move, layer, - quit - - - - - wordlist, prediction, pause-learning.wordlist, language.wordlist, hide.wordlist - - - - - - - m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14, m15 - - - - - - settings - - - diff --git a/recipes-support/onboard/files/NI.theme b/recipes-support/onboard/files/NI.theme deleted file mode 100644 index c611cd9db..000000000 --- a/recipes-support/onboard/files/NI.theme +++ /dev/null @@ -1,16 +0,0 @@ - - - - NI - flat - 20 - 30 - 70 - -3 - - - diff --git a/recipes-support/onboard/files/onboard-defaults.conf b/recipes-support/onboard/files/onboard-defaults.conf deleted file mode 100644 index daaa5b9ff..000000000 --- a/recipes-support/onboard/files/onboard-defaults.conf +++ /dev/null @@ -1,13 +0,0 @@ -[main] -layout=Compact -theme=NI -key-label-font=DejaVu Sans - -[window] -force-to-top=True - -[auto-show] -# Enable autoshow when there's no keyboard detected. -enabled=True -keyboard-device-detection-enabled=True -keyboard-device-detection-exceptions=['::noserial'] diff --git a/recipes-support/onboard/onboard_%.bbappend b/recipes-support/onboard/onboard_%.bbappend deleted file mode 100644 index 6c5344735..000000000 --- a/recipes-support/onboard/onboard_%.bbappend +++ /dev/null @@ -1,38 +0,0 @@ -FILESEXTRAPATHS:prepend := "${THISDIR}/files:" - - -SRC_URI += " \ - file://0001-add-xfce-to-autostart-onlyshowin.patch \ - file://01-gnome-accessibility \ - file://NI.colors \ - file://NI.theme \ - file://onboard-defaults.conf \ -" - - -do_install:append () { - install -d ${D}${sysconfdir}/dconf/db/local.d - install -d ${D}${sysconfdir}/onboard - install -d ${D}${datadir}/onboard/themes - - install -m 644 ${WORKDIR}/01-gnome-accessibility ${D}${sysconfdir}/dconf/db/local.d/ - install -m 644 ${WORKDIR}/onboard-defaults.conf ${D}${sysconfdir}/onboard/ - - install -m 644 ${WORKDIR}/NI.colors ${D}${datadir}/onboard/themes/ - install -m 644 ${WORKDIR}/NI.theme ${D}${datadir}/onboard/themes/ -} - -pkg_postinst:${PN} () { - dconf update -} - - -CONFFILES:${PN}:append := " \ - ${sysconfdir}/onboard/onboard-defaults.conf \ - ${sysconfdir}/dconf/db/local.d/01-gnome-accessibility \ -" - -RDEPENDS:${PN}:append = " dconf" -# Onboard uses unicode glyphs in its key_defs.xml file, which means -# we need a font that has those glyphs present. -RDEPENDS:${PN}:append = " ttf-dejavu-sans" From 081bbf47c8195533ddf9e5f8f92e42e8f9e2b4da Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 13 Dec 2023 03:55:46 -0500 Subject: [PATCH 014/147] uutils-coreutils: remove selinux PACKAGECONFIG As of nanbield, meta-oe's uutils-coreutils recipe adds a DEPENDS on clang-native, in the case that 'selinux' is selected in the recipe PACKAGECONFIG. However, clang-native is not provided by either meta-oe or meat-selinux. It is only available in meta-clang. NILRT doesn't actually use uutils-coreutils, which is an alternative for the normal coreutils recipe. But the fact that NILRT declares selinux support is enough to activate the uutils-coreutils->clang DEPENDS - which breaks the bitbake task tree generation. Manually remove the offending 'selinux' PACKAGECONFIG, so that uutils-coreutils doesn't break. Signed-off-by: Alex Stewart --- recipes-core/uutils-coreutils/uutils-coreutils_%.bbappend | 1 + 1 file changed, 1 insertion(+) create mode 100644 recipes-core/uutils-coreutils/uutils-coreutils_%.bbappend diff --git a/recipes-core/uutils-coreutils/uutils-coreutils_%.bbappend b/recipes-core/uutils-coreutils/uutils-coreutils_%.bbappend new file mode 100644 index 000000000..13cf1fa1e --- /dev/null +++ b/recipes-core/uutils-coreutils/uutils-coreutils_%.bbappend @@ -0,0 +1 @@ +PACKAGECONFIG:remove = "selinux" From e9e77a49ea91d4b3d4a75005d159d3700be919c9 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 13 Dec 2023 05:17:22 -0500 Subject: [PATCH 015/147] curl; upgrade bbappend to 8.4.0 Currency upgrade for OE nanbield. This commit drops 4 threadsafety commits which conflict with a new threadsafety implementation from upstream. Signed-off-by: Alex Stewart --- ...ed-curl_easy_getinfo_va-API-function.patch | 26 +- ...2-Added-curl_formadd_va-API-function.patch | 20 +- ...ed-curl_multi_setopt_va-API-function.patch | 28 ++- ...ed-curl_share_setopt_va-API-function.patch | 28 ++- .../0005-Add-nicurl-wrapper-functions.patch | 28 ++- .../curl/0006-Add-global-init-mutex.patch | 229 ------------------ ...07-Made-curl_global_init-thread-safe.patch | 66 ----- ...Made-curl_global_cleanup-thread-safe.patch | 49 ---- ...0009-Made-curl_easy_init-thread-safe.patch | 38 --- .../{curl_7.%.bbappend => curl_8.%.bbappend} | 4 - 10 files changed, 86 insertions(+), 430 deletions(-) delete mode 100644 recipes-support/curl/curl/0006-Add-global-init-mutex.patch delete mode 100644 recipes-support/curl/curl/0007-Made-curl_global_init-thread-safe.patch delete mode 100644 recipes-support/curl/curl/0008-Made-curl_global_cleanup-thread-safe.patch delete mode 100644 recipes-support/curl/curl/0009-Made-curl_easy_init-thread-safe.patch rename recipes-support/curl/{curl_7.%.bbappend => curl_8.%.bbappend} (66%) diff --git a/recipes-support/curl/curl/0001-Added-curl_easy_getinfo_va-API-function.patch b/recipes-support/curl/curl/0001-Added-curl_easy_getinfo_va-API-function.patch index 4d16290ff..d13ac72cd 100644 --- a/recipes-support/curl/curl/0001-Added-curl_easy_getinfo_va-API-function.patch +++ b/recipes-support/curl/curl/0001-Added-curl_easy_getinfo_va-API-function.patch @@ -1,18 +1,31 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Jonathan David +From bf9638c7d70aecdffc6ded02c9fdd562d47d5e53 Mon Sep 17 00:00:00 2001 +From: Ioan-Adrian Ratiu Date: Mon, 21 Jul 2014 15:02:35 -0500 Subject: [PATCH] Added curl_easy_getinfo_va() API function Added API function with `va_list` variable argument parameter in addition to cURL's function using `...`. -Upstream-Status: Submitted [curl-library@cool.haxx.se] +Upstream-Status: Denied + +https://curl.se/mail/lib-2017-01/0135.html + +This patchset was denied by upstream because they did not agree that our +usecase for these added functions were valid - and we did not have a +defense. + +https://curl.se/mail/lib-2017-01/0146.html + +Signed-off-by: Alex Stewart +--- + lib/easy.c | 17 +++++++++++++---- + 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/easy.c b/lib/easy.c -index a69eb9e56..9a5f09fce 100644 +index 6b4fb8e..8e06b16 100644 --- a/lib/easy.c +++ b/lib/easy.c -@@ -745,18 +745,27 @@ void curl_easy_cleanup(struct Curl_easy *data) +@@ -819,18 +819,27 @@ void curl_easy_cleanup(struct Curl_easy *data) * information from a performed transfer and similar. */ #undef curl_easy_getinfo @@ -44,6 +57,3 @@ index a69eb9e56..9a5f09fce 100644 return result; } --- -2.28.0 - diff --git a/recipes-support/curl/curl/0002-Added-curl_formadd_va-API-function.patch b/recipes-support/curl/curl/0002-Added-curl_formadd_va-API-function.patch index c339eae85..f495507f1 100644 --- a/recipes-support/curl/curl/0002-Added-curl_formadd_va-API-function.patch +++ b/recipes-support/curl/curl/0002-Added-curl_formadd_va-API-function.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From 9414d7a122c50f7834a5783d5ad69b9635870fe2 Mon Sep 17 00:00:00 2001 From: Jonathan David Date: Tue, 15 Mar 2016 14:13:28 -0500 Subject: [PATCH] Added curl_formadd_va() API function @@ -6,10 +6,21 @@ Subject: [PATCH] Added curl_formadd_va() API function Added API function with `va_list` variable argument parameter in addition to cURL's function using `...`. -Upstream-Status: Submitted [curl-library@cool.haxx.se] +Upstream-Status: Denied + +This patchset was denied by upstream because they did not agree that our +usecase for these added functions were valid - and we did not have a +defense. + +https://curl.se/mail/lib-2017-01/0146.html + +Signed-off-by: Alex Stewart +--- + lib/formdata.c | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/formdata.c b/lib/formdata.c -index 1cab2c5c2..1c85b2a1d 100644 +index e40c4bc..11f142f 100644 --- a/lib/formdata.c +++ b/lib/formdata.c @@ -690,6 +690,13 @@ CURLFORMcode FormAdd(struct curl_httppost **httppost, @@ -35,6 +46,3 @@ index 1cab2c5c2..1c85b2a1d 100644 va_end(arg); return result; } --- -2.28.0 - diff --git a/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch b/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch index c73ec124a..60d38abca 100644 --- a/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch +++ b/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From 7646de9036508744501df06a919ca431ffa709c9 Mon Sep 17 00:00:00 2001 From: Jonathan David Date: Mon, 21 Jul 2014 14:32:25 -0500 Subject: [PATCH] Added curl_multi_setopt_va() API function @@ -6,13 +6,26 @@ Subject: [PATCH] Added curl_multi_setopt_va() API function Added API function with `va_list` variable argument parameter in addition to cURL's function using `...`. -Upstream-Status: Submitted [curl-library@cool.haxx.se] +Upstream-Status: Denied + +https://curl.se/mail/lib-2017-01/0138.html + +This patchset was denied by upstream because they did not agree that our +usecase for these added functions were valid - and we did not have a +defense. + +https://curl.se/mail/lib-2017-01/0146.html + +Signed-off-by: Alex Stewart +--- + lib/multi.c | 22 ++++++++++++++++------ + 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/multi.c b/lib/multi.c -index 3c7fb85ed..4cd825d6c 100644 +index ff753ac..6c35014 100644 --- a/lib/multi.c +++ b/lib/multi.c -@@ -2914,12 +2914,10 @@ static CURLMcode multi_socket(struct Curl_multi *multi, +@@ -3290,12 +3290,10 @@ static CURLMcode multi_socket(struct Curl_multi *multi, return result; } @@ -27,7 +40,7 @@ index 3c7fb85ed..4cd825d6c 100644 if(!GOOD_MULTI_HANDLE(multi)) return CURLM_BAD_HANDLE; -@@ -2927,8 +2925,6 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, +@@ -3303,8 +3301,6 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, if(multi->in_callback) return CURLM_RECURSIVE_API_CALL; @@ -36,7 +49,7 @@ index 3c7fb85ed..4cd825d6c 100644 switch(option) { case CURLMOPT_SOCKETFUNCTION: multi->socket_cb = va_arg(param, curl_socket_callback); -@@ -2983,7 +2979,21 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, +@@ -3359,7 +3355,21 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, res = CURLM_UNKNOWN_OPTION; break; } @@ -58,6 +71,3 @@ index 3c7fb85ed..4cd825d6c 100644 return res; } --- -2.28.0 - diff --git a/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch b/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch index 7dfdc4194..c350c5e18 100644 --- a/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch +++ b/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From 16f0b287ad5dfda7df059094464a26620d0d5ff7 Mon Sep 17 00:00:00 2001 From: Jonathan David Date: Mon, 21 Jul 2014 14:25:07 -0500 Subject: [PATCH] Added curl_share_setopt_va() API function @@ -6,13 +6,26 @@ Subject: [PATCH] Added curl_share_setopt_va() API function Added API function with `va_list` variable argument parameter in addition to cURL's function using `...`. -Upstream-Status: Submitted [curl-library@cool.haxx.se] +Upstream-Status: Denied + +https://curl.se/mail/lib-2017-01/0139.html + +This patchset was denied by upstream because they did not agree that our +usecase for these added functions were valid - and we did not have a +defense. + +https://curl.se/mail/lib-2017-01/0146.html + +Signed-off-by: Alex Stewart +--- + lib/share.c | 18 +++++++++++++----- + 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/share.c b/lib/share.c -index a2d896042..ab215daf9 100644 +index c0a8d80..60a4b5d 100644 --- a/lib/share.c +++ b/lib/share.c -@@ -48,11 +48,9 @@ curl_share_init(void) +@@ -49,11 +49,9 @@ curl_share_init(void) return share; } @@ -25,7 +38,7 @@ index a2d896042..ab215daf9 100644 int type; curl_lock_function lockfunc; curl_unlock_function unlockfunc; -@@ -64,8 +62,6 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) +@@ -68,8 +66,6 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) using this share */ return CURLSHE_IN_USE; @@ -34,7 +47,7 @@ index a2d896042..ab215daf9 100644 switch(option) { case CURLSHOPT_SHARE: /* this is a type this share will share */ -@@ -176,6 +172,18 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) +@@ -202,6 +198,18 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) break; } @@ -53,6 +66,3 @@ index a2d896042..ab215daf9 100644 va_end(param); return res; --- -2.28.0 - diff --git a/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch b/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch index 305785c99..0d3fa010e 100644 --- a/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch +++ b/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From d0899b9330a87677eff449f8eddcd544d3eeb1b8 Mon Sep 17 00:00:00 2001 From: Jonathan David Date: Tue, 15 Mar 2016 13:26:34 -0500 Subject: [PATCH] Add nicurl wrapper functions @@ -14,12 +14,19 @@ These functions are called by older NI components, and exist as a means to keep previous code running. Upstream-Status: Inappropriate [NI specific changes] +Signed-off-by: Alex Stewart +--- + lib/easy.c | 21 +++++++++++++++++++++ + lib/formdata.c | 12 ++++++++++++ + lib/multi.c | 11 +++++++++++ + lib/share.c | 11 +++++++++++ + 4 files changed, 55 insertions(+) diff --git a/lib/easy.c b/lib/easy.c -index 9a5f09fce..1e2b0aaa4 100644 +index 8e06b16..0abddf3 100644 --- a/lib/easy.c +++ b/lib/easy.c -@@ -308,6 +308,16 @@ struct Curl_easy *curl_easy_init(void) +@@ -382,6 +382,16 @@ struct Curl_easy *curl_easy_init(void) return data; } @@ -36,7 +43,7 @@ index 9a5f09fce..1e2b0aaa4 100644 #ifdef CURLDEBUG struct socketmonitor { -@@ -769,6 +779,17 @@ CURLcode curl_easy_getinfo(struct Curl_easy *data, CURLINFO info, ...) +@@ -843,6 +853,17 @@ CURLcode curl_easy_getinfo(struct Curl_easy *data, CURLINFO info, ...) return result; } @@ -55,7 +62,7 @@ index 9a5f09fce..1e2b0aaa4 100644 { CURLcode result = CURLE_OK; diff --git a/lib/formdata.c b/lib/formdata.c -index 1c85b2a1d..a5a560acf 100644 +index 11f142f..60de2f8 100644 --- a/lib/formdata.c +++ b/lib/formdata.c @@ -709,6 +709,18 @@ CURLFORMcode curl_formadd(struct curl_httppost **httppost, @@ -78,10 +85,10 @@ index 1c85b2a1d..a5a560acf 100644 * curl_formget() * Serialize a curl_httppost struct. diff --git a/lib/multi.c b/lib/multi.c -index 4cd825d6c..13c8c57ff 100644 +index 6c35014..8828962 100644 --- a/lib/multi.c +++ b/lib/multi.c -@@ -2997,6 +2997,17 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, +@@ -3373,6 +3373,17 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, return res; } @@ -100,10 +107,10 @@ index 4cd825d6c..13c8c57ff 100644 #undef curl_multi_socket diff --git a/lib/share.c b/lib/share.c -index ab215daf9..a57e78b7b 100644 +index 60a4b5d..5acb4de 100644 --- a/lib/share.c +++ b/lib/share.c -@@ -189,6 +189,17 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) +@@ -215,6 +215,17 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) return res; } @@ -121,6 +128,3 @@ index ab215daf9..a57e78b7b 100644 CURLSHcode curl_share_cleanup(struct Curl_share *share) { --- -2.28.0 - diff --git a/recipes-support/curl/curl/0006-Add-global-init-mutex.patch b/recipes-support/curl/curl/0006-Add-global-init-mutex.patch deleted file mode 100644 index 10911b9ab..000000000 --- a/recipes-support/curl/curl/0006-Add-global-init-mutex.patch +++ /dev/null @@ -1,229 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Haris Okanovic -Date: Tue, 17 May 2016 12:57:33 -0500 -Subject: [PATCH] Add global init mutex - -Add global_mutex.cpp/h to allow for thread-safe (un)initialization -of libcurl. - -Upstream-Status: Inappropriate [NIBuild only] - -diff --git a/lib/global_mutex.cpp b/lib/global_mutex.cpp -new file mode 100644 -index 000000000..72f202eec ---- /dev/null -+++ b/lib/global_mutex.cpp -@@ -0,0 +1,186 @@ -+/* Implementation of curl_global_mutex_lock() and curl_global_mutex_unlock(). -+ * -+ * @note This C++ source file MUST NOT throw exceptions, use RTTI, or -+ * declare try/catch blocks, as doing so would result in unpredictable -+ * behavior in this C DLL since it doesn't link to libstdc++. -+ * Nothing in this file is even capable of throwing since every -+ * external function we call is a C API of the operating system. -+ * -+ * ToDo: Rewrite this in C instead of C++. -+ */ -+ -+#include -+#include -+#include "global_mutex.h" -+ -+#ifdef _WIN32 -+ #include -+#elif defined VXWORKS_PPC -+ #include -+#elif defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) || defined(linux) || defined(__linux__) || defined(__gnu_linux__) || defined(__linux) -+ #include -+#else -+ #error "Unsupported platform" -+#endif -+ -+namespace -+{ -+#if defined(linux) || defined(__linux__) || defined(__gnu_linux__) || defined(__linux) -+ static pthread_mutex_t mutex_ = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; -+#endif -+ -+#if defined(_WIN32) || defined(VXWORKS_PPC) || defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) -+ -+ static volatile bool g_cs_initialized = false; -+ -+ struct _CriticalSection -+ { -+#ifdef _WIN32 -+ CRITICAL_SECTION cs_; -+#elif defined VXWORKS_PPC -+ SEM_ID sem_; -+#elif defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) -+ // ToDo: Switch Mac to PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP once -+ // toolchain supports it -+ pthread_mutex_t mutex_; -+#endif -+ -+ inline _CriticalSection() -+ { -+#ifdef _WIN32 -+ InitializeCriticalSection(&cs_); -+#elif defined VXWORKS_PPC -+ sem_ = semMCreate(SEM_Q_PRIORITY | SEM_DELETE_SAFE | SEM_INVERSION_SAFE); -+ if (sem_ == NULL) -+ { -+ perror("cURL fatal error: Error initializing mutex"); -+ exit(-1); -+ } -+#elif defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) -+ /* PTHREAD_RECURSIVE_MUTEX_INITIALIZER not supported by xcode3.2-gcc4.2 -+ * create a mutex that can be locked recursively by the same thread */ -+ pthread_mutexattr_t attr; -+ if (pthread_mutexattr_init(&attr)) -+ { -+ perror("cURL fatal error: Error initializing mutex attribute"); -+ exit(-1); -+ } -+ if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) -+ { -+ perror("cURL fatal error: Error setting mutex recursive attribute"); -+ exit(-1); -+ } -+ if (pthread_mutex_init(&mutex_, &attr)) -+ { -+ perror("cURL fatal error: Error initializing mutex"); -+ exit(-1); -+ } -+ if (pthread_mutexattr_destroy(&attr)) -+ { -+ perror("cURL fatal error: Error destroying mutex attribute"); -+ exit(-1); -+ } -+#endif -+ g_cs_initialized = true; -+ } -+ -+ -+ inline ~_CriticalSection() -+ { -+ g_cs_initialized = false; -+#ifdef _WIN32 -+ DeleteCriticalSection(&cs_); -+#elif defined VXWORKS_PPC -+ if (semDelete(sem_)) -+ { -+ perror("cURL fatal error: Error deleting mutex"); -+ exit(-1); -+ } -+#elif defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) -+ if (pthread_mutex_destroy(&mutex_)) -+ { -+ perror("cURL fatal error: Error deleting mutex"); -+ exit(-1); -+ } -+#endif -+ } -+ -+ inline void lock() -+ { -+#ifdef _WIN32 -+ EnterCriticalSection(&cs_); -+#elif defined VXWORKS_PPC -+ if (semTake(sem_, WAIT_FOREVER)) -+ { -+ perror("cURL fatal error: Error locking mutex"); -+ exit(-1); -+ } -+#elif defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) -+ if (pthread_mutex_lock(&mutex_)) -+ { -+ perror("cURL fatal error: Error locking mutex"); -+ exit(-1); -+ } -+#endif -+ } -+ -+ inline void unlock() -+ { -+#ifdef _WIN32 -+ LeaveCriticalSection(&cs_); -+#elif defined VXWORKS_PPC -+ if (semGive(sem_)) -+ { -+ perror("cURL fatal error: Error unlocking mutex"); -+ exit(-1); -+ } -+#elif defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) -+ if (pthread_mutex_unlock(&mutex_)) -+ { -+ perror("cURL fatal error: Error unlocking mutex"); -+ exit(-1); -+ } -+#endif -+ } -+ }; -+ -+ static _CriticalSection g_cs; -+#endif -+} -+ -+extern "C" -+{ -+ void curl_global_mutex_lock() -+ { -+ // Do NOT add try/catch blocks here. See comments above for details. -+#if defined(_WIN32) || defined(VXWORKS_PPC) || defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) -+ if (g_cs_initialized) -+ { -+ g_cs.lock(); -+ } -+#elif defined(linux) || defined(__linux__) || defined(__gnu_linux__) || defined(__linux) -+ if (pthread_mutex_lock(&mutex_)) -+ { -+ perror("cURL fatal error: Error locking mutex"); -+ exit(-1); -+ } -+#endif -+ } -+ -+ void curl_global_mutex_unlock() -+ { -+ // Do NOT add try/catch blocks here. See comments above for details. -+#if defined(_WIN32) || defined(VXWORKS_PPC) || defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) -+ if (g_cs_initialized) -+ { -+ g_cs.unlock(); -+ } -+#elif defined(linux) || defined(__linux__) || defined(__gnu_linux__) || defined(__linux) -+ if (pthread_mutex_unlock(&mutex_)) -+ { -+ perror("cURL fatal error: Error unlocking mutex"); -+ exit(-1); -+ } -+#endif -+ } -+} -diff --git a/lib/global_mutex.h b/lib/global_mutex.h -new file mode 100644 -index 000000000..87aae31e5 ---- /dev/null -+++ b/lib/global_mutex.h -@@ -0,0 +1,18 @@ -+#ifndef HEADER_CURL_GLOBAL_MUTEX_H -+#define HEADER_CURL_GLOBAL_MUTEX_H -+ -+/* -+ * Implementation of curl_global_mutex_lock() and curl_global_mutex_unlock(). -+ */ -+ -+#ifdef __cplusplus -+extern "C" -+{ -+#endif -+ void curl_global_mutex_lock(); -+ void curl_global_mutex_unlock(); -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* HEADER_CURL_GLOBAL_MUTEX_H */ --- -2.28.0 - diff --git a/recipes-support/curl/curl/0007-Made-curl_global_init-thread-safe.patch b/recipes-support/curl/curl/0007-Made-curl_global_init-thread-safe.patch deleted file mode 100644 index 8fac6dc11..000000000 --- a/recipes-support/curl/curl/0007-Made-curl_global_init-thread-safe.patch +++ /dev/null @@ -1,66 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Ioan-Adrian Ratiu -Date: Wed, 16 Nov 2016 16:07:23 +0200 -Subject: [PATCH] Made curl_global_init() thread safe - -Originally done by J.David but I had to redo the commit because his patch -had a bad sha1 sum, probably because it was based on a non-public commit. - -Upstream-Status: Inappropriate [creates Linux dependency] - -Signed-off-by: Jonathan David -Signed-off-by: Ioan-Adrian Ratiu - -Updated for curl 7.72. - -Signed-off-by: Richard Tollerton - - -diff --git a/lib/easy.c b/lib/easy.c -index 1e2b0aaa4..8464ef347 100644 ---- a/lib/easy.c -+++ b/lib/easy.c -@@ -21,6 +21,7 @@ - ***************************************************************************/ - - #include "curl_setup.h" -+#include "global_mutex.h" - - /* - * See comment in curl_memory.h for the explanation of this sanity check. -@@ -140,8 +141,17 @@ curl_calloc_callback Curl_ccalloc; - */ - static CURLcode global_init(long flags, bool memoryfuncs) - { -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_lock(); -+#endif -+ - if(initialized++) -+ { -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_unlock(); -+#endif - return CURLE_OK; -+ } - - if(memoryfuncs) { - /* Setup the default memory functions here (again) */ -@@ -202,8 +212,14 @@ static CURLcode global_init(long flags, bool memoryfuncs) - -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_unlock(); -+#endif - return CURLE_OK; - - fail: - initialized--; /* undo the increase */ -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_unlock(); -+#endif - return CURLE_FAILED_INIT; - } - --- -2.28.0 - diff --git a/recipes-support/curl/curl/0008-Made-curl_global_cleanup-thread-safe.patch b/recipes-support/curl/curl/0008-Made-curl_global_cleanup-thread-safe.patch deleted file mode 100644 index 9403fc32f..000000000 --- a/recipes-support/curl/curl/0008-Made-curl_global_cleanup-thread-safe.patch +++ /dev/null @@ -1,49 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Jonathan David -Date: Mon, 21 Jul 2014 15:28:15 -0500 -Subject: [PATCH] Made curl_global_cleanup() thread safe - -Upstream-Status: Inappropriate [creates Linux dependency] - -diff --git a/lib/easy.c b/lib/easy.c -index 8464ef347..b6f4627b6 100644 ---- a/lib/easy.c -+++ b/lib/easy.c -@@ -268,11 +268,24 @@ CURLcode curl_global_init_mem(long flags, curl_malloc_callback m, - */ - void curl_global_cleanup(void) - { -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_lock(); -+#endif - if(!initialized) -+ { -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_unlock(); -+#endif - return; -+ } - - if(--initialized) -+ { -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_unlock(); -+#endif - return; -+ } - - Curl_ssl_cleanup(); - Curl_resolver_global_cleanup(); -@@ -290,6 +303,9 @@ void curl_global_cleanup(void) - #endif - - init_flags = 0; -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_unlock(); -+#endif - } - - /* --- -2.28.0 - diff --git a/recipes-support/curl/curl/0009-Made-curl_easy_init-thread-safe.patch b/recipes-support/curl/curl/0009-Made-curl_easy_init-thread-safe.patch deleted file mode 100644 index 22266b00e..000000000 --- a/recipes-support/curl/curl/0009-Made-curl_easy_init-thread-safe.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Jonathan David -Date: Mon, 21 Jul 2014 15:30:46 -0500 -Subject: [PATCH] Made curl_easy_init() thread safe - -Upstream-Status: Inappropriate [creates Linux dependency] - -diff --git a/lib/easy.c b/lib/easy.c -index b6f4627b6..3b210f077 100644 ---- a/lib/easy.c -+++ b/lib/easy.c -@@ -318,14 +318,23 @@ struct Curl_easy *curl_easy_init(void) - struct Curl_easy *data; - - /* Make sure we inited the global SSL stuff */ -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_lock(); -+#endif - if(!initialized) { - result = curl_global_init(CURL_GLOBAL_DEFAULT); - if(result) { - /* something in the global init failed, return nothing */ - DEBUGF(fprintf(stderr, "Error: curl_global_init failed\n")); -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_unlock(); -+#endif - return NULL; - } - } -+#ifdef CURL_THREAD_SAFE_INIT -+ curl_global_mutex_unlock(); -+#endif - - /* We use curl_open() with undefined URL so far */ - result = Curl_open(&data); --- -2.28.0 - diff --git a/recipes-support/curl/curl_7.%.bbappend b/recipes-support/curl/curl_8.%.bbappend similarity index 66% rename from recipes-support/curl/curl_7.%.bbappend rename to recipes-support/curl/curl_8.%.bbappend index 0e8fb2f18..57e6b1781 100644 --- a/recipes-support/curl/curl_7.%.bbappend +++ b/recipes-support/curl/curl_8.%.bbappend @@ -6,10 +6,6 @@ SRC_URI += " \ file://0003-Added-curl_multi_setopt_va-API-function.patch \ file://0004-Added-curl_share_setopt_va-API-function.patch \ file://0005-Add-nicurl-wrapper-functions.patch \ - file://0006-Add-global-init-mutex.patch \ - file://0007-Made-curl_global_init-thread-safe.patch \ - file://0008-Made-curl_global_cleanup-thread-safe.patch \ - file://0009-Made-curl_easy_init-thread-safe.patch \ " SELECTED_OPTIMIZATION += "-Wno-deprecated-declarations" From 86a44caa6b52260eaee08f488086085d02f97212 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 13 Dec 2023 05:24:34 -0500 Subject: [PATCH 016/147] glibc: add upstream-status trailer to patch As of OE nanbield, bitbake will throw an error if a .patch doesn't have an upstream-status trailer. Add it to the glibc patches. Signed-off-by: Alex Stewart --- recipes-core/glibc/glibc/cp936_support.patch | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipes-core/glibc/glibc/cp936_support.patch b/recipes-core/glibc/glibc/cp936_support.patch index 0f17b7246..3f184a085 100644 --- a/recipes-core/glibc/glibc/cp936_support.patch +++ b/recipes-core/glibc/glibc/cp936_support.patch @@ -2,6 +2,8 @@ From a78510f2e89476bd1405ff6ded58c45e65790db9 Mon Sep 17 00:00:00 2001 From: Ben Shelton Date: Fri, 19 Jun 2020 14:15:16 -0500 +Upstream-Status: Inappropriate [NI-specific] + --- iconv/skeleton.c | 4 + iconvdata/Makefile | 2 +- From eab35334c5576b17680e65d5da3e8d966f746c0a Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 13 Dec 2023 05:51:49 -0500 Subject: [PATCH 017/147] util-linux: drop satisfied backport Drop patches which have been merged into util-linux 2.39.2. Signed-off-by: Alex Stewart --- ...-if-kernel-support-for-btrfs-is-miss.patch | 25 ------------------- .../util-linux/util-linux_2.%.bbappend | 2 -- 2 files changed, 27 deletions(-) delete mode 100644 recipes-core/util-linux/util-linux/0001-skip-btrfs-tests-if-kernel-support-for-btrfs-is-miss.patch diff --git a/recipes-core/util-linux/util-linux/0001-skip-btrfs-tests-if-kernel-support-for-btrfs-is-miss.patch b/recipes-core/util-linux/util-linux/0001-skip-btrfs-tests-if-kernel-support-for-btrfs-is-miss.patch deleted file mode 100644 index 0ab4eb870..000000000 --- a/recipes-core/util-linux/util-linux/0001-skip-btrfs-tests-if-kernel-support-for-btrfs-is-miss.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 8780cf56f80853910d1bba89ac551ecba62bf059 Mon Sep 17 00:00:00 2001 -From: Anatoly Pugachev -Date: Wed, 9 Nov 2022 12:00:12 +0300 -Subject: [PATCH] skip btrfs tests if kernel support for btrfs is missing - -Upstream-Status: Backport [https://git.kernel.org/pub/scm/utils/util-linux/util-linux.git/commit/?id=8b9f571d252f921dab6cfd871bd0be20c58162a2] - -Signed-off-by: Chaitanya Vadrevu ---- - tests/ts/mount/fstab-btrfs | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/tests/ts/mount/fstab-btrfs b/tests/ts/mount/fstab-btrfs -index 0003b5d65..4d45d4dd5 100755 ---- a/tests/ts/mount/fstab-btrfs -+++ b/tests/ts/mount/fstab-btrfs -@@ -51,6 +51,8 @@ DEVICE=$TS_LODEV - [ -d "$TS_MOUNTPOINT_BIND" ] || mkdir -p "$TS_MOUNTPOINT_BIND" - mkfs.btrfs -d single -m single $DEVICE &> /dev/null || ts_die "Cannot make btrfs on $DEVICE" - -+btrfs device ready $DEVICE 2>/dev/null || ts_skip "btrfs kernel support is missing" -+ - $TS_CMD_MOUNT -o loop "$DEVICE" "$TS_MOUNTPOINT_CREATE" - pushd . >/dev/null - cd "$TS_MOUNTPOINT_CREATE" diff --git a/recipes-core/util-linux/util-linux_2.%.bbappend b/recipes-core/util-linux/util-linux_2.%.bbappend index e8a02934f..8ccd09889 100644 --- a/recipes-core/util-linux/util-linux_2.%.bbappend +++ b/recipes-core/util-linux/util-linux_2.%.bbappend @@ -1,7 +1,5 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:" -SRC_URI += "file://0001-skip-btrfs-tests-if-kernel-support-for-btrfs-is-miss.patch" - DEPENDS:append:class-target = " shadow-native pseudo-native busybox" RDEPENDS:${PN}-hwclock:append = " niacctbase busybox-hwclock" From f310f3b4ff639d9319b0a678827f967f6363ffdc Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 13 Dec 2023 06:14:26 -0500 Subject: [PATCH 018/147] initscripts: add upstream-status to patches OE nanbield errors if .patches do not have an Upstream-Status trailer. Add those trailers to the bootmisc .patches. Signed-off-by: Alex Stewart --- .../bootmisc_0001_make_hwclock_authoritative.patch | 3 +++ .../initscripts-1.0/bootmisc_0002_add_sanity_checks.patch | 3 +++ 2 files changed, 6 insertions(+) diff --git a/recipes-core/initscripts/initscripts-1.0/bootmisc_0001_make_hwclock_authoritative.patch b/recipes-core/initscripts/initscripts-1.0/bootmisc_0001_make_hwclock_authoritative.patch index f4e10b68d..16eab3376 100644 --- a/recipes-core/initscripts/initscripts-1.0/bootmisc_0001_make_hwclock_authoritative.patch +++ b/recipes-core/initscripts/initscripts-1.0/bootmisc_0001_make_hwclock_authoritative.patch @@ -1,3 +1,6 @@ +Upstream-Status: Inappropriate [NI specific] + +--- --- a/bootmisc.sh +++ b/bootmisc.sh @@ -63,10 +63,14 @@ diff --git a/recipes-core/initscripts/initscripts-1.0/bootmisc_0002_add_sanity_checks.patch b/recipes-core/initscripts/initscripts-1.0/bootmisc_0002_add_sanity_checks.patch index efcf3ab52..6179ce490 100644 --- a/recipes-core/initscripts/initscripts-1.0/bootmisc_0002_add_sanity_checks.patch +++ b/recipes-core/initscripts/initscripts-1.0/bootmisc_0002_add_sanity_checks.patch @@ -1,3 +1,6 @@ +Upstream-Status: Inappropriate [NI specific] + +--- --- a/bootmisc.sh +++ b/bootmisc.sh @@ -60,6 +60,20 @@ fi From 04e712944966daf776f0d82e36fe4fc0a9b14f63 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 13 Dec 2023 06:39:41 -0500 Subject: [PATCH 019/147] docker-functional-tests: correct docker RDEPENDS The correct RDEPENDS for docker is either 'docker' or 'docker-moby'. docker-ce has been deprecated as of nanbield. Signed-off-by: Alex Stewart --- recipes-ni/docker-functional-tests/docker-functional-tests.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-ni/docker-functional-tests/docker-functional-tests.bb b/recipes-ni/docker-functional-tests/docker-functional-tests.bb index 05877ff40..f679673bf 100644 --- a/recipes-ni/docker-functional-tests/docker-functional-tests.bb +++ b/recipes-ni/docker-functional-tests/docker-functional-tests.bb @@ -31,5 +31,5 @@ do_install_ptest:append() { } PACKAGE_ARCH = "${MACHINE_ARCH}" -RDEPENDS:${PN}-ptest += "bash docker-ce" +RDEPENDS:${PN}-ptest += "bash docker" From 2f5b33057e8eed93e7894b4cedc1ca5e2e52d2bb Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 13 Dec 2023 06:46:36 -0500 Subject: [PATCH 020/147] busybox: fixup upstream-status trailer in patches Signed-off-by: Alex Stewart --- ...-ifplugd.c-Increase-buffer-size-for-netlink-binding.patch | 5 +++-- .../zcip-allow-action-script-to-reject-chosen-IP.patch | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/recipes-core/busybox/busybox/0001-ifplugd.c-Increase-buffer-size-for-netlink-binding.patch b/recipes-core/busybox/busybox/0001-ifplugd.c-Increase-buffer-size-for-netlink-binding.patch index e5326c8e0..3de24371d 100644 --- a/recipes-core/busybox/busybox/0001-ifplugd.c-Increase-buffer-size-for-netlink-binding.patch +++ b/recipes-core/busybox/busybox/0001-ifplugd.c-Increase-buffer-size-for-netlink-binding.patch @@ -17,8 +17,9 @@ for x64-based systems. Signed-off-by: Charlie Johnston -Upstream-Status: Inappropriate [Large numbers of adapters -are uncommon on embedded systems] +Upstream-Status: Inappropriate [NI specific] + +Large numbers of adapters are uncommon on embedded systems --- networking/ifplugd.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/recipes-core/busybox/busybox/zcip-allow-action-script-to-reject-chosen-IP.patch b/recipes-core/busybox/busybox/zcip-allow-action-script-to-reject-chosen-IP.patch index df6151f5b..cd06b465e 100644 --- a/recipes-core/busybox/busybox/zcip-allow-action-script-to-reject-chosen-IP.patch +++ b/recipes-core/busybox/busybox/zcip-allow-action-script-to-reject-chosen-IP.patch @@ -13,6 +13,8 @@ delegate the complexity of this problem to the action script, by letting it reject the chosen IP by returning error 67 (ASCII 'C'). It is assumed that the action script has left the interface deconfigured. +Upstream-Status: Inappropriate [NI specific] + Signed-off-by: Richard Tollerton --- networking/zcip.c | 11 ++++++++++- From 780254b9ceaa5e7ffe98424dece626af140cae5e Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 15 Dec 2023 11:23:12 -0500 Subject: [PATCH 021/147] xinput-calibrator: add upstream-status trailer Signed-off-by: Alex Stewart --- ...settings-and-wrapper-to-desktop-file.patch | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/recipes-graphics/xinput-calibrator/xinput-calibrator/add-xfce-settings-and-wrapper-to-desktop-file.patch b/recipes-graphics/xinput-calibrator/xinput-calibrator/add-xfce-settings-and-wrapper-to-desktop-file.patch index 414a07bd7..6d4370f37 100644 --- a/recipes-graphics/xinput-calibrator/xinput-calibrator/add-xfce-settings-and-wrapper-to-desktop-file.patch +++ b/recipes-graphics/xinput-calibrator/xinput-calibrator/add-xfce-settings-and-wrapper-to-desktop-file.patch @@ -2,16 +2,18 @@ commit 11fa3ee8b684fa41cfc50d36ed598a16faa15943 Author: Alex Stewart Date: Wed Feb 15 14:07:55 2017 -0600 - add xfce-settings and wrapper to .desktop file - - XFCE-Settings categories are necessary for the "calibrate touchscreen" entry to - appear in XFCE's settings dialog. They are necessary here for NI's Linux RT gui. - - The Exec line is changed to point to the wrapper script NI uses. This has to be - done in a patch because it needs to be applied before xinput-calibrator - installs the .desktop file to disk. +add xfce-settings and wrapper to .desktop file + +XFCE-Settings categories are necessary for the "calibrate touchscreen" entry to +appear in XFCE's settings dialog. They are necessary here for NI's Linux RT gui. + +The Exec line is changed to point to the wrapper script NI uses. This has to be +done in a patch because it needs to be applied before xinput-calibrator +installs the .desktop file to disk. + +Upstream-Status: Inappropriate [NI specific] - Signed-off-by: Alex Stewart +Signed-off-by: Alex Stewart diff --git a/scripts/xinput_calibrator.desktop b/scripts/xinput_calibrator.desktop index d888899..17d2cd4 100644 From cf083747c458e2cd680ff928bf30ec5ebe00632a Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 15 Dec 2023 11:31:58 -0500 Subject: [PATCH 022/147] grub: add .patch upstream-status trailers Signed-off-by: Alex Stewart --- recipes-bsp/grub/grub/add-inbit-command-to-io-module.patch | 2 ++ .../grub/cmd-test-Add-bitwise-AND-document-the-feature.patch | 2 ++ .../grub/grub/grub-advertise-NI-NILRT-over-GNU-GRUB.patch | 2 ++ 3 files changed, 6 insertions(+) diff --git a/recipes-bsp/grub/grub/add-inbit-command-to-io-module.patch b/recipes-bsp/grub/grub/add-inbit-command-to-io-module.patch index ded519bb5..2ab3a528f 100644 --- a/recipes-bsp/grub/grub/add-inbit-command-to-io-module.patch +++ b/recipes-bsp/grub/grub/add-inbit-command-to-io-module.patch @@ -5,6 +5,8 @@ Subject: [PATCH] add inbit command to io module * grub-core/commands/iorw.c: Add grub_cmd_read_bit and register inbit command + +Upstream-Status: Inappropriate [NI specific] --- grub-core/commands/iorw.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/recipes-bsp/grub/grub/cmd-test-Add-bitwise-AND-document-the-feature.patch b/recipes-bsp/grub/grub/cmd-test-Add-bitwise-AND-document-the-feature.patch index bd8bd9612..3422f079a 100644 --- a/recipes-bsp/grub/grub/cmd-test-Add-bitwise-AND-document-the-feature.patch +++ b/recipes-bsp/grub/grub/cmd-test-Add-bitwise-AND-document-the-feature.patch @@ -11,6 +11,8 @@ Here, we add a more generic ability to test the bitwise AND of a value available to the grub scripting environment. This obviates the need for the inbit command. +Upstream-Status: Inappropriate [NI specific] + Signed-off-by: Brad Mouring --- docs/grub.texi | 2 ++ diff --git a/recipes-bsp/grub/grub/grub-advertise-NI-NILRT-over-GNU-GRUB.patch b/recipes-bsp/grub/grub/grub-advertise-NI-NILRT-over-GNU-GRUB.patch index 5c9db01c7..2e19fd4cd 100644 --- a/recipes-bsp/grub/grub/grub-advertise-NI-NILRT-over-GNU-GRUB.patch +++ b/recipes-bsp/grub/grub/grub-advertise-NI-NILRT-over-GNU-GRUB.patch @@ -8,6 +8,8 @@ they can be entered anyway. Presumably advanced users already know about them and can use them without being told and this discourages behaviour like "uu what does this button do?" from unknowledgeable users. +Upstream-Status: Inappropriate [NI specific] + Signed-off-by: Ioan-Adrian Ratiu --- grub-core/normal/main.c | 2 +- From 7a6b4abe1f4d735876c3970f03c70e4f5e014a57 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 15 Dec 2023 11:32:16 -0500 Subject: [PATCH 023/147] valgrind: add .patch upstream-status trailers Signed-off-by: Alex Stewart --- .../files/0001-Implement-ficom-instruction-for-amd64.patch | 2 ++ .../valgrind/files/0002-Patch-for-FBLD-FBSTP-FTST.patch | 2 ++ 2 files changed, 4 insertions(+) diff --git a/recipes-devtools/valgrind/files/0001-Implement-ficom-instruction-for-amd64.patch b/recipes-devtools/valgrind/files/0001-Implement-ficom-instruction-for-amd64.patch index 76e85601e..b0aad7cc0 100644 --- a/recipes-devtools/valgrind/files/0001-Implement-ficom-instruction-for-amd64.patch +++ b/recipes-devtools/valgrind/files/0001-Implement-ficom-instruction-for-amd64.patch @@ -6,6 +6,8 @@ Subject: [PATCH 1/2] Implement ficom instruction for amd64 This should fix #103594 for amd64 previous commit was for x86 only. +Upstream-Status: Inappropriate [NI specific] + Signed-off-by: Bill Pittman --- VEX/priv/guest_amd64_toIR.c | 79 ++++++++++++++++++++++++++++++++++++++++++++- diff --git a/recipes-devtools/valgrind/files/0002-Patch-for-FBLD-FBSTP-FTST.patch b/recipes-devtools/valgrind/files/0002-Patch-for-FBLD-FBSTP-FTST.patch index 7b58a4b6b..c6b444357 100644 --- a/recipes-devtools/valgrind/files/0002-Patch-for-FBLD-FBSTP-FTST.patch +++ b/recipes-devtools/valgrind/files/0002-Patch-for-FBLD-FBSTP-FTST.patch @@ -6,6 +6,8 @@ Subject: [PATCH 2/2] Patch for FBLD/FBSTP/FTST Pulled from a third party patch set that was not upstreamed. +Upstream-Status: Denied + Signed-off-by: Bill Pittman --- VEX/priv/guest_amd64_defs.h | 3 ++ From cb3cc7e59825bb10d405e149752efd5699ac00a8 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 15 Dec 2023 11:32:44 -0500 Subject: [PATCH 024/147] xserver-xfce-init: remove unnecessary gplv2 .patch The xserver-xfce-init recipe includes some local files which are licensed as GPLv2. A .patch containing the GPLv2 license file is distributed along side the recipe source. This isn't necessary, as we can just set the recipe to use the common GPLv2 license that is distributed with the distro at large. So remove the unecessary .patch. Signed-off-by: Alex Stewart --- .../xserver-xfce-init/xserver-xfce-init.bb | 3 +- .../xserver-xfce-init/gplv2-license.patch | 353 ------------------ 2 files changed, 1 insertion(+), 355 deletions(-) delete mode 100644 recipes-xfce/xserver-xfce-init/xserver-xfce-init/gplv2-license.patch diff --git a/recipes-xfce/xserver-xfce-init/xserver-xfce-init.bb b/recipes-xfce/xserver-xfce-init/xserver-xfce-init.bb index f782d6573..5b7f02d24 100644 --- a/recipes-xfce/xserver-xfce-init/xserver-xfce-init.bb +++ b/recipes-xfce/xserver-xfce-init/xserver-xfce-init.bb @@ -2,7 +2,7 @@ SUMMARY = "NILRT XFCE Initialization." DESCRIPTION = "Used to initialize XFCE when the embedded UI is enabled." SECTION = "x11" LICENSE = "GPL-2.0-only" -LIC_FILES_CHKSUM = "file://COPYING;md5=751419260aa954499f7abaabaa882bbe" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/GPL-2.0-only;md5=801f80980d171dd6425610833a22dbe6" PV = "1.0" @@ -10,7 +10,6 @@ PV = "1.0" SRC_URI = " \ file://xserver-xfce \ - file://gplv2-license.patch \ file://xserver-xfce.service \ file://xserver-xfce.conf \ file://xserver-logrotate.conf \ diff --git a/recipes-xfce/xserver-xfce-init/xserver-xfce-init/gplv2-license.patch b/recipes-xfce/xserver-xfce-init/xserver-xfce-init/gplv2-license.patch deleted file mode 100644 index ec9325334..000000000 --- a/recipes-xfce/xserver-xfce-init/xserver-xfce-init/gplv2-license.patch +++ /dev/null @@ -1,353 +0,0 @@ -COPYING: add GPLv2 license file - -this is a local file recipe and the license file is missing.In order -to pass the license checksum checking, the license file is needed. So -this patch add the GPLv2 license file. - -Signed-off-by: Yu Ke - -diff --git a/COPYING b/COPYING -new file mode 100644 -index 0000000..d511905 ---- /dev/null -+++ b/COPYING -@@ -0,0 +1,339 @@ -+ GNU GENERAL PUBLIC LICENSE -+ Version 2, June 1991 -+ -+ Copyright (C) 1989, 1991 Free Software Foundation, Inc., -+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -+ Everyone is permitted to copy and distribute verbatim copies -+ of this license document, but changing it is not allowed. -+ -+ Preamble -+ -+ The licenses for most software are designed to take away your -+freedom to share and change it. By contrast, the GNU General Public -+License is intended to guarantee your freedom to share and change free -+software--to make sure the software is free for all its users. This -+General Public License applies to most of the Free Software -+Foundation's software and to any other program whose authors commit to -+using it. (Some other Free Software Foundation software is covered by -+the GNU Lesser General Public License instead.) You can apply it to -+your programs, too. -+ -+ When we speak of free software, we are referring to freedom, not -+price. Our General Public Licenses are designed to make sure that you -+have the freedom to distribute copies of free software (and charge for -+this service if you wish), that you receive source code or can get it -+if you want it, that you can change the software or use pieces of it -+in new free programs; and that you know you can do these things. -+ -+ To protect your rights, we need to make restrictions that forbid -+anyone to deny you these rights or to ask you to surrender the rights. -+These restrictions translate to certain responsibilities for you if you -+distribute copies of the software, or if you modify it. -+ -+ For example, if you distribute copies of such a program, whether -+gratis or for a fee, you must give the recipients all the rights that -+you have. You must make sure that they, too, receive or can get the -+source code. And you must show them these terms so they know their -+rights. -+ -+ We protect your rights with two steps: (1) copyright the software, and -+(2) offer you this license which gives you legal permission to copy, -+distribute and/or modify the software. -+ -+ Also, for each author's protection and ours, we want to make certain -+that everyone understands that there is no warranty for this free -+software. If the software is modified by someone else and passed on, we -+want its recipients to know that what they have is not the original, so -+that any problems introduced by others will not reflect on the original -+authors' reputations. -+ -+ Finally, any free program is threatened constantly by software -+patents. We wish to avoid the danger that redistributors of a free -+program will individually obtain patent licenses, in effect making the -+program proprietary. To prevent this, we have made it clear that any -+patent must be licensed for everyone's free use or not licensed at all. -+ -+ The precise terms and conditions for copying, distribution and -+modification follow. -+ -+ GNU GENERAL PUBLIC LICENSE -+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION -+ -+ 0. This License applies to any program or other work which contains -+a notice placed by the copyright holder saying it may be distributed -+under the terms of this General Public License. The "Program", below, -+refers to any such program or work, and a "work based on the Program" -+means either the Program or any derivative work under copyright law: -+that is to say, a work containing the Program or a portion of it, -+either verbatim or with modifications and/or translated into another -+language. (Hereinafter, translation is included without limitation in -+the term "modification".) Each licensee is addressed as "you". -+ -+Activities other than copying, distribution and modification are not -+covered by this License; they are outside its scope. The act of -+running the Program is not restricted, and the output from the Program -+is covered only if its contents constitute a work based on the -+Program (independent of having been made by running the Program). -+Whether that is true depends on what the Program does. -+ -+ 1. You may copy and distribute verbatim copies of the Program's -+source code as you receive it, in any medium, provided that you -+conspicuously and appropriately publish on each copy an appropriate -+copyright notice and disclaimer of warranty; keep intact all the -+notices that refer to this License and to the absence of any warranty; -+and give any other recipients of the Program a copy of this License -+along with the Program. -+ -+You may charge a fee for the physical act of transferring a copy, and -+you may at your option offer warranty protection in exchange for a fee. -+ -+ 2. You may modify your copy or copies of the Program or any portion -+of it, thus forming a work based on the Program, and copy and -+distribute such modifications or work under the terms of Section 1 -+above, provided that you also meet all of these conditions: -+ -+ a) You must cause the modified files to carry prominent notices -+ stating that you changed the files and the date of any change. -+ -+ b) You must cause any work that you distribute or publish, that in -+ whole or in part contains or is derived from the Program or any -+ part thereof, to be licensed as a whole at no charge to all third -+ parties under the terms of this License. -+ -+ c) If the modified program normally reads commands interactively -+ when run, you must cause it, when started running for such -+ interactive use in the most ordinary way, to print or display an -+ announcement including an appropriate copyright notice and a -+ notice that there is no warranty (or else, saying that you provide -+ a warranty) and that users may redistribute the program under -+ these conditions, and telling the user how to view a copy of this -+ License. (Exception: if the Program itself is interactive but -+ does not normally print such an announcement, your work based on -+ the Program is not required to print an announcement.) -+ -+These requirements apply to the modified work as a whole. If -+identifiable sections of that work are not derived from the Program, -+and can be reasonably considered independent and separate works in -+themselves, then this License, and its terms, do not apply to those -+sections when you distribute them as separate works. But when you -+distribute the same sections as part of a whole which is a work based -+on the Program, the distribution of the whole must be on the terms of -+this License, whose permissions for other licensees extend to the -+entire whole, and thus to each and every part regardless of who wrote it. -+ -+Thus, it is not the intent of this section to claim rights or contest -+your rights to work written entirely by you; rather, the intent is to -+exercise the right to control the distribution of derivative or -+collective works based on the Program. -+ -+In addition, mere aggregation of another work not based on the Program -+with the Program (or with a work based on the Program) on a volume of -+a storage or distribution medium does not bring the other work under -+the scope of this License. -+ -+ 3. You may copy and distribute the Program (or a work based on it, -+under Section 2) in object code or executable form under the terms of -+Sections 1 and 2 above provided that you also do one of the following: -+ -+ a) Accompany it with the complete corresponding machine-readable -+ source code, which must be distributed under the terms of Sections -+ 1 and 2 above on a medium customarily used for software interchange; or, -+ -+ b) Accompany it with a written offer, valid for at least three -+ years, to give any third party, for a charge no more than your -+ cost of physically performing source distribution, a complete -+ machine-readable copy of the corresponding source code, to be -+ distributed under the terms of Sections 1 and 2 above on a medium -+ customarily used for software interchange; or, -+ -+ c) Accompany it with the information you received as to the offer -+ to distribute corresponding source code. (This alternative is -+ allowed only for noncommercial distribution and only if you -+ received the program in object code or executable form with such -+ an offer, in accord with Subsection b above.) -+ -+The source code for a work means the preferred form of the work for -+making modifications to it. For an executable work, complete source -+code means all the source code for all modules it contains, plus any -+associated interface definition files, plus the scripts used to -+control compilation and installation of the executable. However, as a -+special exception, the source code distributed need not include -+anything that is normally distributed (in either source or binary -+form) with the major components (compiler, kernel, and so on) of the -+operating system on which the executable runs, unless that component -+itself accompanies the executable. -+ -+If distribution of executable or object code is made by offering -+access to copy from a designated place, then offering equivalent -+access to copy the source code from the same place counts as -+distribution of the source code, even though third parties are not -+compelled to copy the source along with the object code. -+ -+ 4. You may not copy, modify, sublicense, or distribute the Program -+except as expressly provided under this License. Any attempt -+otherwise to copy, modify, sublicense or distribute the Program is -+void, and will automatically terminate your rights under this License. -+However, parties who have received copies, or rights, from you under -+this License will not have their licenses terminated so long as such -+parties remain in full compliance. -+ -+ 5. You are not required to accept this License, since you have not -+signed it. However, nothing else grants you permission to modify or -+distribute the Program or its derivative works. These actions are -+prohibited by law if you do not accept this License. Therefore, by -+modifying or distributing the Program (or any work based on the -+Program), you indicate your acceptance of this License to do so, and -+all its terms and conditions for copying, distributing or modifying -+the Program or works based on it. -+ -+ 6. Each time you redistribute the Program (or any work based on the -+Program), the recipient automatically receives a license from the -+original licensor to copy, distribute or modify the Program subject to -+these terms and conditions. You may not impose any further -+restrictions on the recipients' exercise of the rights granted herein. -+You are not responsible for enforcing compliance by third parties to -+this License. -+ -+ 7. If, as a consequence of a court judgment or allegation of patent -+infringement or for any other reason (not limited to patent issues), -+conditions are imposed on you (whether by court order, agreement or -+otherwise) that contradict the conditions of this License, they do not -+excuse you from the conditions of this License. If you cannot -+distribute so as to satisfy simultaneously your obligations under this -+License and any other pertinent obligations, then as a consequence you -+may not distribute the Program at all. For example, if a patent -+license would not permit royalty-free redistribution of the Program by -+all those who receive copies directly or indirectly through you, then -+the only way you could satisfy both it and this License would be to -+refrain entirely from distribution of the Program. -+ -+If any portion of this section is held invalid or unenforceable under -+any particular circumstance, the balance of the section is intended to -+apply and the section as a whole is intended to apply in other -+circumstances. -+ -+It is not the purpose of this section to induce you to infringe any -+patents or other property right claims or to contest validity of any -+such claims; this section has the sole purpose of protecting the -+integrity of the free software distribution system, which is -+implemented by public license practices. Many people have made -+generous contributions to the wide range of software distributed -+through that system in reliance on consistent application of that -+system; it is up to the author/donor to decide if he or she is willing -+to distribute software through any other system and a licensee cannot -+impose that choice. -+ -+This section is intended to make thoroughly clear what is believed to -+be a consequence of the rest of this License. -+ -+ 8. If the distribution and/or use of the Program is restricted in -+certain countries either by patents or by copyrighted interfaces, the -+original copyright holder who places the Program under this License -+may add an explicit geographical distribution limitation excluding -+those countries, so that distribution is permitted only in or among -+countries not thus excluded. In such case, this License incorporates -+the limitation as if written in the body of this License. -+ -+ 9. The Free Software Foundation may publish revised and/or new versions -+of the General Public License from time to time. Such new versions will -+be similar in spirit to the present version, but may differ in detail to -+address new problems or concerns. -+ -+Each version is given a distinguishing version number. If the Program -+specifies a version number of this License which applies to it and "any -+later version", you have the option of following the terms and conditions -+either of that version or of any later version published by the Free -+Software Foundation. If the Program does not specify a version number of -+this License, you may choose any version ever published by the Free Software -+Foundation. -+ -+ 10. If you wish to incorporate parts of the Program into other free -+programs whose distribution conditions are different, write to the author -+to ask for permission. For software which is copyrighted by the Free -+Software Foundation, write to the Free Software Foundation; we sometimes -+make exceptions for this. Our decision will be guided by the two goals -+of preserving the free status of all derivatives of our free software and -+of promoting the sharing and reuse of software generally. -+ -+ NO WARRANTY -+ -+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -+REPAIR OR CORRECTION. -+ -+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -+POSSIBILITY OF SUCH DAMAGES. -+ -+ END OF TERMS AND CONDITIONS -+ -+ How to Apply These Terms to Your New Programs -+ -+ If you develop a new program, and you want it to be of the greatest -+possible use to the public, the best way to achieve this is to make it -+free software which everyone can redistribute and change under these terms. -+ -+ To do so, attach the following notices to the program. It is safest -+to attach them to the start of each source file to most effectively -+convey the exclusion of warranty; and each file should have at least -+the "copyright" line and a pointer to where the full notice is found. -+ -+ -+ Copyright (C) -+ -+ This program is free software; you can redistribute it and/or modify -+ it under the terms of the GNU General Public License as published by -+ the Free Software Foundation; either version 2 of the License, or -+ (at your option) any later version. -+ -+ This program is distributed in the hope that it will be useful, -+ but WITHOUT ANY WARRANTY; without even the implied warranty of -+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ GNU General Public License for more details. -+ -+ You should have received a copy of the GNU General Public License along -+ with this program; if not, write to the Free Software Foundation, Inc., -+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -+ -+Also add information on how to contact you by electronic and paper mail. -+ -+If the program is interactive, make it output a short notice like this -+when it starts in an interactive mode: -+ -+ Gnomovision version 69, Copyright (C) year name of author -+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. -+ This is free software, and you are welcome to redistribute it -+ under certain conditions; type `show c' for details. -+ -+The hypothetical commands `show w' and `show c' should show the appropriate -+parts of the General Public License. Of course, the commands you use may -+be called something other than `show w' and `show c'; they could even be -+mouse-clicks or menu items--whatever suits your program. -+ -+You should also get your employer (if you work as a programmer) or your -+school, if any, to sign a "copyright disclaimer" for the program, if -+necessary. Here is a sample; alter the names: -+ -+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program -+ `Gnomovision' (which makes passes at compilers) written by James Hacker. -+ -+ , 1 April 1989 -+ Ty Coon, President of Vice -+ -+This General Public License does not permit incorporating your program into -+proprietary programs. If your program is a subroutine library, you may -+consider it more useful to permit linking proprietary applications with the -+library. If this is what you want to do, use the GNU Lesser General -+Public License instead of this License. From c796a09a65be9ecb7bc387bd0d3085540b268d94 Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Wed, 13 Dec 2023 15:25:01 -0600 Subject: [PATCH 025/147] kernel-*-performance-tests: re-factor common files Refactor cyclictest files common between containerized and non-containerized tests to single location. Signed-off-by: Gratian Crisan (cherry picked from commit fedc67539592f130df3a605041c14b83de906121) Signed-off-by: Alex Stewart --- .../common.cfg | 0 .../fio.cfg | 0 .../run-cyclictest | 0 .../upload_cyclictest_results.py | 0 .../common.cfg | 9 - .../kernel-performance-tests-files/fio.cfg | 15 -- .../run-cyclictest | 83 ------- .../upload_cyclictest_results.py | 217 ------------------ 8 files changed, 324 deletions(-) rename recipes-kernel/kernel-tests/{kernel-performance-tests-files => files}/common.cfg (100%) rename recipes-kernel/kernel-tests/{kernel-containerized-performance-tests-files => files}/fio.cfg (100%) rename recipes-kernel/kernel-tests/{kernel-containerized-performance-tests-files => files}/run-cyclictest (100%) rename recipes-kernel/kernel-tests/{kernel-containerized-performance-tests-files => files}/upload_cyclictest_results.py (100%) delete mode 100644 recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/common.cfg delete mode 100644 recipes-kernel/kernel-tests/kernel-performance-tests-files/fio.cfg delete mode 100644 recipes-kernel/kernel-tests/kernel-performance-tests-files/run-cyclictest delete mode 100755 recipes-kernel/kernel-tests/kernel-performance-tests-files/upload_cyclictest_results.py diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/common.cfg b/recipes-kernel/kernel-tests/files/common.cfg similarity index 100% rename from recipes-kernel/kernel-tests/kernel-performance-tests-files/common.cfg rename to recipes-kernel/kernel-tests/files/common.cfg diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/fio.cfg b/recipes-kernel/kernel-tests/files/fio.cfg similarity index 100% rename from recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/fio.cfg rename to recipes-kernel/kernel-tests/files/fio.cfg diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/run-cyclictest b/recipes-kernel/kernel-tests/files/run-cyclictest similarity index 100% rename from recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/run-cyclictest rename to recipes-kernel/kernel-tests/files/run-cyclictest diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/upload_cyclictest_results.py b/recipes-kernel/kernel-tests/files/upload_cyclictest_results.py similarity index 100% rename from recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/upload_cyclictest_results.py rename to recipes-kernel/kernel-tests/files/upload_cyclictest_results.py diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/common.cfg b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/common.cfg deleted file mode 100644 index 70136bf76..000000000 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/common.cfg +++ /dev/null @@ -1,9 +0,0 @@ -# Common configuration for all kernel performance tests -TEST_DURATION=8h - -# Maximum expected latency in useconds. -# The test will fail if cyclictest latency goes beyond this value. -MAX_LATENCY=300 - -# Location for benchmark logs and test results -LOG_DIR="/var/local/ptest-results/kernel-containerized-performance-tests" diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/fio.cfg b/recipes-kernel/kernel-tests/kernel-performance-tests-files/fio.cfg deleted file mode 100644 index 78ff99b68..000000000 --- a/recipes-kernel/kernel-tests/kernel-performance-tests-files/fio.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[global] -directory=/var/cache/fio -numjobs=1 -size=100M -nrfiles=20 -openfiles=10 -direct=0 -verify=sha256 -do_verify=1 -time_based -clocksource=clock_gettime -bs=5k - -[random_rw] -rw=randrw diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-cyclictest b/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-cyclictest deleted file mode 100644 index 4d11def15..000000000 --- a/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-cyclictest +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -source ./common.cfg - -function add_system_info() -{ - { - DESC=$(fw_printenv DeviceDesc) - DEVICE=${DESC#*=} - echo "# Device: $DEVICE" - - echo -n "# Kernel: " - uname -a - - echo -n "# Kernel parameters: " - cat /proc/cmdline - - # save info on the current security mitigations - echo "# Security vulnerabilities settings: " - if [ -d "/sys/devices/system/cpu/vulnerabilities" ]; then - for f in /sys/devices/system/cpu/vulnerabilities/*; do - VULN=$(basename "$f") - if [ -f "$f" ]; then - echo -n -e "# $VULN:\t" - cat "$f" - fi - done - else - echo "# vulnerability information not published" - fi - - # save info on the current C state settings - echo "# C-states settings: " - for CPU in /sys/devices/system/cpu/cpu[0-9]; do - NCPU=$(basename "$CPU") - echo "# [$NCPU]" - for CSTATE in $CPU/cpuidle/state[0-9]; do - NSTATE=$(basename "$CSTATE") - if [ -f "$CSTATE/name" ] && [ -f "$CSTATE/disable" ]; then - NAME=$(cat "$CSTATE/name" 2>/dev/null) - STATUS=$(cat "$CSTATE/disable" 2>/dev/null) - - echo -n "# $NSTATE: $NAME: " - if [ "$STATUS" = "0" ]; then - echo "enabled" - else - echo "disabled" - fi - else - echo "# information not available" - fi - done - done - } >> "$1" -} - -function run_cyclictest() -{ - LOG="$LOG_DIR/cyclictest-$1-`date +'%Y_%m_%d-%H_%M_%S'`.log" - INFLUXDB_INFO="/home/admin/.influxdb.info" - cyclictest --smp --priority=98 --mlockall --interval=997 --quiet --duration="$TEST_DURATION" --histofall=1000 --histfile="$LOG" > /dev/null - add_system_info "$LOG" - if [[ -f "$INFLUXDB_INFO" ]]; then - source "$INFLUXDB_INFO" - python3 upload_cyclictest_results.py "-i $LOG" "-s $INFLUXDB_SERVER" "-p $INFLUXDB_PORT" - else - echo "INFO: No InfluxDB Connection Info. Results will not be published." - fi - - LATENCY=$(grep -sw "# Max Latencies:" "$LOG" | awk '{max=$4; for(i=4; i<=NF; i++) if ($i>max) max=$i; gsub("^0*", "", max); print max}') - if [ "$LATENCY" -le "$MAX_LATENCY" ]; then - echo "cyclictest with $1 load latency: $LATENCY (usec) is less than max latency: $MAX_LATENCY (usec)" - echo "histogram log file: $LOG" - echo "PASS: test_kernel_cyclictest_$1" - CYCLICTEST_RESULT=0 - else - echo "cyclictest with $1 load latency: $LATENCY (usec) is above the expected max latency: $MAX_LATENCY (usec)" - echo "histogram log file: $LOG" - echo "FAIL: test_kernel_cyclictest_$1" - CYCLICTEST_RESULT=1 - fi -} - -mkdir -p "$LOG_DIR" diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/upload_cyclictest_results.py b/recipes-kernel/kernel-tests/kernel-performance-tests-files/upload_cyclictest_results.py deleted file mode 100755 index f46d83563..000000000 --- a/recipes-kernel/kernel-tests/kernel-performance-tests-files/upload_cyclictest_results.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -"""Script for uploading a cyclictest log to influxdb""" - -import argparse -import logging -import mmap -import re -import subprocess -import sys -from pathlib import Path -from datetime import datetime -from influxdb import InfluxDBClient, exceptions -import requests.exceptions - - -def get_current_kernel_version(): - """Get the kernel version currently running on the system""" - try: - process = subprocess.run(['uname', '-r'], - check=True, - stdout=subprocess.PIPE) - kver_full = process.stdout.strip().decode("utf-8") - rgx = re.search(r'([0-9]+\.[0-9]+)\.([0-9]+)', kver_full) - kver = rgx.group(1) - except (FileNotFoundError, AttributeError): - logging.warning("failed to read the current kernel version") - raise - - return kver, kver_full - - -def get_kernel_version(path): - """Get the kernel version used in cyclictest test - - Read 'major.minor' and 'major.minor.patch-rt' kernel version strings from - cyclictest log if found or current running system otherwise. - - """ - try: - with open(path, 'rb', 0) as file, \ - mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as mfile: - kernel = re.search(br'# Kernel: Linux ([^ ]+) ([0-9]+)\.([0-9]+)\.' - br'([0-9]+-rc[0-9]+-rt[0-9]+|[0-9]+-rt[0-9]+)', - mfile) - # ignore build machine name, i.e. group(1) - major = kernel.group(2).decode() - minor = kernel.group(3).decode() - patch = kernel.group(4).decode() - version = major + '.' + minor - full_version = version + '.' + patch - except (ValueError, AttributeError): - logging.warning('kernel version information missing from \'%s\' ' - 'retriving from current system.', path) - version, full_version = get_current_kernel_version() - return version, full_version - - -def get_current_device(): - """Get the device description for the current system (e.g. cRIO-90xx)""" - try: - process = subprocess.run(['fw_printenv', 'DeviceDesc'], - check=True, - stdout=subprocess.PIPE) - dev_str = process.stdout.strip().decode("utf-8") - dev = dev_str.split('=')[-1] - except FileNotFoundError: - logging.warning("failed to read the current device type") - raise - - return dev - - -def get_device(path): - """Get the device name used in cyclictest test - - Read the device name stored in the cyclictest log file or if not available - retrieve the current device name. - - """ - try: - with open(path, 'rb', 0) as file, \ - mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as mfile: - rgx = re.search(br'# Device: ([a-zA-z]+-[0-9]+)', mfile) - device = rgx.group(1).decode() - except (ValueError, AttributeError): - logging.warning('device information missing from \'%s\' ' - 'retriving from current system.', path) - device = get_current_device() - return device - - -def get_test_params(path): - """Get the test name and datetime - - Extract the test name and date/time from cyclictest log name. - - """ - try: - rgx = re.search(r'[^_]*cyclictest-([^-]+)-([^-]+)-([^\.]+).log', - path.name) - test = rgx.group(1) - date = rgx.group(2).replace('_', '-') - time = rgx.group(3).replace('_', ':') - date_time = datetime.fromisoformat(date + ' ' + time) - except AttributeError: - logging.warning('failed to parse test parameters ' - '(i.e. test name, date, time)') - raise - - return test, date_time - - -def get_test_results(path): - """Extract min, avg, max latency from cyclictest log""" - try: - with open(path, 'rb', 0) as file, \ - mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as mfile: - rgx = re.search(br'# Max Latencies: ([^\n]+)', mfile) - max_list = list(map(int, rgx.group(1).decode().split())) - max_latency = max_list[-1] - - rgx = re.search(br'# Min Latencies: ([^\n]+)', mfile) - min_list = list(map(int, rgx.group(1).decode().split())) - min_latency = min(min_list) - - rgx = re.search(br'# Avg Latencies: ([^\n]+)', mfile) - avg_list = list(map(int, rgx.group(1).decode().split())) - avg_latency = sum(avg_list) / len(avg_list) - except (ValueError, AttributeError): - logging.warning("failed to parse test results") - raise - - return min_latency, avg_latency, max_latency - -def get_distro_info(): - """Extract distro version and codename from os-release and return them as a tuple - - Returns 'unknown' for either in case of file read exception or missing fields. - - """ - os_release_stmts = [] - try: - with open('/etc/os-release') as os_release_file: - os_release_stmts = os_release_file.readlines() - except: - return ('unknown', 'unknown') - pairs = dict(map(lambda e: e.split('='), os_release_stmts)) - version = pairs.get('VERSION_ID', 'unknown').strip() - name = pairs.get('DISTRO_CODENAME', 'unknown').strip()[1:-1] - return (version, name) - -def read_data(path): - """Read test data and metadata from a cyclictest log file""" - version, full_version = get_kernel_version(path) - controller = get_device(path) - test, time = get_test_params(path) - min_latency, avg_latency, max_latency = get_test_results(path) - distro_version, distro_name = get_distro_info() - - data = { - "measurement": "latency", - "tags": { - "controller": controller, - "test": test, - "kernel_version": version, - "kernel_full_version": full_version, - "distro_version": distro_version, - "distro_name": distro_name - }, - "fields": { - "min_latency": float(min_latency), - "avg_latency": float(avg_latency), - "max_latency": float(max_latency), - }, - "time": time - } - return data - - -def upload_results(data, server, server_port): - """Upload results to influxdb instance""" - try: - influxdb = InfluxDBClient(host=server, port=server_port) - influxdb.switch_database("rtos_kernel_performance") - if not influxdb.write_points(data): - logging.warning("failed to write data points to influxdb") - influxdb.close() - except (exceptions.InfluxDBClientError, - requests.exceptions.ConnectionError) as err: - logging.warning('influxdb error: %s', err) - - -parser = argparse.ArgumentParser( - description="Parse and upload cyclictest results.") -parser.add_argument("-i", "--input", required=True, - help="cyclictest log input file") -parser.add_argument("-s", "--server", required=True, - help="influxdb server") -parser.add_argument("-p", "--port", type=int, default=8086, - help="influxdb server port") -args = parser.parse_args() - -log_file = Path(args.input.strip()) -if not log_file.is_file(): - logging.error('input file %s does not exist!', log_file) - sys.exit(1) - -try: - result = read_data(log_file) -except (FileNotFoundError, AttributeError, ValueError): - logging.error('skipping malformed or incomplete test results at: %s', - log_file) - sys.exit(1) - -results = [] -results.append(result) -upload_results(results, args.server.strip(), args.port) From 11dd767b7b4bde6d5a968e74b3216e870bc68385 Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Wed, 13 Dec 2023 16:17:33 -0600 Subject: [PATCH 026/147] kernel-containerized-performance-tests: re-factor docker files Move docker related files to common location. Signed-off-by: Gratian Crisan (cherry picked from commit da1ba2b9d44bde2eeed2bf6a45ab04f71e4e26cc) Signed-off-by: Alex Stewart --- .../cyclictest-container/Dockerfile | 0 .../cyclictest-container/call_run_ct.sh | 0 .../parallel-container/Dockerfile | 0 .../parallel-container/run_fio.sh | 0 .../parallel-container/run_hackbench.sh | 0 .../parallel-container/run_iperf.sh | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename recipes-kernel/kernel-tests/{kernel-containerized-performance-tests-files => files}/cyclictest-container/Dockerfile (100%) rename recipes-kernel/kernel-tests/{kernel-containerized-performance-tests-files => files}/cyclictest-container/call_run_ct.sh (100%) rename recipes-kernel/kernel-tests/{kernel-containerized-performance-tests-files => files}/parallel-container/Dockerfile (100%) rename recipes-kernel/kernel-tests/{kernel-containerized-performance-tests-files => files}/parallel-container/run_fio.sh (100%) rename recipes-kernel/kernel-tests/{kernel-containerized-performance-tests-files => files}/parallel-container/run_hackbench.sh (100%) rename recipes-kernel/kernel-tests/{kernel-containerized-performance-tests-files => files}/parallel-container/run_iperf.sh (100%) diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/cyclictest-container/Dockerfile b/recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile similarity index 100% rename from recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/cyclictest-container/Dockerfile rename to recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/cyclictest-container/call_run_ct.sh b/recipes-kernel/kernel-tests/files/cyclictest-container/call_run_ct.sh similarity index 100% rename from recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/cyclictest-container/call_run_ct.sh rename to recipes-kernel/kernel-tests/files/cyclictest-container/call_run_ct.sh diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/parallel-container/Dockerfile b/recipes-kernel/kernel-tests/files/parallel-container/Dockerfile similarity index 100% rename from recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/parallel-container/Dockerfile rename to recipes-kernel/kernel-tests/files/parallel-container/Dockerfile diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/parallel-container/run_fio.sh b/recipes-kernel/kernel-tests/files/parallel-container/run_fio.sh similarity index 100% rename from recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/parallel-container/run_fio.sh rename to recipes-kernel/kernel-tests/files/parallel-container/run_fio.sh diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/parallel-container/run_hackbench.sh b/recipes-kernel/kernel-tests/files/parallel-container/run_hackbench.sh similarity index 100% rename from recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/parallel-container/run_hackbench.sh rename to recipes-kernel/kernel-tests/files/parallel-container/run_hackbench.sh diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/parallel-container/run_iperf.sh b/recipes-kernel/kernel-tests/files/parallel-container/run_iperf.sh similarity index 100% rename from recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/parallel-container/run_iperf.sh rename to recipes-kernel/kernel-tests/files/parallel-container/run_iperf.sh From 13a4bdc81d77b9aa41a1a760e1bfdb5615cbc856 Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Thu, 14 Dec 2023 09:26:40 -0600 Subject: [PATCH 027/147] kernel-containerized-performance-tests: pull container build logic into central script Pull out common container build logic from each containerized performance test file into a single file and run it in run-ptest. Signed-off-by: Gratian Crisan (cherry picked from commit 16be6c6df82b06c97250e5a1b28833826f293eba) Signed-off-by: Alex Stewart --- .../kernel-tests/files/build-containers | 23 ++++++++++++++++ .../run-ptest | 3 ++- ...est_kernel_cyclictest_fio_containerized.sh | 26 ++----------------- ...rnel_cyclictest_hackbench_containerized.sh | 26 ++----------------- ...st_kernel_cyclictest_idle_containerized.sh | 14 +--------- ...t_kernel_cyclictest_iperf_containerized.sh | 26 ++----------------- .../kernel-containerized-performance-tests.bb | 2 ++ 7 files changed, 34 insertions(+), 86 deletions(-) create mode 100644 recipes-kernel/kernel-tests/files/build-containers diff --git a/recipes-kernel/kernel-tests/files/build-containers b/recipes-kernel/kernel-tests/files/build-containers new file mode 100644 index 000000000..88c7285a7 --- /dev/null +++ b/recipes-kernel/kernel-tests/files/build-containers @@ -0,0 +1,23 @@ +#!/bin/bash + +if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then + echo "Building cyclictest-container..." + DOCKER_BUILDKIT=1 \ + docker build -t cyclictest-container --network=host ./cyclictest-container + if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then + echo "Failed to build cyclictest-container" + exit 77 + fi +fi + +if [ "$(docker images -q parallel-container:latest)" = "" ]; then + echo "Building parallel-container..." + DOCKER_BUILDKIT=1 \ + docker build -t parallel-container --network=host ./parallel-container + if [ "$(docker images -q parallel-container:latest)" = "" ]; then + echo "Failed to build parallel-container" + exit 77 + fi +fi + +exit 0 diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/run-ptest b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/run-ptest index 46c4c8eb8..eb813f20a 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/run-ptest +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/run-ptest @@ -1,9 +1,10 @@ #!/bin/bash +./build-containers || exit $? + ./test_kernel_cyclictest_idle_containerized.sh ./test_kernel_cyclictest_hackbench_containerized.sh ./test_kernel_cyclictest_fio_containerized.sh ./test_kernel_cyclictest_iperf_containerized.sh exit 0 - diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh index 1cd101e3c..55f3a0ec1 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh @@ -1,36 +1,14 @@ #!/bin/bash -PTEST_LOCATION=/usr/lib/kernel-containerized-performance-tests/ptest - -# Build the two containers -if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then - echo "Building cyclictest-container..." - DOCKER_BUILDKIT=1 \ - docker build -t cyclictest-container --network=host ${PTEST_LOCATION}/cyclictest-container - if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then - echo "Failed to build cyclictest-container" - exit 77 - fi -fi -if [ "$(docker images -q parallel-container:latest)" = "" ]; then - echo "Building parallel-container..." - DOCKER_BUILDKIT=1 \ - docker build -t parallel-container --network=host ${PTEST_LOCATION}/parallel-container - if [ "$(docker images -q parallel-container:latest)" = "" ]; then - echo "Failed to build parallel-container" - exit 77 - fi -fi - # Start background disk I/O load echo "Starting fio load..." -LOAD_CONT=$(docker run -d --privileged -v ${PTEST_LOCATION}:/ptests -t parallel-container \ +LOAD_CONT=$(docker run -d --privileged -v ${PWD}:/ptests -t parallel-container \ bash run_fio.sh) # Run cyclictest echo "Running cyclictest in docker container..." RESULT=$(docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} -v ${PTEST_LOCATION}:/ptests -v /home/admin:/home/admin \ + -v ${LOG_DIR}:${LOG_DIR} -v ${PWD}:/ptests -v /home/admin:/home/admin \ -v /usr/share/fw_printenv:/usr/share/fw_printenv -v /sbin/fw_printenv:/sbin/fw_printenv \ -v /usr/share/nisysinfo:/usr/share/nisysinfo -v /dev:/dev \ -t cyclictest-container \ diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh index 25e3ccc27..1a8d802de 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh @@ -1,36 +1,14 @@ #!/bin/bash -PTEST_LOCATION=/usr/lib/kernel-containerized-performance-tests/ptest - -# Build the two containers -if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then - echo "Building cyclictest-container..." - DOCKER_BUILDKIT=1 \ - docker build -t cyclictest-container --network=host ${PTEST_LOCATION}/cyclictest-container - if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then - echo "Failed to build cyclictest-container" - exit 77 - fi -fi -if [ "$(docker images -q parallel-container:latest)" = "" ]; then - echo "Building parallel-container..." - DOCKER_BUILDKIT=1 \ - docker build -t parallel-container --network=host ${PTEST_LOCATION}/parallel-container - if [ "$(docker images -q parallel-container:latest)" = "" ]; then - echo "Failed to build parallel-container" - exit 77 - fi -fi - # Start background scheduler load echo "Starting hackbench load..." -LOAD_CONT=$(docker run -d --privileged -v ${PTEST_LOCATION}:/ptests -t parallel-container \ +LOAD_CONT=$(docker run -d --privileged -v ${PWD}:/ptests -t parallel-container \ bash run_hackbench.sh) # Run cyclictest echo "Running cyclictest in docker container..." RESULT=$(docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} -v ${PTEST_LOCATION}:/ptests -v /home/admin:/home/admin \ + -v ${LOG_DIR}:${LOG_DIR} -v ${PWD}:/ptests -v /home/admin:/home/admin \ -v /usr/share/fw_printenv:/usr/share/fw_printenv -v /sbin/fw_printenv:/sbin/fw_printenv \ -v /usr/share/nisysinfo:/usr/share/nisysinfo -v /dev:/dev \ -t cyclictest-container \ diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh index 9cf74f401..33d63648c 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh @@ -1,23 +1,11 @@ #!/bin/bash -PTEST_LOCATION=/usr/lib/kernel-containerized-performance-tests/ptest LOG_DIR="/var/local/ptest-results/kernel-containerized-performance-tests" -# Build the two containers -if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then - echo "Building cyclictest-container..." - DOCKER_BUILDKIT=1 \ - docker build -t cyclictest-container --network=host ${PTEST_LOCATION}/cyclictest-container - if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then - echo "Failed to build cyclictest-container" - exit 77 - fi -fi - # Run cyclictest echo "Running cyclictest in docker container..." RESULT=$(docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} -v ${PTEST_LOCATION}:/ptests -v /home/admin:/home/admin \ + -v ${LOG_DIR}:${LOG_DIR} -v ${PWD}:/ptests -v /home/admin:/home/admin \ -v /usr/share/fw_printenv:/usr/share/fw_printenv -v /sbin/fw_printenv:/sbin/fw_printenv \ -v /usr/share/nisysinfo:/usr/share/nisysinfo -v /dev:/dev \ -t cyclictest-container \ diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh index 18f8c6f9d..c20d9fd92 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh @@ -1,7 +1,5 @@ #!/bin/bash -PTEST_LOCATION=/usr/lib/kernel-containerized-performance-tests/ptest - source /home/admin/.iperf.info if [ -z "$IPERF_SERVER" ]; then echo "Warning: iperf server not configured; skipping iperf based network load test." @@ -18,36 +16,16 @@ if [ ! -z "$IPERF_PORT" ]; then fi fi -# Build the two containers -if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then - echo "Building cyclictest-container..." - DOCKER_BUILDKIT=1 \ - docker build -t cyclictest-container --network=host ${PTEST_LOCATION}/cyclictest-container - if [ "$(docker images -q cyclictest-container:latest)" = "" ]; then - echo "Failed to build cyclictest-container" - exit 77 - fi -fi -if [ "$(docker images -q parallel-container:latest)" = "" ]; then - echo "Building parallel-container..." - DOCKER_BUILDKIT=1 \ - docker build -t parallel-container --network=host ${PTEST_LOCATION}/parallel-container - if [ "$(docker images -q parallel-container:latest)" = "" ]; then - echo "Failed to build parallel-container" - exit 77 - fi -fi - # Start background network load echo "Starting iperf load..." LOAD_CONT=$(docker run -d --privileged \ - -v ${PTEST_LOCATION}:/ptests -v /home/admin:/home/admin --network=host \ + -v ${PWD}:/ptests -v /home/admin:/home/admin --network=host \ -t parallel-container bash run_iperf.sh) # Run cyclictest echo "Running cyclictest in docker container..." RESULT=$(docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} -v ${PTEST_LOCATION}:/ptests -v /home/admin:/home/admin \ + -v ${LOG_DIR}:${LOG_DIR} -v ${PWD}:/ptests -v /home/admin:/home/admin \ -v /usr/share/fw_printenv:/usr/share/fw_printenv -v /sbin/fw_printenv:/sbin/fw_printenv \ -v /usr/share/nisysinfo:/usr/share/nisysinfo -v /dev:/dev \ -t cyclictest-container \ diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb index 58828e18f..049630e8c 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb @@ -18,6 +18,7 @@ RDEPENDS:${PN}-ptest:append:armv7a = " u-boot-fw-utils" ALLOW_EMPTY:${PN} = "1" SRC_URI += "\ + file://build-containers \ file://run-ptest \ file://run-cyclictest \ file://upload_cyclictest_results.py \ @@ -36,6 +37,7 @@ SRC_URI += "\ " do_install_ptest:append() { + install -m 0755 ${S}/build-containers ${D}${PTEST_PATH} install -m 0755 ${S}/run-ptest ${D}${PTEST_PATH} install -m 0755 ${S}/run-cyclictest ${D}${PTEST_PATH} install -m 0755 ${S}/upload_cyclictest_results.py ${D}${PTEST_PATH} From 677d057771c5a3481f6b9d7001c3e71e5728f40a Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Thu, 14 Dec 2023 16:31:07 -0600 Subject: [PATCH 028/147] kernel-containerized-performance-tests: re-factor the need for wrapper script Simplify container test logic - Remove need for intermediate log files to capture container output and remove associated wrapper script - Modify logging directory location in common.cfg to be applicable for all tests - Imporve sourcing of common.cfg - Improve exit code handling for clarity Signed-off-by: Gratian Crisan (cherry picked from commit 0a4cf8bf9d2abe98a70952ea41502bbf2eb661ce) Signed-off-by: Alex Stewart --- recipes-kernel/kernel-tests/files/common.cfg | 5 +++- .../files/cyclictest-container/Dockerfile | 4 --- .../files/cyclictest-container/call_run_ct.sh | 7 ------ .../kernel-tests/files/run-cyclictest | 7 +++++- ...est_kernel_cyclictest_fio_containerized.sh | 23 +++++++++-------- ...rnel_cyclictest_hackbench_containerized.sh | 22 ++++++++-------- ...st_kernel_cyclictest_idle_containerized.sh | 25 +++++++++---------- ...t_kernel_cyclictest_iperf_containerized.sh | 22 ++++++++-------- .../kernel-containerized-performance-tests.bb | 5 ++-- 9 files changed, 60 insertions(+), 60 deletions(-) delete mode 100755 recipes-kernel/kernel-tests/files/cyclictest-container/call_run_ct.sh diff --git a/recipes-kernel/kernel-tests/files/common.cfg b/recipes-kernel/kernel-tests/files/common.cfg index 324119a36..0fae9bb06 100644 --- a/recipes-kernel/kernel-tests/files/common.cfg +++ b/recipes-kernel/kernel-tests/files/common.cfg @@ -1,9 +1,12 @@ # Common configuration for all kernel performance tests TEST_DURATION=8h +TEST_DIR="$(dirname "$(readlink -f "$0")")" +TEST_NAME=$(basename "$(dirname "$TEST_DIR")") + # Maximum expected latency in useconds. # The test will fail if cyclictest latency goes beyond this value. MAX_LATENCY=300 # Location for benchmark logs and test results -LOG_DIR="/var/local/ptest-results/kernel-performance-tests" +LOG_DIR="/var/local/ptest-results/$TEST_NAME" diff --git a/recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile b/recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile index 1c087e913..ea2e71d14 100644 --- a/recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile +++ b/recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile @@ -10,7 +10,3 @@ RUN DEBIAN_FRONTEND=noninteractive apt update && apt install -y \ # Need to send to InfluxDB RUN pip install influxdb - -# Script to call "run_cyclictest" from outside Docker container -ADD --chmod=0775 call_run_ct.sh ./call_run_ct.sh - diff --git a/recipes-kernel/kernel-tests/files/cyclictest-container/call_run_ct.sh b/recipes-kernel/kernel-tests/files/cyclictest-container/call_run_ct.sh deleted file mode 100755 index cd897943c..000000000 --- a/recipes-kernel/kernel-tests/files/cyclictest-container/call_run_ct.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -cd /ptests -source ./run-cyclictest -TEST_LOG=${LOG_DIR}/run_cyclictest-${1}.log -run_cyclictest "${1}" > ${TEST_LOG} 2>&1 -echo "${CYCLICTEST_RESULT}" - diff --git a/recipes-kernel/kernel-tests/files/run-cyclictest b/recipes-kernel/kernel-tests/files/run-cyclictest index 4d11def15..46c8fcef9 100644 --- a/recipes-kernel/kernel-tests/files/run-cyclictest +++ b/recipes-kernel/kernel-tests/files/run-cyclictest @@ -1,5 +1,5 @@ #!/bin/bash -source ./common.cfg +source "$(dirname "$0")"/common.cfg function add_system_info() { @@ -81,3 +81,8 @@ function run_cyclictest() } mkdir -p "$LOG_DIR" + +if [ "$#" -ge 1 ]; then + run_cyclictest "${@:1}" + exit $CYCLICTEST_RESULT +fi diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh index 55f3a0ec1..c2ca01dae 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh @@ -1,4 +1,5 @@ #!/bin/bash +source "$(dirname "$0")"/common.cfg # Start background disk I/O load echo "Starting fio load..." @@ -7,19 +8,19 @@ LOAD_CONT=$(docker run -d --privileged -v ${PWD}:/ptests -t parallel-container \ # Run cyclictest echo "Running cyclictest in docker container..." -RESULT=$(docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} -v ${PWD}:/ptests -v /home/admin:/home/admin \ - -v /usr/share/fw_printenv:/usr/share/fw_printenv -v /sbin/fw_printenv:/sbin/fw_printenv \ - -v /usr/share/nisysinfo:/usr/share/nisysinfo -v /dev:/dev \ - -t cyclictest-container \ - bash call_run_ct.sh "fio_containerized" \ - | tr -d '\r' | tr -d '\n') - -# Make sure we print the PASS/FAIL message -cat ${LOG_DIR}/run_cyclictest-fio_containerized.log +docker run --privileged --network=host \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -v /usr/share/fw_printenv:/usr/share/fw_printenv \ + -v /sbin/fw_printenv:/sbin/fw_printenv \ + -v /usr/share/nisysinfo:/usr/share/nisysinfo \ + -v /dev:/dev \ + -t cyclictest-container \ + bash "${TEST_DIR}/run-cyclictest" "fio_containerized" +RESULT=$? # Clean up the background container docker exec ${LOAD_CONT} bash -c "killall -INT fio > /dev/null 2>&1" exit ${RESULT} - diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh index 1a8d802de..1ce127fdc 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh @@ -1,4 +1,5 @@ #!/bin/bash +source "$(dirname "$0")"/common.cfg # Start background scheduler load echo "Starting hackbench load..." @@ -7,16 +8,17 @@ LOAD_CONT=$(docker run -d --privileged -v ${PWD}:/ptests -t parallel-container \ # Run cyclictest echo "Running cyclictest in docker container..." -RESULT=$(docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} -v ${PWD}:/ptests -v /home/admin:/home/admin \ - -v /usr/share/fw_printenv:/usr/share/fw_printenv -v /sbin/fw_printenv:/sbin/fw_printenv \ - -v /usr/share/nisysinfo:/usr/share/nisysinfo -v /dev:/dev \ - -t cyclictest-container \ - bash call_run_ct.sh "hackbench_containerized" \ - | tr -d '\r' | tr -d '\n') - -# Make sure we print the PASS/FAIL message -cat ${LOG_DIR}/run_cyclictest-hackbench_containerized.log +docker run --privileged --network=host \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -v /usr/share/fw_printenv:/usr/share/fw_printenv \ + -v /sbin/fw_printenv:/sbin/fw_printenv \ + -v /usr/share/nisysinfo:/usr/share/nisysinfo \ + -v /dev:/dev \ + -t cyclictest-container \ + bash "${TEST_DIR}/run-cyclictest" "hackbench_containerized" +RESULT=$? # Clean up the background container docker exec ${LOAD_CONT} \ diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh index 33d63648c..182521f2d 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh @@ -1,19 +1,18 @@ #!/bin/bash - -LOG_DIR="/var/local/ptest-results/kernel-containerized-performance-tests" +source "$(dirname "$0")"/common.cfg # Run cyclictest echo "Running cyclictest in docker container..." -RESULT=$(docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} -v ${PWD}:/ptests -v /home/admin:/home/admin \ - -v /usr/share/fw_printenv:/usr/share/fw_printenv -v /sbin/fw_printenv:/sbin/fw_printenv \ - -v /usr/share/nisysinfo:/usr/share/nisysinfo -v /dev:/dev \ - -t cyclictest-container \ - bash call_run_ct.sh "idle_containerized" \ - | tr -d '\r' | tr -d '\n') - -# Make sure we print the PASS/FAIL message -cat ${LOG_DIR}/run_cyclictest-idle_containerized.log +docker run --privileged --network=host \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -v /usr/share/fw_printenv:/usr/share/fw_printenv \ + -v /sbin/fw_printenv:/sbin/fw_printenv \ + -v /usr/share/nisysinfo:/usr/share/nisysinfo \ + -v /dev:/dev \ + -t cyclictest-container \ + bash "${TEST_DIR}/run-cyclictest" "idle_containerized" +RESULT=$? exit ${RESULT} - diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh index c20d9fd92..0238f717e 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh @@ -1,4 +1,5 @@ #!/bin/bash +source "$(dirname "$0")"/common.cfg source /home/admin/.iperf.info if [ -z "$IPERF_SERVER" ]; then @@ -24,16 +25,17 @@ LOAD_CONT=$(docker run -d --privileged \ # Run cyclictest echo "Running cyclictest in docker container..." -RESULT=$(docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} -v ${PWD}:/ptests -v /home/admin:/home/admin \ - -v /usr/share/fw_printenv:/usr/share/fw_printenv -v /sbin/fw_printenv:/sbin/fw_printenv \ - -v /usr/share/nisysinfo:/usr/share/nisysinfo -v /dev:/dev \ - -t cyclictest-container \ - bash call_run_ct.sh "iperf_containerized" \ - | tr -d '\r' | tr -d '\n') - -# Make sure we print the PASS/FAIL message -cat ${LOG_DIR}/run_cyclictest-iperf_containerized.log +docker run --privileged --network=host \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -v /usr/share/fw_printenv:/usr/share/fw_printenv \ + -v /sbin/fw_printenv:/sbin/fw_printenv \ + -v /usr/share/nisysinfo:/usr/share/nisysinfo \ + -v /dev:/dev \ + -t cyclictest-container \ + bash "${TEST_DIR}/run-cyclictest" "iperf_containerized" +RESULT=$? # Clean up the background container docker exec ${LOAD_CONT} \ diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb index 049630e8c..1aba3283a 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb @@ -29,7 +29,6 @@ SRC_URI += "\ file://test_kernel_cyclictest_fio_containerized.sh \ file://test_kernel_cyclictest_iperf_containerized.sh \ file://cyclictest-container/Dockerfile \ - file://cyclictest-container/call_run_ct.sh \ file://parallel-container/Dockerfile \ file://parallel-container/run_hackbench.sh \ file://parallel-container/run_fio.sh \ @@ -47,10 +46,10 @@ do_install_ptest:append() { install -m 0755 ${S}/test_kernel_cyclictest_hackbench_containerized.sh ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_fio_containerized.sh ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_iperf_containerized.sh ${D}${PTEST_PATH} + mkdir -p ${D}${PTEST_PATH}/cyclictest-container install -m 0755 ${S}/cyclictest-container/Dockerfile ${D}${PTEST_PATH}/cyclictest-container - install -m 0755 ${S}/cyclictest-container/call_run_ct.sh \ - ${D}${PTEST_PATH}/cyclictest-container/call_run_ct.sh + mkdir -p ${D}${PTEST_PATH}/parallel-container install -m 0755 ${S}/parallel-container/Dockerfile ${D}${PTEST_PATH}/parallel-container install -m 0755 ${S}/parallel-container/run_hackbench.sh \ From 696352b474c562c7f589f40966a6ccf144fca099 Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Fri, 15 Dec 2023 14:58:40 -0600 Subject: [PATCH 029/147] kernel-*-performance-tests: re-factor load wrapper scripts Simplify tetst logic for containerized and non-containerized tests - Remove unnecessary load wrapper scripts in containerized tests - Single source hackbench, fio, iperf start/stop logic in individual tests by moving into common scripts - Remove superfluous syntax Signed-off-by: Gratian Crisan (cherry picked from commit 30a3fdcbfffa11edfa9ce7012c13f160fb3be232) Signed-off-by: Alex Stewart --- recipes-kernel/kernel-tests/files/fio-load | 19 +++++++++ .../kernel-tests/files/hackbench-load | 17 ++++++++ recipes-kernel/kernel-tests/files/iperf-load | 40 +++++++++++++++++++ .../files/parallel-container/Dockerfile | 6 --- .../files/parallel-container/run_fio.sh | 8 ---- .../files/parallel-container/run_hackbench.sh | 6 --- .../files/parallel-container/run_iperf.sh | 12 ------ ...est_kernel_cyclictest_fio_containerized.sh | 13 ++++-- ...rnel_cyclictest_hackbench_containerized.sh | 14 ++++--- ...st_kernel_cyclictest_idle_containerized.sh | 2 +- ...t_kernel_cyclictest_iperf_containerized.sh | 33 +++++---------- .../kernel-containerized-performance-tests.bb | 25 +++++------- .../test_kernel_cyclictest_fio.sh | 6 +-- .../test_kernel_cyclictest_hackbench.sh | 4 +- .../test_kernel_cyclictest_iperf.sh | 25 ++---------- .../kernel-tests/kernel-performance-tests.bb | 6 +++ 16 files changed, 127 insertions(+), 109 deletions(-) create mode 100644 recipes-kernel/kernel-tests/files/fio-load create mode 100644 recipes-kernel/kernel-tests/files/hackbench-load create mode 100644 recipes-kernel/kernel-tests/files/iperf-load delete mode 100644 recipes-kernel/kernel-tests/files/parallel-container/run_fio.sh delete mode 100644 recipes-kernel/kernel-tests/files/parallel-container/run_hackbench.sh delete mode 100644 recipes-kernel/kernel-tests/files/parallel-container/run_iperf.sh diff --git a/recipes-kernel/kernel-tests/files/fio-load b/recipes-kernel/kernel-tests/files/fio-load new file mode 100644 index 000000000..f84e34555 --- /dev/null +++ b/recipes-kernel/kernel-tests/files/fio-load @@ -0,0 +1,19 @@ +#!/bin/bash +source "$(dirname "$0")"/common.cfg + +case "$1" in + start) + mkdir -p /var/cache/fio + fio fio.cfg --ioengine="sync" --runtime="$TEST_DURATION" --ramp_time=1m --output="$LOG_DIR/fio-sync-`date +'%Y_%m_%d-%H_%M_%S'`.log" > /dev/null & + ;; + stop) + killall -INT fio > /dev/null 2>&1 + rm -rf /var/cache/fio + ;; + *) + echo "Usage: $0 {start|stop}" + exit 1 + ;; +esac + +exit 0 diff --git a/recipes-kernel/kernel-tests/files/hackbench-load b/recipes-kernel/kernel-tests/files/hackbench-load new file mode 100644 index 000000000..43b6df4f1 --- /dev/null +++ b/recipes-kernel/kernel-tests/files/hackbench-load @@ -0,0 +1,17 @@ +#!/bin/bash +source "$(dirname "$0")"/common.cfg + +case "$1" in + start) + hackbench -l 36000000 -g 10 2>/dev/null >"$LOG_DIR/hackbench-`date +'%Y_%m_%d-%H_%M_%S'`.log" & + ;; + stop) + killall -INT hackbench > /dev/null 2>&1 + ;; + *) + echo "Usage: $0 {start|stop}" + exit 1 + ;; +esac + +exit 0 diff --git a/recipes-kernel/kernel-tests/files/iperf-load b/recipes-kernel/kernel-tests/files/iperf-load new file mode 100644 index 000000000..edcf44f50 --- /dev/null +++ b/recipes-kernel/kernel-tests/files/iperf-load @@ -0,0 +1,40 @@ +#!/bin/bash +source "$(dirname "$0")"/common.cfg +source /home/admin/.iperf.info + +case "$1" in + check) + if [ -z "$IPERF_SERVER" ]; then + echo "Warning: iperf server not configured; skipping iperf based network load test." + echo "Create/edit /home/admin/.iperf.info file with IPERF_SERVER= and IPERF_PORT= to configure a server to connect to for this test." + echo "SKIP: test_kernel_cyclictest_iperf" + exit 77 + fi + if [ ! -z "$IPERF_PORT" ]; then + iperf3 -c "$IPERF_SERVER" -p "$IPERF_PORT" -t 1 > /dev/null 2>&1 + else + iperf3 -c "$IPERF_SERVER" -t 1 > /dev/null 2>&1 + fi + if [ $? -ne 0 ]; then + echo "ERROR: iperf server not reachable; skipping iperf based network load test." + echo "SKIP: test_kernel_cyclictest_iperf" + exit 77 + fi + ;; + start) + if [ ! -z "$IPERF_PORT" ]; then + iperf3 -c "$IPERF_SERVER" -p "$IPERF_PORT" -t 36000 --logfile "$LOG_DIR/iperf-`date +'%Y_%m_%d-%H_%M_%S'`.log" & + else + iperf3 -c "$IPERF_SERVER" -t 36000 --logfile "$LOG_DIR/iperf-`date +'%Y_%m_%d-%H_%M_%S'`.log" & + fi + ;; + stop) + killall -INT iperf3 > /dev/null 2>&1 + ;; + *) + echo "Usage: $0 {check|start|stop}" + exit 1 + ;; +esac + +exit 0 diff --git a/recipes-kernel/kernel-tests/files/parallel-container/Dockerfile b/recipes-kernel/kernel-tests/files/parallel-container/Dockerfile index 51a0e87bf..b2d2c9003 100644 --- a/recipes-kernel/kernel-tests/files/parallel-container/Dockerfile +++ b/recipes-kernel/kernel-tests/files/parallel-container/Dockerfile @@ -6,9 +6,3 @@ RUN DEBIAN_FRONTEND=noninteractive apt update && apt install -y \ fio \ iperf3 \ psmisc - -# Scripts to start loads from outside container -ADD --chmod=0775 run_hackbench.sh ./run_hackbench.sh -ADD --chmod=0775 run_fio.sh ./run_fio.sh -ADD --chmod=0775 run_iperf.sh ./run_iperf.sh - diff --git a/recipes-kernel/kernel-tests/files/parallel-container/run_fio.sh b/recipes-kernel/kernel-tests/files/parallel-container/run_fio.sh deleted file mode 100644 index 0f3396d1a..000000000 --- a/recipes-kernel/kernel-tests/files/parallel-container/run_fio.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -cd /ptests -source run-cyclictest -mkdir -p /var/cache/fio -fio fio.cfg --ioengine="sync" --runtime="${TEST_DURATION}" --ramp_time=1m \ - --output="${LOG_DIR}/fio-sync-`date +'%Y_%m_%d-%H_%M_%S'`.log" \ - > /dev/null - diff --git a/recipes-kernel/kernel-tests/files/parallel-container/run_hackbench.sh b/recipes-kernel/kernel-tests/files/parallel-container/run_hackbench.sh deleted file mode 100644 index c52cf307c..000000000 --- a/recipes-kernel/kernel-tests/files/parallel-container/run_hackbench.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -cd /ptests -source run-cyclictest -hackbench -l 36000000 -g 10 \ - 2>/dev/null > "${LOG_DIR}/hackbench-`date +'%Y_%m_%d-%H_%M_%S\'`.log" - diff --git a/recipes-kernel/kernel-tests/files/parallel-container/run_iperf.sh b/recipes-kernel/kernel-tests/files/parallel-container/run_iperf.sh deleted file mode 100644 index 413c91b63..000000000 --- a/recipes-kernel/kernel-tests/files/parallel-container/run_iperf.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -cd /ptests -source run-cyclictest -source /home/admin/.iperf.info -if [ ! -z "${IPERF_PORT}" ]; then - iperf3 -c "${IPERF_SERVER}" -p "${IPERF_PORT}" -t 36000 \ - --logfile "${LOG_DIR}/iperf-`date +'%Y_%m_%d-%H_%M_%S'`.log" -else - iperf3 -c "${IPERF_SERVER}" -t 36000 \ - --logfile "${LOG_DIR}/iperf-`date +'%Y_%m_%d-%H_%M_%S'`.log" -fi - diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh index c2ca01dae..2a5adb0d3 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh @@ -3,8 +3,12 @@ source "$(dirname "$0")"/common.cfg # Start background disk I/O load echo "Starting fio load..." -LOAD_CONT=$(docker run -d --privileged -v ${PWD}:/ptests -t parallel-container \ - bash run_fio.sh) +LOAD_CONT=$(docker run -d --privileged --network=host \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -t parallel-container) +docker exec -d ${LOAD_CONT} "${TEST_DIR}/fio-load" "start" # Run cyclictest echo "Running cyclictest in docker container..." @@ -17,10 +21,11 @@ docker run --privileged --network=host \ -v /usr/share/nisysinfo:/usr/share/nisysinfo \ -v /dev:/dev \ -t cyclictest-container \ - bash "${TEST_DIR}/run-cyclictest" "fio_containerized" + "${TEST_DIR}/run-cyclictest" "fio_containerized" RESULT=$? # Clean up the background container -docker exec ${LOAD_CONT} bash -c "killall -INT fio > /dev/null 2>&1" +docker exec ${LOAD_CONT} "${TEST_DIR}/fio-load" "stop" +docker stop ${LOAD_CONT} exit ${RESULT} diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh index 1ce127fdc..7e61b44bf 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh @@ -3,8 +3,12 @@ source "$(dirname "$0")"/common.cfg # Start background scheduler load echo "Starting hackbench load..." -LOAD_CONT=$(docker run -d --privileged -v ${PWD}:/ptests -t parallel-container \ - bash run_hackbench.sh) +LOAD_CONT=$(docker run -d --privileged --network=host \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -t parallel-container) +docker exec -d ${LOAD_CONT} "${TEST_DIR}/hackbench-load" "start" # Run cyclictest echo "Running cyclictest in docker container..." @@ -17,12 +21,12 @@ docker run --privileged --network=host \ -v /usr/share/nisysinfo:/usr/share/nisysinfo \ -v /dev:/dev \ -t cyclictest-container \ - bash "${TEST_DIR}/run-cyclictest" "hackbench_containerized" + "${TEST_DIR}/run-cyclictest" "hackbench_containerized" RESULT=$? # Clean up the background container -docker exec ${LOAD_CONT} \ - bash -c "killall -INT hackbench > /dev/null 2>&1" +docker exec ${LOAD_CONT} "${TEST_DIR}/hackbench-load" "stop" +docker stop ${LOAD_CONT} exit ${RESULT} diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh index 182521f2d..ede130a9b 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh @@ -12,7 +12,7 @@ docker run --privileged --network=host \ -v /usr/share/nisysinfo:/usr/share/nisysinfo \ -v /dev:/dev \ -t cyclictest-container \ - bash "${TEST_DIR}/run-cyclictest" "idle_containerized" + "${TEST_DIR}/run-cyclictest" "idle_containerized" RESULT=$? exit ${RESULT} diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh index 0238f717e..36fba043b 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh @@ -1,27 +1,15 @@ #!/bin/bash source "$(dirname "$0")"/common.cfg -source /home/admin/.iperf.info -if [ -z "$IPERF_SERVER" ]; then - echo "Warning: iperf server not configured; skipping iperf based network load test." - echo "Create/edit /home/admin/.iperf.info file with IPERF_SERVER= and IPERF_PORT= to configure a server to connect to for this test." - echo "SKIP: test_kernel_cyclictest_iperf_containerized" - exit 77 -fi -if [ ! -z "$IPERF_PORT" ]; then - iperf3 -c "$IPERF_SERVER" -p "$IPERF_PORT" -t 1 > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "ERROR: iperf server not reachable; skipping iperf based network load test." - echo "SKIP: test_kernel_cyclictest_iperf_containerized" - exit 77 - fi -fi - # Start background network load echo "Starting iperf load..." -LOAD_CONT=$(docker run -d --privileged \ - -v ${PWD}:/ptests -v /home/admin:/home/admin --network=host \ - -t parallel-container bash run_iperf.sh) +LOAD_CONT=$(docker run -d --privileged --network=host \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -t parallel-container) +docker exec ${LOAD_CONT} "${TEST_DIR}/iperf-load" "check" || exit $? +docker exec -d ${LOAD_CONT} "${TEST_DIR}/iperf-load" "start" # Run cyclictest echo "Running cyclictest in docker container..." @@ -34,12 +22,11 @@ docker run --privileged --network=host \ -v /usr/share/nisysinfo:/usr/share/nisysinfo \ -v /dev:/dev \ -t cyclictest-container \ - bash "${TEST_DIR}/run-cyclictest" "iperf_containerized" + "${TEST_DIR}/run-cyclictest" "iperf_containerized" RESULT=$? # Clean up the background container -docker exec ${LOAD_CONT} \ - bash -c "killall -INT iperf3 > /dev/null 2>&1" +docker exec ${LOAD_CONT} "${TEST_DIR}/iperf-load" "stop" +docker stop ${LOAD_CONT} exit ${RESULT} - diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb index 1aba3283a..3f0f6ce13 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb @@ -11,7 +11,7 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}-files:" S = "${WORKDIR}" DEPENDS = "virtual/kernel" -RDEPENDS:${PN}-ptest += "bash rt-tests fio iperf3 python3 python3-pip docker-ce" +RDEPENDS:${PN}-ptest += "bash python3 docker-ce" RDEPENDS:${PN}-ptest:append:x64 = " fw-printenv" RDEPENDS:${PN}-ptest:append:armv7a = " u-boot-fw-utils" @@ -24,15 +24,15 @@ SRC_URI += "\ file://upload_cyclictest_results.py \ file://common.cfg \ file://fio.cfg \ + file://fio-load \ + file://hackbench-load \ + file://iperf-load \ file://test_kernel_cyclictest_idle_containerized.sh \ file://test_kernel_cyclictest_hackbench_containerized.sh \ file://test_kernel_cyclictest_fio_containerized.sh \ file://test_kernel_cyclictest_iperf_containerized.sh \ file://cyclictest-container/Dockerfile \ file://parallel-container/Dockerfile \ - file://parallel-container/run_hackbench.sh \ - file://parallel-container/run_fio.sh \ - file://parallel-container/run_iperf.sh \ " do_install_ptest:append() { @@ -42,26 +42,19 @@ do_install_ptest:append() { install -m 0755 ${S}/upload_cyclictest_results.py ${D}${PTEST_PATH} install -m 0644 ${S}/common.cfg ${D}${PTEST_PATH} install -m 0644 ${S}/fio.cfg ${D}${PTEST_PATH} + install -m 0755 ${S}/fio-load ${D}${PTEST_PATH} + install -m 0755 ${S}/hackbench-load ${D}${PTEST_PATH} + install -m 0755 ${S}/iperf-load ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_idle_containerized.sh ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_hackbench_containerized.sh ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_fio_containerized.sh ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_iperf_containerized.sh ${D}${PTEST_PATH} mkdir -p ${D}${PTEST_PATH}/cyclictest-container - install -m 0755 ${S}/cyclictest-container/Dockerfile ${D}${PTEST_PATH}/cyclictest-container + install -m 0644 ${S}/cyclictest-container/Dockerfile ${D}${PTEST_PATH}/cyclictest-container mkdir -p ${D}${PTEST_PATH}/parallel-container - install -m 0755 ${S}/parallel-container/Dockerfile ${D}${PTEST_PATH}/parallel-container - install -m 0755 ${S}/parallel-container/run_hackbench.sh \ - ${D}${PTEST_PATH}/parallel-container/run_hackbench.sh - install -m 0755 ${S}/parallel-container/run_fio.sh \ - ${D}${PTEST_PATH}/parallel-container/run_fio.sh - install -m 0755 ${S}/parallel-container/run_iperf.sh \ - ${D}${PTEST_PATH}/parallel-container/run_iperf.sh -} - -pkg_postinst_ontarget:${PN}-ptest:append() { - python3 -m pip install influxdb + install -m 0644 ${S}/parallel-container/Dockerfile ${D}${PTEST_PATH}/parallel-container } PACKAGE_ARCH = "${MACHINE_ARCH}" diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_fio.sh b/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_fio.sh index d80cca94f..438a3924b 100644 --- a/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_fio.sh +++ b/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_fio.sh @@ -2,14 +2,12 @@ source ./run-cyclictest # start background disk I/O load -mkdir -p /var/cache/fio -fio fio.cfg --ioengine="sync" --runtime="$TEST_DURATION" --ramp_time=1m --output="$LOG_DIR/fio-sync-`date +'%Y_%m_%d-%H_%M_%S'`.log" > /dev/null & +./fio-load start # measure the system latency under disk load run_cyclictest "fio" # clean-up -killall -INT fio > /dev/null 2>&1 -rm -rf /var/cache/fio +./fio-load stop exit $CYCLICTEST_RESULT diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_hackbench.sh b/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_hackbench.sh index 856334643..19668a725 100644 --- a/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_hackbench.sh +++ b/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_hackbench.sh @@ -2,12 +2,12 @@ source ./run-cyclictest # start background scheduler load -hackbench -l 36000000 -g 10 2>/dev/null >"$LOG_DIR/hackbench-`date +'%Y_%m_%d-%H_%M_%S'`.log" & +./hackbench-load start # measure the system latency under scheduler load run_cyclictest "hackbench" # clean-up -killall -INT hackbench > /dev/null 2>&1 +./hackbench-load stop exit $CYCLICTEST_RESULT diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_iperf.sh b/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_iperf.sh index 75c313e86..fee98420e 100644 --- a/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_iperf.sh +++ b/recipes-kernel/kernel-tests/kernel-performance-tests-files/test_kernel_cyclictest_iperf.sh @@ -1,31 +1,12 @@ #!/bin/bash source ./run-cyclictest -# start background network load -source /home/admin/.iperf.info -if [ -z "$IPERF_SERVER" ]; then - echo "Warning: iperf server not configured; skipping iperf based network load test." - echo "Create/edit /home/admin/.iperf.info file with IPERF_SERVER= and IPERF_PORT= to configure a server to connect to for this test." - echo "SKIP: test_kernel_cyclictest_iperf" - exit 77 -fi - -if [ ! -z "$IPERF_PORT" ]; then - iperf3 -c "$IPERF_SERVER" -p "$IPERF_PORT" -t 1 > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "ERROR: iperf server not reachable; skipping iperf based network load test." - echo "SKIP: test_kernel_cyclictest_iperf" - exit 77 - fi - iperf3 -c "$IPERF_SERVER" -p "$IPERF_PORT" -t 36000 --logfile "$LOG_DIR/iperf-`date +'%Y_%m_%d-%H_%M_%S'`.log" & -else - iperf3 -c "$IPERF_SERVER" -t 36000 --logfile "$LOG_DIR/iperf-`date +'%Y_%m_%d-%H_%M_%S'`.log" & -fi +./iperf-load check || exit $? +./iperf-load start # measure the system latency under network load run_cyclictest "iperf" -# clean-up -killall -INT iperf3 > /dev/null 2>&1 +./iperf-load stop exit $CYCLICTEST_RESULT diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests.bb b/recipes-kernel/kernel-tests/kernel-performance-tests.bb index b06816ac7..9bc2dd998 100644 --- a/recipes-kernel/kernel-tests/kernel-performance-tests.bb +++ b/recipes-kernel/kernel-tests/kernel-performance-tests.bb @@ -23,6 +23,9 @@ SRC_URI += "\ file://upload_cyclictest_results.py \ file://common.cfg \ file://fio.cfg \ + file://fio-load \ + file://hackbench-load \ + file://iperf-load \ file://test_kernel_cyclictest_idle.sh \ file://test_kernel_cyclictest_hackbench.sh \ file://test_kernel_cyclictest_fio.sh \ @@ -37,6 +40,9 @@ do_install_ptest:append() { install -m 0755 ${S}/upload_cyclictest_results.py ${D}${PTEST_PATH} install -m 0644 ${S}/common.cfg ${D}${PTEST_PATH} install -m 0644 ${S}/fio.cfg ${D}${PTEST_PATH} + install -m 0755 ${S}/fio-load ${D}${PTEST_PATH} + install -m 0755 ${S}/hackbench-load ${D}${PTEST_PATH} + install -m 0755 ${S}/iperf-load ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_idle.sh ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_hackbench.sh ${D}${PTEST_PATH} install -m 0755 ${S}/test_kernel_cyclictest_fio.sh ${D}${PTEST_PATH} From 9fea47f3f45bd9bf92b1bf26967f05cb9effc1b1 Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Fri, 15 Dec 2023 15:59:28 -0600 Subject: [PATCH 030/147] kernel-containerized-performance-tests: factor out docker calls - Single source cyclictest docker calls, params into a script - Remove superfluous syntax Signed-off-by: Gratian Crisan (cherry picked from commit 20867641d49a240886cebe3833baf89c6c173b48) Signed-off-by: Alex Stewart --- .../kernel-tests/files/run-docker-cyclictest | 19 +++++++++++++++ ...est_kernel_cyclictest_fio_containerized.sh | 22 ++++------------- ...rnel_cyclictest_hackbench_containerized.sh | 23 ++++-------------- ...st_kernel_cyclictest_idle_containerized.sh | 15 ++---------- ...t_kernel_cyclictest_iperf_containerized.sh | 24 +++++-------------- .../kernel-containerized-performance-tests.bb | 2 ++ 6 files changed, 39 insertions(+), 66 deletions(-) create mode 100644 recipes-kernel/kernel-tests/files/run-docker-cyclictest diff --git a/recipes-kernel/kernel-tests/files/run-docker-cyclictest b/recipes-kernel/kernel-tests/files/run-docker-cyclictest new file mode 100644 index 000000000..8a4aad3c4 --- /dev/null +++ b/recipes-kernel/kernel-tests/files/run-docker-cyclictest @@ -0,0 +1,19 @@ +#!/bin/bash +source "$(dirname "$0")"/common.cfg + +if [ "$#" -lt 1 ]; then + echo echo "Usage: $0 {test_name}" + exit 1 +fi + +docker run --privileged --network=host \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -v /usr/share/fw_printenv:/usr/share/fw_printenv \ + -v /sbin/fw_printenv:/sbin/fw_printenv \ + -v /usr/share/nisysinfo:/usr/share/nisysinfo \ + -v /dev:/dev \ + -t cyclictest-container \ + "${TEST_DIR}/run-cyclictest" "$1" +CYCLICTEST_RESULT=$? diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh index 2a5adb0d3..8216f2656 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_fio_containerized.sh @@ -2,30 +2,18 @@ source "$(dirname "$0")"/common.cfg # Start background disk I/O load -echo "Starting fio load..." LOAD_CONT=$(docker run -d --privileged --network=host \ -v ${LOG_DIR}:${LOG_DIR} \ -v ${TEST_DIR}:${TEST_DIR} \ -v /home/admin:/home/admin \ -t parallel-container) -docker exec -d ${LOAD_CONT} "${TEST_DIR}/fio-load" "start" +docker exec -d $LOAD_CONT "$TEST_DIR/fio-load" "start" # Run cyclictest -echo "Running cyclictest in docker container..." -docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} \ - -v ${TEST_DIR}:${TEST_DIR} \ - -v /home/admin:/home/admin \ - -v /usr/share/fw_printenv:/usr/share/fw_printenv \ - -v /sbin/fw_printenv:/sbin/fw_printenv \ - -v /usr/share/nisysinfo:/usr/share/nisysinfo \ - -v /dev:/dev \ - -t cyclictest-container \ - "${TEST_DIR}/run-cyclictest" "fio_containerized" -RESULT=$? +"$TEST_DIR/run-docker-cyclictest" "fio_containerized" # Clean up the background container -docker exec ${LOAD_CONT} "${TEST_DIR}/fio-load" "stop" -docker stop ${LOAD_CONT} +docker exec $LOAD_CONT "$TEST_DIR/fio-load" "stop" +docker stop $LOAD_CONT -exit ${RESULT} +exit $CYCLICTEST_RESULT diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh index 7e61b44bf..5e6610dc9 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh @@ -2,31 +2,18 @@ source "$(dirname "$0")"/common.cfg # Start background scheduler load -echo "Starting hackbench load..." LOAD_CONT=$(docker run -d --privileged --network=host \ -v ${LOG_DIR}:${LOG_DIR} \ -v ${TEST_DIR}:${TEST_DIR} \ -v /home/admin:/home/admin \ -t parallel-container) -docker exec -d ${LOAD_CONT} "${TEST_DIR}/hackbench-load" "start" +docker exec -d $LOAD_CONT "$TEST_DIR/hackbench-load" "start" # Run cyclictest -echo "Running cyclictest in docker container..." -docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} \ - -v ${TEST_DIR}:${TEST_DIR} \ - -v /home/admin:/home/admin \ - -v /usr/share/fw_printenv:/usr/share/fw_printenv \ - -v /sbin/fw_printenv:/sbin/fw_printenv \ - -v /usr/share/nisysinfo:/usr/share/nisysinfo \ - -v /dev:/dev \ - -t cyclictest-container \ - "${TEST_DIR}/run-cyclictest" "hackbench_containerized" -RESULT=$? +"$TEST_DIR/run-docker-cyclictest" "hackbench_containerized" # Clean up the background container -docker exec ${LOAD_CONT} "${TEST_DIR}/hackbench-load" "stop" -docker stop ${LOAD_CONT} - -exit ${RESULT} +docker exec $LOAD_CONT "$TEST_DIR/hackbench-load" "stop" +docker stop $LOAD_CONT +exit $CYCLICTEST_RESULT diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh index ede130a9b..7e4f26b3e 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_idle_containerized.sh @@ -2,17 +2,6 @@ source "$(dirname "$0")"/common.cfg # Run cyclictest -echo "Running cyclictest in docker container..." -docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} \ - -v ${TEST_DIR}:${TEST_DIR} \ - -v /home/admin:/home/admin \ - -v /usr/share/fw_printenv:/usr/share/fw_printenv \ - -v /sbin/fw_printenv:/sbin/fw_printenv \ - -v /usr/share/nisysinfo:/usr/share/nisysinfo \ - -v /dev:/dev \ - -t cyclictest-container \ - "${TEST_DIR}/run-cyclictest" "idle_containerized" -RESULT=$? +"$TEST_DIR/run-docker-cyclictest" "idle_containerized" -exit ${RESULT} +exit $CYCLICTEST_RESULT diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh index 36fba043b..fdb5403c6 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_iperf_containerized.sh @@ -2,31 +2,19 @@ source "$(dirname "$0")"/common.cfg # Start background network load -echo "Starting iperf load..." LOAD_CONT=$(docker run -d --privileged --network=host \ -v ${LOG_DIR}:${LOG_DIR} \ -v ${TEST_DIR}:${TEST_DIR} \ -v /home/admin:/home/admin \ -t parallel-container) -docker exec ${LOAD_CONT} "${TEST_DIR}/iperf-load" "check" || exit $? -docker exec -d ${LOAD_CONT} "${TEST_DIR}/iperf-load" "start" +docker exec $LOAD_CONT "$TEST_DIR/iperf-load" "check" || exit $? +docker exec -d $LOAD_CONT "$TEST_DIR/iperf-load" "start" # Run cyclictest -echo "Running cyclictest in docker container..." -docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} \ - -v ${TEST_DIR}:${TEST_DIR} \ - -v /home/admin:/home/admin \ - -v /usr/share/fw_printenv:/usr/share/fw_printenv \ - -v /sbin/fw_printenv:/sbin/fw_printenv \ - -v /usr/share/nisysinfo:/usr/share/nisysinfo \ - -v /dev:/dev \ - -t cyclictest-container \ - "${TEST_DIR}/run-cyclictest" "iperf_containerized" -RESULT=$? +"$TEST_DIR/run-docker-cyclictest" "iperf_containerized" # Clean up the background container -docker exec ${LOAD_CONT} "${TEST_DIR}/iperf-load" "stop" -docker stop ${LOAD_CONT} +docker exec $LOAD_CONT "$TEST_DIR/iperf-load" "stop" +docker stop $LOAD_CONT -exit ${RESULT} +exit $CYCLICTEST_RESULT diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb index 3f0f6ce13..9001fe5fe 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb @@ -21,6 +21,7 @@ SRC_URI += "\ file://build-containers \ file://run-ptest \ file://run-cyclictest \ + file://run-docker-cyclictest \ file://upload_cyclictest_results.py \ file://common.cfg \ file://fio.cfg \ @@ -39,6 +40,7 @@ do_install_ptest:append() { install -m 0755 ${S}/build-containers ${D}${PTEST_PATH} install -m 0755 ${S}/run-ptest ${D}${PTEST_PATH} install -m 0755 ${S}/run-cyclictest ${D}${PTEST_PATH} + install -m 0755 ${S}/run-docker-cyclictest ${D}${PTEST_PATH} install -m 0755 ${S}/upload_cyclictest_results.py ${D}${PTEST_PATH} install -m 0644 ${S}/common.cfg ${D}${PTEST_PATH} install -m 0644 ${S}/fio.cfg ${D}${PTEST_PATH} From 43ffcb3c08b707b663faa7a1a9f023d94097e29e Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Fri, 15 Dec 2023 16:21:08 -0600 Subject: [PATCH 031/147] kernel-containerized-performance-tests: run two cyclictest containers in parallel Modify hackbench test to run two cyclictest containers in parallel instead of one. Signed-off-by: Gratian Crisan Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 22d2e57830a0d5452cb40e3ec1b441af04277992) Signed-off-by: Alex Stewart --- .../kernel-tests/files/run-docker-cyclictest | 50 +++++++++++++------ ...rnel_cyclictest_hackbench_containerized.sh | 23 ++++++++- 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/recipes-kernel/kernel-tests/files/run-docker-cyclictest b/recipes-kernel/kernel-tests/files/run-docker-cyclictest index 8a4aad3c4..afa796e09 100644 --- a/recipes-kernel/kernel-tests/files/run-docker-cyclictest +++ b/recipes-kernel/kernel-tests/files/run-docker-cyclictest @@ -1,19 +1,37 @@ #!/bin/bash source "$(dirname "$0")"/common.cfg -if [ "$#" -lt 1 ]; then - echo echo "Usage: $0 {test_name}" - exit 1 -fi - -docker run --privileged --network=host \ - -v ${LOG_DIR}:${LOG_DIR} \ - -v ${TEST_DIR}:${TEST_DIR} \ - -v /home/admin:/home/admin \ - -v /usr/share/fw_printenv:/usr/share/fw_printenv \ - -v /sbin/fw_printenv:/sbin/fw_printenv \ - -v /usr/share/nisysinfo:/usr/share/nisysinfo \ - -v /dev:/dev \ - -t cyclictest-container \ - "${TEST_DIR}/run-cyclictest" "$1" -CYCLICTEST_RESULT=$? +case "$#" in + 1) + docker run --privileged --network=host \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -v /usr/share/fw_printenv:/usr/share/fw_printenv \ + -v /sbin/fw_printenv:/sbin/fw_printenv \ + -v /usr/share/nisysinfo:/usr/share/nisysinfo \ + -v /dev:/dev \ + -w ${TEST_DIR} \ + -t cyclictest-container \ + "${TEST_DIR}/run-cyclictest" "$1" + CYCLICTEST_RESULT=$? + ;; + 2) + docker run --privileged --network=host \ + --cpuset-cpus "$2" \ + -v ${LOG_DIR}:${LOG_DIR} \ + -v ${TEST_DIR}:${TEST_DIR} \ + -v /home/admin:/home/admin \ + -v /usr/share/fw_printenv:/usr/share/fw_printenv \ + -v /sbin/fw_printenv:/sbin/fw_printenv \ + -v /usr/share/nisysinfo:/usr/share/nisysinfo \ + -v /dev:/dev \ + -w ${TEST_DIR} \ + -t cyclictest-container \ + "${TEST_DIR}/run-cyclictest" "$1" + ;; + *) + echo echo "Usage: $0 {test_name} [{cpu}]" + exit 1 + ;; +esac diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh index 5e6610dc9..0386feab5 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests-files/test_kernel_cyclictest_hackbench_containerized.sh @@ -1,6 +1,13 @@ #!/bin/bash source "$(dirname "$0")"/common.cfg +CPUS=`nproc --all` +if [ $CPUS -lt 2 ]; then + echo "ERROR: the containerized hackbench test requires a system with at least 2 CPUs" + echo "SKIP: test_kernel_hackbench_containerized" + exit 77 +fi + # Start background scheduler load LOAD_CONT=$(docker run -d --privileged --network=host \ -v ${LOG_DIR}:${LOG_DIR} \ @@ -9,8 +16,20 @@ LOAD_CONT=$(docker run -d --privileged --network=host \ -t parallel-container) docker exec -d $LOAD_CONT "$TEST_DIR/hackbench-load" "start" -# Run cyclictest -"$TEST_DIR/run-docker-cyclictest" "hackbench_containerized" +# Run cyclictest containers on the last 2 CPUs on the system + +"$TEST_DIR/run-docker-cyclictest" "hackbench_containerized_container_1" $((CPUS - 1)) & +PID1=$! + +"$TEST_DIR/run-docker-cyclictest" "hackbench_containerized_container_2" $((CPUS - 2)) +CYCLICTEST_RESULT_2=$? + +# Wait for first cyclictest to finish +wait $PID1 +CYCLICTEST_RESULT_1=$? + +# Aggregate results from both cyclictests +CYCLICTEST_RESULT=$(( $CYCLICTEST_RESULT_1 + $CYCLICTEST_RESULT_2 )) # Clean up the background container docker exec $LOAD_CONT "$TEST_DIR/hackbench-load" "stop" From bb2109e5d99a955fbc1cb5c7653113adca7e8b86 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 2 Jan 2024 12:46:39 -0500 Subject: [PATCH 032/147] nilrt.conf: dump distro and feed versions for 24Q3 Signed-off-by: Alex Stewart --- conf/distro/nilrt.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/distro/nilrt.conf b/conf/distro/nilrt.conf index d948cd9c1..dec02d255 100644 --- a/conf/distro/nilrt.conf +++ b/conf/distro/nilrt.conf @@ -1,10 +1,10 @@ DISTRO_NAME = "NI Linux Real-Time" -DISTRO_VERSION = "10.1" +DISTRO_VERSION = "11.0" DISTRO_CODENAME = "${LAYERSERIES_COMPAT_meta-nilrt}" -NILRT_FEED_NAME ?= "2024Q1" +NILRT_FEED_NAME ?= "2024Q3" DISTRO_FEATURES:append:x64 = "\ x11 \ From 8ab7ed0bcaa0f8ce084e58b8e1c70b460065af0b Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 3 Jan 2024 15:09:50 -0600 Subject: [PATCH 033/147] fio.cfg: Enable options to reduce memory footprint fio keeps consuming more memory the longer it is run and at somepoint, runs out of memory on some targets and gets killed. This generates a huge coredump which on some targets contributes to filling up diskspace. So set 'experimental_verify', 'random_generator' options as suggested in https://lore.kernel.org/all/50F12401.8090606@kernel.dk/T/ which causes not more than a few MB of memory to be used. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit b9a13fccfb21417734290f629524efaec6ae2bf1) Signed-off-by: Alex Stewart --- recipes-kernel/kernel-tests/files/fio.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipes-kernel/kernel-tests/files/fio.cfg b/recipes-kernel/kernel-tests/files/fio.cfg index 78ff99b68..beffbe542 100644 --- a/recipes-kernel/kernel-tests/files/fio.cfg +++ b/recipes-kernel/kernel-tests/files/fio.cfg @@ -7,6 +7,8 @@ openfiles=10 direct=0 verify=sha256 do_verify=1 +experimental_verify=1 +random_generator=lfsr time_based clocksource=clock_gettime bs=5k From bd0ff5220c91c3c2c03fe6c6b618705b26dcd6bd Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Thu, 4 Jan 2024 11:43:38 -0600 Subject: [PATCH 034/147] Revert "Revert "linux-nilrt-next: Bump version to latest 6.6 development branch"" This reverts commit 85ae8d37e24dd854a0617d1dcf345257bcb08f94. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 493597072689a6028ca2415e91c7efa0dd444031) Signed-off-by: Alex Stewart --- recipes-kernel/linux/linux-nilrt-next_git.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/linux/linux-nilrt-next_git.bb b/recipes-kernel/linux/linux-nilrt-next_git.bb index 4f8c18f97..41f773111 100644 --- a/recipes-kernel/linux/linux-nilrt-next_git.bb +++ b/recipes-kernel/linux/linux-nilrt-next_git.bb @@ -1,6 +1,6 @@ DESCRIPTION = "NILRT linux kernel next development build" NI_RELEASE_VERSION = "master" -LINUX_VERSION = "6.1" +LINUX_VERSION = "6.6" LINUX_KERNEL_TYPE = "next" require linux-nilrt-alternate.inc From 432f03c210f1acbeb3a842507f395337c25008fc Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Mon, 8 Jan 2024 11:49:16 -0600 Subject: [PATCH 035/147] kernel-performance-tests: Remove stopped containers Container load test starts a docker container every few secconds which creates more and more files on disk until inodes are exhausted. So run 'docker container prune -f' after each container run. Also remove sleep between container runs as 'docker container prune -f' itself adds some delay. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit acdcd5416af422596b78d76ca1815243535ee0ec) Signed-off-by: Alex Stewart --- .../kernel-performance-tests-files/run-container-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-container-load b/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-container-load index 05b435450..8904ee498 100644 --- a/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-container-load +++ b/recipes-kernel/kernel-tests/kernel-performance-tests-files/run-container-load @@ -9,6 +9,6 @@ function start_container_load() { while true; do docker run hello-world > /dev/null - sleep 1 + docker container prune -f > /dev/null done } From fb3dafe65b77ec575df6c107752cd8d65db463ed Mon Sep 17 00:00:00 2001 From: James Minor Date: Wed, 10 Jan 2024 16:46:50 -0600 Subject: [PATCH 036/147] linux-nilrt-nohz: add guidance for faster boots on some 9030s Some devices like 9030 (stepping=3) have been shown to have long boot times and/or freezes when using the NOHZ kernel. Due to the variety of issues around cstate handling, add some guidance to help would-be users and have them change the idle mechanism to poll or set the minimum cstate. Signed-off-by: James Minor (cherry picked from commit f4b81e46c24848f4e84cf59c08a0ac0e0a8c269e) Signed-off-by: Alex Stewart --- recipes-kernel/linux/nilrt-nohz/README.nohz | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/recipes-kernel/linux/nilrt-nohz/README.nohz b/recipes-kernel/linux/nilrt-nohz/README.nohz index ae18c3447..749ac7ee7 100644 --- a/recipes-kernel/linux/nilrt-nohz/README.nohz +++ b/recipes-kernel/linux/nilrt-nohz/README.nohz @@ -92,6 +92,20 @@ Application Design Considerations echo 0 > "/sys/kernel/debug/tracing/tracing_on" + * Significant delays and freezes have been observed using the NOHZ kernel on + some early versions of the cRIO-9030 (Intel Atom E3825). These delays and + freezes are present only on older revisions of that processor, and appear + to be caused by the processor attempting to enter a lower power cstate. To + see your processor revision, examine the cpuinfo ("cat /proc/cpuinfo"). If + your processor is an E3825 with "stepping : 3", you can work around this + issue issue by adding either "idle=poll" or "intel_idle.max_cstate=1" to + the kernel command line. For example: + + fw_setenv othbootargs idle=poll + + This kernel command-line option may also result in higher performance for + some applications regardless of the processor being used. + Additional considerations for C applications: From dd34b15a5eb538ad496b568a0ea9a3465fe65c91 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Thu, 11 Jan 2024 08:37:06 -0600 Subject: [PATCH 037/147] kernel-tests: Add options to get current dmesg log from database These options allow the test to be run from a dev machine instead of a NILRT target for the purpose of debugging the test. Signed-off-by: Jeffrey Pautler (cherry picked from commit bbac52898cacf79562a1aafa3952c8f28bad02aa) Signed-off-by: Alex Stewart --- .../kernel-tests-files/kernel_dmesg_diff.py | 84 ++++++++++++++----- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py index b48a0ae5d..8a767b3b0 100644 --- a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py +++ b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py @@ -1,4 +1,5 @@ import argparse +import copy import datetime import difflib import hashlib @@ -62,8 +63,11 @@ def __init__(self, versionString): self.build_metadata = version.group('buildmetadata') class KernelVersion(SemanticVersion): - def __init__(self): - version =run_cmd(['uname', '-r']) + def __init__(self, versionString=None): + if versionString == None: + version = run_cmd(['uname', '-r']) + else: + version = versionString SemanticVersion.__init__(self, version) self.type = 'next' if self.build_metadata else 'current' prerelease_version = int(re.search(r'0|[1-9]\d*', self.prerelease).group()) @@ -108,23 +112,25 @@ def upload_log(db, dmesg_log, logger): def strip_headers(dmesg_log): return ''.join(line + '\n' for line in dmesg_log.splitlines() if not line.startswith('#')) -def get_old_dmesg_log(db, logger): - query = {} - kernel_version = KernelVersion() - os_version = OsVersion() - query['kernel_version'] = { '$lt': kernel_version.version_dict } - query['device_desc'] = get_device_desc() - query['kernel_version.major'] = kernel_version.major - query['kernel_version.minor'] = kernel_version.minor - query['os_version_major_minor'] = os_version.major_minor - +def get_old_dmesg_log(db, logger, query=None, kernel_version=None, log_type="previous"): + if query == None: + query = {} + kernel_version = KernelVersion() + os_version = OsVersion() + query['kernel_version'] = { '$lt': kernel_version.version_dict } + query['device_desc'] = get_device_desc() + query['kernel_version.major'] = kernel_version.major + query['kernel_version.minor'] = kernel_version.minor + query['os_version_major_minor'] = os_version.major_minor + + log_dmesg_diff_line = True if log_type == "previous" else False if db.count_documents(query): results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) else: # If there are no results, there may not be a run from the current os version. # Remove that requirement. del query['os_version_major_minor'] - logger.log('INFO: No prior logs from this OS Version found. Using previous version.') + logger.log('INFO: No prior logs from this OS Version found. Allowing other OS versions.') if db.count_documents(query): results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) @@ -134,22 +140,25 @@ def get_old_dmesg_log(db, logger): del query['kernel_version.major'] del query['kernel_version.minor'] query['kernel_type'] = kernel_version.type + logger.log('INFO: No prior logs from this kernel version. Allowing other kernel versions.') if db.count_documents(query): results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) else: # Keep this log in first line to help streak indexer group results. Diff hash will be populated at end. - logger.first_log('INFO: dmesg_diff: {} against , diff hash '.format(kernel_version.full)) + if log_dmesg_diff_line: + logger.first_log('INFO: dmesg_diff: {} against , diff hash '.format(kernel_version.full)) - logger.log('INFO: No suitable previous dmesg log found') + logger.log('INFO: No suitable {} dmesg log found'.format(log_type)) return '' result = next(results) # Keep this log in first line to help streak indexer group results. Diff hash will be populated at end. - logger.first_log('INFO: dmesg_diff: {} against older log of {}, diff hash '.format(kernel_version.full, result['kernel_version_full'])) + if log_dmesg_diff_line: + logger.first_log('INFO: dmesg_diff: {} against older log of {}, diff hash '.format(kernel_version.full, result['kernel_version_full'])) - logger.log('INFO: Using previous dmesg log of "{}" kernel {} from {} with _id {}'.format(result['kernel_type'], result['kernel_version_full'], result['date'], result['_id'])) + logger.log('INFO: Using {} dmesg log of "{}" kernel {} from {} with _id {}'.format(log_type, result['kernel_type'], result['kernel_version_full'], result['date'], result['_id'])) dmesg_log = result['dmesg_log'] return strip_headers(dmesg_log) @@ -223,16 +232,47 @@ def parse_args(): parser.add_argument('--server', required=True, help='Mongo server hostname') parser.add_argument('--user', required=True, help='Mongo server username') parser.add_argument('--password', required=True, help='Mongo server password') - - return parser.parse_args() + parser.add_argument('--get_current_log_from_db', + help='Specify this flag to use a log from the database as the current dmesg log. This is useful for debugging. Specify the details of the log you want with additional options.', + action="store_true") + parser.add_argument('--get_current_log_kernel_version', + help='The kernel version of the log to use as the current dmesg log. Should be of the format "6.1.59-rt17" or "6.1.59-rt17-next". Use in conjunction with --get_current_log_from_db.') + parser.add_argument('--get_current_log_device_desc', + help='The device description of the log to use as the current dmesg log. Should be of the format "cRIO-9035" or "PXI-8881". Use in conjunction with --get_current_log_from_db.') + parser.add_argument('--get_current_log_os_version', + help='The kernel version of the log to use as the current dmesg log. Should be of the format "24.0". Use in conjunction with --get_current_log_from_db.') + args = parser.parse_args() + if args.get_current_log_from_db: + if args.get_current_log_kernel_version == None or args.get_current_log_device_desc == None or args.get_current_log_os_version == None: + parser.error('When passing --get_current_log_from_db, you must also specify --get_current_log_kernel_version, --get_current_log_device_desc, and --get_current_log_os_version.') + return args logger = Logger() args = parse_args() db = DB(args.server, args.user, args.password) -old_dmesg_log = get_old_dmesg_log(db, logger) -dmesg_log = get_dmesg_log() -upload_log(db, dmesg_log, logger) +if args.get_current_log_from_db: + previous_query = {} + kernel_version = KernelVersion(args.get_current_log_kernel_version) + previous_query['kernel_version'] = { '$lt': kernel_version.version_dict } + previous_query['device_desc'] = args.get_current_log_device_desc + previous_query['kernel_version.major'] = kernel_version.major + previous_query['kernel_version.minor'] = kernel_version.minor + previous_query['os_version_major_minor'] = args.get_current_log_os_version + + current_query = copy.copy(previous_query) + dmesg_log = get_old_dmesg_log(db, logger, current_query, kernel_version, "current") + assert dmesg_log, "Could not find matching current log from database." + + # Add +1 to the prerelease version so the query (which looks for an earlier version) will return the version we actually want. + kernel_version.version_dict["prerelease"] += 1 + old_dmesg_log = get_old_dmesg_log(db, logger, previous_query, kernel_version) +else: + dmesg_log = get_dmesg_log() + old_dmesg_log = get_old_dmesg_log(db, logger) + +if not args.get_current_log_from_db: + upload_log(db, dmesg_log, logger) result = diff_logs(dmesg_log, old_dmesg_log, logger) From d041d7dca27bf84215753c7eccc6de32326e270b Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Fri, 12 Jan 2024 10:40:22 -0600 Subject: [PATCH 038/147] kernel-tests: Update debugging options Retrieving current dmesg log from database by date is more convenient. Add additional options to aide debugging. Signed-off-by: Jeffrey Pautler (cherry picked from commit 04f9ea1e0eed5eae5f1808e432e8f94ed63eefe8) Signed-off-by: Alex Stewart --- .../kernel-tests-files/kernel_dmesg_diff.py | 132 +++++++++--------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py index 8a767b3b0..b064ea3fc 100644 --- a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py +++ b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py @@ -1,5 +1,4 @@ import argparse -import copy import datetime import difflib import hashlib @@ -87,14 +86,12 @@ def get_dmesg_log(): dmesg_log = run_cmd(['dmesg']) return dmesg_log -def upload_log(db, dmesg_log, logger): +def upload_log(db, dmesg_log, kernel_version, os_version, device_desc, logger): data = {} - kernel_version = KernelVersion() - os_version = OsVersion() data['kernel_version_full'] = kernel_version.full data['kernel_version'] = kernel_version.version_dict data['kernel_type'] = kernel_version.type - data['device_desc'] = get_device_desc() + data['device_desc'] = device_desc data['architecture'] = get_architecture() data['os_version'] = os_version.full data['os_version_major_minor'] = os_version.major_minor @@ -112,25 +109,37 @@ def upload_log(db, dmesg_log, logger): def strip_headers(dmesg_log): return ''.join(line + '\n' for line in dmesg_log.splitlines() if not line.startswith('#')) -def get_old_dmesg_log(db, logger, query=None, kernel_version=None, log_type="previous"): - if query == None: - query = {} - kernel_version = KernelVersion() - os_version = OsVersion() - query['kernel_version'] = { '$lt': kernel_version.version_dict } - query['device_desc'] = get_device_desc() - query['kernel_version.major'] = kernel_version.major - query['kernel_version.minor'] = kernel_version.minor - query['os_version_major_minor'] = os_version.major_minor - - log_dmesg_diff_line = True if log_type == "previous" else False +def get_dmesg_record_by_date(db, date, logger): + query = {} + query['date'] = date + count = db.count_documents(query) + if count == 1: + results = db.find(query).limit(1) + result = next(results) + logger.log('INFO: Found dmesg record with date: {}'.format(date)) + return result + elif count > 1: + logger.log('INFO: Found multiple dmesg records with the same date: {}'.format(date)) + return False + else: + logger.log('INFO: Could not find dmesg record with date: {}'.format(date)) + return False + +def get_previous_dmesg_record(db, kernel_version, os_version_major_minor, device_desc, logger): + query = {} + query['kernel_version'] = { '$lt': kernel_version.version_dict } + query['device_desc'] = device_desc + query['kernel_version.major'] = kernel_version.major + query['kernel_version.minor'] = kernel_version.minor + query['os_version_major_minor'] = os_version_major_minor + if db.count_documents(query): results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) else: # If there are no results, there may not be a run from the current os version. # Remove that requirement. del query['os_version_major_minor'] - logger.log('INFO: No prior logs from this OS Version found. Allowing other OS versions.') + logger.log('INFO: No previous log found from OS version {}. Allowing other OS versions.'.format(os_version_major_minor)) if db.count_documents(query): results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) @@ -140,27 +149,25 @@ def get_old_dmesg_log(db, logger, query=None, kernel_version=None, log_type="pre del query['kernel_version.major'] del query['kernel_version.minor'] query['kernel_type'] = kernel_version.type - logger.log('INFO: No prior logs from this kernel version. Allowing other kernel versions.') + logger.log('INFO: No previous log found from kernel version {}.{}. Allowing other kernel versions.'.format(kernel_version.major, kernel_version.minor)) if db.count_documents(query): results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) else: # Keep this log in first line to help streak indexer group results. Diff hash will be populated at end. - if log_dmesg_diff_line: - logger.first_log('INFO: dmesg_diff: {} against , diff hash '.format(kernel_version.full)) + logger.first_log('INFO: dmesg_diff: {} against , diff hash '.format(kernel_version.full)) - logger.log('INFO: No suitable {} dmesg log found'.format(log_type)) - return '' + logger.log('INFO: No suitable previous log found. The following query was used:') + logger.log('INFO: {}'.format(query)) + return False result = next(results) # Keep this log in first line to help streak indexer group results. Diff hash will be populated at end. - if log_dmesg_diff_line: - logger.first_log('INFO: dmesg_diff: {} against older log of {}, diff hash '.format(kernel_version.full, result['kernel_version_full'])) + logger.first_log('INFO: dmesg_diff: {} against older log of {}, diff hash '.format(kernel_version.full, result['kernel_version_full'])) - logger.log('INFO: Using {} dmesg log of "{}" kernel {} from {} with _id {}'.format(log_type, result['kernel_type'], result['kernel_version_full'], result['date'], result['_id'])) - dmesg_log = result['dmesg_log'] - return strip_headers(dmesg_log) + logger.log('INFO: Using previous dmesg log of "{}" kernel {} from OS version {} dated {} with _id {}'.format(result['kernel_type'], result['kernel_version_full'], result['os_version'], result['date'], result['_id'])) + return result def strip_timestamps(log): return re.sub(r'^\[.+?\] ', '', log, flags=re.MULTILINE) @@ -207,7 +214,7 @@ def prepare_log_for_diff(log): log.sort() return log -def diff_logs(current_log, old_log, logger): +def diff_logs(current_log, old_log, suppress_diff_output, logger): current_log = prepare_log_for_diff(current_log) old_log = prepare_log_for_diff(old_log) diff = list(difflib.unified_diff(old_log, current_log, n=0)) @@ -216,8 +223,11 @@ def diff_logs(current_log, old_log, logger): logger.logs[0] += hashlib.md5(''.join(diff).encode('utf-8')).hexdigest() logger.log('INFO: Starting diff') - for line in diff: - logger.log(line) + if suppress_diff_output: + logger.log('') + else: + for line in diff: + logger.log(line) logger.log('INFO: End of diff') return False @@ -232,49 +242,39 @@ def parse_args(): parser.add_argument('--server', required=True, help='Mongo server hostname') parser.add_argument('--user', required=True, help='Mongo server username') parser.add_argument('--password', required=True, help='Mongo server password') - parser.add_argument('--get_current_log_from_db', - help='Specify this flag to use a log from the database as the current dmesg log. This is useful for debugging. Specify the details of the log you want with additional options.', - action="store_true") - parser.add_argument('--get_current_log_kernel_version', - help='The kernel version of the log to use as the current dmesg log. Should be of the format "6.1.59-rt17" or "6.1.59-rt17-next". Use in conjunction with --get_current_log_from_db.') - parser.add_argument('--get_current_log_device_desc', - help='The device description of the log to use as the current dmesg log. Should be of the format "cRIO-9035" or "PXI-8881". Use in conjunction with --get_current_log_from_db.') - parser.add_argument('--get_current_log_os_version', - help='The kernel version of the log to use as the current dmesg log. Should be of the format "24.0". Use in conjunction with --get_current_log_from_db.') - args = parser.parse_args() - if args.get_current_log_from_db: - if args.get_current_log_kernel_version == None or args.get_current_log_device_desc == None or args.get_current_log_os_version == None: - parser.error('When passing --get_current_log_from_db, you must also specify --get_current_log_kernel_version, --get_current_log_device_desc, and --get_current_log_os_version.') - return args + parser.add_argument('--current_log_db_date', metavar="", + help='Use this flag to supply a date string that will be used to locate a dmesg log in the database. That log will be used as the current dmesg log. '\ + 'Should be of the format "2023-03-30 15:32:43.203476". Date strings for previous dmesg logs can be found in the output of previous runs of this '\ + 'test or can be extracted from the Mongo database using a viewer tool like Compass. This flag also skips the upload of the dmesg record to the '\ + 'database, so the --skip_upload flag is not needed. This flag is useful for debugging issues with this test.') + parser.add_argument('--skip_upload', help='Skip upload of dmesg record to database. Useful when debugging.', action="store_true") + parser.add_argument('--suppress_diff_output', help='Reduces test output noise by removing the dmesg log diff from the output. Useful when debugging.', action="store_true") + return parser.parse_args() logger = Logger() args = parse_args() db = DB(args.server, args.user, args.password) -if args.get_current_log_from_db: - previous_query = {} - kernel_version = KernelVersion(args.get_current_log_kernel_version) - previous_query['kernel_version'] = { '$lt': kernel_version.version_dict } - previous_query['device_desc'] = args.get_current_log_device_desc - previous_query['kernel_version.major'] = kernel_version.major - previous_query['kernel_version.minor'] = kernel_version.minor - previous_query['os_version_major_minor'] = args.get_current_log_os_version - - current_query = copy.copy(previous_query) - dmesg_log = get_old_dmesg_log(db, logger, current_query, kernel_version, "current") - assert dmesg_log, "Could not find matching current log from database." - - # Add +1 to the prerelease version so the query (which looks for an earlier version) will return the version we actually want. - kernel_version.version_dict["prerelease"] += 1 - old_dmesg_log = get_old_dmesg_log(db, logger, previous_query, kernel_version) +if args.current_log_db_date: + current_dmesg_record = get_dmesg_record_by_date(db, args.current_log_db_date, logger) + assert current_dmesg_record, "Could not find matching current log record from database." + current_dmesg_log = strip_headers(current_dmesg_record['dmesg_log']) + + previous_dmesg_record = get_previous_dmesg_record(db, KernelVersion(current_dmesg_record['kernel_version_full']), current_dmesg_record['os_version_major_minor'], current_dmesg_record['device_desc'], logger) else: - dmesg_log = get_dmesg_log() - old_dmesg_log = get_old_dmesg_log(db, logger) + current_dmesg_log = get_dmesg_log() + + kernel_version = KernelVersion() + os_version = OsVersion() + device_desc = get_device_desc() + previous_dmesg_record = get_previous_dmesg_record(db, kernel_version, os_version.major_minor, device_desc, logger) + +previous_dmesg_log = strip_headers(previous_dmesg_record['dmesg_log']) if previous_dmesg_record else '' -if not args.get_current_log_from_db: - upload_log(db, dmesg_log, logger) +if not args.current_log_db_date and not args.skip_upload: + upload_log(db, current_dmesg_log, kernel_version, os_version, device_desc, logger) -result = diff_logs(dmesg_log, old_dmesg_log, logger) +result = diff_logs(current_dmesg_log, previous_dmesg_log, args.suppress_diff_output, logger) logger.report() From 508d0171959866d78e79b97e914e0e66bdf5b249 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Thu, 18 Jan 2024 11:17:44 -0600 Subject: [PATCH 039/147] kernel-tests: fix detection of current vs next kernels We are not currently using this field from the database, but fixing it in case we decide to use it in the future. Signed-off-by: Jeffrey Pautler (cherry picked from commit 6128a762808e8979bafdafdc2c3f1fd55170d2f9) Signed-off-by: Alex Stewart --- .../kernel-tests/kernel-tests-files/kernel_dmesg_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py index b064ea3fc..68c94d405 100644 --- a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py +++ b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py @@ -68,7 +68,7 @@ def __init__(self, versionString=None): else: version = versionString SemanticVersion.__init__(self, version) - self.type = 'next' if self.build_metadata else 'current' + self.type = 'next' if 'next' in self.prerelease else 'current' prerelease_version = int(re.search(r'0|[1-9]\d*', self.prerelease).group()) self.version_dict = { 'major': self.major, 'minor': self.minor, "patch": self.patch, 'prerelease': prerelease_version } From de2a05084965a2cde3008b3db5d13bd330a02732 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Thu, 18 Jan 2024 08:03:37 -0600 Subject: [PATCH 040/147] kernel-tests: fix previous dmesg record selection logic Signed-off-by: Jeffrey Pautler (cherry picked from commit 922dff67a748bc84f9ab8136dba705ed3324b712) Signed-off-by: Alex Stewart --- .../kernel-tests-files/kernel_dmesg_diff.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py index 68c94d405..4254ba3c6 100644 --- a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py +++ b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py @@ -116,7 +116,7 @@ def get_dmesg_record_by_date(db, date, logger): if count == 1: results = db.find(query).limit(1) result = next(results) - logger.log('INFO: Found dmesg record with date: {}'.format(date)) + logger.log('INFO: Found dmesg record with date: {} with kernel version: {} and OS version: {}'.format(date, result['kernel_version_full'], result['os_version'])) return result elif count > 1: logger.log('INFO: Found multiple dmesg records with the same date: {}'.format(date)) @@ -129,37 +129,37 @@ def get_previous_dmesg_record(db, kernel_version, os_version_major_minor, device query = {} query['kernel_version'] = { '$lt': kernel_version.version_dict } query['device_desc'] = device_desc - query['kernel_version.major'] = kernel_version.major - query['kernel_version.minor'] = kernel_version.minor - query['os_version_major_minor'] = os_version_major_minor if db.count_documents(query): - results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) - else: - # If there are no results, there may not be a run from the current os version. - # Remove that requirement. - del query['os_version_major_minor'] - logger.log('INFO: No previous log found from OS version {}. Allowing other OS versions.'.format(os_version_major_minor)) + # The goal of this test is to automate the identification of any dmesg differences due to a kernel upgrade, so they can be reviewed by a developer. + # Reviewing those differences is easier when the differences are small. This tends to be true when we diff between closer kernel versions. + # Therefore, this logic prefers the highest kernel version less than the kernel version under test, above all other factors. I'll refer to this as the DESIRED version. + # Some dmesg differences are due to a change in the OS rather than the kernel. We would like to reduce these differences, but not at the cost of increasing the differences between kernel versions. + # Therefore, if a record exists that matches against the DESIRED kernel version and a matching OS major.minor version, then we choose the most recently dated record that matches both. + # If no such record exists, then we simply choose the most recently dated record that matches against only the DESIRED kernel version. + results = db.find(query).sort('kernel_version', pymongo.DESCENDING).limit(1) + result = next(results) + previous_kernel_version = result['kernel_version'] + previous_kernel_version_str = "{}.{}.{}-{}".format(previous_kernel_version['major'], previous_kernel_version['minor'], previous_kernel_version['patch'], previous_kernel_version['prerelease']) + query['kernel_version'] = { '$eq': previous_kernel_version } + logger.log('INFO: Most recent previous kernel version in database is {}.'.format(previous_kernel_version_str)) + # If a record exists that also matches the OS major.minor version, then give preference to that in order to minimize differences due to non-kernel OS changes. Use the most recently dated matching record. + query['os_version_major_minor'] = os_version_major_minor if db.count_documents(query): results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) else: - # If there still aren't any results matching the kernel major minor version, broaden the - # search but limit it to same kernel type - del query['kernel_version.major'] - del query['kernel_version.minor'] - query['kernel_type'] = kernel_version.type - logger.log('INFO: No previous log found from kernel version {}.{}. Allowing other kernel versions.'.format(kernel_version.major, kernel_version.minor)) - - if db.count_documents(query): - results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) - else: - # Keep this log in first line to help streak indexer group results. Diff hash will be populated at end. - logger.first_log('INFO: dmesg_diff: {} against , diff hash '.format(kernel_version.full)) - - logger.log('INFO: No suitable previous log found. The following query was used:') - logger.log('INFO: {}'.format(query)) - return False + # If no record exists that also matches the OS major.minor version, then drop that requirement. Use the most recently dated matching record. + del query['os_version_major_minor'] + logger.log('INFO: No database record found for {} kernel version with OS version {}. Allowing other OS versions.'.format(previous_kernel_version_str, os_version_major_minor)) + assert db.count_documents(query), "Could not find previously located log record in database." + results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) + else: + # Keep this log in first line to help streak indexer group results. Diff hash will be populated at end. + logger.first_log('INFO: dmesg_diff: {} against , diff hash '.format(kernel_version.full)) + logger.log('INFO: No suitable previous log found. The following query was used:') + logger.log('INFO: {}'.format(query)) + return False result = next(results) From 312186a66e52cf8d4ccd8c383653886d1c3d9a9a Mon Sep 17 00:00:00 2001 From: James Minor Date: Fri, 12 Jan 2024 17:21:46 -0600 Subject: [PATCH 041/147] fw_printenv: Refactor variable printing and avoid duplicates The existing command line usage of fw_printenv will correctly pick an order and return only a single value from the BIOS, EFI or GRUB. However, when fw_printenv is called without any , all values are printed from all tables. This can lead to duplicates being printed, and doesn't match the behavior of the single-variable case. This is also an opportunity to clean up all of the conditionals in each of the show functions so that only a common code path is used to print the variables in the "show all" and "show single variable" cases. Signed-off-by: James Minor (cherry picked from commit 0becea6ba1e2eba46ccb125331b76e5ef837597c) Signed-off-by: Alex Stewart --- recipes-ni/fw-printenv/files/fw_printenv | 110 ++++++++++------------- 1 file changed, 47 insertions(+), 63 deletions(-) diff --git a/recipes-ni/fw-printenv/files/fw_printenv b/recipes-ni/fw-printenv/files/fw_printenv index 7d8e8c598..4287b7247 100755 --- a/recipes-ni/fw-printenv/files/fw_printenv +++ b/recipes-ni/fw-printenv/files/fw_printenv @@ -23,6 +23,9 @@ File syntax example: In this case, var1 will be deleted, var2 and other_value will be set with the values passed. The value can contain any number of spaces. + +Variables defined in multiple places will have the +following preference order: BIOS -> EFI -> GRUB EOF } @@ -30,32 +33,23 @@ EOF #they will write the value of the "$1" variable name show_variable_grub(){ - if [ $# -eq 0 ] ; then - #show all variables - grub-editenv list - if [ $? -ne 0 ]; then - return 1 - fi - return 0 - else - set -o pipefail - value=$(grub-editenv - list | grep -- "$1=" | while read -r line; do - key=$(echo "$line" | cut -d "=" -f 1) - #if the $1 matches exactly the key, extract the value - if [ "$key" = "$1" ] ; then - value_tmp=$(echo "$line" | cut -d "=" -f 2-) - echo "$value_tmp" - break - fi - done) - if [ $? -ne 0 ]; then - return 1 - fi - if [[ $value ]]; then - return 0 + set -o pipefail + value=$(grub-editenv - list | grep -- "$1=" | while read -r line; do + key=$(echo "$line" | cut -d "=" -f 1) + #if the $1 matches exactly the key, extract the value + if [ "$key" = "$1" ] ; then + value_tmp=$(echo "$line" | cut -d "=" -f 2-) + echo "$value_tmp" + break fi + done) + if [ $? -ne 0 ]; then return 1 fi + if [[ $value ]]; then + return 0 + fi + return 1 } readonly_grub_variable() @@ -82,17 +76,7 @@ if [[ "$DISTRIB_ID" == "nilrt-nxg" ]] ; then fi show_variable_smbios(){ - # We show all variables when no args passed - if [ $# -eq 0 ]; then - if is_ni_bios || is_ni_device; then - for var in ${!smbios_funcs[@]}; do - echo ${var}=`${smbios_funcs[${var}]}` - done - fi - # else, return nothing and exit successfully - return 0 - fi - # Otherwise, return the value that was requested only on NI targets + # Return the value that was requested only on NI targets if is_ni_bios || is_ni_device; then if [ "${smbios_funcs[$1]}" ]; then value=`${smbios_funcs[$1]}` @@ -107,20 +91,7 @@ declare -A efi_funcs=( ["IsNILinuxRTBoot"]=get_set_nilrt_boot) show_variable_efi(){ - # We show all variables when no args passed - if [ $# -eq 0 ]; then - for var in ${!efi_funcs[@]}; do - local key=${var} - local val=`${efi_funcs[${var}]}` - - if [ -n "$val" ]; then - echo $key=$val - fi - done - # else, return nothing and exit successfully - return 0 - fi - # Otherwise, return the value that was requested only on NI targets + # Return the value that was requested only on NI targets if [ "${efi_funcs[$1]}" ]; then ${efi_funcs[$1]} >/dev/null return $? @@ -128,23 +99,36 @@ show_variable_efi(){ return 1 } +show_all_variables(){ + # Show all variables by compiling list of all available variables and + # printing individually (ensures that the same priority order is + # enforced as a single variable print case) + all_variables=$(grub-editenv - list | awk -F= '{print $1}') + # Add a newline since one is missing from the above command + all_variables+=$'\n' + for var in ${!smbios_funcs[@]}; do all_variables+=$var; all_variables+=$'\n'; done + for var in ${!efi_funcs[@]}; do all_variables+=$var; all_variables+=$'\n'; done + all_variables=$(echo $all_variables | tr " " "\n" | sort -u) + for var in $all_variables; do + var_name=$var + show_variable_smbios $var || show_variable_efi $var || show_variable_grub $var + if [ -n "$value" ]; then + echo $var_name=$value + fi + done +} + #wraps together smbios, efi and grub variable query show_variable(){ - if [ $# -eq 0 ]; then - #show all variables - show_variable_smbios && show_variable_efi && show_variable_grub - return $? + value="" + #check for single variable + if show_variable_smbios $1 || show_variable_efi $1 || show_variable_grub $1; then + (( n_opt )) || printf '%s=' "$var_name" + printf '%s\n' "$value" + return 0 else - value="" - #check for single variable - if show_variable_smbios $1 || show_variable_efi $1 || show_variable_grub $1; then - (( n_opt )) || printf '%s=' "$var_name" - printf '%s\n' "$value" - return 0 - else - echo "## Error: \"$var_name\" not defined " >&2 - exit 1 - fi + echo "## Error: \"$var_name\" not defined " >&2 + exit 1 fi } @@ -277,7 +261,7 @@ parse_opt $@ if (( ! is_setenv )) ; then if [ $# -eq 0 ] ; then # if no arguments, list the variables - show_variable || exit $? + show_all_variables || exit $? else #show all variables passed by name for var_name in $varnames From c36c84c8cb5d6e3407a9a071d01985880d60c384 Mon Sep 17 00:00:00 2001 From: James Minor Date: Fri, 19 Jan 2024 09:17:53 -0600 Subject: [PATCH 042/147] fw_printenv: Simplify GRUB variable printing Signed-off-by: James Minor (cherry picked from commit c6626094f86a69baad98f1d5cdb3b7d3798798e9) Signed-off-by: Alex Stewart --- recipes-ni/fw-printenv/files/fw_printenv | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/recipes-ni/fw-printenv/files/fw_printenv b/recipes-ni/fw-printenv/files/fw_printenv index 4287b7247..3a802df62 100755 --- a/recipes-ni/fw-printenv/files/fw_printenv +++ b/recipes-ni/fw-printenv/files/fw_printenv @@ -34,15 +34,7 @@ EOF show_variable_grub(){ set -o pipefail - value=$(grub-editenv - list | grep -- "$1=" | while read -r line; do - key=$(echo "$line" | cut -d "=" -f 1) - #if the $1 matches exactly the key, extract the value - if [ "$key" = "$1" ] ; then - value_tmp=$(echo "$line" | cut -d "=" -f 2-) - echo "$value_tmp" - break - fi - done) + value=$(grub-editenv - list | awk -F= -v key="$1" '{ if ($1 == key) { print $2; exit }}') if [ $? -ne 0 ]; then return 1 fi From 7af720c91faca01b4e29c75471e1743cd9c193ff Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 19 Jan 2024 13:38:42 -0500 Subject: [PATCH 043/147] python3-schema: add recipe schema is a library for validating Python data structures, such as those obtained from config-files, forms, external services or command-line parsing, converted from JSON/YAML (or something else) to Python data-types. This package is a dependency of ni/grpc-device, as of 2.0. Signed-off-by: Alex Stewart --- recipes-devtools/python/python3-schema_0.7.5.bb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 recipes-devtools/python/python3-schema_0.7.5.bb diff --git a/recipes-devtools/python/python3-schema_0.7.5.bb b/recipes-devtools/python/python3-schema_0.7.5.bb new file mode 100644 index 000000000..79cdc1677 --- /dev/null +++ b/recipes-devtools/python/python3-schema_0.7.5.bb @@ -0,0 +1,17 @@ +SUMMARY = "Schema validation just got Pythonic" +DESCRIPTION = "\ +schema is a library for validating Python data structures, such as those \ +obtained from config-files, forms, external services or command-line parsing, \ +converted from JSON/YAML (or something else) to Python data-types.\ +" +HOMEPAGE = "https://github.com/keleshev/schema" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://LICENSE-MIT;md5=09b77fb74986791a3d4a0e746a37d88f" + + +SRC_URI[sha256sum] = "f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197" + + +inherit pypi setuptools3 + +BBCLASSEXTEND = "native nativesdk" From 6f0b527589b48d8a2828996e635974bba0da2b8f Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 19 Jan 2024 13:39:44 -0500 Subject: [PATCH 044/147] utf8cpp: add recipe The utfcpp/utf8cpp project is a library for transcoding UTF8 strings in C++. This package is a direct dependency of ni-grpc-device. Signed-off-by: Alex Stewart --- recipes-devtools/utf8cpp/utf8cpp_4.0.5.bb | 40 +++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 recipes-devtools/utf8cpp/utf8cpp_4.0.5.bb diff --git a/recipes-devtools/utf8cpp/utf8cpp_4.0.5.bb b/recipes-devtools/utf8cpp/utf8cpp_4.0.5.bb new file mode 100644 index 000000000..2784532ad --- /dev/null +++ b/recipes-devtools/utf8cpp/utf8cpp_4.0.5.bb @@ -0,0 +1,40 @@ +SUMMARY = "UTF-8 with C++ in a Portable Way" +DESCRIPTION = "\ +C++ developers still miss an easy and portable way of handling Unicode encoded \ +strings. The original C++ standard (known as C++98 or C++03) is Unicode \ +agnostic. Some progress has been made in the later editions of the standard, \ +but it is still hard to work with Unicode using only the standard facilities.\ +\ +I came up with a small, C++98 compatible generic library in order to \ +handle UTF-8 encoded strings. For anybody used to work with STL algorithms and \ +iterators, it should be easy and natural to use. The code is freely available \ +for any purpose - check out the license. The library has been used a lot since \ +the first release in 2006 both in commercial and open-source projects and \ +proved to be stable and useful." +HOMEPAGE = "https://github.com/nemtrif/utfcpp" +SECTION = "libs" +LICENSE = "BSL-1.0" +LIC_FILES_CHKSUM = "file://LICENSE;md5=e4224ccaecb14d942c71d31bef20d78c" + +DEPENDS = "" + +SRC_URI = "\ + git://github.com/nemtrif/utfcpp;branch=master;protocol=https \ +" +SRCREV = "6be08bbea14ffa0a5c594257fb6285a054395cd7" + +S = "${WORKDIR}/git" + + +inherit cmake + + +# For a library, the header files should be in the base package. The cmake +# configs should be in -dev. +FILES:${PN}:append = " ${includedir}" +FILES:${PN}-dev:remove = "${includedir}" +FILES:${PN}-dev:append = " ${datadir}/${BPN}/cmake" + +RDEPENDS:${PN}-dev:append = "${PN}" + +BBCLASSEXTEND = "native" From 32843da03af0d2a335fab79fd8e5e5fc3eef542a Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 22 Jan 2024 10:44:10 -0500 Subject: [PATCH 045/147] ni-grpc-device: upgrade to 2.4.0 Upstream grpc-device's CMakeLists has been substantially reworked since version 1, breaking its functionality in OE in new and exciting ways. Repatch the file with new fixes, many of which can be upstreamed. Signed-off-by: Alex Stewart --- .../0001-CMakeLists-find-gpr-library.patch | 35 ++++++ ...s.txt-remove-local-protobuf-includes.patch | 28 ----- ...akeLists.txt-remove-local-protobuf-i.patch | 35 ++++++ ...ix-undefined-condition_variable-type.patch | 34 ----- ...MakeLists-fixup-utf8cpp-library-link.patch | 29 +++++ ...Lists-use-find_package-for-gtest-dep.patch | 35 ------ ...akeLists-use-native-python3-binaries.patch | 54 ++++++++ .../ni-grpc-device/ni-grpc-device_git.bb | 116 +++++++++++------- 8 files changed, 222 insertions(+), 144 deletions(-) create mode 100644 recipes-ni/ni-grpc-device/files/0001-CMakeLists-find-gpr-library.patch delete mode 100644 recipes-ni/ni-grpc-device/files/0001-CMakeLists.txt-remove-local-protobuf-includes.patch create mode 100644 recipes-ni/ni-grpc-device/files/0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch delete mode 100644 recipes-ni/ni-grpc-device/files/0001-semaphore.h-fix-undefined-condition_variable-type.patch create mode 100644 recipes-ni/ni-grpc-device/files/0002-CMakeLists-fixup-utf8cpp-library-link.patch delete mode 100644 recipes-ni/ni-grpc-device/files/0002-CMakeLists-use-find_package-for-gtest-dep.patch create mode 100644 recipes-ni/ni-grpc-device/files/0003-CMakeLists-use-native-python3-binaries.patch diff --git a/recipes-ni/ni-grpc-device/files/0001-CMakeLists-find-gpr-library.patch b/recipes-ni/ni-grpc-device/files/0001-CMakeLists-find-gpr-library.patch new file mode 100644 index 000000000..99f8f3402 --- /dev/null +++ b/recipes-ni/ni-grpc-device/files/0001-CMakeLists-find-gpr-library.patch @@ -0,0 +1,35 @@ +From e74df62e09595d32ef8c52a359fcfb29f432d7e2 Mon Sep 17 00:00:00 2001 +From: Alex Stewart +Date: Tue, 23 Jan 2024 15:57:22 -0500 +Subject: [PATCH] CMakeLists: find gpr library + +The ni_grpc_device_server must be linked to the grpc-provided gpr.so +library during compilation. + +Modify the CMakeLists logic to find and link gpr.so. + +Signed-off-by: Alex Stewart +--- + CMakeLists.txt | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 6c966a2b..67074c65 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -18,6 +18,7 @@ endif() + if(CMAKE_CROSSCOMPILING) + find_program(_PROTOBUF_PROTOC protoc) + find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin) ++ find_library(_GRPC_GPR gpr) + + if(NOT _GRPC_DEVICE_NILRT_LEGACY_TOOLCHAIN) + find_package(gRPC REQUIRED) +@@ -441,6 +442,7 @@ endif() + target_link_libraries(ni_grpc_device_server + ${_REFLECTION} + ${_GRPC_GRPCPP} ++ ${_GRPC_GPR} + ${_PROTOBUF_LIBPROTOBUF} + ${CMAKE_DL_LIBS} + nlohmann_json::nlohmann_json diff --git a/recipes-ni/ni-grpc-device/files/0001-CMakeLists.txt-remove-local-protobuf-includes.patch b/recipes-ni/ni-grpc-device/files/0001-CMakeLists.txt-remove-local-protobuf-includes.patch deleted file mode 100644 index 7bad34451..000000000 --- a/recipes-ni/ni-grpc-device/files/0001-CMakeLists.txt-remove-local-protobuf-includes.patch +++ /dev/null @@ -1,28 +0,0 @@ -From 5beb04fda1a29cd1c82a3177564415845bf40ba5 Mon Sep 17 00:00:00 2001 -From: Alex Stewart -Date: Mon, 10 May 2021 23:09:38 +0000 -Subject: [PATCH] CMakeLists.txt: remove local protobuf includes - -When building grpc from within OE, protobuf is supplied by the native -sysroot. Do not add an includes path to the un-checked-out submodule. - -Signed-off-by: Alex Stewart - -Upstream-Status: Inappropriate [Would break non-cross-compile builds] - ---- - CMakeLists.txt | 1 - - 1 file changed, 1 deletion(-) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 656909d..4be9fd3 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -141,7 +141,6 @@ function(GenerateGrpcSources proto_file proto_path proto_srcs proto_hdrs grpc_sr - ARGS --grpc_out "${proto_out_path}" - --cpp_out "${proto_out_path}" - -I "${proto_path}" -- -I ${CMAKE_SOURCE_DIR}/third_party/grpc/third_party/protobuf/src/ - -I ${CMAKE_SOURCE_DIR}/source/protobuf - --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" - "${proto_file}" diff --git a/recipes-ni/ni-grpc-device/files/0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch b/recipes-ni/ni-grpc-device/files/0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch new file mode 100644 index 000000000..9dd86b091 --- /dev/null +++ b/recipes-ni/ni-grpc-device/files/0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch @@ -0,0 +1,35 @@ +From aca928177133eea00462fa720034bc203a959add Mon Sep 17 00:00:00 2001 +From: Alex Stewart +Date: Thu, 18 Jan 2024 14:37:36 -0500 +Subject: [PATCH] Subject: [PATCH] CMakeLists.txt: remove local protobuf + includes + +When building grpc from within OE, protobuf is supplied by the native +sysroot. Do not add an includes path to the un-checked-out submodule. + +Upstream-Status: Inappropriate [Would break non-cross-compile builds] + +Signed-off-by: Alex Stewart +--- + CMakeLists.txt | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index d52498d7..5471d6cd 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -250,7 +250,6 @@ function(GenerateGrpcSources) + ARGS --grpc_out "${proto_out_path}" + --cpp_out "${proto_out_path}" + -I "${proto_path}" +- -I ${CMAKE_SOURCE_DIR}/third_party/grpc/third_party/protobuf/src/ + -I ${CMAKE_SOURCE_DIR}/imports/protobuf + --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" + "${proto_file}" +@@ -787,4 +786,4 @@ endif() + # add_test( NAME UnitTests COMMAND UnitTestsRunner ) + add_test( NAME UnitTests COMMAND UnitTestsRunner ) + add_test( NAME IntegrationTests COMMAND IntegrationTestsRunner ) +-add_test( NAME SystemTests COMMAND SystemTestsRunner ) +\ No newline at end of file ++add_test( NAME SystemTests COMMAND SystemTestsRunner ) diff --git a/recipes-ni/ni-grpc-device/files/0001-semaphore.h-fix-undefined-condition_variable-type.patch b/recipes-ni/ni-grpc-device/files/0001-semaphore.h-fix-undefined-condition_variable-type.patch deleted file mode 100644 index f96cdfd21..000000000 --- a/recipes-ni/ni-grpc-device/files/0001-semaphore.h-fix-undefined-condition_variable-type.patch +++ /dev/null @@ -1,34 +0,0 @@ -From d636ef8cfefdaa564e1363373adc77d80529e0c3 Mon Sep 17 00:00:00 2001 -From: Alex Stewart -Date: Sat, 5 Nov 2022 16:56:59 -0500 -Subject: [PATCH] semaphore.h: fix undefined condition_variable type - -Building the grpc server component on Debian 11 and NILRT 10 throws an -error in semaphore.h about `std::condition_variable` being undefined. -g++ recommends explicitly adding the `condition_variable` header. - -Add the header. - -Signed-off-by: Alex Stewart - -Upstream-Status: Submitted [https://github.com/ni/grpc-device/pull/786] - ---- - source/server/semaphore.h | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/source/server/semaphore.h b/source/server/semaphore.h -index bd25461..ac8411f 100644 ---- a/source/server/semaphore.h -+++ b/source/server/semaphore.h -@@ -2,6 +2,7 @@ - #define NIDEVICE_GRPC_SEMAPHORE_H - - #include -+#include - - namespace nidevice_grpc { - --- -2.38.1 - diff --git a/recipes-ni/ni-grpc-device/files/0002-CMakeLists-fixup-utf8cpp-library-link.patch b/recipes-ni/ni-grpc-device/files/0002-CMakeLists-fixup-utf8cpp-library-link.patch new file mode 100644 index 000000000..71c2e2110 --- /dev/null +++ b/recipes-ni/ni-grpc-device/files/0002-CMakeLists-fixup-utf8cpp-library-link.patch @@ -0,0 +1,29 @@ +From 7de2f0751b126851a003392f2cc08901e84bfd5e Mon Sep 17 00:00:00 2001 +From: Alex Stewart +Date: Fri, 19 Jan 2024 14:45:14 -0500 +Subject: [PATCH] CMakeLists: fixup utf8cpp library link + +The utf8cpp cmake library namespace is incorrectly identified as +'utf8cpp', instead of the proper 'utf8cpp:utf8cpp'. As a result, cmake +does not link the utf8.h header and compilation fails. + +Correct the library namespace. + +Signed-off-by: Alex Stewart +--- + CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 5471d6cd..48b5c07e 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -452,7 +452,7 @@ target_link_libraries(ni_grpc_device_server + ${_PROTOBUF_LIBPROTOBUF} + ${CMAKE_DL_LIBS} + nlohmann_json::nlohmann_json +- utf8cpp ++ utf8cpp::utf8cpp + ) + + set_target_properties(ni_grpc_device_server PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE) diff --git a/recipes-ni/ni-grpc-device/files/0002-CMakeLists-use-find_package-for-gtest-dep.patch b/recipes-ni/ni-grpc-device/files/0002-CMakeLists-use-find_package-for-gtest-dep.patch deleted file mode 100644 index 9643d023d..000000000 --- a/recipes-ni/ni-grpc-device/files/0002-CMakeLists-use-find_package-for-gtest-dep.patch +++ /dev/null @@ -1,35 +0,0 @@ -From ac8145c7646b2783db4625069ddeb67083656a17 Mon Sep 17 00:00:00 2001 -From: Alex Stewart -Date: Fri, 15 Apr 2022 15:41:23 -0500 -Subject: [PATCH] CMakeLists: use find_package for gtest dep - -Within the context of an OpenEmbedded build, `find_library(gtest...` -fails to detect googletest in the recipe-sysroot-native, blocking -configuration. - -Switch the find_ operation to `find_package()`, which functions -properly. - -Signed-off-by: Alex Stewart - -Upstream-Status: Pending [https://github.com/ni/grpc-device/pull/660] ---- - CMakeLists.txt | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 573b7e94d8bb69de203907cb8eee60de11ebbd32..406590b4dedb9cb9e12efc1bf0c74840719c4cf3 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -227,7 +227,7 @@ add_custom_command( - #---------------------------------------------------------------------- - if(CMAKE_CROSSCOMPILING AND NOT _GRPC_DEVICE_NILRT_LEGACY_TOOLCHAIN) - find_package(nlohmann_json REQUIRED) -- find_library(gtest REQUIRED) -+ find_package(GTest REQUIRED) - else() - add_subdirectory(third_party/json ${CMAKE_CURRENT_BINARY_DIR}/json EXCLUDE_FROM_ALL) - enable_testing() --- -2.35.1 - diff --git a/recipes-ni/ni-grpc-device/files/0003-CMakeLists-use-native-python3-binaries.patch b/recipes-ni/ni-grpc-device/files/0003-CMakeLists-use-native-python3-binaries.patch new file mode 100644 index 000000000..45ca6b279 --- /dev/null +++ b/recipes-ni/ni-grpc-device/files/0003-CMakeLists-use-native-python3-binaries.patch @@ -0,0 +1,54 @@ +From 6536cafe3d6e753fdc4de70b3cfc3360e39cdfb5 Mon Sep 17 00:00:00 2001 +From: Alex Stewart +Date: Mon, 22 Jan 2024 10:37:19 -0500 +Subject: [PATCH] CMakeLists: use native python3 binaries + +The existing CMakeLists implementation assumes that you want to setup a +python virtual environment, instead of using what is provided by the +local system. For OE, this isn't sensible since part of the missions is +to build the whole native toolchain. + +So remove this functionality. + +Signed-off-by: Alex Stewart +--- + CMakeLists.txt | 12 ++---------- + 1 file changed, 2 insertions(+), 10 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 48b5c07e..6c966a2b 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -41,14 +41,7 @@ else() + endif() + + #---------------------------------------------------------------------- +-CreateVirtualEnvironment(virtual_environment +- REQUIREMENTS_TXT +- ${CMAKE_SOURCE_DIR}/python_build_requirements.txt +- ENV_NAME +- venv +- OUT_PYTHON_EXE +- PYTHON_EXE +-) ++set(PYTHON_EXE ${Python3_EXECUTABLE}) + + #---------------------------------------------------------------------- + # Use the utfcpp targets directly from this build, only when not cross-compiling. +@@ -191,7 +184,7 @@ foreach(api ${nidrivers}) + ${nidriver_client_srcs} + "${service_output_dir}/${api}/${api}_client.cpp") + endif() +- set(proto_dependencies ${codegen_dependencies} ${codegen_scripts} virtual_environment) ++ set(proto_dependencies ${codegen_dependencies} ${codegen_scripts}) + add_custom_command(OUTPUT ${output_files} + ${gen_command} + COMMENT "Generating proto file and service for ${api}" +@@ -211,7 +204,6 @@ add_custom_command( + DEPENDS + ${all_codegen_dependencies} + ${codegen_scripts} +- virtual_environment + ) + + set(nidriver_service_srcs diff --git a/recipes-ni/ni-grpc-device/ni-grpc-device_git.bb b/recipes-ni/ni-grpc-device/ni-grpc-device_git.bb index 60446d2ca..1be102878 100644 --- a/recipes-ni/ni-grpc-device/ni-grpc-device_git.bb +++ b/recipes-ni/ni-grpc-device/ni-grpc-device_git.bb @@ -4,8 +4,8 @@ HOMEPAGE = "https://github.com/ni/grpc-device" SECTION = "base" LICENSE = "MIT & Apache-2.0" LIC_FILES_CHKSUM = "\ - file://LICENSE;md5=08ed4de411f83eee0363c9d12e31e92d \ - file://ThirdPartyNotices.txt;md5=6def9ca42f278e76d89243699fae2b67 \ + file://LICENSE;md5=f7f2c0314387ea903bd40965abfad353 \ + file://ThirdPartyNotices.txt;md5=57283cec49f9773bb10bac48da2e56df \ " @@ -17,19 +17,22 @@ DEPENDS += "\ python3-grpcio-tools-native \ python3-mako-native \ python3-native \ + utf8cpp-native \ + python3-schema-native \ " -PV = "1.1.0" +PV = "2.4.0" SRC_URI = "\ git://github.com/ni/grpc-device.git;name=grpc-device;branch=main;protocol=https \ - file://0001-CMakeLists.txt-remove-local-protobuf-includes.patch \ - file://0001-semaphore.h-fix-undefined-condition_variable-type.patch \ - file://0002-CMakeLists-use-find_package-for-gtest-dep.patch \ file://ptest \ + file://0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch \ + file://0002-CMakeLists-fixup-utf8cpp-library-link.patch \ + file://0003-CMakeLists-use-native-python3-binaries.patch \ + file://0001-CMakeLists-find-gpr-library.patch \ " -SRCREV_grpc-device = "15508bba63a6c289a32445ffb7f8986c99a2f286" +SRCREV_grpc-device = "aeef9995eb634ed5dc4d87dc5adbfd7c66d9fa64" SRCREV_FORMAT = "grpc-device" S = "${WORKDIR}/git" @@ -37,59 +40,78 @@ S = "${WORKDIR}/git" inherit cmake python3native -EXTRA_OECMAKE += "-DCMAKE_BUILD_TYPE=Release" +EXTRA_OECMAKE += "-DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release" OECMAKE_TARGET_COMPILE = "ni_grpc_device_server" OECMAKE_GENERATOR = "Unix Makefiles" -inherit ptest - -BUILD_PTEST = "${B}/ptest" -RDEPENDS:${PN}-ptest += "\ - ${PN} \ - bash \ - python3-grpcio \ -" - -do_compile_ptest:append () { - install -d ${BUILD_PTEST} - python3 -m grpc_tools.protoc \ - -I${S}/source/protobuf \ - --python_out=${BUILD_PTEST} \ - --grpc_python_out=${BUILD_PTEST} \ - ${S}/source/protobuf/session.proto -} - -do_install_ptest:append () { - install -d ${D}${PTEST_PATH} - install -m 0755 ${WORKDIR}/ptest/run-ptest ${D}${PTEST_PATH}/ - - install -m 0644 ${S}/examples/session/enumerate-device.py ${D}${PTEST_PATH}/ - - # These files are generated by the do_compile_ptest task. - install -m 0644 ${BUILD_PTEST}/session_pb2_grpc.py ${D}${PTEST_PATH}/ - install -m 0644 ${BUILD_PTEST}/session_pb2.py ${D}${PTEST_PATH}/ -} - - +#inherit ptest +# +#BUILD_PTEST = "${B}/ptest" +#RDEPENDS:${PN}-ptest += "\ +# ${PN} \ +# bash \ +# python3-grpcio \ +#" +# +#do_compile_ptest:append () { +# install -d ${BUILD_PTEST} +# python3 -m grpc_tools.protoc \ +# -I${S}/source/protobuf \ +# --python_out=${BUILD_PTEST} \ +# --grpc_python_out=${BUILD_PTEST} \ +# ${S}/source/protobuf/session.proto +#} +# +#do_install_ptest:append () { +# install -d ${D}${PTEST_PATH} +# install -m 0755 ${WORKDIR}/ptest/run-ptest ${D}${PTEST_PATH}/ +# +# install -m 0644 ${S}/examples/session/enumerate-device.py ${D}${PTEST_PATH}/ +# +# # These files are generated by the do_compile_ptest task. +# install -m 0644 ${BUILD_PTEST}/session_pb2_grpc.py ${D}${PTEST_PATH}/ +# install -m 0644 ${BUILD_PTEST}/session_pb2.py ${D}${PTEST_PATH}/ +#} + + +serverdir = "${D}${libdir}/${BPN}" # grpc-device does not provide an 'install' target (yet); so overwrite the whole # of do_install, to keep cmake from building anything further. do_install () { - # install server binaries - install -d ${D}${bindir} - install --mode=0755 ${B}/ni_grpc_device_server ${D}${bindir} + # install server components + install -d ${serverdir} + install --mode=0755 ${B}/ni_grpc_device_server ${serverdir} + install --mode=0644 ${B}/server_config.json ${serverdir}/server_config.json.example + install --mode=0644 ${B}/server_capabilities.json ${serverdir} - # install default server_config + # package .proto files for use by developers install -d ${D}${datadir}/${BPN} - install --mode=0644 ${B}/server_config.json ${D}${datadir}/${BPN}/server_config.json.example + install --mode=0644 ${S}/generated/**/*.proto ${D}${datadir}/${BPN} + install --mode=0644 ${S}/imports/protobuf/*.proto ${D}${datadir}/${BPN} - # package .proto files for use by developers - install -d ${D}${includedir}/${BPN} - install --mode=0644 ${S}/generated/**/*.proto ${D}${includedir}/${BPN} - install --mode=0644 ${S}/source/protobuf/session.proto ${D}${includedir}/${BPN} + # install example files + install -d ${D}${datadir}/${BPN} + cp -r ${S}/examples ${D}${datadir}/${BPN} } + +PACKAGE_BEFORE_PN = "${PN}-examples" + +# ${PN} RDEPENDS:${PN} += "\ grpc \ protobuf \ " + +# ${PN}-dev +FILES:${PN}-dev:append = "${datadir}/${BPN}/*.proto" +RDEPENDS:${PN}-dev:append = "\ + grpc-dev \ +" + +# ${PN}-examples +FILES:${PN}-examples = "${datadir}/${BPN}/examples" +RDEPENDS:${PN}-examples = "\ + python3 \ +" From 7b6f77df52d6486f3e2e7f5b27cf0d1c563a2504 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Thu, 25 Jan 2024 17:08:50 -0500 Subject: [PATCH 046/147] nilrt-recovery-media: fixup ISO symlink In nanbield, the recovery media ISO symlink should point to a target with 'rootfs' in its name. Fixup the generated link. Signed-off-by: Alex Stewart --- recipes-core/images/nilrt-recovery-media.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-core/images/nilrt-recovery-media.bb b/recipes-core/images/nilrt-recovery-media.bb index 6a6ccc520..7329427e8 100644 --- a/recipes-core/images/nilrt-recovery-media.bb +++ b/recipes-core/images/nilrt-recovery-media.bb @@ -38,7 +38,7 @@ bootimg_fixup() { } symlink_iso () { - ln -sf ${PN}-${MACHINE}.wic ${DEPLOY_DIR_IMAGE}/${PN}-${MACHINE}.iso + ln -sf ${PN}-${MACHINE}.rootfs.wic ${DEPLOY_DIR_IMAGE}/${PN}-${MACHINE}.iso } ROOTFS_POSTPROCESS_COMMAND += "symlink_iso;" From 4997272ac1a566b1190b15cb9226556c23cedf6b Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 26 Jan 2024 07:59:53 -0500 Subject: [PATCH 047/147] nilrt-base-system-image: fixup runmode rootfs path In nanbield, the runmode rootfs deploy file has an additional `.rootfs` in its name. Correct the bootimg_fixup() method to accommodate this change. Signed-off-by: Alex Stewart --- recipes-core/images/nilrt-base-system-image.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-core/images/nilrt-base-system-image.bb b/recipes-core/images/nilrt-base-system-image.bb index dc47ccfe2..6005c049b 100644 --- a/recipes-core/images/nilrt-base-system-image.bb +++ b/recipes-core/images/nilrt-base-system-image.bb @@ -18,7 +18,7 @@ ROOTFS_IMAGE = "nilrt-runmode-rootfs" do_rootfs[depends] += "${ROOTFS_IMAGE}:do_image_complete" bootimg_fixup() { - install -m 0644 "${DEPLOY_DIR_IMAGE}/${ROOTFS_IMAGE}-${MACHINE}.tar.gz" "${IMAGE_ROOTFS}/data.tar.gz" + install -m 0644 "${DEPLOY_DIR_IMAGE}/${ROOTFS_IMAGE}-${MACHINE}.rootfs.tar.gz" "${IMAGE_ROOTFS}/data.tar.gz" install -m 0755 "${THISDIR}/files/${BPN}.postinst" "${IMAGE_ROOTFS}/postinst" # Remove everything that is not data.tar.gz nor the postinst From 5440523f6ecd7ba90f9e33298ea68189b6829240 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 26 Jan 2024 10:21:36 -0500 Subject: [PATCH 048/147] kernel-ctn-perf-tests: fixup docker RDEPENDS The proper RDEPENDS for docker is 'docker', not 'docker-ce'. Signed-off-by: Alex Stewart --- .../kernel-tests/kernel-containerized-performance-tests.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb index 9001fe5fe..6dd29bcb2 100644 --- a/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb +++ b/recipes-kernel/kernel-tests/kernel-containerized-performance-tests.bb @@ -11,7 +11,7 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}-files:" S = "${WORKDIR}" DEPENDS = "virtual/kernel" -RDEPENDS:${PN}-ptest += "bash python3 docker-ce" +RDEPENDS:${PN}-ptest += "bash python3 docker" RDEPENDS:${PN}-ptest:append:x64 = " fw-printenv" RDEPENDS:${PN}-ptest:append:armv7a = " u-boot-fw-utils" From 83920732aad0fe5092f789c140fb867385264801 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 26 Jan 2024 10:22:16 -0500 Subject: [PATCH 049/147] pkggrp-ni-desirable: remove ntpdate ntpdate has been dropped by upstream meta-oe, because the tool is totally deprecated in favor of native functions in ntpd. Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagegroup-ni-desirable.bb | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes-core/packagegroups/packagegroup-ni-desirable.bb b/recipes-core/packagegroups/packagegroup-ni-desirable.bb index 9bec7bdcc..49c53f796 100644 --- a/recipes-core/packagegroups/packagegroup-ni-desirable.bb +++ b/recipes-core/packagegroups/packagegroup-ni-desirable.bb @@ -33,7 +33,6 @@ RDEPENDS:${PN} += "\ ldd \ ltrace \ ntp \ - ntpdate \ nvme-cli \ openssl-dev \ perf \ From fcafbc89a74428967d4deb947d24c38e93bb7052 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 26 Jan 2024 13:46:42 -0500 Subject: [PATCH 050/147] pkggrp-ni-internal-deps: rm ni-grpc-device The ni-grpc-device recipe doesn't compile with grpc > 1.51.1. Rather than pulling the whole distro back to an older grpc version, temporarily disable ni-grpc-device until we can improve its compatibility. Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagegroup-ni-internal-deps.bb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb index 1a198469c..c257b8466 100644 --- a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb +++ b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb @@ -24,9 +24,9 @@ RDEPENDS:${PN} += "\ " # ni-sync -RDEPENDS:${PN} += "\ - ni-grpc-device \ -" +#RDEPENDS:${PN} += "\ +# ni-grpc-device \ +#" # Required components for Veristand. # Engineering contact: Marcelo Izaguirre From 65070b903bc57164e6e87a16ffae4a991fd1e16f Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 29 Jan 2024 12:33:47 -0500 Subject: [PATCH 051/147] ni-hw-scripts-common: fixup allarch variables When setting a package as "all" architecture, it is important to inherit 'allarch.bbclass', in order to set many secondary variables for that case. ni-hw-scripts-common does not inherit 'allarch'. As a result, its spdx sstate hash information is namespaced as 'all/', instead of the expected 'allarch/'. As a result, the SPDX operations in do_rootfs fail to find its SPDX snippet. Fixup the recipe by inheriting allarch.bbclass. Signed-off-by: Alex Stewart --- recipes-ni/ni-hw-scripts/ni-hw-scripts-common.bb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/recipes-ni/ni-hw-scripts/ni-hw-scripts-common.bb b/recipes-ni/ni-hw-scripts/ni-hw-scripts-common.bb index c86cb5f6c..a552ef1d8 100644 --- a/recipes-ni/ni-hw-scripts/ni-hw-scripts-common.bb +++ b/recipes-ni/ni-hw-scripts/ni-hw-scripts-common.bb @@ -16,6 +16,11 @@ SRC_URI += "\ S = "${WORKDIR}" +inherit allarch +PACKAGE_ARCH = "all" +PACKAGES:remove = "${PN}-staticdev ${PN}-dev ${PN}-dbg" + + do_install () { install -d ${D}${sysconfdir}/init.d/ @@ -46,9 +51,6 @@ pkg_postrm:${PN} () { } -PACKAGE_ARCH = "all" -PACKAGES:remove = "${PN}-staticdev ${PN}-dev ${PN}-dbg" - FILES:${PN} += "\ ${sysconfdir}/init.d/ni-rename-ifaces \ ${sysconfdir}/init.d/nisetserialnumber \ From fbf97fc1c8f7cddefc74a45a233376697222cf1d Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 29 Jan 2024 12:45:24 -0500 Subject: [PATCH 052/147] ni-rtfeatures: fixup allarch variables When setting a package as "all" architecture, it is important to inherit 'allarch.bbclass', in order to set many secondary variables for that case. ni-rtfeatures does not inherit 'allarch'. As a result, its spdx sstate hash information is namespaced as 'all/', instead of the expected 'allarch/'. As a result, the SPDX operations in do_rootfs fail to find its SPDX snippet. Fixup the recipe by inheriting allarch.bbclass. Signed-off-by: Alex Stewart --- recipes-ni/ni-rtfeatures/ni-rtfeatures.bb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/recipes-ni/ni-rtfeatures/ni-rtfeatures.bb b/recipes-ni/ni-rtfeatures/ni-rtfeatures.bb index ee030a53b..719132baa 100644 --- a/recipes-ni/ni-rtfeatures/ni-rtfeatures.bb +++ b/recipes-ni/ni-rtfeatures/ni-rtfeatures.bb @@ -20,6 +20,11 @@ SRC_URI += "\ S = "${WORKDIR}" +inherit allarch +PACKAGE_ARCH = "all" +PACKAGES:remove = "${PN}-staticdev ${PN}-dev ${PN}-dbg" + + do_install:append () { install -d ${D}${sysconfdir}/init.d/ install -m 0755 ${S}/handle_cpld_ip_reset.initd ${D}${sysconfdir}/init.d/handle_cpld_ip_reset @@ -50,9 +55,6 @@ pkg_postrm:${PN} () { } -PACKAGE_ARCH = "all" -PACKAGES:remove = "${PN}-staticdev ${PN}-dev ${PN}-dbg" - FILES:${PN} += "\ ${sysconfdir}/init.d/* \ ${sysconfdir}/udev \ From 201124fda6125fdcbda613192501fef59fce681b Mon Sep 17 00:00:00 2001 From: James Minor Date: Mon, 22 Jan 2024 17:21:08 -0600 Subject: [PATCH 053/147] fixup! fw_printenv: Simplify GRUB variable printing The previous change to simplify the GRUB variable printing broke the printing of variables with extra "=" in them. Fix that. Signed-off-by: James Minor (cherry picked from commit fff07bc01a95b5e6c387c7100c65dca4611a4f3b) Signed-off-by: Alex Stewart --- recipes-ni/fw-printenv/files/fw_printenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-ni/fw-printenv/files/fw_printenv b/recipes-ni/fw-printenv/files/fw_printenv index 3a802df62..4b9edcdde 100755 --- a/recipes-ni/fw-printenv/files/fw_printenv +++ b/recipes-ni/fw-printenv/files/fw_printenv @@ -34,7 +34,7 @@ EOF show_variable_grub(){ set -o pipefail - value=$(grub-editenv - list | awk -F= -v key="$1" '{ if ($1 == key) { print $2; exit }}') + value=$(grub-editenv - list | sed -n -e "/^$1=/p" | sed -e "s/$1=//") if [ $? -ne 0 ]; then return 1 fi From 5bad074adad1a4a27192daf75b8abe61e7e87683 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Mon, 22 Jan 2024 15:20:45 -0600 Subject: [PATCH 054/147] kernel-tests: add replacement pattern for filesystem UUIDs Signed-off-by: Jeffrey Pautler (cherry picked from commit e91448abaa395d6484c89b3fef1680ed758c6580) Signed-off-by: Alex Stewart --- .../kernel-tests/kernel-tests-files/kernel_dmesg_diff.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py index 4254ba3c6..ae053625b 100644 --- a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py +++ b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py @@ -199,7 +199,8 @@ def strip_timestamps(log): [r'tsc: Refined TSC clocksource calibration: \d+.\d+ MHz', 'tsc: Refined TSC clocksource calibration: MHz'], [r'software IO TLB: mapped mem .*', 'software IO TLB: mapped mem'], [r'eth\d:', 'ethX:'], - [r'renamed from eth\d', 'renamed from ethX'] + [r'renamed from eth\d', 'renamed from ethX'], + [r'mounted filesystem [0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}', 'mounted filesystem xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'] ] def strip_known_differences(log): From d3787a1ce3ca1c6e17fff7c30c1cd120c317e398 Mon Sep 17 00:00:00 2001 From: James Minor Date: Fri, 26 Jan 2024 14:25:13 -0600 Subject: [PATCH 055/147] fs_permissions_diff: Ignore all version changes The primary intention of this test was to look for changes to the filesystem that we didn't notice. All version changes should be ignored for this purpose, as they will happen almost constantly as we take in updates for both NI internal and external packages. We also have a separate review process for that, so no need to review it here also. Signed-off-by: James Minor (cherry picked from commit f7b14172df38d80aa3de03af75b6dd7804a684eb) Signed-off-by: Alex Stewart --- .../files/fs_permissions_diff.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index 9b04424e2..ddeec3f59 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -9,6 +9,7 @@ import subprocess import sys import fs_permissions_shared +import re class DB: mongo_db_name = 'rtos' @@ -164,9 +165,30 @@ def run_query(label, query): return basis_manifest, recent_manifest +def strip_versions_from_path(path): + # Strip all things that look like version numbers to simplify the diff + # + # The below order matters, since the last sub will prevent previous ones from working correctly + + # Strip all versions for salt (2 forms of this) + stripped_path = re.sub(r"salt-\d+\.\d+_\d+_g[0-9a-f]+", "salt-SALTVERSION", path) + stripped_path = re.sub(r"salt-\d+\.\d+\+\d+\.g[0-9a-f]+", "salt-SALTVERSION", stripped_path) + # Strip everything that looks like an RT kernel version (so module and kernel version changes are ignored) + stripped_path = re.sub(r"\d+\.\d+\.\d+-rt\d+", "VERSION.VERSION.VERSION-rtVERSION", stripped_path) + # Strip everything that looks like 1.2.3 + stripped_path = re.sub(r"\d+\.\d+\.\d+", "VERSION.VERSION.VERSION", stripped_path) + # Strip everything that looks like 1.2 + stripped_path = re.sub(r"\d+\.\d+", "VERSION.VERSION", stripped_path) + + return stripped_path + def prepare_manifest_for_diff(manifest): reader = csv.DictReader(manifest.splitlines(), fieldnames=fs_manifest_columns, delimiter='\t') - return {row['path']: row for row in reader} + result = {} + for row in reader: + stripped_path = strip_versions_from_path(row['path']) + result[stripped_path] = row + return result class IntermediateDiff: def __init__(self, old, mid, new): From 32bd887a67723f588182113fdd83cb7e603987be Mon Sep 17 00:00:00 2001 From: James Minor Date: Fri, 26 Jan 2024 14:31:29 -0600 Subject: [PATCH 056/147] fw_permissions_diff: Add command-line option for previous maniests It's helpful when debugging to be able to recall specific manifests for both the basis and current manifest. Add command-line parameters similar to the dmesg diff test. We may also decide to use this mechanism to reset the basis manifest when the changes have been reviewed. That would be a "production" feature, not just for debug. Signed-off-by: James Minor (cherry picked from commit 2fa9a3f8c16377a9ecb04cc5ef63fd7906d80154) Signed-off-by: Alex Stewart --- .../files/fs_permissions_diff.py | 67 ++++++++++++++++--- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index ddeec3f59..009bbf947 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -104,7 +104,32 @@ def upload_manifest(db, fs_manifest, logger): def strip_headers(fs_manifest): return ''.join(line + '\n' for line in fs_manifest.splitlines() if not line.startswith('#')) -def get_old_fs_manifests(db, logger): +def get_previous_fs_manifest_as_current(db, previous_date, logger): + query = {} + query['date'] = previous_date + count = db.count_documents(query) + logger.log('!!!!!OVERRIDING CURRENT MANIFEST!!!!!\n'.format(count)) + logger.log('Found {} records\n'.format(count)) + if count == 1: + results = db.find(query).limit(1) + result = next(results) + logger.log('INFO: Found fs_manifest record with these details') + log_version_info(logger, + 'current', + result['os_version_codename'], + result['os_version_full'], + result['date'], + result['_id'] + ) + return strip_headers(result['fs_permissions']) + elif count > 1: + logger.log('INFO: Found multiple fs manifest records with the same date: {}'.format(previous_date)) + exit(1) + else: + logger.log('INFO: Could not find fs manifest record with date: {}'.format(previous_date)) + exit(1) + +def get_old_fs_manifests(db, logger, basis_override): def run_query(label, query): nonlocal db nonlocal logger @@ -116,8 +141,9 @@ def run_query(label, query): else: # If there are no results, there must not be a run from the current os codename. # Remove that requirement. - del query['os_version_codename'] - logger.log('INFO: No prior fs permissions from this OS codename found. Relaxing that constraint.') + if 'os_version_codename' in query: + del query['os_version_codename'] + logger.log('INFO: No prior fs permissions from this OS codename found. Relaxing that constraint.') if db.count_documents(query): results = db.find(query).sort(sort_order).limit(1) @@ -140,7 +166,7 @@ def run_query(label, query): os_version = OsVersion() - query_build = lambda build: { + query_version_build = lambda build: { 'os_version_codename': os_version.codename, 'os_version': { '$lt': { @@ -152,10 +178,17 @@ def run_query(label, query): } } + query_date_build = lambda date: { + 'date': date + } + # The "basis" manifest is from the latest version with a lesser MAJOR.MINOR.PATCH - basis_manifest, basis_version = run_query('basis', query_build(0)) + if basis_override is not None: + basis_manifest, basis_version = run_query('basis', query_date_build(basis_override)) + else: + basis_manifest, basis_version = run_query('basis', query_version_build(0)) # The "recent" manifest is from the latest version, likely the last run with the same MAJOR.MINOR.PATCH - recent_manifest, recent_version = run_query('recent', query_build(os_version.build)) + recent_manifest, recent_version = run_query('recent', query_version_build(os_version.build)) # Keep this log in first line to help streak indexer group results. Overall hash will be populated at end. logger.prefix_log(f'INFO: fs_permissions_diff: current against {basis_version} ') @@ -341,6 +374,16 @@ def parse_args(): parser.add_argument('--server', required=True, help='Mongo server hostname') parser.add_argument('--user', required=True, help='Mongo server username') parser.add_argument('--password', required=True, help='Mongo server password') + parser.add_argument('--current_log_db_date', metavar="", + help='Use this flag to supply a date string that will be used to locate a filesystem manifest in the database. That log will be used as the current filesystem manifest. '\ + 'Should be of the format "2023-03-30 15:32:43.203476". Date strings for previous filesystem manifest can be found in the output of previous runs of this '\ + 'test or can be extracted from the Mongo database using a viewer tool like Compass. This flag also skips the upload of the filesystem manifest record to the '\ + 'database, so the --skip_upload flag is not needed. This flag is useful for debugging issues with this test.') + parser.add_argument('--basis_log_db_date', metavar="", + help='Use this flag to supply a date string that will be used to locate a filesystem manifest in the database. That log will be used as the basis filesystem manifest. '\ + 'Should be of the format "2023-03-30 15:32:43.203476". Date strings for previous filesystem manifest can be found in the output of previous runs of this '\ + 'test or can be extracted from the Mongo database using a viewer tool like Compass. This flag is useful for resetting the comparison baseline for the test.') + parser.add_argument('--skip_upload', help='Skip upload of fs manifest record to database. Useful when debugging.', action="store_true") return parser.parse_args() @@ -348,10 +391,16 @@ def main(): logger = fs_permissions_shared.Logger() args = parse_args() db = DB(args.server, args.user, args.password) - basis_fs_manifest, recent_fs_manifest = get_old_fs_manifests(db, logger) - fs_manifest = get_fs_manifest() - upload_manifest(db, fs_manifest, logger) + basis_fs_manifest, recent_fs_manifest = get_old_fs_manifests(db, logger, args.basis_log_db_date) + + if args.current_log_db_date: + fs_manifest = get_previous_fs_manifest_as_current(db, args.current_log_db_date, logger) + else: + fs_manifest = get_fs_manifest() + + if not args.current_log_db_date and not args.skip_upload: + upload_manifest(db, fs_manifest, logger) result = diff_manifests(fs_manifest, basis_fs_manifest, recent_fs_manifest, logger) From f978d8a9e07a43ddfa7bee8eba49dac7d096278c Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Mon, 29 Jan 2024 12:45:42 -0600 Subject: [PATCH 057/147] Revert "linux-nilrt-nohz: Downgrade to 5.15 kernel" This reverts commit afb05b05939c4459eb10fdd51d0ba8148cea217d. We have determined that, at this time, the current shipping products and client teams using the nohz kernel (NO_HZ_FULL) variant are not affected by the small regression in performance observed in the 6.1 nohz stress test[1]. Re-upgrade the nohz kernel to 6.1 in order to keep it in sync and up to date with the rest of the shipping kernel variants. Root cause analysis and testing will continue if/when needed. [1] https://dev.azure.com/ni/DevCentral/_wiki/wikis/AppCentral.wiki/103057/NO_HZ-Kernel-Uses Natinst-AZDO-ID: 2488177 Signed-off-by: Gratian Crisan (cherry picked from commit 56f55e764347317f731e0b8b7eb00b6041caa5d3) Signed-off-by: Alex Stewart --- recipes-kernel/linux/linux-nilrt-nohz_git.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/linux/linux-nilrt-nohz_git.bb b/recipes-kernel/linux/linux-nilrt-nohz_git.bb index e51a80897..6e639fcc2 100644 --- a/recipes-kernel/linux/linux-nilrt-nohz_git.bb +++ b/recipes-kernel/linux/linux-nilrt-nohz_git.bb @@ -1,6 +1,6 @@ DESCRIPTION = "NILRT linux kernel full dynamic ticks (NO_HZ_FULL) build" NI_RELEASE_VERSION = "master" -LINUX_VERSION = "5.15" +LINUX_VERSION = "6.1" LINUX_KERNEL_TYPE = "nohz" require linux-nilrt-alternate.inc From 68546d2d248d74179c8c00048ee796712817d5ea Mon Sep 17 00:00:00 2001 From: James Minor Date: Wed, 31 Jan 2024 10:47:37 -0600 Subject: [PATCH 058/147] fw_permissions_diff: Run logrotate before running the test logrotate runs via cron every 5 mins, which means that sometimes logrotate.status will be there and sometimes it won't during runs. We do want to check that it's permissions are correct, so just manually poke it the same way that cron will do it before running the test. Given that there may be other things we want to do before grabbing the manifest, go ahead and add a function for it also. Signed-off-by: James Minor (cherry picked from commit 32df14210f972e1f317dbf0f384127f740ea69fa) Signed-off-by: Alex Stewart --- .../ni-base-system-image-tests/files/fs_permissions_diff.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index 009bbf947..3d561868a 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -369,6 +369,11 @@ def diff_manifests(current_manifest, basis_manifest, recent_manifest, logger): return not any_differences +def prepare_system_for_manifest(): + # logrotate may not have run yet, so run it now to generate + # the logrotate.status + subprocess.run(["/usr/sbin/logrotate", "/etc/logrotate.conf"]) + def parse_args(): parser = argparse.ArgumentParser(description='diff fs permissions with previous') parser.add_argument('--server', required=True, help='Mongo server hostname') @@ -397,6 +402,7 @@ def main(): if args.current_log_db_date: fs_manifest = get_previous_fs_manifest_as_current(db, args.current_log_db_date, logger) else: + prepare_system_for_manifest() fs_manifest = get_fs_manifest() if not args.current_log_db_date and not args.skip_upload: From d59e7f1912f77448a336f5659a3844fe4f3b003c Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Wed, 31 Jan 2024 10:00:59 -0600 Subject: [PATCH 059/147] kernel-test-nohz: tweak latency limits for kernels >5.15 PREEMPT_RT kernels configured for NO_HZ_FULL show a small regression in the `kernel-test-nohz-ptest` stress benchmark for 5.16 and later kernels. This regression is not currently affecting any products/customers. Tweak the test limits to avoid unnecessary test failure noise: - PXIe-8880 - observed new max latency is ~6.5 uS; bump limit to 8 uS. - PXIe-8881 - observed new max latency is ~25 uS; increase max to 27 uS. - observed new 99.9999 percentile is 14.5 uS; increase limit to 16 uS. AB#2488177 Signed-off-by: Gratian Crisan (cherry picked from commit a2f8ce2f4c88f1b124c72f3b35722a0a1135446c) Signed-off-by: Alex Stewart --- .../kernel-tests/kernel-test-nohz-files/PXIe-8880.conf | 2 +- .../kernel-tests/kernel-test-nohz-files/PXIe-8881.conf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/recipes-kernel/kernel-tests/kernel-test-nohz-files/PXIe-8880.conf b/recipes-kernel/kernel-tests/kernel-test-nohz-files/PXIe-8880.conf index 5d026e0fe..5c533e6ac 100644 --- a/recipes-kernel/kernel-tests/kernel-test-nohz-files/PXIe-8880.conf +++ b/recipes-kernel/kernel-tests/kernel-test-nohz-files/PXIe-8880.conf @@ -1,3 +1,3 @@ -MAX_LATENCY=6000 +MAX_LATENCY=8000 P3NINES=250 P4NINES=3000 diff --git a/recipes-kernel/kernel-tests/kernel-test-nohz-files/PXIe-8881.conf b/recipes-kernel/kernel-tests/kernel-test-nohz-files/PXIe-8881.conf index 666dda5d7..145ad3e26 100644 --- a/recipes-kernel/kernel-tests/kernel-test-nohz-files/PXIe-8881.conf +++ b/recipes-kernel/kernel-tests/kernel-test-nohz-files/PXIe-8881.conf @@ -1,3 +1,3 @@ -MAX_LATENCY=22000 +MAX_LATENCY=27000 P3NINES=300 -P4NINES=10000 +P4NINES=16000 From b1a02572be29c232dbccec514af71e94f5815884 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Wed, 31 Jan 2024 14:46:10 -0600 Subject: [PATCH 060/147] kernel_tests: add rc patch version logic and fix rt-patch version logic Signed-off-by: Jeffrey Pautler (cherry picked from commit d3b6971ab1017e41269893370f4dc5a63ad97ae8) Signed-off-by: Alex Stewart --- .../kernel-tests-files/kernel_dmesg_diff.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py index ae053625b..08e578cdd 100644 --- a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py +++ b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py @@ -61,6 +61,8 @@ def __init__(self, versionString): self.prerelease = version.group('prerelease') self.build_metadata = version.group('buildmetadata') +RELEASE_KERNEL_RC_VALUE = 999 + class KernelVersion(SemanticVersion): def __init__(self, versionString=None): if versionString == None: @@ -69,8 +71,16 @@ def __init__(self, versionString=None): version = versionString SemanticVersion.__init__(self, version) self.type = 'next' if 'next' in self.prerelease else 'current' - prerelease_version = int(re.search(r'0|[1-9]\d*', self.prerelease).group()) - self.version_dict = { 'major': self.major, 'minor': self.minor, "patch": self.patch, 'prerelease': prerelease_version } + re_match = re.search(r'rc(0|[1-9]\d*)', self.prerelease) + if re_match: + self.rc = int(re_match.group(1)) + else: + # If no rcX value is present in the prerelease field of the kernel version string, then this should be a released kernel. + # Use 999 as the rc value in this case, as this value should be larger than any actual rcX value. This means that sorting + # by this field will correctly order a release version of a specific M.m.p version as higher than any actual rc version of M.m.p. + self.rc = RELEASE_KERNEL_RC_VALUE + self.rt_patch = int(re.search(r'rt(0|[1-9]\d*)', self.prerelease).group(1)) + self.version_dict = { 'major': self.major, 'minor': self.minor, 'patch': self.patch, 'rc': self.rc, 'rt_patch': self.rt_patch } class OsVersion: def __init__(self): @@ -140,7 +150,10 @@ def get_previous_dmesg_record(db, kernel_version, os_version_major_minor, device results = db.find(query).sort('kernel_version', pymongo.DESCENDING).limit(1) result = next(results) previous_kernel_version = result['kernel_version'] - previous_kernel_version_str = "{}.{}.{}-{}".format(previous_kernel_version['major'], previous_kernel_version['minor'], previous_kernel_version['patch'], previous_kernel_version['prerelease']) + if previous_kernel_version['rc'] == RELEASE_KERNEL_RC_VALUE: + previous_kernel_version_str = "{}.{}.{}-rt{}".format(previous_kernel_version['major'], previous_kernel_version['minor'], previous_kernel_version['patch'], previous_kernel_version['rt_patch']) + else: + previous_kernel_version_str = "{}.{}.{}-rc{}-rt{}".format(previous_kernel_version['major'], previous_kernel_version['minor'], previous_kernel_version['patch'], previous_kernel_version['rc'], previous_kernel_version['rt_patch']) query['kernel_version'] = { '$eq': previous_kernel_version } logger.log('INFO: Most recent previous kernel version in database is {}.'.format(previous_kernel_version_str)) From d27a45b86cd2fe75b5d97f1a34c54e067278165c Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Mon, 5 Feb 2024 10:39:27 -0600 Subject: [PATCH 061/147] kernel-tests: shorten and simplify first line of test output We use the first 100 characters of the test output to group test failures. Simplify the kernel versions by only including the components we use for sorts and querys. Also shorten the output to keep it within this 100 character limit. Signed-off-by: Jeffrey Pautler (cherry picked from commit f1e10cbd0d6ce8492be7cb44f1b43044623c7b3a) Signed-off-by: Alex Stewart --- .../kernel-tests-files/kernel_dmesg_diff.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py index 08e578cdd..c77e1b1d6 100644 --- a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py +++ b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py @@ -63,6 +63,12 @@ def __init__(self, versionString): RELEASE_KERNEL_RC_VALUE = 999 +def get_short_kernel_version(version_dict): + if version_dict['rc'] == RELEASE_KERNEL_RC_VALUE: + return "{}.{}.{}-rt{}".format(version_dict['major'], version_dict['minor'], version_dict['patch'], version_dict['rt_patch']) + else: + return "{}.{}.{}-rc{}-rt{}".format(version_dict['major'], version_dict['minor'], version_dict['patch'], version_dict['rc'], version_dict['rt_patch']) + class KernelVersion(SemanticVersion): def __init__(self, versionString=None): if versionString == None: @@ -139,6 +145,7 @@ def get_previous_dmesg_record(db, kernel_version, os_version_major_minor, device query = {} query['kernel_version'] = { '$lt': kernel_version.version_dict } query['device_desc'] = device_desc + kernel_version_short = get_short_kernel_version(kernel_version.version_dict) if db.count_documents(query): # The goal of this test is to automate the identification of any dmesg differences due to a kernel upgrade, so they can be reviewed by a developer. @@ -150,12 +157,9 @@ def get_previous_dmesg_record(db, kernel_version, os_version_major_minor, device results = db.find(query).sort('kernel_version', pymongo.DESCENDING).limit(1) result = next(results) previous_kernel_version = result['kernel_version'] - if previous_kernel_version['rc'] == RELEASE_KERNEL_RC_VALUE: - previous_kernel_version_str = "{}.{}.{}-rt{}".format(previous_kernel_version['major'], previous_kernel_version['minor'], previous_kernel_version['patch'], previous_kernel_version['rt_patch']) - else: - previous_kernel_version_str = "{}.{}.{}-rc{}-rt{}".format(previous_kernel_version['major'], previous_kernel_version['minor'], previous_kernel_version['patch'], previous_kernel_version['rc'], previous_kernel_version['rt_patch']) + previous_kernel_version_short = get_short_kernel_version(previous_kernel_version) query['kernel_version'] = { '$eq': previous_kernel_version } - logger.log('INFO: Most recent previous kernel version in database is {}.'.format(previous_kernel_version_str)) + logger.log('INFO: Most recent previous kernel version in database is {}.'.format(previous_kernel_version_short)) # If a record exists that also matches the OS major.minor version, then give preference to that in order to minimize differences due to non-kernel OS changes. Use the most recently dated matching record. query['os_version_major_minor'] = os_version_major_minor @@ -164,12 +168,12 @@ def get_previous_dmesg_record(db, kernel_version, os_version_major_minor, device else: # If no record exists that also matches the OS major.minor version, then drop that requirement. Use the most recently dated matching record. del query['os_version_major_minor'] - logger.log('INFO: No database record found for {} kernel version with OS version {}. Allowing other OS versions.'.format(previous_kernel_version_str, os_version_major_minor)) + logger.log('INFO: No database record found for {} kernel version with OS version {}. Allowing other OS versions.'.format(previous_kernel_version_short, os_version_major_minor)) assert db.count_documents(query), "Could not find previously located log record in database." results = db.find(query).sort('date', pymongo.DESCENDING).limit(1) else: # Keep this log in first line to help streak indexer group results. Diff hash will be populated at end. - logger.first_log('INFO: dmesg_diff: {} against , diff hash '.format(kernel_version.full)) + logger.first_log('INFO: dmesg_diff: {} against , diff hash '.format(kernel_version_short)) logger.log('INFO: No suitable previous log found. The following query was used:') logger.log('INFO: {}'.format(query)) return False @@ -177,7 +181,7 @@ def get_previous_dmesg_record(db, kernel_version, os_version_major_minor, device result = next(results) # Keep this log in first line to help streak indexer group results. Diff hash will be populated at end. - logger.first_log('INFO: dmesg_diff: {} against older log of {}, diff hash '.format(kernel_version.full, result['kernel_version_full'])) + logger.first_log('INFO: dmesg_diff: {} against {}, diff '.format(kernel_version_short, previous_kernel_version_short)) logger.log('INFO: Using previous dmesg log of "{}" kernel {} from OS version {} dated {} with _id {}'.format(result['kernel_type'], result['kernel_version_full'], result['os_version'], result['date'], result['_id'])) return result From ecc3aaf9d3c68e22206afce4bc56a3208d21a39f Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 30 Jan 2024 11:15:05 -0500 Subject: [PATCH 062/147] salt: remove deprecated RDEPENDS Remove python3-distutils from the salt RDEPENDS, as it as been deprecated as of OE-nanbield in favor of the setuptools implementations in python3-pip. Signed-off-by: Alex Stewart --- recipes-support/salt/salt_3000.2.bb | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes-support/salt/salt_3000.2.bb b/recipes-support/salt/salt_3000.2.bb index 72f781dd1..4feac943b 100644 --- a/recipes-support/salt/salt_3000.2.bb +++ b/recipes-support/salt/salt_3000.2.bb @@ -72,7 +72,6 @@ RDEPENDS:${PN}-common += " \ python3-configparser \ python3-dateutil \ python3-difflib \ - python3-distutils \ python3-misc \ python3-multiprocessing \ python3-profile \ From 888e74cc0de3091227c957a00006d9d966944c47 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 30 Jan 2024 11:16:01 -0500 Subject: [PATCH 063/147] pkggrp-ni-runmode: remove deprecated packages Remove several python3 modules which have been deprecated in upstream. Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagegroup-ni-runmode.bb | 2 -- 1 file changed, 2 deletions(-) diff --git a/recipes-core/packagegroups/packagegroup-ni-runmode.bb b/recipes-core/packagegroups/packagegroup-ni-runmode.bb index 5c8a92fc8..c23940a88 100644 --- a/recipes-core/packagegroups/packagegroup-ni-runmode.bb +++ b/recipes-core/packagegroups/packagegroup-ni-runmode.bb @@ -67,7 +67,6 @@ RDEPENDS:${PN} += "\ python3-dateutil \ python3-dbus \ python3-difflib \ - python3-distutils \ python3-email \ python3-fcntl \ python3-html \ @@ -112,7 +111,6 @@ RDEPENDS:${PN} += "\ python3-setuptools \ python3-shell \ python3-six \ - python3-smtpd \ python3-stringold \ python3-terminal \ python3-threading \ From 1a3394d034a9b4f8537397e2074799cf1c3e4abe Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 12 Feb 2024 15:50:53 -0500 Subject: [PATCH 064/147] curl: rebase patches for curl 8.6.0 NI-specific curl patches fail to apply to curl 8.6.0. Rebase them. Signed-off-by: Alex Stewart --- ...ed-curl_multi_setopt_va-API-function.patch | 65 ++++++++----------- ...ed-curl_share_setopt_va-API-function.patch | 4 +- .../0005-Add-nicurl-wrapper-functions.patch | 16 ++--- 3 files changed, 37 insertions(+), 48 deletions(-) diff --git a/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch b/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch index 60d38abca..3d25a512a 100644 --- a/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch +++ b/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch @@ -1,4 +1,4 @@ -From 7646de9036508744501df06a919ca431ffa709c9 Mon Sep 17 00:00:00 2001 +From 77eb39432190cb6e58dfdb8a4865c578643c4968 Mon Sep 17 00:00:00 2001 From: Jonathan David Date: Mon, 21 Jul 2014 14:32:25 -0500 Subject: [PATCH] Added curl_multi_setopt_va() API function @@ -16,31 +16,42 @@ defense. https://curl.se/mail/lib-2017-01/0146.html +Signed-off-by: Alex Stewart + +Reapplied to cURL 8.6.0. + Signed-off-by: Alex Stewart --- - lib/multi.c | 22 ++++++++++++++++------ - 1 file changed, 16 insertions(+), 6 deletions(-) + lib/multi.c | 16 +++++++++++++--- + 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/multi.c b/lib/multi.c -index ff753ac..6c35014 100644 +index 0926b0d..d78f8b2 100644 --- a/lib/multi.c +++ b/lib/multi.c -@@ -3290,12 +3290,10 @@ static CURLMcode multi_socket(struct Curl_multi *multi, - return result; - } - --#undef curl_multi_setopt --CURLMcode curl_multi_setopt(struct Curl_multi *multi, -- CURLMoption option, ...) +@@ -3316,8 +3316,20 @@ static CURLMcode multi_socket(struct Curl_multi *multi, + CURLMcode curl_multi_setopt(struct Curl_multi *multi, + CURLMoption option, ...) + { +- CURLMcode res = CURLM_OK; ++ CURLMcode res; + va_list param; ++ va_start(param, option); ++ ++ res = curl_multi_setopt_va(multi, option, param); ++ va_end(param); ++ return res; ++} ++ ++/* NI-specific function, with va_list param, instead of extended args (...) */ +CURLMcode curl_multi_setopt_va(struct Curl_multi *multi, + CURLMoption option, va_list param) - { - CURLMcode res = CURLM_OK; -- va_list param; ++{ ++ CURLMcode res = CURLM_OK; + unsigned long uarg; if(!GOOD_MULTI_HANDLE(multi)) - return CURLM_BAD_HANDLE; -@@ -3303,8 +3301,6 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, +@@ -3326,8 +3338,6 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, if(multi->in_callback) return CURLM_RECURSIVE_API_CALL; @@ -49,25 +60,3 @@ index ff753ac..6c35014 100644 switch(option) { case CURLMOPT_SOCKETFUNCTION: multi->socket_cb = va_arg(param, curl_socket_callback); -@@ -3359,7 +3355,21 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, - res = CURLM_UNKNOWN_OPTION; - break; - } -+ -+ return res; -+} -+ -+#undef curl_multi_setopt -+CURLMcode curl_multi_setopt(struct Curl_multi *multi, -+ CURLMoption option, ...) -+{ -+ CURLMcode res; -+ va_list param; -+ va_start(param, option); -+ -+ res = curl_multi_setopt_va(multi, option, param); - va_end(param); -+ - return res; - } - diff --git a/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch b/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch index c350c5e18..1be9cc4fa 100644 --- a/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch +++ b/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch @@ -1,4 +1,4 @@ -From 16f0b287ad5dfda7df059094464a26620d0d5ff7 Mon Sep 17 00:00:00 2001 +From 63db75b1009c15ba646e439b8c055879cb11e479 Mon Sep 17 00:00:00 2001 From: Jonathan David Date: Mon, 21 Jul 2014 14:25:07 -0500 Subject: [PATCH] Added curl_share_setopt_va() API function @@ -22,7 +22,7 @@ Signed-off-by: Alex Stewart 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/share.c b/lib/share.c -index c0a8d80..60a4b5d 100644 +index 8fa5cda..d3121ff 100644 --- a/lib/share.c +++ b/lib/share.c @@ -49,11 +49,9 @@ curl_share_init(void) diff --git a/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch b/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch index 0d3fa010e..88fb09e9c 100644 --- a/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch +++ b/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch @@ -1,4 +1,4 @@ -From d0899b9330a87677eff449f8eddcd544d3eeb1b8 Mon Sep 17 00:00:00 2001 +From 855ee1bb2b597617c2421f4429a4359c7de4999e Mon Sep 17 00:00:00 2001 From: Jonathan David Date: Tue, 15 Mar 2016 13:26:34 -0500 Subject: [PATCH] Add nicurl wrapper functions @@ -23,10 +23,10 @@ Signed-off-by: Alex Stewart 4 files changed, 55 insertions(+) diff --git a/lib/easy.c b/lib/easy.c -index 8e06b16..0abddf3 100644 +index fd75910..3ab98f7 100644 --- a/lib/easy.c +++ b/lib/easy.c -@@ -382,6 +382,16 @@ struct Curl_easy *curl_easy_init(void) +@@ -374,6 +374,16 @@ struct Curl_easy *curl_easy_init(void) return data; } @@ -43,7 +43,7 @@ index 8e06b16..0abddf3 100644 #ifdef CURLDEBUG struct socketmonitor { -@@ -843,6 +853,17 @@ CURLcode curl_easy_getinfo(struct Curl_easy *data, CURLINFO info, ...) +@@ -839,6 +849,17 @@ CURLcode curl_easy_getinfo(struct Curl_easy *data, CURLINFO info, ...) return result; } @@ -62,7 +62,7 @@ index 8e06b16..0abddf3 100644 { CURLcode result = CURLE_OK; diff --git a/lib/formdata.c b/lib/formdata.c -index 11f142f..60de2f8 100644 +index d4e0303..18f384f 100644 --- a/lib/formdata.c +++ b/lib/formdata.c @@ -709,6 +709,18 @@ CURLFORMcode curl_formadd(struct curl_httppost **httppost, @@ -85,10 +85,10 @@ index 11f142f..60de2f8 100644 * curl_formget() * Serialize a curl_httppost struct. diff --git a/lib/multi.c b/lib/multi.c -index 6c35014..8828962 100644 +index d78f8b2..6d4d365 100644 --- a/lib/multi.c +++ b/lib/multi.c -@@ -3373,6 +3373,17 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, +@@ -3398,6 +3398,17 @@ CURLMcode curl_multi_setopt_va(struct Curl_multi *multi, return res; } @@ -107,7 +107,7 @@ index 6c35014..8828962 100644 #undef curl_multi_socket diff --git a/lib/share.c b/lib/share.c -index 60a4b5d..5acb4de 100644 +index d3121ff..8113567 100644 --- a/lib/share.c +++ b/lib/share.c @@ -215,6 +215,17 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) From 107bd1554a39aba9c8a01e9bf2a94623f85b746b Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 12 Feb 2024 15:54:30 -0500 Subject: [PATCH 065/147] glibc: reapply patches for glibc_2.39-22 Rebase and reapply NI-specific .patches. Signed-off-by: Alex Stewart --- ...tch => 0001-glibc-add-cp936-support.patch} | 12 ++++++----- ...P936-charmap-and-zh_CN.CP936-locale.patch} | 20 ++++++++++--------- recipes-core/glibc/glibc_2.%.bbappend | 4 ++-- 3 files changed, 20 insertions(+), 16 deletions(-) rename recipes-core/glibc/glibc/{cp936_support.patch => 0001-glibc-add-cp936-support.patch} (99%) rename recipes-core/glibc/glibc/{cp936_charmap_support.patch => 0002-glibc-add-CP936-charmap-and-zh_CN.CP936-locale.patch} (99%) diff --git a/recipes-core/glibc/glibc/cp936_support.patch b/recipes-core/glibc/glibc/0001-glibc-add-cp936-support.patch similarity index 99% rename from recipes-core/glibc/glibc/cp936_support.patch rename to recipes-core/glibc/glibc/0001-glibc-add-cp936-support.patch index 3f184a085..11e0bcd31 100644 --- a/recipes-core/glibc/glibc/cp936_support.patch +++ b/recipes-core/glibc/glibc/0001-glibc-add-cp936-support.patch @@ -1,9 +1,11 @@ -From a78510f2e89476bd1405ff6ded58c45e65790db9 Mon Sep 17 00:00:00 2001 +From 04db7826a114d2a3f506155a22a781694549381c Mon Sep 17 00:00:00 2001 From: Ben Shelton Date: Fri, 19 Jun 2020 14:15:16 -0500 +Subject: [PATCH] glibc: add cp936 support Upstream-Status: Inappropriate [NI-specific] +Signed-off-by: Alex Stewart --- iconv/skeleton.c | 4 + iconvdata/Makefile | 2 +- @@ -12,10 +14,10 @@ Upstream-Status: Inappropriate [NI-specific] create mode 100644 iconvdata/cp936.c diff --git a/iconv/skeleton.c b/iconv/skeleton.c -index 1dc642e2fc..4c46b079db 100644 +index 9b74a3853c..003e997b3a 100644 --- a/iconv/skeleton.c +++ b/iconv/skeleton.c -@@ -168,6 +168,10 @@ +@@ -167,6 +167,10 @@ # endif #endif @@ -27,10 +29,10 @@ index 1dc642e2fc..4c46b079db 100644 #ifndef MAX_NEEDED_FROM # define MAX_NEEDED_FROM MIN_NEEDED_FROM diff --git a/iconvdata/Makefile b/iconvdata/Makefile -index c83962f351..0015fde589 100644 +index ea019ce5c0..88602a05e5 100644 --- a/iconvdata/Makefile +++ b/iconvdata/Makefile -@@ -61,7 +61,7 @@ modules := ISO8859-1 ISO8859-2 ISO8859-3 ISO8859-4 ISO8859-5 \ +@@ -62,7 +62,7 @@ modules := ISO8859-1 ISO8859-2 ISO8859-3 ISO8859-4 ISO8859-5 \ IBM5347 IBM9030 IBM9066 IBM9448 IBM12712 IBM16804 \ IBM1364 IBM1371 IBM1388 IBM1390 IBM1399 ISO_11548-1 MIK BRF \ MAC-CENTRALEUROPE KOI8-RU ISO8859-9E \ diff --git a/recipes-core/glibc/glibc/cp936_charmap_support.patch b/recipes-core/glibc/glibc/0002-glibc-add-CP936-charmap-and-zh_CN.CP936-locale.patch similarity index 99% rename from recipes-core/glibc/glibc/cp936_charmap_support.patch rename to recipes-core/glibc/glibc/0002-glibc-add-CP936-charmap-and-zh_CN.CP936-locale.patch index f61f1f0e1..747619139 100644 --- a/recipes-core/glibc/glibc/cp936_charmap_support.patch +++ b/recipes-core/glibc/glibc/0002-glibc-add-CP936-charmap-and-zh_CN.CP936-locale.patch @@ -1,8 +1,9 @@ -From 50acca0fba51a116e108b6549f4a88ce38d8ac80 Mon Sep 17 00:00:00 2001 +From 317d15193531d2beafaffcb7d8dff1f43951578f Mon Sep 17 00:00:00 2001 From: Charlie Johnston Date: Wed, 25 May 2022 10:37:37 -0500 -Subject: [PATCH] Adding CP936 charmap and zh_CN.CP936 locale. These are - equivalent to the GBK charmap and zh_CN.GBK locale. +Subject: [PATCH] glibc: add CP936 charmap and zh_CN.CP936 locale + +These are equivalent to the GBK charmap and zh_CN.GBK locale. Some NI Software expects that the charmap for Chinese language specifically be CP936. While this is equivalent to the GBK charmap, the naming appears @@ -10,6 +11,10 @@ to matter. This patch adds in a CP936 charmap and equivalent locale with the expected values. Upstream-Status: Inappropriate [NI-specific changes] + +Reapplied to glibc-2.39-22-g0244a37d2c. + +Signed-off-by: Alex Stewart --- localedata/SUPPORTED | 1 + localedata/charmaps/CP936 | 22037 ++++++++++++++++++++++++++++++++++++ @@ -17,13 +22,13 @@ Upstream-Status: Inappropriate [NI-specific changes] create mode 100644 localedata/charmaps/CP936 diff --git a/localedata/SUPPORTED b/localedata/SUPPORTED -index 1ee5b5e8c8..0bf665834d 100644 +index f3ea3267f3..c7efe08ae4 100644 --- a/localedata/SUPPORTED +++ b/localedata/SUPPORTED -@@ -475,6 +475,7 @@ yi_US/CP1255 \ - yo_NG/UTF-8 \ +@@ -485,6 +485,7 @@ yo_NG/UTF-8 \ yue_HK/UTF-8 \ yuw_PG/UTF-8 \ + zgh_MA/UTF-8 \ +zh_CN.CP936/CP936 \ zh_CN.GB18030/GB18030 \ zh_CN.GBK/GBK \ @@ -22071,6 +22076,3 @@ index 0000000000..a469702bee +WIDTH +... 2 +END WIDTH --- -2.30.2 - diff --git a/recipes-core/glibc/glibc_2.%.bbappend b/recipes-core/glibc/glibc_2.%.bbappend index 165a49729..34ef4ca15 100644 --- a/recipes-core/glibc/glibc_2.%.bbappend +++ b/recipes-core/glibc/glibc_2.%.bbappend @@ -4,15 +4,15 @@ FILESEXTRAPATHS:prepend := "${THISDIR}:${THISDIR}/${PN}:" # Chinese support. #file://cp936-gconv-modules.patch SRC_URI =+ " \ - file://cp936_support.patch \ file://gconv-modules-extra-remove-CP936-to-GBK-mapping.patch \ + file://0001-glibc-add-cp936-support.patch \ " # Add patches to build the ja_JP.WINDOWS-31J and zh_CN.CP936 locales for # LabVIEW Japanese and Chinese language support. SRC_URI =+ " \ file://windows-31j_support.patch \ - file://cp936_charmap_support.patch \ + file://0002-glibc-add-CP936-charmap-and-zh_CN.CP936-locale.patch \ " # Add patch to alias custom LabVIEW locales to the equivalent From 01ac041e848b67553001cee00336637eefb9384a Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 14 Feb 2024 18:30:48 -0500 Subject: [PATCH 066/147] curl: drop nicurl patches and bbappends The remaining meta-nilrt bbappend changes to curl amount to creating a secondary API for some functions, patterned like nicurl_*(). We had to patch in these functions into OE, because the NILRT curl package was the source for the downstream nicurl component. These functions have been explicitly rejected by upstream (in their upstreamable parts), and they are constantly breaking during curl upstream merges. Further, the nicurl component has been reworked to pull its curl source directly from upstream. So there is no longer a reason to apply these changes at the OE level. Drop the nicurl patchsets. And remove the whole curl bbappend, since it is no longer substantial. Signed-off-by: Alex Stewart --- ...ed-curl_easy_getinfo_va-API-function.patch | 59 -------- ...2-Added-curl_formadd_va-API-function.patch | 48 ------- ...ed-curl_multi_setopt_va-API-function.patch | 62 --------- ...ed-curl_share_setopt_va-API-function.patch | 68 --------- .../0005-Add-nicurl-wrapper-functions.patch | 130 ------------------ recipes-support/curl/curl_8.%.bbappend | 11 -- 6 files changed, 378 deletions(-) delete mode 100644 recipes-support/curl/curl/0001-Added-curl_easy_getinfo_va-API-function.patch delete mode 100644 recipes-support/curl/curl/0002-Added-curl_formadd_va-API-function.patch delete mode 100644 recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch delete mode 100644 recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch delete mode 100644 recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch delete mode 100644 recipes-support/curl/curl_8.%.bbappend diff --git a/recipes-support/curl/curl/0001-Added-curl_easy_getinfo_va-API-function.patch b/recipes-support/curl/curl/0001-Added-curl_easy_getinfo_va-API-function.patch deleted file mode 100644 index d13ac72cd..000000000 --- a/recipes-support/curl/curl/0001-Added-curl_easy_getinfo_va-API-function.patch +++ /dev/null @@ -1,59 +0,0 @@ -From bf9638c7d70aecdffc6ded02c9fdd562d47d5e53 Mon Sep 17 00:00:00 2001 -From: Ioan-Adrian Ratiu -Date: Mon, 21 Jul 2014 15:02:35 -0500 -Subject: [PATCH] Added curl_easy_getinfo_va() API function - -Added API function with `va_list` variable argument parameter in -addition to cURL's function using `...`. - -Upstream-Status: Denied - -https://curl.se/mail/lib-2017-01/0135.html - -This patchset was denied by upstream because they did not agree that our -usecase for these added functions were valid - and we did not have a -defense. - -https://curl.se/mail/lib-2017-01/0146.html - -Signed-off-by: Alex Stewart ---- - lib/easy.c | 17 +++++++++++++---- - 1 file changed, 13 insertions(+), 4 deletions(-) - -diff --git a/lib/easy.c b/lib/easy.c -index 6b4fb8e..8e06b16 100644 ---- a/lib/easy.c -+++ b/lib/easy.c -@@ -819,18 +819,27 @@ void curl_easy_cleanup(struct Curl_easy *data) - * information from a performed transfer and similar. - */ - #undef curl_easy_getinfo --CURLcode curl_easy_getinfo(struct Curl_easy *data, CURLINFO info, ...) -+CURLcode curl_easy_getinfo_va(struct Curl_easy *data, CURLINFO info, va_list arg) - { -- va_list arg; - void *paramp; - CURLcode result; - -- va_start(arg, info); - paramp = va_arg(arg, void *); -- - result = Curl_getinfo(data, info, paramp); - -+ return result; -+} -+ -+#undef curl_easy_getinfo -+CURLcode curl_easy_getinfo(struct Curl_easy *data, CURLINFO info, ...) -+{ -+ CURLcode result; -+ va_list arg; -+ va_start(arg, info); -+ -+ result = curl_easy_getinfo_va(data, info, arg); - va_end(arg); -+ - return result; - } - diff --git a/recipes-support/curl/curl/0002-Added-curl_formadd_va-API-function.patch b/recipes-support/curl/curl/0002-Added-curl_formadd_va-API-function.patch deleted file mode 100644 index f495507f1..000000000 --- a/recipes-support/curl/curl/0002-Added-curl_formadd_va-API-function.patch +++ /dev/null @@ -1,48 +0,0 @@ -From 9414d7a122c50f7834a5783d5ad69b9635870fe2 Mon Sep 17 00:00:00 2001 -From: Jonathan David -Date: Tue, 15 Mar 2016 14:13:28 -0500 -Subject: [PATCH] Added curl_formadd_va() API function - -Added API function with `va_list` variable argument parameter in -addition to cURL's function using `...`. - -Upstream-Status: Denied - -This patchset was denied by upstream because they did not agree that our -usecase for these added functions were valid - and we did not have a -defense. - -https://curl.se/mail/lib-2017-01/0146.html - -Signed-off-by: Alex Stewart ---- - lib/formdata.c | 9 ++++++++- - 1 file changed, 8 insertions(+), 1 deletion(-) - -diff --git a/lib/formdata.c b/lib/formdata.c -index e40c4bc..11f142f 100644 ---- a/lib/formdata.c -+++ b/lib/formdata.c -@@ -690,6 +690,13 @@ CURLFORMcode FormAdd(struct curl_httppost **httppost, - * @unittest: 1308 - */ - -+CURLFORMcode curl_formadd_va(struct curl_httppost **httppost, -+ struct curl_httppost **last_post, -+ va_list arg) -+{ -+ return FormAdd(httppost, last_post, arg); -+} -+ - CURLFORMcode curl_formadd(struct curl_httppost **httppost, - struct curl_httppost **last_post, - ...) -@@ -697,7 +704,7 @@ CURLFORMcode curl_formadd(struct curl_httppost **httppost, - va_list arg; - CURLFORMcode result; - va_start(arg, last_post); -- result = FormAdd(httppost, last_post, arg); -+ result = curl_formadd_va(httppost, last_post, arg); - va_end(arg); - return result; - } diff --git a/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch b/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch deleted file mode 100644 index 3d25a512a..000000000 --- a/recipes-support/curl/curl/0003-Added-curl_multi_setopt_va-API-function.patch +++ /dev/null @@ -1,62 +0,0 @@ -From 77eb39432190cb6e58dfdb8a4865c578643c4968 Mon Sep 17 00:00:00 2001 -From: Jonathan David -Date: Mon, 21 Jul 2014 14:32:25 -0500 -Subject: [PATCH] Added curl_multi_setopt_va() API function - -Added API function with `va_list` variable argument parameter in -addition to cURL's function using `...`. - -Upstream-Status: Denied - -https://curl.se/mail/lib-2017-01/0138.html - -This patchset was denied by upstream because they did not agree that our -usecase for these added functions were valid - and we did not have a -defense. - -https://curl.se/mail/lib-2017-01/0146.html - -Signed-off-by: Alex Stewart - -Reapplied to cURL 8.6.0. - -Signed-off-by: Alex Stewart ---- - lib/multi.c | 16 +++++++++++++--- - 1 file changed, 13 insertions(+), 3 deletions(-) - -diff --git a/lib/multi.c b/lib/multi.c -index 0926b0d..d78f8b2 100644 ---- a/lib/multi.c -+++ b/lib/multi.c -@@ -3316,8 +3316,20 @@ static CURLMcode multi_socket(struct Curl_multi *multi, - CURLMcode curl_multi_setopt(struct Curl_multi *multi, - CURLMoption option, ...) - { -- CURLMcode res = CURLM_OK; -+ CURLMcode res; - va_list param; -+ va_start(param, option); -+ -+ res = curl_multi_setopt_va(multi, option, param); -+ va_end(param); -+ return res; -+} -+ -+/* NI-specific function, with va_list param, instead of extended args (...) */ -+CURLMcode curl_multi_setopt_va(struct Curl_multi *multi, -+ CURLMoption option, va_list param) -+{ -+ CURLMcode res = CURLM_OK; - unsigned long uarg; - - if(!GOOD_MULTI_HANDLE(multi)) -@@ -3326,8 +3338,6 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, - if(multi->in_callback) - return CURLM_RECURSIVE_API_CALL; - -- va_start(param, option); -- - switch(option) { - case CURLMOPT_SOCKETFUNCTION: - multi->socket_cb = va_arg(param, curl_socket_callback); diff --git a/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch b/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch deleted file mode 100644 index 1be9cc4fa..000000000 --- a/recipes-support/curl/curl/0004-Added-curl_share_setopt_va-API-function.patch +++ /dev/null @@ -1,68 +0,0 @@ -From 63db75b1009c15ba646e439b8c055879cb11e479 Mon Sep 17 00:00:00 2001 -From: Jonathan David -Date: Mon, 21 Jul 2014 14:25:07 -0500 -Subject: [PATCH] Added curl_share_setopt_va() API function - -Added API function with `va_list` variable argument parameter in -addition to cURL's function using `...`. - -Upstream-Status: Denied - -https://curl.se/mail/lib-2017-01/0139.html - -This patchset was denied by upstream because they did not agree that our -usecase for these added functions were valid - and we did not have a -defense. - -https://curl.se/mail/lib-2017-01/0146.html - -Signed-off-by: Alex Stewart ---- - lib/share.c | 18 +++++++++++++----- - 1 file changed, 13 insertions(+), 5 deletions(-) - -diff --git a/lib/share.c b/lib/share.c -index 8fa5cda..d3121ff 100644 ---- a/lib/share.c -+++ b/lib/share.c -@@ -49,11 +49,9 @@ curl_share_init(void) - return share; - } - --#undef curl_share_setopt - CURLSHcode --curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) -+curl_share_setopt_va(struct Curl_share *share, CURLSHoption option, va_list param) - { -- va_list param; - int type; - curl_lock_function lockfunc; - curl_unlock_function unlockfunc; -@@ -68,8 +66,6 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) - using this share */ - return CURLSHE_IN_USE; - -- va_start(param, option); -- - switch(option) { - case CURLSHOPT_SHARE: - /* this is a type this share will share */ -@@ -202,6 +198,18 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) - break; - } - -+ return res; -+} -+ -+#undef curl_share_setopt -+CURLSHcode -+curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) -+{ -+ CURLSHcode res; -+ va_list param; -+ va_start(param, option); -+ -+ res = curl_share_setopt_va(share, option, param); - va_end(param); - - return res; diff --git a/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch b/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch deleted file mode 100644 index 88fb09e9c..000000000 --- a/recipes-support/curl/curl/0005-Add-nicurl-wrapper-functions.patch +++ /dev/null @@ -1,130 +0,0 @@ -From 855ee1bb2b597617c2421f4429a4359c7de4999e Mon Sep 17 00:00:00 2001 -From: Jonathan David -Date: Tue, 15 Mar 2016 13:26:34 -0500 -Subject: [PATCH] Add nicurl wrapper functions - -Added the following API wrapper functions: -nicurl_easy_getinfo -nicurl_easy_setopt -nicurl_formadd -nicurl_multi_setopt -nicurl_share_setopt - -These functions are called by older NI components, and exist as -a means to keep previous code running. - -Upstream-Status: Inappropriate [NI specific changes] -Signed-off-by: Alex Stewart ---- - lib/easy.c | 21 +++++++++++++++++++++ - lib/formdata.c | 12 ++++++++++++ - lib/multi.c | 11 +++++++++++ - lib/share.c | 11 +++++++++++ - 4 files changed, 55 insertions(+) - -diff --git a/lib/easy.c b/lib/easy.c -index fd75910..3ab98f7 100644 ---- a/lib/easy.c -+++ b/lib/easy.c -@@ -374,6 +374,16 @@ struct Curl_easy *curl_easy_init(void) - return data; - } - -+#ifdef ENABLE_NICURL_API -+CURL_EXTERN CURLcode nicurl_easy_setopt(CURL *curl, CURLoption tag, va_list arg) -+{ -+ CURLcode result; -+ result = Curl_vsetopt(curl, tag, arg); -+ va_end(arg); -+ return result; -+} -+#endif -+ - #ifdef CURLDEBUG - - struct socketmonitor { -@@ -839,6 +849,17 @@ CURLcode curl_easy_getinfo(struct Curl_easy *data, CURLINFO info, ...) - return result; - } - -+#ifdef ENABLE_NICURL_API -+CURL_EXTERN CURLcode nicurl_easy_getinfo(CURL *curl, CURLINFO info, va_list arg) -+{ -+/* Unlike other nicurl functions, this one did not previously call va_start() -+ * nor va_end(). Thus, it can behave as expected of variadic functions that -+ * are passed va_list. -+ */ -+ return curl_easy_getinfo_va(curl, info, arg); -+} -+#endif -+ - static CURLcode dupset(struct Curl_easy *dst, struct Curl_easy *src) - { - CURLcode result = CURLE_OK; -diff --git a/lib/formdata.c b/lib/formdata.c -index d4e0303..18f384f 100644 ---- a/lib/formdata.c -+++ b/lib/formdata.c -@@ -709,6 +709,18 @@ CURLFORMcode curl_formadd(struct curl_httppost **httppost, - return result; - } - -+#ifdef ENABLE_NICURL_API -+CURL_EXTERN CURLFORMcode nicurl_formadd(struct curl_httppost **httppost, -+ struct curl_httppost **last_post, -+ va_list arg) -+{ -+ CURLFORMcode result; -+ result = curl_formadd_va(httppost, last_post, arg); -+ va_end(arg); -+ return result; -+} -+#endif -+ - /* - * curl_formget() - * Serialize a curl_httppost struct. -diff --git a/lib/multi.c b/lib/multi.c -index d78f8b2..6d4d365 100644 ---- a/lib/multi.c -+++ b/lib/multi.c -@@ -3398,6 +3398,17 @@ CURLMcode curl_multi_setopt_va(struct Curl_multi *multi, - return res; - } - -+#ifdef ENABLE_NICURL_API -+CURL_EXTERN CURLMcode nicurl_multi_setopt(CURLM *multi_handle, -+ CURLMoption option, va_list param) -+{ -+ CURLMcode result; -+ result = curl_multi_setopt_va(multi_handle, option, param); -+ va_end(param); -+ return result; -+} -+#endif -+ - /* we define curl_multi_socket() in the public multi.h header */ - #undef curl_multi_socket - -diff --git a/lib/share.c b/lib/share.c -index d3121ff..8113567 100644 ---- a/lib/share.c -+++ b/lib/share.c -@@ -215,6 +215,17 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) - return res; - } - -+#ifdef ENABLE_NICURL_API -+CURLSHcode -+CURL_EXTERN nicurl_share_setopt(CURLSH *sh, CURLSHoption option, va_list param) -+{ -+ CURLSHcode result; -+ result = curl_share_setopt_va(sh, option, param); -+ va_end(param); -+ return result; -+} -+#endif -+ - CURLSHcode - curl_share_cleanup(struct Curl_share *share) - { diff --git a/recipes-support/curl/curl_8.%.bbappend b/recipes-support/curl/curl_8.%.bbappend deleted file mode 100644 index 57e6b1781..000000000 --- a/recipes-support/curl/curl_8.%.bbappend +++ /dev/null @@ -1,11 +0,0 @@ -FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:" - -SRC_URI += " \ - file://0001-Added-curl_easy_getinfo_va-API-function.patch \ - file://0002-Added-curl_formadd_va-API-function.patch \ - file://0003-Added-curl_multi_setopt_va-API-function.patch \ - file://0004-Added-curl_share_setopt_va-API-function.patch \ - file://0005-Add-nicurl-wrapper-functions.patch \ -" - -SELECTED_OPTIMIZATION += "-Wno-deprecated-declarations" From fcc19e8ea7ded418d0ef2ba25587adcfa158996b Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 14 Feb 2024 19:24:13 -0500 Subject: [PATCH 067/147] grub: reapply patches for grub 2.12 Signed-off-by: Alex Stewart --- .../grub/add-inbit-command-to-io-module.patch | 8 +++----- ...-Add-bitwise-AND-document-the-feature.patch | 12 +++++------- ...grub-advertise-NI-NILRT-over-GNU-GRUB.patch | 18 ++++++++---------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/recipes-bsp/grub/grub/add-inbit-command-to-io-module.patch b/recipes-bsp/grub/grub/add-inbit-command-to-io-module.patch index 2ab3a528f..a0233a45b 100644 --- a/recipes-bsp/grub/grub/add-inbit-command-to-io-module.patch +++ b/recipes-bsp/grub/grub/add-inbit-command-to-io-module.patch @@ -1,4 +1,4 @@ -From 48de0b2faec205e874099e0afdc14ab70919e702 Mon Sep 17 00:00:00 2001 +From 8267085891ed1efd3a8f46aff8ae172f94caaab7 Mon Sep 17 00:00:00 2001 From: Adrian Papp Date: Thu, 20 Feb 2014 15:01:05 +0200 Subject: [PATCH] add inbit command to io module @@ -7,12 +7,13 @@ Subject: [PATCH] add inbit command to io module command Upstream-Status: Inappropriate [NI specific] + --- grub-core/commands/iorw.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/grub-core/commands/iorw.c b/grub-core/commands/iorw.c -index 584baec8f..45a79ed7f 100644 +index 584baec..45a79ed 100644 --- a/grub-core/commands/iorw.c +++ b/grub-core/commands/iorw.c @@ -27,6 +27,7 @@ @@ -83,6 +84,3 @@ index 584baec8f..45a79ed7f 100644 grub_unregister_command (cmd_write_byte); grub_unregister_command (cmd_write_word); grub_unregister_command (cmd_write_dword); --- -2.25.1 - diff --git a/recipes-bsp/grub/grub/cmd-test-Add-bitwise-AND-document-the-feature.patch b/recipes-bsp/grub/grub/cmd-test-Add-bitwise-AND-document-the-feature.patch index 3422f079a..667ddcc6f 100644 --- a/recipes-bsp/grub/grub/cmd-test-Add-bitwise-AND-document-the-feature.patch +++ b/recipes-bsp/grub/grub/cmd-test-Add-bitwise-AND-document-the-feature.patch @@ -1,4 +1,4 @@ -From 1dfdfdfa97910081e50e70df9b3e78162987130a Mon Sep 17 00:00:00 2001 +From 2fa7700d70f71fa96061b78dabefbda536e63d9d Mon Sep 17 00:00:00 2001 From: Brad Mouring Date: Thu, 8 Dec 2016 11:03:42 -0600 Subject: [PATCH] cmd: test: Add bitwise AND, document the feature @@ -14,16 +14,17 @@ need for the inbit command. Upstream-Status: Inappropriate [NI specific] Signed-off-by: Brad Mouring + --- docs/grub.texi | 2 ++ grub-core/commands/test.c | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/docs/grub.texi b/docs/grub.texi -index f829684ec..37aef949b 100644 +index a225f9a..51ae3e0 100644 --- a/docs/grub.texi +++ b/docs/grub.texi -@@ -5034,6 +5034,8 @@ the strings are not equal +@@ -5817,6 +5817,8 @@ the strings are not equal @var{integer1} is less than @var{integer2} @item @var{integer1} @code{-ne} @var{integer2} @var{integer1} is not equal to @var{integer2} @@ -33,7 +34,7 @@ index f829684ec..37aef949b 100644 @var{integer1} is greater than @var{integer2} after stripping off common non-numeric @var{prefix}. @item @var{prefix}@var{integer1} @code{-plt} @var{prefix}@var{integer2} diff --git a/grub-core/commands/test.c b/grub-core/commands/test.c -index 5f06642f6..af4fad728 100644 +index 62d3fb3..a34a8ed 100644 --- a/grub-core/commands/test.c +++ b/grub-core/commands/test.c @@ -290,6 +290,16 @@ test_parse (char **args, int *argn, int argc) @@ -53,6 +54,3 @@ index 5f06642f6..af4fad728 100644 /* -nt and -ot tests. GRUB extension: when doing -?t bias will be added to the first mtime. */ if (grub_memcmp (args[*argn + 1], "-nt", 3) == 0 --- -2.25.1 - diff --git a/recipes-bsp/grub/grub/grub-advertise-NI-NILRT-over-GNU-GRUB.patch b/recipes-bsp/grub/grub/grub-advertise-NI-NILRT-over-GNU-GRUB.patch index 2e19fd4cd..ce09569a8 100644 --- a/recipes-bsp/grub/grub/grub-advertise-NI-NILRT-over-GNU-GRUB.patch +++ b/recipes-bsp/grub/grub/grub-advertise-NI-NILRT-over-GNU-GRUB.patch @@ -1,4 +1,4 @@ -From 83f4667406917c3ea089f7cbb4da3f6e8a48c1fe Mon Sep 17 00:00:00 2001 +From 464ac648e9df45779754fadccbc0088042a3c441 Mon Sep 17 00:00:00 2001 From: Ioan-Adrian Ratiu Date: Thu, 15 Dec 2016 14:04:40 +0200 Subject: [PATCH] grub: advertise NI NILRT over GNU GRUB @@ -11,26 +11,27 @@ like "uu what does this button do?" from unknowledgeable users. Upstream-Status: Inappropriate [NI specific] Signed-off-by: Ioan-Adrian Ratiu + --- grub-core/normal/main.c | 2 +- grub-core/normal/menu_text.c | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/grub-core/normal/main.c b/grub-core/normal/main.c -index 78a70a8bf..0a93eef30 100644 +index bd44310..d556012 100644 --- a/grub-core/normal/main.c +++ b/grub-core/normal/main.c -@@ -208,7 +208,7 @@ grub_normal_init_page (struct grub_term_output *term, - +@@ -209,7 +209,7 @@ grub_normal_init_page (struct grub_term_output *term, + grub_term_cls (term); - msg_formatted = grub_xasprintf (_("GNU GRUB version %s"), PACKAGE_VERSION); + msg_formatted = grub_xasprintf (_("NI Linux Real-Time Boot Options")); if (!msg_formatted) return; - + diff --git a/grub-core/normal/menu_text.c b/grub-core/normal/menu_text.c -index e22bb91f6..d2411750b 100644 +index b1321eb..50a77b7 100644 --- a/grub-core/normal/menu_text.c +++ b/grub-core/normal/menu_text.c @@ -181,17 +181,13 @@ command-line or ESC to discard edits and return to the GRUB menu."), @@ -51,8 +52,5 @@ index e22bb91f6..d2411750b 100644 - "or `c' for a command-line."), + (_("Press enter to boot the selected OS. "), STANDARD_MARGIN, STANDARD_MARGIN, term, dry_run); - } + } } --- -2.25.1 - From 36ea91b69ba3b185e79d70372593ec85a7e89c8b Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 14 Feb 2024 23:45:05 -0500 Subject: [PATCH 068/147] salt: remove python3-pytest-salt RDEP The python3-pytest-salt recipe from meta-cloud-services is not directly attached to this salt recipe, and it doesn't seem to compile in recent OE releases. Probably no one has been maintaining it and it needs some reworking. Rather than doing that, just remove it from the RDEPENDS here until we can more broadly work on the salt recipes. Signed-off-by: Alex Stewart --- recipes-support/salt/salt_3000.2.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-support/salt/salt_3000.2.bb b/recipes-support/salt/salt_3000.2.bb index 4feac943b..3f500333c 100644 --- a/recipes-support/salt/salt_3000.2.bb +++ b/recipes-support/salt/salt_3000.2.bb @@ -206,7 +206,7 @@ FILES:${PN}-cloud = "${bindir}/${PN}-cloud ${sysconfdir}/${PN}/cloud.conf.d/ ${s SUMMARY:${PN}-tests = "salt stack test suite" DESCRIPTION:${PN}-tests ="${DESCRIPTION_COMMON} This particular package provides the salt unit test suite." -RDEPENDS:${PN}-tests = "${PN}-common python3-pytest-salt python3-pyzmq python3-six python3-tests python3-image bash" +RDEPENDS:${PN}-tests = "${PN}-common python3-pyzmq python3-six python3-tests python3-image bash" FILES:${PN}-tests = "${PYTHON_SITEPACKAGES_DIR}/salt-tests/tests/" RDEPENDS:${PN}-ptest += "salt-tests python3-distro python3-mock" From 50b8e7a9387e1346ff52d4bac701b1f71682f095 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Thu, 22 Feb 2024 08:42:41 -0600 Subject: [PATCH 069/147] initrdscripts: remove efifix script This unused script was originally added for a new RAUC/NXG bootflow which is no longer being pursued. Removing now as it contains calls to egrep which are being removed due to obsolescence in the context of the recovery tool. Signed-off-by: Jeffrey Pautler --- recipes-core/initrdscripts/files/efifix | 67 ------------------- .../initrdscripts/init-restore-mode.bb | 4 +- 2 files changed, 1 insertion(+), 70 deletions(-) delete mode 100755 recipes-core/initrdscripts/files/efifix diff --git a/recipes-core/initrdscripts/files/efifix b/recipes-core/initrdscripts/files/efifix deleted file mode 100755 index 156c75a63..000000000 --- a/recipes-core/initrdscripts/files/efifix +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -if [ -e /ni_provisioning.common ]; then - source /ni_provisioning.common -else - printf "\n***Error: File ni_provisioning.common not found!\n" - printf "PROVISIONING FAILED!" - exit 1 -fi - -die() -{ - echo -e "${RED}\n***Error: $1\nEFI Recovery FAILED!${NC}" - cleanup_and_exit 1 -} - -show_help() -{ - >&3 echo -e "\nUsage: $0 [-hv] [-t DEVICE]\n" - >&3 echo -e "Options\n" - >&3 echo " -h : Show help." - >&3 echo " -v : Verbose mode." - >&3 echo " -t DEVICE : DEVICE is the device name for the disk to update. (e.g /dev/sda)" - >&3 echo " If DEVICE is not specificed, the first non-removable block device" - >&3 echo " that is not the recovery tool is used." -} - -check_partition() -{ - local label=`lsblk ${TARGET_DISK}$1 -n -o LABEL 2>/dev/null` - if [[ "$label" != "$2" ]]; then - (( verbose_mode )) && die "Expected ${TARGET_DISK}$1 to have disk label $2." - fi - - mkdir -p /mnt/partition - NILRT_ERROR=`mount -t vfat ${TARGET_DISK}$1 /mnt/partition 2>&1` || die "$NILRT_ERROR" - if ! [[ -f /mnt/partition/efi/nilrt/bootx64.efi ]]; then - NILRT_ERROR="File not found: /efi/nilrt/bootx64.efi on ${TARGET_DISK}$1" - fi - umount /mnt/partition - rmdir /mnt/partition - [ -z $NILRT_ERROR ] || die "$NILRT_ERROR" -} - -early_setup "$@" - -# Verify disk labels on selected device -check_partition 1 niboota -check_partition 2 nibootb - -# Delete any existing entries matching a known spec -for ENTRY in $(efibootmgr | egrep -i '(LabVIEW RT)|(niboota)|(nibootb)' | egrep -o '[0-9A-Fa-f]{4}' || true); -do - do_silent echo "Drop entry $ENTRY" - EFIMGR=$(efibootmgr -b "$ENTRY" -B 2>&1) || print_warning "efibootmgr -b $ENTRY -B failed with: $EFIMGR" -done - -# Add nibootX entries reversed order so niboota is booted first -do_silent efibootmgr -c -d "$TARGET_DISK" -p 2 -L 'nibootb' -l '\efi\nilrt\bootx64.efi' -do_silent efibootmgr -c -d "$TARGET_DISK" -p 1 -L 'niboota' -l '\efi\nilrt\bootx64.efi' - -echo "EFI boot configuration:" -efibootmgr -v -echo "---" - -echo -e ${GREEN}"EFI Recovery SUCCESSFUL!"${NC} 1>&3 2>&4 - diff --git a/recipes-core/initrdscripts/init-restore-mode.bb b/recipes-core/initrdscripts/init-restore-mode.bb index c7c454a9b..bb4be62e3 100644 --- a/recipes-core/initrdscripts/init-restore-mode.bb +++ b/recipes-core/initrdscripts/init-restore-mode.bb @@ -9,7 +9,6 @@ PV = "1.0" SRC_URI = "\ file://init-restore-mode.sh \ file://00-init-restore-mode.sh \ - file://efifix \ file://mmc_storage_device_codes.allow \ file://ni_provisioning \ file://ni_provisioning.common \ @@ -33,7 +32,6 @@ do_install() { install -m 0644 ${WORKDIR}/mmc_storage_device_codes.allow ${D}${sysconfdir}/ni-provisioning/ install -m 0755 ${WORKDIR}/init-restore-mode.sh ${D}/init - install -m 0755 ${WORKDIR}/efifix ${D}/ install -m 0755 ${WORKDIR}/ni_provisioning ${D}/ install -m 0644 ${WORKDIR}/ni_provisioning.common ${D}/ install -m 0644 ${WORKDIR}/ni_provisioning.answers.default ${D}/ @@ -54,5 +52,5 @@ do_install:append:xilinx-zynqhf() { PACKAGE_ARCH = "${MACHINE_ARCH}" -FILES:${PN} += " /init /ni_provisioning* /disk_config /efifix /etc/profile.d/00-init-restore-mode.sh" +FILES:${PN} += " /init /ni_provisioning* /disk_config /etc/profile.d/00-init-restore-mode.sh" FILES:${PN}:append:x64 = " /grub.cfg " From 971c3d372ee0f7a31baf1b76bd3bb8aef1549342 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Thu, 22 Feb 2024 07:47:55 -0600 Subject: [PATCH 070/147] Replace egrep with grep -E in recovery tool files The version of egrep present in the recovery tool now outputs "egrep is obsolescent; using grep -E" on every call. Replacing egrep with grep -E in all locations present in the recovery tool. Signed-off-by: Jeffrey Pautler --- recipes-core/base-files/base-files/safemode-ps1.sh | 2 +- recipes-core/initrdscripts/files/ni_provisioning | 2 +- .../initrdscripts/files/ni_provisioning.common | 12 ++++++------ .../initrdscripts/files/ni_provisioning.safemode | 2 +- recipes-ni/ni-smbios-helper/files/smbios_helper | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/recipes-core/base-files/base-files/safemode-ps1.sh b/recipes-core/base-files/base-files/safemode-ps1.sh index cf7e0a7a5..892c91cfb 100644 --- a/recipes-core/base-files/base-files/safemode-ps1.sh +++ b/recipes-core/base-files/base-files/safemode-ps1.sh @@ -1,3 +1,3 @@ -if [ -f /etc/natinst/safemode ] || /sbin/runlevel | egrep -q ' 4$' ; then +if [ -f /etc/natinst/safemode ] || /sbin/runlevel | grep -Eq ' 4$' ; then PS1="(safemode) $PS1" fi diff --git a/recipes-core/initrdscripts/files/ni_provisioning b/recipes-core/initrdscripts/files/ni_provisioning index 33f78835b..205b4abd6 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning +++ b/recipes-core/initrdscripts/files/ni_provisioning @@ -3,7 +3,7 @@ INVALID_NILRT_ID_MSG="Invalid value for PROVISION_PART_NILRT_ID. Use UUID= or PARTUUID= or auto." # verify required tools are installed -for toolName in poweroff reboot mount umount sync ls rm ln mkdir cp echo printf dirname basename find grep egrep tar bzip2 bunzip2 lsblk cut udevadm date sleep head; do +for toolName in poweroff reboot mount umount sync ls rm ln mkdir cp echo printf dirname basename find grep tar bzip2 bunzip2 lsblk cut udevadm date sleep head; do if ! type "$toolName" >/dev/null; then printf "\n***Error: Missing $toolName\n" printf "PROVISIONING FAILED!" diff --git a/recipes-core/initrdscripts/files/ni_provisioning.common b/recipes-core/initrdscripts/files/ni_provisioning.common index 302031a49..32f7257e8 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning.common +++ b/recipes-core/initrdscripts/files/ni_provisioning.common @@ -346,7 +346,7 @@ elif [ "$ARCH" = "x86_64" ]; then echo "Configuring EFI for A/B image boot..." # Delete existing NILRT entries with "-B" option - for ENTRY in $(efibootmgr | egrep -i '(LabVIEW RT)|(niboota)|(nibootb)' | egrep -o '[0-9A-Fa-f]{4}' || true); + for ENTRY in $(efibootmgr | grep -Ei '(LabVIEW RT)|(niboota)|(nibootb)' | grep -Eo '[0-9A-Fa-f]{4}' || true); do do_silent echo "Drop entry $ENTRY" EFIMGR=$(efibootmgr -b "$ENTRY" -B 2>&1) || print_warning "efibootmgr -b $ENTRY -B failed with: $EFIMGR" @@ -383,19 +383,19 @@ elif [ "$ARCH" = "x86_64" ]; then # first two boot entries and niboota is BootNext echo -n "Check EFI boot configuration: " - boot_order=$(echo "$efi_boot_config" | egrep "^BootOrder: [0-9,]+" | cut -d" " -f2) + boot_order=$(echo "$efi_boot_config" | grep -E "^BootOrder: [0-9,]+" | cut -d" " -f2) boot_A_numb=$(echo "$boot_order" | cut -d"," -f1) boot_B_numb=$(echo "$boot_order" | cut -d"," -f2) - if ! echo "$efi_boot_config" | egrep -q "^Boot${boot_A_numb}.* niboota"$'\t'"+HD\(1,"; then + if ! echo "$efi_boot_config" | grep -Eq "^Boot${boot_A_numb}.* niboota"$'\t'"+HD\(1,"; then die "niboota is not at Boot${boot_A_numb}, boot_order=$boot_order" fi - if ! echo "$efi_boot_config" | egrep -q "^Boot${boot_B_numb}.* nibootb"$'\t'"+HD\(2,"; then + if ! echo "$efi_boot_config" | grep -Eq "^Boot${boot_B_numb}.* nibootb"$'\t'"+HD\(2,"; then die "nibootb is not at Boot${boot_B_numb}, boot_order=$boot_order" fi - boot_next=$(echo "$efi_boot_config" | egrep "^BootNext: [0-9]+ *$" | cut -d" " -f2) + boot_next=$(echo "$efi_boot_config" | grep -E "^BootNext: [0-9]+ *$" | cut -d" " -f2) [ "$boot_next" == "$boot_A_numb" ] || die "BootNext=$boot_next, expecting niboota at $boot_A_numb" echo "OK" @@ -599,7 +599,7 @@ check_answer_file() if [ -r "$filePath" ]; then # Answer file may only contain lines beginning with '#' or 'PROVISION_*=' variables if [ "`head -1 "$filePath"`" == "#NI_PROVISIONING_ANSWERS_V1" ]; then - if ! egrep -q -v '(^$|^#|^PROVISION_[A-Z0-9_]+=)' "$filePath"; then + if ! grep -E -q -v '(^$|^#|^PROVISION_[A-Z0-9_]+=)' "$filePath"; then return 0 fi fi diff --git a/recipes-core/initrdscripts/files/ni_provisioning.safemode b/recipes-core/initrdscripts/files/ni_provisioning.safemode index aba18fdfc..36ef6a236 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning.safemode +++ b/recipes-core/initrdscripts/files/ni_provisioning.safemode @@ -211,7 +211,7 @@ install_grub() GRUB_TARGET=$(uname -m) cp /boot/EFI/BOOT/bootx64.efi $GRUB_TARGET_DIR # Delete existing NILRT entries with "-B" option - for ENTRY in $(efibootmgr | egrep -i '(LabVIEW RT)|(niboota)|(nibootb)' | egrep -o '[0-9A-Fa-f]{4}' || true); + for ENTRY in $(efibootmgr | grep -Ei '(LabVIEW RT)|(niboota)|(nibootb)' | grep -Eo '[0-9A-Fa-f]{4}' || true); do print_info " Drop entry $ENTRY." EFIMGR=$(efibootmgr -b "$ENTRY" -B 2>&1) || print_warning "efibootmgr -b $ENTRY -B failed with: $EFIMGR" diff --git a/recipes-ni/ni-smbios-helper/files/smbios_helper b/recipes-ni/ni-smbios-helper/files/smbios_helper index 7f75ef773..a1e3654a8 100644 --- a/recipes-ni/ni-smbios-helper/files/smbios_helper +++ b/recipes-ni/ni-smbios-helper/files/smbios_helper @@ -70,7 +70,7 @@ get_serial_number() local sn="`$DMIDECODE -t 1 | grep "Serial Number:"`" # Filter out invalid sn strings like "Not Applicable" and "Not Specified" - if echo "$sn" | egrep -q "Serial Number: Not "; then + if echo "$sn" | grep -Eq "Serial Number: Not "; then sn="" fi From 1d80fbda55823c8f2f08cfbfce74803b0129762b Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Fri, 23 Feb 2024 09:42:32 -0600 Subject: [PATCH 071/147] ni_provisioning: fix efibootmgr output parsing The output of efibootmgr changed to include the PARTUUID of each boot entry. This broke the parsing code which previously scanned boot entries for a sequence of four digits. Signed-off-by: Jeffrey Pautler --- recipes-core/initrdscripts/files/ni_provisioning.common | 7 ++++--- recipes-core/initrdscripts/files/ni_provisioning.safemode | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/recipes-core/initrdscripts/files/ni_provisioning.common b/recipes-core/initrdscripts/files/ni_provisioning.common index 32f7257e8..26f6e3fe5 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning.common +++ b/recipes-core/initrdscripts/files/ni_provisioning.common @@ -346,10 +346,11 @@ elif [ "$ARCH" = "x86_64" ]; then echo "Configuring EFI for A/B image boot..." # Delete existing NILRT entries with "-B" option - for ENTRY in $(efibootmgr | grep -Ei '(LabVIEW RT)|(niboota)|(nibootb)' | grep -Eo '[0-9A-Fa-f]{4}' || true); + for ENTRY in $(efibootmgr | grep -Ei '(LabVIEW RT)|(niboota)|(nibootb)' | grep -Eo 'Boot[0-9A-Fa-f]{4}' || true); do - do_silent echo "Drop entry $ENTRY" - EFIMGR=$(efibootmgr -b "$ENTRY" -B 2>&1) || print_warning "efibootmgr -b $ENTRY -B failed with: $EFIMGR" + ENTRY_NUMBER=${ENTRY: 4:4} + do_silent echo "Drop entry $ENTRY_NUMBER" + EFIMGR=$(efibootmgr -b "$ENTRY_NUMBER" -B 2>&1) || print_warning "efibootmgr -b $ENTRY_NUMBER -B failed with: $EFIMGR" done # Add nibootX entries with "-c" option diff --git a/recipes-core/initrdscripts/files/ni_provisioning.safemode b/recipes-core/initrdscripts/files/ni_provisioning.safemode index 36ef6a236..44d5797ce 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning.safemode +++ b/recipes-core/initrdscripts/files/ni_provisioning.safemode @@ -211,10 +211,11 @@ install_grub() GRUB_TARGET=$(uname -m) cp /boot/EFI/BOOT/bootx64.efi $GRUB_TARGET_DIR # Delete existing NILRT entries with "-B" option - for ENTRY in $(efibootmgr | grep -Ei '(LabVIEW RT)|(niboota)|(nibootb)' | grep -Eo '[0-9A-Fa-f]{4}' || true); + for ENTRY in $(efibootmgr | grep -Ei '(LabVIEW RT)|(niboota)|(nibootb)' | grep -Eo 'Boot[0-9A-Fa-f]{4}' || true); do - print_info " Drop entry $ENTRY." - EFIMGR=$(efibootmgr -b "$ENTRY" -B 2>&1) || print_warning "efibootmgr -b $ENTRY -B failed with: $EFIMGR" + ENTRY_NUMBER=${ENTRY: 4:4} + print_info " Drop entry $ENTRY_NUMBER." + EFIMGR=$(efibootmgr -b "$ENTRY_NUMBER" -B 2>&1) || print_warning "efibootmgr -b $ENTRY_NUMBER -B failed with: $EFIMGR" done efibootmgr $VERBOSE_ARGS -c -d ${TARGET_DISK} -p 1 -L 'LabVIEW RT' -l '\efi\boot\bootx64.efi' print_done From 24944710329aa46e398714f73b003d56a8f0e1bf Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 8 Apr 2024 09:56:16 -0400 Subject: [PATCH 072/147] pkggrp-ni-internal-deps: add ntfs-3g dep to PXI PS The PXI PS ATS depends on the `ntfs-3g` package, which provides write access to NTFS storage devices. Add this dep to the internal-deps pkggrp, so that the package doesn't silently drop from the feeds. Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagegroup-ni-internal-deps.bb | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb index c257b8466..9b0c49695 100644 --- a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb +++ b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb @@ -93,6 +93,7 @@ RDEPENDS:${PN} += "\ # Contact: Kevin Khai-Wern Lim RDEPENDS:${PN} += "\ memtester \ + ntfs-3g-ntfsprogs \ " # Required by aim-arinc-664 From 6e6771ff449dcbb285188754590b556512bb8e8a Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 8 Apr 2024 16:31:43 -0400 Subject: [PATCH 073/147] opkg-utils: revert break to opkg-feeds A recent patch went into the opkg-utils_0.6.2:opkg-feeds script that fixed operations when feed names overlapped with architecture names. But the patch also broke the `list` operation - which is used by NI MAX to manage feeds. Revert the offending opkg-utils fix until a proper fix can be developed. The fix is not needed on NILRT. Signed-off-by: Alex Stewart --- ...-Fix-adding-feeds-with-same-name-as-.patch | 34 +++++++++++++++++++ .../opkg-utils/opkg-utils_%.bbappend | 4 +++ 2 files changed, 38 insertions(+) create mode 100644 recipes-devtools/opkg-utils/opkg-utils/0001-Revert-opkg-feed-Fix-adding-feeds-with-same-name-as-.patch create mode 100644 recipes-devtools/opkg-utils/opkg-utils_%.bbappend diff --git a/recipes-devtools/opkg-utils/opkg-utils/0001-Revert-opkg-feed-Fix-adding-feeds-with-same-name-as-.patch b/recipes-devtools/opkg-utils/opkg-utils/0001-Revert-opkg-feed-Fix-adding-feeds-with-same-name-as-.patch new file mode 100644 index 000000000..9155b60e7 --- /dev/null +++ b/recipes-devtools/opkg-utils/opkg-utils/0001-Revert-opkg-feed-Fix-adding-feeds-with-same-name-as-.patch @@ -0,0 +1,34 @@ +From c93b3c56725efc80b51d628fe3446eeeac970227 Mon Sep 17 00:00:00 2001 +From: Alex Stewart +Date: Mon, 8 Apr 2024 16:16:08 -0400 +Subject: [PATCH] Revert "opkg-feed: Fix adding feeds with same name as + architecture" + +The new regexes added by the target commit invalidate the regex group +indexes used to extract the feed source information. As a result, +`opkg-feed list` operations fail to find feeds, in all cases. + +Revert this change until a fix can be developed. + +This reverts commit 67994e62dc598282830385da75ba9b1abbbda941. + +Signed-off-by: Alex Stewart + +Upstream-Status: Inappropriate [upstream ticket] +--- + opkg-feed | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/opkg-feed b/opkg-feed +index 36d4463..25ef879 100755 +--- a/opkg-feed ++++ b/opkg-feed +@@ -171,7 +171,7 @@ createFeedLineRegex() + # 1 = full source type with any quotes + # 2 = source type without quotes if quoted + # 3 = source type if unquoted +- sourceTypePattern='("([^"](src|dist)(/gz)?)"|(src|dist)(/gz)?)\s+'; ++ sourceTypePattern='("([^"]*)"|(\S+))\s+'; + + # Feed name capture groups (4, 5, 6) + # 4 = full feed name with any quotes diff --git a/recipes-devtools/opkg-utils/opkg-utils_%.bbappend b/recipes-devtools/opkg-utils/opkg-utils_%.bbappend new file mode 100644 index 000000000..356b7d3e8 --- /dev/null +++ b/recipes-devtools/opkg-utils/opkg-utils_%.bbappend @@ -0,0 +1,4 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:" + +SRC_URI += "file://0001-Revert-opkg-feed-Fix-adding-feeds-with-same-name-as-.patch" + From ef5ff03b9eb729a8430ee8ed3e03c74bedb9b3e1 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 10 Apr 2024 15:32:53 -0500 Subject: [PATCH 074/147] layer.conf: Set layer compatibility to scarthgap Signed-off-by: Chaitanya Vadrevu --- conf/layer.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/layer.conf b/conf/layer.conf index dfc47a16a..8e412e87d 100644 --- a/conf/layer.conf +++ b/conf/layer.conf @@ -9,4 +9,4 @@ BBFILE_COLLECTIONS += "meta-nilrt" BBFILE_PATTERN_meta-nilrt = "^${LAYERDIR}/" BBFILE_PRIORITY_meta-nilrt = "25" -LAYERSERIES_COMPAT_meta-nilrt = "nanbield" +LAYERSERIES_COMPAT_meta-nilrt = "scarthgap" From d4a2cc5a9e6fac2aa10395eb3d9c97850c1a202e Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Tue, 16 Apr 2024 16:26:38 -0500 Subject: [PATCH 075/147] fs_permissions_diff: Modify the test to run from a developer machine When reproducing test failures or debugging the test itself, it can be useful to run the test from a non-NILRT developer machine. This is not currently possible because the logic to determine the basis/recent manifests uses the NILRT os version as obtained by querying the system running the test. This fails on a non-NILRT machine. This change obtains the os version from the database record when the test is run with a command-line option to get the current manifest from the database. Otherwise, when the current manifest is obtained from the system running the test the os version is also obtained by querying the system running the test. Signed-off-by: Jeffrey Pautler (cherry picked from commit 8cb1b3be7ec447f276a375ab5fa2753436e0ada0) Signed-off-by: Alex Stewart --- .../files/fs_permissions_diff.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index 3d561868a..44670d138 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -33,13 +33,19 @@ def count_documents(self, query): return self.fs_permissions_collection.count_documents(query) class OsVersion: - def __init__(self): - with open('/etc/os-release', 'rb') as file, mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as mfile: - build_id = re.search(br'BUILD_ID=\"((\d+\.\d+).*)\"', mfile) - self.full = build_id.group(1).decode() - self.major_minor = build_id.group(2).decode() - codename = re.search(br'VERSION_CODENAME=\"(.*)\"', mfile) - self.codename = codename.group(1).decode() + def __init__(self, os_version_full=None, os_version_codename=None): + if os_version_full is not None: + self.full = os_version_full + major_minor_version = re.match(r'(\d+.\d+).*', self.full) + self.major_minor = major_minor_version.group(1) + self.codename = os_version_codename + else: + with open('/etc/os-release', 'rb') as file, mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as mfile: + build_id = re.search(br'BUILD_ID=\"((\d+\.\d+).*)\"', mfile) + self.full = build_id.group(1).decode() + self.major_minor = build_id.group(2).decode() + codename = re.search(br'VERSION_CODENAME=\"(.*)\"', mfile) + self.codename = codename.group(1).decode() decomposed_version = re.match(r'(\d+).(\d+).(\d+)([dabf])(\d+)', self.full) self.major = int(decomposed_version.group(1)) @@ -76,9 +82,8 @@ def get_fs_manifest(): def log_version_info(logger, label, codename, full_version, date, db_id): logger.log(f'INFO: {label} = {codename} {full_version} from {date} with _id {db_id}') -def upload_manifest(db, fs_manifest, logger): +def upload_manifest(db, fs_manifest, os_version, logger): data = {} - os_version = OsVersion() data['os_version_codename'] = os_version.codename data['os_version_full'] = os_version.full data['os_version'] = os_version.version_dict @@ -121,7 +126,7 @@ def get_previous_fs_manifest_as_current(db, previous_date, logger): result['date'], result['_id'] ) - return strip_headers(result['fs_permissions']) + return strip_headers(result['fs_permissions']), result['os_version_full'], result['os_version_codename'] elif count > 1: logger.log('INFO: Found multiple fs manifest records with the same date: {}'.format(previous_date)) exit(1) @@ -129,7 +134,7 @@ def get_previous_fs_manifest_as_current(db, previous_date, logger): logger.log('INFO: Could not find fs manifest record with date: {}'.format(previous_date)) exit(1) -def get_old_fs_manifests(db, logger, basis_override): +def get_old_fs_manifests(db, logger, os_version, basis_override): def run_query(label, query): nonlocal db nonlocal logger @@ -164,8 +169,6 @@ def run_query(label, query): return strip_headers(result['fs_permissions']), result['os_version_full'] - os_version = OsVersion() - query_version_build = lambda build: { 'os_version_codename': os_version.codename, 'os_version': { @@ -397,13 +400,15 @@ def main(): args = parse_args() db = DB(args.server, args.user, args.password) - basis_fs_manifest, recent_fs_manifest = get_old_fs_manifests(db, logger, args.basis_log_db_date) - if args.current_log_db_date: - fs_manifest = get_previous_fs_manifest_as_current(db, args.current_log_db_date, logger) + fs_manifest, os_version_full, os_version_codename = get_previous_fs_manifest_as_current(db, args.current_log_db_date, logger) + os_version = OsVersion(os_version_full, os_version_codename) else: prepare_system_for_manifest() fs_manifest = get_fs_manifest() + os_version = OsVersion() + + basis_fs_manifest, recent_fs_manifest = get_old_fs_manifests(db, logger, os_version, args.basis_log_db_date) if not args.current_log_db_date and not args.skip_upload: upload_manifest(db, fs_manifest, logger) From 5cd749fddf2cadc3a9be4bae46c08c78cc40e112 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Fri, 19 Apr 2024 09:06:53 -0500 Subject: [PATCH 076/147] fs_permissions_diff: Display current manifest details in all cases Current manifest details, including the os version, are not being displayed when the current manifest is obtained from the system running the test and when the skip_upload option is specified. This change displays the the current manifest details in all cases. Signed-off-by: Jeffrey Pautler (cherry picked from commit 84bf75c1de18ae8ccaacc866d322ed887e20a094) Signed-off-by: Alex Stewart --- .../files/fs_permissions_diff.py | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index 44670d138..3cff61047 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -96,15 +96,7 @@ def upload_manifest(db, fs_manifest, os_version, logger): data['fs_permissions'] = header + fs_manifest record = db.insert(data) - - log_version_info( - logger, - 'current', - data['os_version_codename'], - data['os_version_full'], - data['date'], - record.inserted_id - ) + return data['date'], record.inserted_id def strip_headers(fs_manifest): return ''.join(line + '\n' for line in fs_manifest.splitlines() if not line.startswith('#')) @@ -119,14 +111,7 @@ def get_previous_fs_manifest_as_current(db, previous_date, logger): results = db.find(query).limit(1) result = next(results) logger.log('INFO: Found fs_manifest record with these details') - log_version_info(logger, - 'current', - result['os_version_codename'], - result['os_version_full'], - result['date'], - result['_id'] - ) - return strip_headers(result['fs_permissions']), result['os_version_full'], result['os_version_codename'] + return strip_headers(result['fs_permissions']), result['os_version_full'], result['os_version_codename'], result['date'], result['_id'] elif count > 1: logger.log('INFO: Found multiple fs manifest records with the same date: {}'.format(previous_date)) exit(1) @@ -271,7 +256,6 @@ def differences(self): difference['path'] = path yield difference - def diff_manifests(current_manifest, basis_manifest, recent_manifest, logger): hash_and_prep = lambda manifest: (hashlib.md5(manifest.encode('utf-8')).hexdigest(), prepare_manifest_for_diff(manifest)) @@ -401,17 +385,21 @@ def main(): db = DB(args.server, args.user, args.password) if args.current_log_db_date: - fs_manifest, os_version_full, os_version_codename = get_previous_fs_manifest_as_current(db, args.current_log_db_date, logger) + fs_manifest, os_version_full, os_version_codename, db_date, db_id = get_previous_fs_manifest_as_current(db, args.current_log_db_date, logger) os_version = OsVersion(os_version_full, os_version_codename) else: prepare_system_for_manifest() fs_manifest = get_fs_manifest() os_version = OsVersion() + db_date = "" + db_id = "" basis_fs_manifest, recent_fs_manifest = get_old_fs_manifests(db, logger, os_version, args.basis_log_db_date) if not args.current_log_db_date and not args.skip_upload: - upload_manifest(db, fs_manifest, logger) + db_date, db_id = upload_manifest(db, fs_manifest, logger) + + log_version_info(logger, 'current', os_version.codename, os_version.full, db_date, db_id) result = diff_manifests(fs_manifest, basis_fs_manifest, recent_fs_manifest, logger) From bd0b331b3c9efd663823cad9d174346e51af23c4 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Mon, 22 Apr 2024 08:42:17 -0500 Subject: [PATCH 077/147] fs_permissions_diff: Fix run-time error caused by missing parameter A recent commit left out a parameter in a function call. This change fixes the issue. Signed-off-by: Jeffrey Pautler (cherry picked from commit b19781464841a0fe111dae6b23e82d3ca4cd5b47) Signed-off-by: Alex Stewart --- .../ni-base-system-image-tests/files/fs_permissions_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index 3cff61047..6ee11e37b 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -397,7 +397,7 @@ def main(): basis_fs_manifest, recent_fs_manifest = get_old_fs_manifests(db, logger, os_version, args.basis_log_db_date) if not args.current_log_db_date and not args.skip_upload: - db_date, db_id = upload_manifest(db, fs_manifest, logger) + db_date, db_id = upload_manifest(db, fs_manifest, os_version, logger) log_version_info(logger, 'current', os_version.codename, os_version.full, db_date, db_id) From 476d19bb36e124a1585ab9f65837c8eacb874baa Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Mon, 22 Apr 2024 12:00:43 -0500 Subject: [PATCH 078/147] fs_permissions_diff: Ensure recent manifest is not earlier than basis manifest If the basis manifest is set using the command-line option, it is possible that the basis version will match the current version. In this case, the current method to determine the recent manifest can result in a recent manifest that is earlier than the basis manifest. This should never happen. In this case, make the recent manifest the same as the basis manifest. Signed-off-by: Jeffrey Pautler (cherry picked from commit 24a44a241a29c4fe97f3085e202d1851b99aaba8) Signed-off-by: Alex Stewart --- .../files/fs_permissions_diff.py | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index 6ee11e37b..5156827af 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -139,7 +139,7 @@ def run_query(label, query): results = db.find(query).sort(sort_order).limit(1) else: logger.log('INFO: No suitable previous fs permissions found') - return '', '' + return '', '', '' result = next(results) @@ -152,7 +152,7 @@ def run_query(label, query): result['_id'] ) - return strip_headers(result['fs_permissions']), result['os_version_full'] + return strip_headers(result['fs_permissions']), result['os_version_full'], result['date'] query_version_build = lambda build: { 'os_version_codename': os_version.codename, @@ -170,19 +170,30 @@ def run_query(label, query): 'date': date } - # The "basis" manifest is from the latest version with a lesser MAJOR.MINOR.PATCH if basis_override is not None: - basis_manifest, basis_version = run_query('basis', query_date_build(basis_override)) + logger.log(f'INFO: Basis manifest was specified to have date: {basis_override}') + basis_manifest, basis_version_full, basis_date = run_query('basis', query_date_build(basis_override)) else: - basis_manifest, basis_version = run_query('basis', query_version_build(0)) - # The "recent" manifest is from the latest version, likely the last run with the same MAJOR.MINOR.PATCH - recent_manifest, recent_version = run_query('recent', query_version_build(os_version.build)) + # If not specified, the "basis" manifest is the most recent version with a lesser MAJOR.MINOR.PATCH than the current manifest. + # In general, it should be the last final build of the previous MAJOR.MINOR.PATCH version. + basis_manifest, basis_version_full, basis_date = run_query('basis', query_version_build(0)) + + # If the current version and basis version are the same (most likely scenario is if basis version is pinned to the most recent build available), + # then just set recent to the same version as well. The normal mechanism of querying for the most recent MAJOR.MINOR.PATCH.BUILD less than the + # current version is going to result in a recent version that is *older* than the basis version, which doesn't make any sense. + if os_version.full == basis_version_full: + recent_manifest, recent_version_full, unused = run_query('recent', query_date_build(basis_date)) + else: + # If not specified, the "recent" manifest is the most recent version less than the current manifest. + # In general, it will be the previous build with the same MAJOR.MINOR.PATCH. Or if current is the first build of a new MAJOR.MINOR.PATCH, + # then it will be the same as the basis version. + recent_manifest, recent_version_full, unused = run_query('recent', query_version_build(os_version.build)) # Keep this log in first line to help streak indexer group results. Overall hash will be populated at end. - logger.prefix_log(f'INFO: fs_permissions_diff: current against {basis_version} ') + logger.prefix_log(f'INFO: fs_permissions_diff: current against {basis_version_full} ') # Keep this log in second line to avoid using it for grouping, but keep it for tracking. # Individual hashes to be populated at end. - logger.prefix_log(f'INFO: and {recent_version} ') + logger.prefix_log(f'INFO: and {recent_version_full} ') return basis_manifest, recent_manifest From 2d6a68754c87069afc2a0c8d11796b910f21ab1a Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Mon, 22 Apr 2024 12:37:18 -0500 Subject: [PATCH 079/147] fixup! fs_permissions_diff: Ensure recent manifest is not earlier than basis manifest (cherry picked from commit 07b122a235dcefe5a16eff0d21873aa6ebc6c2f0) Signed-off-by: Alex Stewart --- .../ni-base-system-image-tests/files/fs_permissions_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index 5156827af..f437f29ac 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -184,7 +184,7 @@ def run_query(label, query): if os_version.full == basis_version_full: recent_manifest, recent_version_full, unused = run_query('recent', query_date_build(basis_date)) else: - # If not specified, the "recent" manifest is the most recent version less than the current manifest. + # The "recent" manifest is the most recent version less than the current manifest. # In general, it will be the previous build with the same MAJOR.MINOR.PATCH. Or if current is the first build of a new MAJOR.MINOR.PATCH, # then it will be the same as the basis version. recent_manifest, recent_version_full, unused = run_query('recent', query_version_build(os_version.build)) From dfe607da803ccddeb5c6d46ae1845025b745d45f Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 17 Apr 2024 15:00:23 -0500 Subject: [PATCH 080/147] ni-systemimage: Remove unnecessary dependencies Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 89310ce5f8411a6facc77cf340a3145368d2d56d) Signed-off-by: Alex Stewart --- recipes-ni/ni-systemimage/files/nisystemimage | 2 -- recipes-ni/ni-systemimage/ni-systemimage.bb | 1 - 2 files changed, 3 deletions(-) diff --git a/recipes-ni/ni-systemimage/files/nisystemimage b/recipes-ni/ni-systemimage/files/nisystemimage index bfc52a923..35a7bca3e 100644 --- a/recipes-ni/ni-systemimage/files/nisystemimage +++ b/recipes-ni/ni-systemimage/files/nisystemimage @@ -5,8 +5,6 @@ set -eEu -o pipefail shopt -s dotglob nullglob -. /etc/natinst/networking/functions.common - ## ## Globals ## diff --git a/recipes-ni/ni-systemimage/ni-systemimage.bb b/recipes-ni/ni-systemimage/ni-systemimage.bb index d4fa50bfa..0fc8f659d 100644 --- a/recipes-ni/ni-systemimage/ni-systemimage.bb +++ b/recipes-ni/ni-systemimage/ni-systemimage.bb @@ -31,6 +31,5 @@ RDEPENDS:${PN} += "\ bash \ findutils \ ni-netcfgutil \ - ni-utils \ niacctbase \ " From 5f564d99d3c59999bb20641f2769d828eef274a0 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 17 Apr 2024 17:30:47 -0500 Subject: [PATCH 081/147] ni-systemimage: Add getall Add support for getting full disk image i.e., including nigrub and safemode files. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 5038afc2927158a564c335c2338282585dcbd7a8) Signed-off-by: Alex Stewart --- .../packagegroup-ni-restoremode.bb | 1 + recipes-ni/ni-systemimage/files/nisystemimage | 93 ++++++++++++++++--- recipes-ni/ni-systemimage/ni-systemimage.bb | 2 + 3 files changed, 82 insertions(+), 14 deletions(-) diff --git a/recipes-core/packagegroups/packagegroup-ni-restoremode.bb b/recipes-core/packagegroups/packagegroup-ni-restoremode.bb index 01c163da9..b34685cf2 100644 --- a/recipes-core/packagegroups/packagegroup-ni-restoremode.bb +++ b/recipes-core/packagegroups/packagegroup-ni-restoremode.bb @@ -29,6 +29,7 @@ RDEPENDS:${PN} += "\ init-restore-mode \ kmod \ ni-smbios-helper \ + ni-systemimage \ parted \ procps \ sed \ diff --git a/recipes-ni/ni-systemimage/files/nisystemimage b/recipes-ni/ni-systemimage/files/nisystemimage index 35a7bca3e..69e41fb70 100644 --- a/recipes-ni/ni-systemimage/files/nisystemimage +++ b/recipes-ni/ni-systemimage/files/nisystemimage @@ -49,6 +49,13 @@ BLACKLIST_LISTS=() BLACKLIST_PATHS=( # syslog-ng socket (sockets can't be tar'd) /var/syslog-ng.ctl + /var/volatile + # Systemlink Configuration Files + /etc/salt/minion_id + /etc/salt/pki +) + +BLACKLIST_PATHS_get=( # Safe mode /boot/.safe # Grub variables @@ -57,14 +64,14 @@ BLACKLIST_PATHS=( /boot/.defbit # User bitfile /boot/user.bit* - /var/volatile # Restore-mode fail-safe files /.restore - # Systemlink Configuration Files - /etc/salt/minion_id - /etc/salt/pki ) +if [[ "${MODE}" == "get" ]]; then + BLACKLIST_PATHS+=(${BLACKLIST_PATHS_get[*]}) +fi + # All excludes to pass to `tar`. Includes BLACKLIST_PATHS and contents of all # files in BLACKLIST_LISTS. The contents of EXCLUDES are paths relative to # /mnt/userfs; the contents of BLACKLIST_PATHS and the contents of the files in @@ -92,9 +99,16 @@ IMAGE_FILE_TEMP= ORIG_USERFS=/mnt/userfs ORIG_BOOTFS=/boot ORIG_CONFIGFS=/etc/natinst/share + +USERFS_LABEL=nirootfs +BOOTFS_LABEL=nibootfs +CONFIGFS_LABEL=niconfig +GRUBFS_LABEL=nigrub + WORKING_USERFS=/mnt/.userfs.tmp WORKING_BOOTFS=${WORKING_USERFS}${ORIG_BOOTFS} WORKING_CONFIGFS=${WORKING_USERFS}${ORIG_CONFIGFS} +WORKING_GRUBFS=${WORKING_USERFS}/${GRUBFS_LABEL} # tar xattr extract defaults to only extract user xattrs, not system xattrs. # Override this with --xattrs-include=*. @@ -147,7 +161,7 @@ declare -Ar EXITCODES=( [FILE_NOT_FOUND]=11 [FS_PREP_FAILED]=12 [NETCFG_FAIL]=13 - [ILLTIMED]=14 + [ILLTIMED]=14 ) # $@: text @@ -234,6 +248,10 @@ in_safemode () { [[ -f /etc/natinst/safemode ]] } +in_restoremode () { + [[ -f /ni_provisioning ]] +} + ## ## Network configuration management ## @@ -366,6 +384,15 @@ init_fs_get () { fi } +init_fs_getall () { + init_fs_get + + is_mounted $WORKING_GRUBFS && run_cmd_always umount -f $WORKING_GRUBFS || true + run_cmd_always mkdir -p $WORKING_GRUBFS + run_cmd_always mount -L $GRUBFS_LABEL $WORKING_GRUBFS -o ro \ + || die FS_PREP_FAILED "Couldn't mount $GRUBFS_LABEL to $WORKING_GRUBFS" +} + init_fd_get () { if [[ $IMAGE_FILE ]] ; then rm -f -- "$IMAGE_FILE" || die UNKNOWN "Couldn't clear '$IMAGE_FILE'" @@ -377,6 +404,10 @@ init_fd_get () { fi } +init_fd_getall () { + init_fd_get +} + image_get () { image_get_${IMAGE_TYPE} if [[ $IMAGE_FILE ]]; then @@ -385,6 +416,10 @@ image_get () { fi } +image_getall () { + image_get +} + ## ## Set Image ## @@ -601,6 +636,27 @@ init_fs () { init_fs_$MODE } +init_fs_restoremode () { + # If the util did not previously exit cleanly, it's possible that the + # working mount points didn't get unmount them. To avoid double-mounts, + # try to unmount them first + is_mounted $WORKING_BOOTFS && run_cmd_always umount -f $WORKING_BOOTFS || true + is_mounted $WORKING_CONFIGFS && run_cmd_always umount -f $WORKING_CONFIGFS || true + is_mounted $WORKING_USERFS && run_cmd_always umount -f $WORKING_USERFS || true + + run_cmd_always mkdir -p $WORKING_USERFS + run_cmd_always mount -L $USERFS_LABEL $WORKING_USERFS \ + || die FS_PREP_FAILED "Couldn't mount label $USERFS_LABEL to $WORKING_USERFS" + run_cmd_always mkdir -p $WORKING_BOOTFS + run_cmd_always mount -L $BOOTFS_LABEL $WORKING_BOOTFS \ + || die FS_PREP_FAILED "Couldn't mount label $BOOTFS_LABEL to $WORKING_BOOTFS" + run_cmd_always mkdir -p $WORKING_CONFIGFS + run_cmd_always mount -L $CONFIGFS_LABEL $WORKING_CONFIGFS \ + || die FS_PREP_FAILED "Couldn't mount label CONFIGFS_LABEL to $WORKING_CONFIGFS" + + init_fs_$MODE +} + ## ## Cleanup ## @@ -608,13 +664,16 @@ cleanup_fs () { cd $ORIG_PWD || true echo_verbose 1 "starting fs cleanup: PWD=$PWD" - # For set, mount -o mount,rw appears to have the indirect effect - # of forcing a sync - run_cmd_always mount -o remount,rw $ORIG_USERFS || true - run_cmd_always mount -o remount,rw $ORIG_BOOTFS || true + if ! in_restoremode; then + # For set, mount -o mount,rw appears to have the indirect effect + # of forcing a sync + run_cmd_always mount -o remount,rw $ORIG_USERFS || true + run_cmd_always mount -o remount,rw $ORIG_BOOTFS || true + fi is_mounted $WORKING_BOOTFS && run_cmd_always umount -f $WORKING_BOOTFS || true is_mounted $WORKING_CONFIGFS && run_cmd_always umount -f $WORKING_CONFIGFS || true + is_mounted $WORKING_GRUBFS && run_cmd_always umount -f $WORKING_GRUBFS || true is_mounted $WORKING_USERFS && run_cmd_always umount -f $WORKING_USERFS || true [[ -d $WORKING_USERFS ]] && run_cmd_always rmdir $WORKING_USERFS || true [[ -f $IMAGE_FILE_TEMP ]] && run_cmd_always rm -f "$IMAGE_FILE_TEMP" || true @@ -673,7 +732,7 @@ parse_settings () { check_settings () { case "$MODE" in - get|set|wipelist) ;; + get|set|wipelist|getall) ;; "-h") print_help; exit 0 ;; "") die_with_help "Mode must be specified" ;; *) die_with_help "Unknown mode '$MODE'";; @@ -784,9 +843,11 @@ trap 'handle_err ${BASH_SOURCE} ${LINENO} ${FUNCNAME:-unknown} $?' ERR [[ -f $SCRIPT_ROOTFS ]] && . "$SCRIPT_ROOTFS" [[ -f $SCRIPT_PRE ]] && . "$SCRIPT_PRE" -case "$(runlevel)" in -*\ 6|*\ 0) die ILLTIMED "Called during shutdown; aborting." ;; -esac +if ! in_restoremode; then + case "$(runlevel)" in + *\ 6|*\ 0) die ILLTIMED "Called during shutdown; aborting." ;; + esac +fi # Process settings run_hook hook_settings_pre @@ -797,7 +858,11 @@ dump_settings run_hook hook_settings_post # Initialize filesystem and file descriptor state -init_fs +if in_restoremode; then + init_fs_restoremode +else + init_fs +fi init_fd # Perform the image operation diff --git a/recipes-ni/ni-systemimage/ni-systemimage.bb b/recipes-ni/ni-systemimage/ni-systemimage.bb index 0fc8f659d..0893a38eb 100644 --- a/recipes-ni/ni-systemimage/ni-systemimage.bb +++ b/recipes-ni/ni-systemimage/ni-systemimage.bb @@ -30,6 +30,8 @@ FILES:${PN} += "\ RDEPENDS:${PN} += "\ bash \ findutils \ + gzip \ ni-netcfgutil \ niacctbase \ + tar \ " From 5187018fe39392bbb3da00260b26bf381757ae41 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Wed, 24 Apr 2024 15:36:53 -0500 Subject: [PATCH 082/147] fs_permissions_diff: Make diff hash stable despite stripped versions This test ignores version number changes in file paths by stripping the version numbers out of the paths before diffing the file manifests. However, the code that calculates a combined diff hash based on the contents of the basis and current manifests is still using manifests that contain the unstripped paths. This causes the combined diff hash to change when there are changes in version numbers in file paths, even though those changes are not affecting the actual diff of the manifests. This change makes the combined diff hash stable in this situation by removing the unstripped paths from the manifests before calculating the hash. Signed-off-by: Jeffrey Pautler (cherry picked from commit 914c1d85d5dfcee4ef3d1acc4cc8f2ae88818433) Signed-off-by: Alex Stewart --- .../files/fs_permissions_diff.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index f437f29ac..b13c0eb13 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -219,6 +219,7 @@ def prepare_manifest_for_diff(manifest): result = {} for row in reader: stripped_path = strip_versions_from_path(row['path']) + del row['path'] result[stripped_path] = row return result @@ -267,9 +268,11 @@ def differences(self): difference['path'] = path yield difference +def hash_and_prep(manifest): + manifest = prepare_manifest_for_diff(manifest) + return hashlib.md5(str(manifest).encode('utf-8')).hexdigest(), manifest + def diff_manifests(current_manifest, basis_manifest, recent_manifest, logger): - hash_and_prep = lambda manifest: (hashlib.md5(manifest.encode('utf-8')).hexdigest(), - prepare_manifest_for_diff(manifest)) current_hash, current_manifest = hash_and_prep(current_manifest) basis_hash, basis_manifest = hash_and_prep(basis_manifest) recent_hash, recent_manifest = hash_and_prep(recent_manifest) From 2f515222227d52f88d6ccc634dd365836a94a477 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 24 Apr 2024 18:03:32 -0500 Subject: [PATCH 083/147] ni-systeimage: Fix set functionality "set" MODE requires the same list of BLACKLIST_PATHS as "get" MODE. This commit restores "set" functionality broken in 5038afc. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 5586c09eed6c01e7766d59e2b0cbc26d7062c3c0) Signed-off-by: Alex Stewart --- recipes-ni/ni-systemimage/files/nisystemimage | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipes-ni/ni-systemimage/files/nisystemimage b/recipes-ni/ni-systemimage/files/nisystemimage index 69e41fb70..b1ac57e12 100644 --- a/recipes-ni/ni-systemimage/files/nisystemimage +++ b/recipes-ni/ni-systemimage/files/nisystemimage @@ -55,7 +55,7 @@ BLACKLIST_PATHS=( /etc/salt/pki ) -BLACKLIST_PATHS_get=( +BLACKLIST_PATHS_get_set=( # Safe mode /boot/.safe # Grub variables @@ -68,8 +68,8 @@ BLACKLIST_PATHS_get=( /.restore ) -if [[ "${MODE}" == "get" ]]; then - BLACKLIST_PATHS+=(${BLACKLIST_PATHS_get[*]}) +if [[ "${MODE}" == "get" || "${MODE}" == "set" ]]; then + BLACKLIST_PATHS+=(${BLACKLIST_PATHS_get_set[*]}) fi # All excludes to pass to `tar`. Includes BLACKLIST_PATHS and contents of all From e2bc4f0271e3319ef44ff981884d5de6f9afc5bf Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 24 Apr 2024 18:35:05 -0500 Subject: [PATCH 084/147] ni-systemimage: Remove nigrubfs dir in cleanup nigrubfs directory is created in nirootfs for getall (and setall). Remove it in cleanup. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 9a81140219129c0bdad59bf8a245f15378ab917f) Signed-off-by: Alex Stewart --- recipes-ni/ni-systemimage/files/nisystemimage | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes-ni/ni-systemimage/files/nisystemimage b/recipes-ni/ni-systemimage/files/nisystemimage index b1ac57e12..540689d2f 100644 --- a/recipes-ni/ni-systemimage/files/nisystemimage +++ b/recipes-ni/ni-systemimage/files/nisystemimage @@ -674,6 +674,7 @@ cleanup_fs () { is_mounted $WORKING_BOOTFS && run_cmd_always umount -f $WORKING_BOOTFS || true is_mounted $WORKING_CONFIGFS && run_cmd_always umount -f $WORKING_CONFIGFS || true is_mounted $WORKING_GRUBFS && run_cmd_always umount -f $WORKING_GRUBFS || true + [[ -d $WORKING_GRUBFS ]] && run_cmd_always rmdir $WORKING_GRUBFS || true is_mounted $WORKING_USERFS && run_cmd_always umount -f $WORKING_USERFS || true [[ -d $WORKING_USERFS ]] && run_cmd_always rmdir $WORKING_USERFS || true [[ -f $IMAGE_FILE_TEMP ]] && run_cmd_always rm -f "$IMAGE_FILE_TEMP" || true From 6d9cbd27d5b5ffa7edaece46ab0418a3c4bc1042 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Fri, 19 Apr 2024 13:27:09 -0500 Subject: [PATCH 085/147] ni-systemimage: Add setall Add support for setting full disk image i.e., including nigrub and safemode files. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 372667334db9b3132250fce33805fd137fbd4f6a) Signed-off-by: Alex Stewart --- recipes-ni/ni-systemimage/files/nisystemimage | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/recipes-ni/ni-systemimage/files/nisystemimage b/recipes-ni/ni-systemimage/files/nisystemimage index 540689d2f..ee86d22c2 100644 --- a/recipes-ni/ni-systemimage/files/nisystemimage +++ b/recipes-ni/ni-systemimage/files/nisystemimage @@ -210,6 +210,10 @@ is_set () { [[ "${MODE}" == "set" ]] } +is_setall () { + [[ "${MODE}" == "setall" ]] +} + # $1: mount point is_mounted () { if (( $VERBOSE >= 2 )); then @@ -570,6 +574,10 @@ image_set () { netcfg_reset } +image_setall () { + image_set_tgz +} + init_fd_set () { if [[ $IMAGE_FILE ]] ; then exec {FD_IMAGE}<"$IMAGE_FILE" \ @@ -579,6 +587,10 @@ init_fd_set () { fi } +init_fd_setall () { + init_fd_set +} + init_fs_set () { # working mountpts will inherit the ro-ness of the bound mountpoint, so # make sure everything is rw @@ -593,6 +605,15 @@ init_fs_set () { fi } +init_fs_setall () { + init_fs_set + + is_mounted $WORKING_GRUBFS && run_cmd_always umount -f $WORKING_GRUBFS || true + run_cmd_always mkdir -p $WORKING_GRUBFS + run_cmd_always mount -L $GRUBFS_LABEL $WORKING_GRUBFS -o rw \ + || die FS_PREP_FAILED "Couldn't mount $GRUBFS_LABEL to $WORKING_GRUBFS" +} + # Undocumented debugging mode: Print NUL-delimited list of files that would get # deleted on set image_wipelist () { @@ -733,13 +754,13 @@ parse_settings () { check_settings () { case "$MODE" in - get|set|wipelist|getall) ;; + get|set|wipelist|getall|setall) ;; "-h") print_help; exit 0 ;; "") die_with_help "Mode must be specified" ;; *) die_with_help "Unknown mode '$MODE'";; esac - if ! is_set; then + if ! is_set && ! is_setall; then if [[ $PRIMARY_SETTING || $SECONDARY_SETTING ]]; then die_with_help "-p, -s only allowed on nisystemimage set" fi @@ -764,7 +785,7 @@ check_settings () { esac - if ! is_set && [[ $IMAGE_FILE ]]; then + if ! is_set && ! is_setall && [[ $IMAGE_FILE ]]; then # Convert relative image path to absolute path [[ "$IMAGE_FILE" =~ ^/ ]] \ || IMAGE_FILE="$PWD/$IMAGE_FILE" From 5da3cf3e012bb16022702ebd9df5c13da15353d2 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Mon, 22 Apr 2024 18:52:19 -0500 Subject: [PATCH 086/147] ni_provisioning.safemode: Split common code Refactor code that is common to usb replication usecase to ni_provisioning.safemode.common. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 413e0c7b4b7cd772664fd461317e46c10b4f0fef) Fixed a conflict between this code-movement commit and a prior eifbootmgr fixup to the ni_provisioning scripts commited by jpautler. Signed-off-by: Alex Stewart --- .../files/ni_provisioning.safemode | 420 +---------------- .../files/ni_provisioning.safemode.common | 423 ++++++++++++++++++ .../initrdscripts/init-restore-mode.bb | 2 + 3 files changed, 426 insertions(+), 419 deletions(-) create mode 100755 recipes-core/initrdscripts/files/ni_provisioning.safemode.common diff --git a/recipes-core/initrdscripts/files/ni_provisioning.safemode b/recipes-core/initrdscripts/files/ni_provisioning.safemode index 44d5797ce..a79185ae9 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning.safemode +++ b/recipes-core/initrdscripts/files/ni_provisioning.safemode @@ -1,426 +1,8 @@ #!/bin/bash set -e -ask_at_every_error="N" -grubenv_consoleoutenable=${grubenv_consoleoutenable:-""} -grubenv_bootdelay=${grubenv_bootdelay:-""} +source /ni_provisioning.safemode.common -PART1_LABEL=nigrub -PART2_LABEL=nibootfs -PART3_LABEL=niconfig -PART4_LABEL=nirootfs - -# GPT partitions must assign a partition name -PART_STYLE=gpt -PART1_NAME=$PART1_LABEL -PART2_NAME=$PART2_LABEL -PART3_NAME=$PART3_LABEL -PART4_NAME=$PART4_LABEL - -. /usr/share/nisysinfo/smbios_helper - -print_info() -{ - if [ $verbose_mode -eq 1 ]; then - >&3 echo $1 - else - >&3 echo -n $1 - fi -} - -print_error() -{ - >&4 echo -e ${RED}"\n***Error: $1\n"${NC} - local continue_on_error="N" - if [[ "$ask_at_every_error" == "y" ]]; then - exec 2>&4 - read -p "Do you want to ignore this error?[y/N]" continue_on_error - exec 4>&2 - fi - if [[ "$continue_on_error" == "N" || "$continue_on_error" == "n" || "$continue_on_error" == "" ]]; then - >&4 echo -e ${RED}"PROVISIONING FAILED!"${NC} - cleanup_and_exit 1 - fi -} - -die() -{ - >&4 echo -e ${RED}"\n***Fatal Error: $1"${NC} - >&4 echo -e ${RED}"PROVISIONING FAILED!"${NC} - cleanup_and_exit 1 -} - -print_done() -{ - if [ $verbose_mode -eq 0 ]; then - >&3 echo "Done" - fi -} - -disable_automount() -{ - echo -e $TARGET_DISK >> /etc/udev/mount.blacklist - AUTOMOUNT_DISABLED=1 -} - -enable_automount() -{ - sed -ie "\#$TARGET_DISK#d" /etc/udev/mount.blacklist - AUTOMOUNT_DISABLED=0 -} -#waits until the four partitions are visible -wait_for_partitions() -{ - DEVICE_TMP=$1 - for i in 1 2 3 4 - do - #wait max 3 seconds for each partition - print_verbose "Waiting for $DEVICE_TMP$i " - MaxTries=30 - while [ ! -b ${DEVICE_TMP}${PART_SEPARATOR}$i -a $MaxTries -gt 0 ]; - do - print_verbose "." - #sleep 0.1 seconds - usleep 100000 - MaxTries=$(($MaxTries - 1)) - done - if [ $MaxTries -eq 0 ]; then - return 1 - fi - done -} - -handle_err() -{ - TMP_EVAL=`eval echo $5` - print_error "$1:$2 (fn=$3): Unexpected status code $4, while running command: '$TMP_EVAL'" ${NC} -} - -check_all_used_binaries() -{ - commands=(awk dmidecode e2label hexdump lsblk mkfs.ext4 mkfs.vfat parted sed sfdisk sgdisk udevadm modprobe dd) - for ind_command in ${commands[@]}; do - one_command=`command -v ${ind_command}` - if [[ -z $one_command ]]; then - die "Command \"${ind_command}\" not found. Please install ${ind_command}." - fi - done - -} - -override_primaryport_grubenv() -{ - # Add target-specific override of primary ethernet port if the lowest ifIndex is not to be used - local device_code=$(get_target_id) - # CVS-1458RT - if [[ x"$device_code" == x"77AA" ]]; then - # Set eth0 (note: renamed to eth0 via udev) as the primary port. Do not let the primary - # port selection logic decide since it selects the 'eth' port with the lowest ifIndex - # value. (Renaming the port via udev does not alter ifIndex, so eth0 is not the lowest) - grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "ethaddr=`cat /sys/class/net/eth0/address`" - fi -} - -add_USB_gadget_args_to_grubenv() -{ - local MAC=$(get_usbdev_mac) - local VID=$(get_usbdev_vid) - local PID=$(get_usbdev_pid) - local SN=$(get_serial_number) - local TID=$(get_target_id) - local PROD=$(get_target_name) - - if is_usbdev_target; then - - if [ -n "$MAC" -a -n "$VID" -a -n "$PID" -a -n "$SN" -a -n "$TID" -a -n "$PROD" ]; then - cat << EOF > $GRUB_MOUNTPOINT/grubvar_readonly -set usbgadgetethaddr=$MAC -set USBVendorID=$VID -set USBProductID=$PID -set SerialNum=$SN -set USBDevice=$TID -set USBProduct=$PROD -EOF - else - print_error "Details required for USB device mode are missing from SMBIOS! UPGRADE your BIOS if using NI Target. USB Gadget will not be functional." - return - fi - fi -} - -set_serial_port() -{ - if [ -d /sys/class/tty/ttyS0 ] && [ -f /sys/class/tty/ttyS0/type ] && [ `cat /sys/class/tty/ttyS0/type` -ne 0 ]; then - echo "set serial_port="`cat /sys/class/tty/ttyS0/port` >> $GRUB_MOUNTPOINT/grubvar_readonly - - local base_clock - if base_clock=$(get_serial_base_clock); then - echo "set GRUB_SERIAL_BASE_CLOCK=$base_clock" >> $GRUB_MOUNTPOINT/grubvar_readonly - fi - fi - -} - -partitioning_disk() -{ - umount "$TARGET_DISK"? 2>/dev/null || true - print_info "Partitioning TARGET_DISK=$TARGET_DISK (PART_STYLE=$PART_STYLE)..." - PARTED_ERROR=`parted -s $TARGET_DISK mklabel $PART_STYLE 2>&1` || die "$PARTED_ERROR" - PARTED_ERROR=`parted -s --align optimal $TARGET_DISK mkpart $PART1_NAME 1MB 16MB 2>&1` || die "$PARTED_ERROR" - PARTED_ERROR=`parted -s --align optimal $TARGET_DISK mkpart $PART2_NAME 16MB 200MB 2>&1` || die "$PARTED_ERROR" - PARTED_ERROR=`parted -s --align optimal $TARGET_DISK mkpart $PART3_NAME 200MB 216MB 2>&1` || die "$PARTED_ERROR" - PARTED_ERROR=`parted -s --align optimal $TARGET_DISK mkpart $PART4_NAME 216MB 100% 2>&1` || die "$PARTED_ERROR" - print_done - - print_info "Assigning EFI System Partition..." - # Per https://www.freedesktop.org/wiki/Specifications/DiscoverablePartitionsSpec/ - # C12A7328-F81F-11D2-BA4B-00A0C93EC93B -- EFI System Partition - sgdisk --typecode="1:C12A7328-F81F-11D2-BA4B-00A0C93EC93B" $TARGET_DISK - print_done - - print_info "Probing partitions of TARGET_DISK=$TARGET_DISK ..." - partprobe -s "$TARGET_DISK" - print_done -} - -create_filesystems() -{ - print_info "Creating filesystems ..." - - MKFS_ERROR=`mkfs.vfat -n $PART1_LABEL ${TARGET_DISK}${PART_SEPARATOR}1 2>&1` || die "$MKFS_ERROR" - do_silent mkfs.ext4 $MKFS_ARGS ${TARGET_DISK}${PART_SEPARATOR}2 || die "Format failed!" - do_silent mkfs.ext4 $MKFS_ARGS ${TARGET_DISK}${PART_SEPARATOR}3 || die "Format failed!" - do_silent mkfs.ext4 $MKFS_ARGS ${TARGET_DISK}${PART_SEPARATOR}4 || die "Format failed!" - - LABEL_ERROR=`e2label ${TARGET_DISK}${PART_SEPARATOR}2 $PART2_LABEL 2>&1` || die "$LABEL_ERROR" - LABEL_ERROR=`e2label ${TARGET_DISK}${PART_SEPARATOR}3 $PART3_LABEL 2>&1` || die "$LABEL_ERROR" - LABEL_ERROR=`e2label ${TARGET_DISK}${PART_SEPARATOR}4 $PART4_LABEL 2>&1` || die "$LABEL_ERROR" - - print_done -} - -install_grub() -{ - GRUB_MOUNTPOINT=/var/volatile/grub - mkdir $GRUB_MOUNTPOINT -p - MOUNT_ERROR=`mount -L $PART1_LABEL $GRUB_MOUNTPOINT 2>&1` || die "$MOUNT_ERROR" - - print_info "Configuring EFI grub2..." - GRUB_TARGET_DIR=$GRUB_MOUNTPOINT/efi/boot - mkdir -p $GRUB_TARGET_DIR - GRUB_TARGET=$(uname -m) - cp /boot/EFI/BOOT/bootx64.efi $GRUB_TARGET_DIR - # Delete existing NILRT entries with "-B" option - for ENTRY in $(efibootmgr | grep -Ei '(LabVIEW RT)|(niboota)|(nibootb)' | grep -Eo 'Boot[0-9A-Fa-f]{4}' || true); - do - ENTRY_NUMBER=${ENTRY: 4:4} - print_info " Drop entry $ENTRY_NUMBER." - EFIMGR=$(efibootmgr -b "$ENTRY_NUMBER" -B 2>&1) || print_warning "efibootmgr -b $ENTRY_NUMBER -B failed with: $EFIMGR" - done - efibootmgr $VERBOSE_ARGS -c -d ${TARGET_DISK} -p 1 -L 'LabVIEW RT' -l '\efi\boot\bootx64.efi' - print_done - - print_info "Installing grub.cfg..." - - cp $SOURCE_DIR/grub.cfg $GRUB_TARGET_DIR/grub.cfg - mkdir -p $GRUB_TARGET_DIR/fonts - cp $SOURCE_DIR/fonts/unicode.pf2 $GRUB_TARGET_DIR/fonts/ - print_done -} - -install_safemode() -{ - print_info "Installing safemode kernel and ramdisk..." - - BOOTFS_MOUNTPOINT=/var/volatile/bootfs - mkdir $BOOTFS_MOUNTPOINT -p - BOOTFS_ERROR=`mount -L $PART2_LABEL $BOOTFS_MOUNTPOINT 2>&1` || die "$BOOTFS_ERROR" - mkdir -p $BOOTFS_MOUNTPOINT/.safe - - cp "$SOURCE_DIR"/*Image "$BOOTFS_MOUNTPOINT/.safe/" - cp "$SOURCE_DIR"/ramdisk.* "$BOOTFS_MOUNTPOINT/.safe/" - cp "$SOURCE_DIR"/bootimage.* "$BOOTFS_MOUNTPOINT/.safe/" - - print_done -} - -install_bootmode_file() -{ - # write initial bootmode file - print_info "Installing bootmode file..." - echo "set BOOT_MODE=safemode" >$BOOTFS_MOUNTPOINT/bootmode - print_done -} - -install_grubenv() -{ - NI_TARGET="false" - - mkdir $BOOTFS_MOUNTPOINT/grub - touch $BOOTFS_MOUNTPOINT/grub/grubenv - touch $BOOTFS_MOUNTPOINT/.safe/SMBIOS_NI_vars - touch $BOOTFS_MOUNTPOINT/.safe/EFI_NI_vars - touch $BOOTFS_MOUNTPOINT/.safe/GRUB_NI_readonly_vars - if is_ni_device ; then - NI_TARGET="true" - cp $SOURCE_DIR/grubenv $BOOTFS_MOUNTPOINT/grub - cp $SOURCE_DIR/SMBIOS_NI_vars $BOOTFS_MOUNTPOINT/.safe - cp $SOURCE_DIR/EFI_NI_vars $BOOTFS_MOUNTPOINT/.safe - add_USB_gadget_args_to_grubenv - override_primaryport_grubenv - else - cp $SOURCE_DIR/grubenv_non_ni_target $BOOTFS_MOUNTPOINT/grub/grubenv - fi - - set_serial_port - - grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "BIOSBootMode=efi" - grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "NITarget=$NI_TARGET" - - # Mark aforementioned grub vars read-only - for varname in "BIOSBootMode" "NITarget" ; do - echo "$varname" >> "$BOOTFS_MOUNTPOINT/.safe/GRUB_NI_readonly_vars" - done - - # save rootfs UUID for grub to reference - ROOTUUID=`lsblk ${TARGET_DISK}${PART_SEPARATOR}4 -n -o PARTUUID` - echo set rootuuid=$ROOTUUID >> $GRUB_MOUNTPOINT/grubvar_readonly - - # set proper permissions on and backup firmware variable files - chown 0:500 $BOOTFS_MOUNTPOINT/grub/grubenv - chmod ug=rw,o=r $BOOTFS_MOUNTPOINT/grub/grubenv - cp -p $BOOTFS_MOUNTPOINT/grub/grubenv $BOOTFS_MOUNTPOINT/grub/grubenv.bak - chmod a=r $BOOTFS_MOUNTPOINT/.safe/SMBIOS_NI_vars - chmod a=r $BOOTFS_MOUNTPOINT/.safe/EFI_NI_vars - chmod a=r $BOOTFS_MOUNTPOINT/.safe/GRUB_NI_readonly_vars - - # Additional grubenv configuration, _NOT_ mirrored in grubenv.bak - - # If the target is a qemu vm, enable consoleout - if dmidecode |grep -qi "virtual machine" ; then - grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "consoleoutenable=True" - fi - - # If an initial value for console out was specified, write it to grubenv - if [ -n "$grubenv_consoleoutenable" ]; then - grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "consoleoutenable=$grubenv_consoleoutenable" - fi - - # Set quietbootdelay to show the boot menu on PXI controllers - # - # The reasoning behind this is that, for NI Linux RT support, Controller - # Software (the BIOS team) wants to keep the business logic out of - # BIOS as much as possible, and make the BIOS agnostic of the actual - # OS it is booting as much as possible (one exception being setting - # HT based on the EFI variable since this can only, practically, be - # handled in BIOS). - - if [ "`get_target_class`" == "PXI" ]; then - grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "quietbootdelay=3" - fi - - # If an initial value for bootdelay was specified, write it to grubenv - if [ -n "$grubenv_bootdelay" ]; then - grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "bootdelay=$grubenv_bootdelay" - fi -} - -set_versions() -{ - BUILD_IDENTIFIER=$(get_image_info BUILD_IDENTIFIER) - - echo "set ni_recoverytool_version=${BUILD_IDENTIFIER}" > $BOOTFS_MOUNTPOINT/grub/recoverytool-ni-version - - chmod 444 $BOOTFS_MOUNTPOINT/grub/recoverytool-ni-version -} - -fixup_configfs() -{ - CONFIG_MOUNTPOINT=/var/volatile/configfs - - mkdir -p $CONFIG_MOUNTPOINT - mount -L $PART3_LABEL $CONFIG_MOUNTPOINT - - # Set root dir ownership to lvuser:ni - chown 500:500 $CONFIG_MOUNTPOINT - chmod ug=rwx,o=rx $CONFIG_MOUNTPOINT - - # Set hostname, if configured - if [ -n "$PROVISION_HOSTNAME" ]; then - print_info "Setting hostname=\"$PROVISION_HOSTNAME\" in ni-rt.ini..." - - echo >$CONFIG_MOUNTPOINT/ni-rt.ini "[systemsettings]" - echo >>$CONFIG_MOUNTPOINT/ni-rt.ini "Host_Name=\"$PROVISION_HOSTNAME\"" - - # Set ownership to lvuser:ni - chown 500:500 $CONFIG_MOUNTPOINT/ni-rt.ini - chmod ug=rw,o=r $CONFIG_MOUNTPOINT/ni-rt.ini - - print_done - fi - - umount $CONFIG_MOUNTPOINT -} - -sanity_check() -{ - print_info "Sanity check TARGET_DISK=$TARGET_DISK..." - - # correct partition table type - parted -s --list "$TARGET_DISK" | grep -q "^Partition Table: $PART_STYLE" || die "Wrong partition style reported" - - attrib_list="NAME KNAME PARTUUID UUID LABEL" - if [[ "$PART_STYLE" == "gpt" ]]; then - attrib_list="$attrib_list PARTLABEL" - fi - - # unique attributes - for attrib_name in $attrib_list; do - sanity_list="`lsblk -nro "$attrib_name" "$TARGET_DISK"`" - sanity_list_len="`echo "$sanity_list" | wc -l`" - sanity_list_uniq_len="`echo "$sanity_list" | uniq | wc -l`" - [ "$sanity_list_len" -eq 5 ] || die "There should be 4 $attrib_name's (+1 blank line) on TARGET_DISK=$TARGET_DISK; found $sanity_list_len" - [ "$sanity_list_len" -eq "$sanity_list_uniq_len" ] || die "$attrib_name's not unique on TARGET_DISK=$TARGET_DISK; $sanity_list_len != $sanity_list_uniq_len" - done - - # can search by file system label - for fs_label in "$PART1_LABEL" "$PART2_LABEL" "$PART3_LABEL" "$PART4_LABEL"; do - fs_node_name="`lsblk -nro LABEL,NAME "$TARGET_DISK" | grep "^$fs_label " | cut -d" " -f2`" - [ "`echo "$fs_node_name" | wc -l`" -eq 1 ] || die "Invalid number of devnodes found" - [ -e "/dev/$fs_node_name" ] || die "No devnode for LABEL $fs_label" - done - - # can search by partition label, on gpt only - if [ "$PART_STYLE" == "gpt" ]; then - for part_label in "$PART1_NAME" "$PART2_NAME" "$PART3_NAME" "$PART4_NAME"; do - part_node_name="`lsblk -nro PARTLABEL,NAME "$TARGET_DISK" | grep "^$part_label " | cut -d" " -f2`" - [ "`echo "$part_node_name" | wc -l`" -eq 1 ] || die "Invalid number of devnodes found" - [ -e "/dev/$part_node_name" ] || die "No devnode for PARTLABEL $part_label" - done - fi - - # just in case meta-data is updated - sync - print_done -} - -check_all_used_binaries - -echo "Installing safemode to: $TARGET_DISK." -echo 6 > /proc/sys/kernel/printk - -SOURCE_DIR=/payload - -print_info "Disabling automount..." -disable_automount -print_done - -partitioning_disk -wait_for_partitions $TARGET_DISK -create_filesystems - -prune_efi_crash_vars -install_grub install_safemode install_bootmode_file install_grubenv diff --git a/recipes-core/initrdscripts/files/ni_provisioning.safemode.common b/recipes-core/initrdscripts/files/ni_provisioning.safemode.common new file mode 100755 index 000000000..cbbd53556 --- /dev/null +++ b/recipes-core/initrdscripts/files/ni_provisioning.safemode.common @@ -0,0 +1,423 @@ +#!/bin/bash +set -e + +ask_at_every_error="N" +grubenv_consoleoutenable=${grubenv_consoleoutenable:-""} +grubenv_bootdelay=${grubenv_bootdelay:-""} + +PART1_LABEL=nigrub +PART2_LABEL=nibootfs +PART3_LABEL=niconfig +PART4_LABEL=nirootfs + +# GPT partitions must assign a partition name +PART_STYLE=gpt +PART1_NAME=$PART1_LABEL +PART2_NAME=$PART2_LABEL +PART3_NAME=$PART3_LABEL +PART4_NAME=$PART4_LABEL + +. /usr/share/nisysinfo/smbios_helper + +print_info() +{ + if [ $verbose_mode -eq 1 ]; then + >&3 echo $1 + else + >&3 echo -n $1 + fi +} + +print_error() +{ + >&4 echo -e ${RED}"\n***Error: $1\n"${NC} + local continue_on_error="N" + if [[ "$ask_at_every_error" == "y" ]]; then + exec 2>&4 + read -p "Do you want to ignore this error?[y/N]" continue_on_error + exec 4>&2 + fi + if [[ "$continue_on_error" == "N" || "$continue_on_error" == "n" || "$continue_on_error" == "" ]]; then + >&4 echo -e ${RED}"PROVISIONING FAILED!"${NC} + cleanup_and_exit 1 + fi +} + +die() +{ + >&4 echo -e ${RED}"\n***Fatal Error: $1"${NC} + >&4 echo -e ${RED}"PROVISIONING FAILED!"${NC} + cleanup_and_exit 1 +} + +print_done() +{ + if [ $verbose_mode -eq 0 ]; then + >&3 echo "Done" + fi +} + +disable_automount() +{ + echo -e $TARGET_DISK >> /etc/udev/mount.blacklist + AUTOMOUNT_DISABLED=1 +} + +enable_automount() +{ + sed -ie "\#$TARGET_DISK#d" /etc/udev/mount.blacklist + AUTOMOUNT_DISABLED=0 +} +#waits until the four partitions are visible +wait_for_partitions() +{ + DEVICE_TMP=$1 + for i in 1 2 3 4 + do + #wait max 3 seconds for each partition + print_verbose "Waiting for $DEVICE_TMP$i " + MaxTries=30 + while [ ! -b ${DEVICE_TMP}${PART_SEPARATOR}$i -a $MaxTries -gt 0 ]; + do + print_verbose "." + #sleep 0.1 seconds + usleep 100000 + MaxTries=$(($MaxTries - 1)) + done + if [ $MaxTries -eq 0 ]; then + return 1 + fi + done +} + +handle_err() +{ + TMP_EVAL=`eval echo $5` + print_error "$1:$2 (fn=$3): Unexpected status code $4, while running command: '$TMP_EVAL'" ${NC} +} + +check_all_used_binaries() +{ + commands=(awk dmidecode e2label hexdump lsblk mkfs.ext4 mkfs.vfat parted sed sfdisk sgdisk udevadm modprobe dd) + for ind_command in ${commands[@]}; do + one_command=`command -v ${ind_command}` + if [[ -z $one_command ]]; then + die "Command \"${ind_command}\" not found. Please install ${ind_command}." + fi + done + +} + +override_primaryport_grubenv() +{ + # Add target-specific override of primary ethernet port if the lowest ifIndex is not to be used + local device_code=$(get_target_id) + # CVS-1458RT + if [[ x"$device_code" == x"77AA" ]]; then + # Set eth0 (note: renamed to eth0 via udev) as the primary port. Do not let the primary + # port selection logic decide since it selects the 'eth' port with the lowest ifIndex + # value. (Renaming the port via udev does not alter ifIndex, so eth0 is not the lowest) + grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "ethaddr=`cat /sys/class/net/eth0/address`" + fi +} + +add_USB_gadget_args_to_grubenv() +{ + local MAC=$(get_usbdev_mac) + local VID=$(get_usbdev_vid) + local PID=$(get_usbdev_pid) + local SN=$(get_serial_number) + local TID=$(get_target_id) + local PROD=$(get_target_name) + + if is_usbdev_target; then + + if [ -n "$MAC" -a -n "$VID" -a -n "$PID" -a -n "$SN" -a -n "$TID" -a -n "$PROD" ]; then + cat << EOF > $GRUB_MOUNTPOINT/grubvar_readonly +set usbgadgetethaddr=$MAC +set USBVendorID=$VID +set USBProductID=$PID +set SerialNum=$SN +set USBDevice=$TID +set USBProduct=$PROD +EOF + else + print_error "Details required for USB device mode are missing from SMBIOS! UPGRADE your BIOS if using NI Target. USB Gadget will not be functional." + return + fi + fi +} + +set_serial_port() +{ + if [ -d /sys/class/tty/ttyS0 ] && [ -f /sys/class/tty/ttyS0/type ] && [ `cat /sys/class/tty/ttyS0/type` -ne 0 ]; then + echo "set serial_port="`cat /sys/class/tty/ttyS0/port` >> $GRUB_MOUNTPOINT/grubvar_readonly + + local base_clock + if base_clock=$(get_serial_base_clock); then + echo "set GRUB_SERIAL_BASE_CLOCK=$base_clock" >> $GRUB_MOUNTPOINT/grubvar_readonly + fi + fi + +} + +partitioning_disk() +{ + umount "$TARGET_DISK"? 2>/dev/null || true + print_info "Partitioning TARGET_DISK=$TARGET_DISK (PART_STYLE=$PART_STYLE)..." + PARTED_ERROR=`parted -s $TARGET_DISK mklabel $PART_STYLE 2>&1` || die "$PARTED_ERROR" + PARTED_ERROR=`parted -s --align optimal $TARGET_DISK mkpart $PART1_NAME 1MB 16MB 2>&1` || die "$PARTED_ERROR" + PARTED_ERROR=`parted -s --align optimal $TARGET_DISK mkpart $PART2_NAME 16MB 200MB 2>&1` || die "$PARTED_ERROR" + PARTED_ERROR=`parted -s --align optimal $TARGET_DISK mkpart $PART3_NAME 200MB 216MB 2>&1` || die "$PARTED_ERROR" + PARTED_ERROR=`parted -s --align optimal $TARGET_DISK mkpart $PART4_NAME 216MB 100% 2>&1` || die "$PARTED_ERROR" + print_done + + print_info "Assigning EFI System Partition..." + # Per https://www.freedesktop.org/wiki/Specifications/DiscoverablePartitionsSpec/ + # C12A7328-F81F-11D2-BA4B-00A0C93EC93B -- EFI System Partition + sgdisk --typecode="1:C12A7328-F81F-11D2-BA4B-00A0C93EC93B" $TARGET_DISK + print_done + + print_info "Probing partitions of TARGET_DISK=$TARGET_DISK ..." + partprobe -s "$TARGET_DISK" + print_done +} + +create_filesystems() +{ + print_info "Creating filesystems ..." + + MKFS_ERROR=`mkfs.vfat -n $PART1_LABEL ${TARGET_DISK}${PART_SEPARATOR}1 2>&1` || die "$MKFS_ERROR" + do_silent mkfs.ext4 $MKFS_ARGS ${TARGET_DISK}${PART_SEPARATOR}2 || die "Format failed!" + do_silent mkfs.ext4 $MKFS_ARGS ${TARGET_DISK}${PART_SEPARATOR}3 || die "Format failed!" + do_silent mkfs.ext4 $MKFS_ARGS ${TARGET_DISK}${PART_SEPARATOR}4 || die "Format failed!" + + LABEL_ERROR=`e2label ${TARGET_DISK}${PART_SEPARATOR}2 $PART2_LABEL 2>&1` || die "$LABEL_ERROR" + LABEL_ERROR=`e2label ${TARGET_DISK}${PART_SEPARATOR}3 $PART3_LABEL 2>&1` || die "$LABEL_ERROR" + LABEL_ERROR=`e2label ${TARGET_DISK}${PART_SEPARATOR}4 $PART4_LABEL 2>&1` || die "$LABEL_ERROR" + + print_done +} + +install_grub() +{ + GRUB_MOUNTPOINT=/var/volatile/grub + mkdir $GRUB_MOUNTPOINT -p + MOUNT_ERROR=`mount -L $PART1_LABEL $GRUB_MOUNTPOINT 2>&1` || die "$MOUNT_ERROR" + + print_info "Configuring EFI grub2..." + GRUB_TARGET_DIR=$GRUB_MOUNTPOINT/efi/boot + mkdir -p $GRUB_TARGET_DIR + GRUB_TARGET=$(uname -m) + cp /boot/EFI/BOOT/bootx64.efi $GRUB_TARGET_DIR + # Delete existing NILRT entries with "-B" option + for ENTRY in $(efibootmgr | grep -Ei '(LabVIEW RT)|(niboota)|(nibootb)' | grep -Eo 'Boot[0-9A-Fa-f]{4}' || true); + do + ENTRY_NUMBER=${ENTRY: 4:4} + print_info " Drop entry $ENTRY_NUMBER." + EFIMGR=$(efibootmgr -b "$ENTRY_NUMBER" -B 2>&1) || print_warning "efibootmgr -b $ENTRY_NUMBER -B failed with: $EFIMGR" + done + efibootmgr $VERBOSE_ARGS -c -d ${TARGET_DISK} -p 1 -L 'LabVIEW RT' -l '\efi\boot\bootx64.efi' + print_done + + print_info "Installing grub.cfg..." + + cp $SOURCE_DIR/grub.cfg $GRUB_TARGET_DIR/grub.cfg + mkdir -p $GRUB_TARGET_DIR/fonts + cp $SOURCE_DIR/fonts/unicode.pf2 $GRUB_TARGET_DIR/fonts/ + print_done +} + +install_safemode() +{ + print_info "Installing safemode kernel and ramdisk..." + + BOOTFS_MOUNTPOINT=/var/volatile/bootfs + mkdir $BOOTFS_MOUNTPOINT -p + BOOTFS_ERROR=`mount -L $PART2_LABEL $BOOTFS_MOUNTPOINT 2>&1` || die "$BOOTFS_ERROR" + mkdir -p $BOOTFS_MOUNTPOINT/.safe + + cp "$SOURCE_DIR"/*Image "$BOOTFS_MOUNTPOINT/.safe/" + cp "$SOURCE_DIR"/ramdisk.* "$BOOTFS_MOUNTPOINT/.safe/" + cp "$SOURCE_DIR"/bootimage.* "$BOOTFS_MOUNTPOINT/.safe/" + + print_done +} + +install_bootmode_file() +{ + # write initial bootmode file + print_info "Installing bootmode file..." + echo "set BOOT_MODE=safemode" >$BOOTFS_MOUNTPOINT/bootmode + print_done +} + +install_grubenv() +{ + NI_TARGET="false" + + mkdir $BOOTFS_MOUNTPOINT/grub + touch $BOOTFS_MOUNTPOINT/grub/grubenv + touch $BOOTFS_MOUNTPOINT/.safe/SMBIOS_NI_vars + touch $BOOTFS_MOUNTPOINT/.safe/EFI_NI_vars + touch $BOOTFS_MOUNTPOINT/.safe/GRUB_NI_readonly_vars + if is_ni_device ; then + NI_TARGET="true" + cp $SOURCE_DIR/grubenv $BOOTFS_MOUNTPOINT/grub + cp $SOURCE_DIR/SMBIOS_NI_vars $BOOTFS_MOUNTPOINT/.safe + cp $SOURCE_DIR/EFI_NI_vars $BOOTFS_MOUNTPOINT/.safe + add_USB_gadget_args_to_grubenv + override_primaryport_grubenv + else + cp $SOURCE_DIR/grubenv_non_ni_target $BOOTFS_MOUNTPOINT/grub/grubenv + fi + + set_serial_port + + grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "BIOSBootMode=efi" + grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "NITarget=$NI_TARGET" + + # Mark aforementioned grub vars read-only + for varname in "BIOSBootMode" "NITarget" ; do + echo "$varname" >> "$BOOTFS_MOUNTPOINT/.safe/GRUB_NI_readonly_vars" + done + + # save rootfs UUID for grub to reference + ROOTUUID=`lsblk ${TARGET_DISK}${PART_SEPARATOR}4 -n -o PARTUUID` + echo set rootuuid=$ROOTUUID >> $GRUB_MOUNTPOINT/grubvar_readonly + + # set proper permissions on and backup firmware variable files + chown 0:500 $BOOTFS_MOUNTPOINT/grub/grubenv + chmod ug=rw,o=r $BOOTFS_MOUNTPOINT/grub/grubenv + cp -p $BOOTFS_MOUNTPOINT/grub/grubenv $BOOTFS_MOUNTPOINT/grub/grubenv.bak + chmod a=r $BOOTFS_MOUNTPOINT/.safe/SMBIOS_NI_vars + chmod a=r $BOOTFS_MOUNTPOINT/.safe/EFI_NI_vars + chmod a=r $BOOTFS_MOUNTPOINT/.safe/GRUB_NI_readonly_vars + + # Additional grubenv configuration, _NOT_ mirrored in grubenv.bak + + # If the target is a qemu vm, enable consoleout + if dmidecode |grep -qi "virtual machine" ; then + grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "consoleoutenable=True" + fi + + # If an initial value for console out was specified, write it to grubenv + if [ -n "$grubenv_consoleoutenable" ]; then + grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "consoleoutenable=$grubenv_consoleoutenable" + fi + + # Set quietbootdelay to show the boot menu on PXI controllers + # + # The reasoning behind this is that, for NI Linux RT support, Controller + # Software (the BIOS team) wants to keep the business logic out of + # BIOS as much as possible, and make the BIOS agnostic of the actual + # OS it is booting as much as possible (one exception being setting + # HT based on the EFI variable since this can only, practically, be + # handled in BIOS). + + if [ "`get_target_class`" == "PXI" ]; then + grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "quietbootdelay=3" + fi + + # If an initial value for bootdelay was specified, write it to grubenv + if [ -n "$grubenv_bootdelay" ]; then + grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "bootdelay=$grubenv_bootdelay" + fi +} + +set_versions() +{ + BUILD_IDENTIFIER=$(get_image_info BUILD_IDENTIFIER) + + echo "set ni_recoverytool_version=${BUILD_IDENTIFIER}" > $BOOTFS_MOUNTPOINT/grub/recoverytool-ni-version + + chmod 444 $BOOTFS_MOUNTPOINT/grub/recoverytool-ni-version +} + +fixup_configfs() +{ + CONFIG_MOUNTPOINT=/var/volatile/configfs + + mkdir -p $CONFIG_MOUNTPOINT + mount -L $PART3_LABEL $CONFIG_MOUNTPOINT + + # Set root dir ownership to lvuser:ni + chown 500:500 $CONFIG_MOUNTPOINT + chmod ug=rwx,o=rx $CONFIG_MOUNTPOINT + + # Set hostname, if configured + if [ -n "$PROVISION_HOSTNAME" ]; then + print_info "Setting hostname=\"$PROVISION_HOSTNAME\" in ni-rt.ini..." + + echo >$CONFIG_MOUNTPOINT/ni-rt.ini "[systemsettings]" + echo >>$CONFIG_MOUNTPOINT/ni-rt.ini "Host_Name=\"$PROVISION_HOSTNAME\"" + + # Set ownership to lvuser:ni + chown 500:500 $CONFIG_MOUNTPOINT/ni-rt.ini + chmod ug=rw,o=r $CONFIG_MOUNTPOINT/ni-rt.ini + + print_done + fi + + umount $CONFIG_MOUNTPOINT +} + +sanity_check() +{ + print_info "Sanity check TARGET_DISK=$TARGET_DISK..." + + # correct partition table type + parted -s --list "$TARGET_DISK" | grep -q "^Partition Table: $PART_STYLE" || die "Wrong partition style reported" + + attrib_list="NAME KNAME PARTUUID UUID LABEL" + if [[ "$PART_STYLE" == "gpt" ]]; then + attrib_list="$attrib_list PARTLABEL" + fi + + # unique attributes + for attrib_name in $attrib_list; do + sanity_list="`lsblk -nro "$attrib_name" "$TARGET_DISK"`" + sanity_list_len="`echo "$sanity_list" | wc -l`" + sanity_list_uniq_len="`echo "$sanity_list" | uniq | wc -l`" + [ "$sanity_list_len" -eq 5 ] || die "There should be 4 $attrib_name's (+1 blank line) on TARGET_DISK=$TARGET_DISK; found $sanity_list_len" + [ "$sanity_list_len" -eq "$sanity_list_uniq_len" ] || die "$attrib_name's not unique on TARGET_DISK=$TARGET_DISK; $sanity_list_len != $sanity_list_uniq_len" + done + + # can search by file system label + for fs_label in "$PART1_LABEL" "$PART2_LABEL" "$PART3_LABEL" "$PART4_LABEL"; do + fs_node_name="`lsblk -nro LABEL,NAME "$TARGET_DISK" | grep "^$fs_label " | cut -d" " -f2`" + [ "`echo "$fs_node_name" | wc -l`" -eq 1 ] || die "Invalid number of devnodes found" + [ -e "/dev/$fs_node_name" ] || die "No devnode for LABEL $fs_label" + done + + # can search by partition label, on gpt only + if [ "$PART_STYLE" == "gpt" ]; then + for part_label in "$PART1_NAME" "$PART2_NAME" "$PART3_NAME" "$PART4_NAME"; do + part_node_name="`lsblk -nro PARTLABEL,NAME "$TARGET_DISK" | grep "^$part_label " | cut -d" " -f2`" + [ "`echo "$part_node_name" | wc -l`" -eq 1 ] || die "Invalid number of devnodes found" + [ -e "/dev/$part_node_name" ] || die "No devnode for PARTLABEL $part_label" + done + fi + + # just in case meta-data is updated + sync + print_done +} + +check_all_used_binaries + +echo "Installing safemode to: $TARGET_DISK." +echo 6 > /proc/sys/kernel/printk + +SOURCE_DIR=/payload + +print_info "Disabling automount..." +disable_automount +print_done + +partitioning_disk +wait_for_partitions $TARGET_DISK +create_filesystems + +prune_efi_crash_vars +install_grub diff --git a/recipes-core/initrdscripts/init-restore-mode.bb b/recipes-core/initrdscripts/init-restore-mode.bb index bb4be62e3..7a0c14d8b 100644 --- a/recipes-core/initrdscripts/init-restore-mode.bb +++ b/recipes-core/initrdscripts/init-restore-mode.bb @@ -20,6 +20,7 @@ SRC_URI:append:xilinx-zynqhf = "\ " SRC_URI:append:x64 = "\ + file://ni_provisioning.safemode.common \ file://ni_provisioning.safemode \ file://disk_config_x64 \ file://grub.cfg \ @@ -41,6 +42,7 @@ do_install() { } do_install:append:x64() { + install -m 0644 ${WORKDIR}/ni_provisioning.safemode.common ${D}/ install -m 0644 ${WORKDIR}/ni_provisioning.safemode ${D}/ install -m 0755 ${WORKDIR}/disk_config_x64 ${D}/disk_config install -m 0644 ${WORKDIR}/grub.cfg ${D}/ From 14e3a31b2e660f5e868af9ebd2a84d6ba12cb92a Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Mon, 22 Apr 2024 18:53:50 -0500 Subject: [PATCH 087/147] ni-systemreplication: Add for USB replication Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 5443bb694aa5098945d794e5291a436b86f46862) Signed-off-by: Alex Stewart --- .../packagegroup-ni-restoremode.bb | 2 +- .../files/nisystemreplication | 23 +++++++++++++++++ .../ni-systemreplication.bb | 25 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 recipes-ni/ni-systemreplication/files/nisystemreplication create mode 100644 recipes-ni/ni-systemreplication/ni-systemreplication.bb diff --git a/recipes-core/packagegroups/packagegroup-ni-restoremode.bb b/recipes-core/packagegroups/packagegroup-ni-restoremode.bb index b34685cf2..532723605 100644 --- a/recipes-core/packagegroups/packagegroup-ni-restoremode.bb +++ b/recipes-core/packagegroups/packagegroup-ni-restoremode.bb @@ -29,7 +29,7 @@ RDEPENDS:${PN} += "\ init-restore-mode \ kmod \ ni-smbios-helper \ - ni-systemimage \ + ni-systemreplication \ parted \ procps \ sed \ diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication new file mode 100644 index 000000000..f610b2439 --- /dev/null +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +# MODE determines the operation to be performed +# get or set +declare -r MODE="${1:-}" + +image_get () { + nisystemimage getall -x tgz -f getall.tgz +} + +image_set () { + source /ni_provisioning.common + early_setup + source /ni_provisioning.safemode.common + nisystemimage setall -x tgz -f getall.tgz -p reset -s reset + + # save rootfs UUID for grub to reference + ROOTUUID=`lsblk ${TARGET_DISK}${PART_SEPARATOR}4 -n -o PARTUUID` + echo set rootuuid=$ROOTUUID >> $GRUB_MOUNTPOINT/grubvar_readonly +} + +image_${MODE} diff --git a/recipes-ni/ni-systemreplication/ni-systemreplication.bb b/recipes-ni/ni-systemreplication/ni-systemreplication.bb new file mode 100644 index 000000000..25e33fa94 --- /dev/null +++ b/recipes-ni/ni-systemreplication/ni-systemreplication.bb @@ -0,0 +1,25 @@ +SUMMARY = "A system replication utility for NI LinuxRT" +DESCRIPTION = "Installs the nisystemreplication utility" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" +SECTION = "base" + +SRC_URI = "\ + file://nisystemreplication \ +" + +S = "${WORKDIR}" + +do_install () { + install -d ${D}${base_sbindir} + install -m 0550 ${S}/nisystemreplication ${D}${base_sbindir} +} + + +FILES:${PN} += "\ + ${base_sbindir}/nisystemreplication \ +" +RDEPENDS:${PN} += "\ + bash \ + ni-systemimage \ +" From 12ae6623637a87b177e4703852fa81d96f5f1cc2 Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Fri, 26 Apr 2024 12:33:20 -0500 Subject: [PATCH 088/147] fs_permissions_diff: Rename command line args to specify manifests Signed-off-by: Jeffrey Pautler (cherry picked from commit 35549a24d71e00f36b95b0617108f2a74eff51f5) Signed-off-by: Alex Stewart --- .../files/fs_permissions_diff.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index b13c0eb13..374c7b86c 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -380,13 +380,13 @@ def parse_args(): parser.add_argument('--server', required=True, help='Mongo server hostname') parser.add_argument('--user', required=True, help='Mongo server username') parser.add_argument('--password', required=True, help='Mongo server password') - parser.add_argument('--current_log_db_date', metavar="", - help='Use this flag to supply a date string that will be used to locate a filesystem manifest in the database. That log will be used as the current filesystem manifest. '\ + parser.add_argument('--current_manifest_db_date', metavar="", + help='Use this flag to supply a date string that will be used to locate a filesystem manifest in the database. That manifest will be used as the current filesystem manifest. '\ 'Should be of the format "2023-03-30 15:32:43.203476". Date strings for previous filesystem manifest can be found in the output of previous runs of this '\ 'test or can be extracted from the Mongo database using a viewer tool like Compass. This flag also skips the upload of the filesystem manifest record to the '\ 'database, so the --skip_upload flag is not needed. This flag is useful for debugging issues with this test.') - parser.add_argument('--basis_log_db_date', metavar="", - help='Use this flag to supply a date string that will be used to locate a filesystem manifest in the database. That log will be used as the basis filesystem manifest. '\ + parser.add_argument('--basis_manifest_db_date', metavar="", + help='Use this flag to supply a date string that will be used to locate a filesystem manifest in the database. That manifest will be used as the basis filesystem manifest. '\ 'Should be of the format "2023-03-30 15:32:43.203476". Date strings for previous filesystem manifest can be found in the output of previous runs of this '\ 'test or can be extracted from the Mongo database using a viewer tool like Compass. This flag is useful for resetting the comparison baseline for the test.') parser.add_argument('--skip_upload', help='Skip upload of fs manifest record to database. Useful when debugging.', action="store_true") @@ -398,8 +398,8 @@ def main(): args = parse_args() db = DB(args.server, args.user, args.password) - if args.current_log_db_date: - fs_manifest, os_version_full, os_version_codename, db_date, db_id = get_previous_fs_manifest_as_current(db, args.current_log_db_date, logger) + if args.current_manifest_db_date: + fs_manifest, os_version_full, os_version_codename, db_date, db_id = get_previous_fs_manifest_as_current(db, args.current_manifest_db_date, logger) os_version = OsVersion(os_version_full, os_version_codename) else: prepare_system_for_manifest() @@ -408,9 +408,9 @@ def main(): db_date = "" db_id = "" - basis_fs_manifest, recent_fs_manifest = get_old_fs_manifests(db, logger, os_version, args.basis_log_db_date) + basis_fs_manifest, recent_fs_manifest = get_old_fs_manifests(db, logger, os_version, args.basis_manifest_db_date) - if not args.current_log_db_date and not args.skip_upload: + if not args.current_manifest_db_date and not args.skip_upload: db_date, db_id = upload_manifest(db, fs_manifest, os_version, logger) log_version_info(logger, 'current', os_version.codename, os_version.full, db_date, db_id) From aa4fe56387fd977c93f8e37efe53f04fea5ea24e Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Mon, 29 Apr 2024 11:57:54 -0500 Subject: [PATCH 089/147] fs_permissions_diff: Cleanup/improve logging output Signed-off-by: Jeffrey Pautler (cherry picked from commit b869fdc5f534d79845ee7faa48aba88e2af499c2) Signed-off-by: Alex Stewart --- .../files/fs_permissions_diff.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py index 374c7b86c..705783656 100644 --- a/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py +++ b/recipes-ni/ni-base-system-image-tests/files/fs_permissions_diff.py @@ -80,7 +80,7 @@ def get_fs_manifest(): return '\n'.join(sorted(fs_manifest.splitlines())) def log_version_info(logger, label, codename, full_version, date, db_id): - logger.log(f'INFO: {label} = {codename} {full_version} from {date} with _id {db_id}') + logger.log(f'INFO: {label: <7} = {codename} {full_version: <14} from {date} with _id {db_id}') def upload_manifest(db, fs_manifest, os_version, logger): data = {} @@ -105,18 +105,19 @@ def get_previous_fs_manifest_as_current(db, previous_date, logger): query = {} query['date'] = previous_date count = db.count_documents(query) - logger.log('!!!!!OVERRIDING CURRENT MANIFEST!!!!!\n'.format(count)) - logger.log('Found {} records\n'.format(count)) + logger.log('!!!!! OVERRIDING CURRENT MANIFEST !!!!!'.format(count)) if count == 1: results = db.find(query).limit(1) result = next(results) - logger.log('INFO: Found fs_manifest record with these details') + logger.log(f'INFO: Found manifest with the specified date: {previous_date}') return strip_headers(result['fs_permissions']), result['os_version_full'], result['os_version_codename'], result['date'], result['_id'] elif count > 1: - logger.log('INFO: Found multiple fs manifest records with the same date: {}'.format(previous_date)) + logger.log(f'INFO: Found multiple manifests with the specified date ({previous_date})') + logger.report() exit(1) else: - logger.log('INFO: Could not find fs manifest record with date: {}'.format(previous_date)) + logger.log(f'INFO: Could not find manifest with the specified date ({previous_date})') + logger.report() exit(1) def get_old_fs_manifests(db, logger, os_version, basis_override): @@ -133,12 +134,12 @@ def run_query(label, query): # Remove that requirement. if 'os_version_codename' in query: del query['os_version_codename'] - logger.log('INFO: No prior fs permissions from this OS codename found. Relaxing that constraint.') + logger.log('INFO: No prior manifest from this OS codename found. Relaxing that constraint.') if db.count_documents(query): results = db.find(query).sort(sort_order).limit(1) else: - logger.log('INFO: No suitable previous fs permissions found') + logger.log('INFO: No suitable previous manifest found') return '', '', '' result = next(results) @@ -193,7 +194,7 @@ def run_query(label, query): logger.prefix_log(f'INFO: fs_permissions_diff: current against {basis_version_full} ') # Keep this log in second line to avoid using it for grouping, but keep it for tracking. # Individual hashes to be populated at end. - logger.prefix_log(f'INFO: and {recent_version_full} ') + logger.prefix_log(f'INFO: Manifest hashes: ') return basis_manifest, recent_manifest @@ -282,7 +283,7 @@ def diff_manifests(current_manifest, basis_manifest, recent_manifest, logger): # Add hash to first line of log to allow review queue to distinguish runs, logger.prefix_logs[0] += overall_hash # and add each hash on second line for tracking - logger.prefix_logs[1] += f'(basis={basis_hash}, recent={recent_hash}, current={current_hash})' + logger.prefix_logs[1] += f'basis={basis_hash}, recent={recent_hash}, current={current_hash}' diff = IntermediateDiff(basis_manifest, recent_manifest, current_manifest) From 9627f480a88ac29c5b4f701a67dca6161f05b5dd Mon Sep 17 00:00:00 2001 From: Jeffrey Pautler Date: Wed, 1 May 2024 10:12:11 -0500 Subject: [PATCH 090/147] fs_permissions_diff: Pass dynamic command-line arguments In some situations, it is useful to pass additional command-line arguments along to the test that are determined dynamically each time a test is run. An example would be to pass a specific manifest to use as the basis of comparison. This can be used to move forward the basis of comparison each time manifest differences reported by a prior test failure have been confirmed as valid changes. Signed-off-by: Jeffrey Pautler (cherry picked from commit 58d30de9cbd08d4a22b6f5398ca3590741c794c0) Signed-off-by: Alex Stewart --- .../files/test_fs_permissions_diff.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/recipes-ni/ni-base-system-image-tests/files/test_fs_permissions_diff.sh b/recipes-ni/ni-base-system-image-tests/files/test_fs_permissions_diff.sh index 8de821b96..c565dd62d 100755 --- a/recipes-ni/ni-base-system-image-tests/files/test_fs_permissions_diff.sh +++ b/recipes-ni/ni-base-system-image-tests/files/test_fs_permissions_diff.sh @@ -5,7 +5,11 @@ source $(dirname "$0")/ptest-format.sh ptest_change_test $(basename "$0" ".sh") "" "Diff filesystem permissions with previous" source /home/admin/.mongodb.creds -python3 fs_permissions_diff.py --server $MONGO_SERVER --user $MONGO_USER --password $MONGO_PASSWORD +if [ -e /home/admin/.test.fs_permissions_diff.args ]; then + source /home/admin/.test.fs_permissions_diff.args +fi + +python3 fs_permissions_diff.py --server $MONGO_SERVER --user $MONGO_USER --password $MONGO_PASSWORD "${FS_PERMISSIONS_DIFF_TEST_EXTRA_ARGS[@]}" if [ $? -eq 0 ]; then ptest_pass From 4432c89c48e49d2d94f6f1f7e73ef9f3650c2bfd Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Thu, 9 May 2024 14:02:53 -0400 Subject: [PATCH 091/147] recipes-devtools: Add python3-backports-ssl-match-hostname Salt 3000.2 depends on the ssl_match_hostname module. In python3<3.12, this module was a part of the core libs. But later releases have removed it. Add a recipe to forward-port the python_3.5 module implementation as a standalone package for use in python_3.12. Signed-off-by: Alex Stewart --- ...n3-backports-ssl-match-hostname_3.7.0.1.bb | 23 +++++++++++++++++++ recipes-support/salt/salt_3000.2.bb | 1 + 2 files changed, 24 insertions(+) create mode 100644 recipes-devtools/python/python3-backports-ssl-match-hostname_3.7.0.1.bb diff --git a/recipes-devtools/python/python3-backports-ssl-match-hostname_3.7.0.1.bb b/recipes-devtools/python/python3-backports-ssl-match-hostname_3.7.0.1.bb new file mode 100644 index 000000000..4efc82a9f --- /dev/null +++ b/recipes-devtools/python/python3-backports-ssl-match-hostname_3.7.0.1.bb @@ -0,0 +1,23 @@ +SUMMARY = "The ssl.match_hostname() function from Python 3.5" +DESCRIPTION = "\ +The Secure Sockets layer is only actually secure if you check the hostname in \ +the certificate returned by the server to which you are connecting, and verify \ +that it matches to hostname that you are trying to reach. But the matching \ +logic, defined in RFC2818, can be a bit tricky to implement on your own. So \ +the ssl package in the Standard Library of Python 3.2 and greater now includes \ +a match_hostname() function for performing this check instead of requiring \ +every application to implement the check separately. This backport brings \ +match_hostname() to users of earlier versions of Python" +AUTHOR = "Toshio Kuratomi " +HOMEPAGE = "https://pypi.org/project/backports.ssl_match_hostname/" +LICENSE = "Python-2.0" +LIC_FILES_CHKSUM = "file://PKG-INFO;md5=b2adbe8bfdeb625c9a01afd9aaa66619" + +SRC_URI[md5sum] = "32d2f593af01a046bec3d2f5181a420a" +SRC_URI[sha256sum] = "bb82e60f9fbf4c080eabd957c39f0641f0fc247d9a16e31e26d594d8f42b9fd2" + +PYPI_PACKAGE = "backports.ssl_match_hostname" + +inherit pypi setuptools3 + +RDEPENDS:${PN} += "${PYTHON_PN}-pkgutil" diff --git a/recipes-support/salt/salt_3000.2.bb b/recipes-support/salt/salt_3000.2.bb index 3f500333c..300b8326d 100644 --- a/recipes-support/salt/salt_3000.2.bb +++ b/recipes-support/salt/salt_3000.2.bb @@ -141,6 +141,7 @@ SUMMARY:${PN}-common = "shared libraries that salt requires for all packages" DESCRIPTION:${PN}-common ="${DESCRIPTION_COMMON} This particular package provides shared libraries that \ salt-master, salt-minion, and salt-syndic require to function." RDEPENDS:${PN}-common = "\ + python3-backports-ssl-match-hostname \ python3-core \ python3-dateutil \ python3-fcntl \ From 266c8bf0a845b87c0e23a4b868d9db4744e5b948 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 10 May 2024 16:32:05 -0400 Subject: [PATCH 092/147] salt: RDEPEND on charset-normalizer Some salt calls require the charset-normalizer module. This was previously being installed implicitly by another package. Signed-off-by: Alex Stewart --- recipes-support/salt/salt_3000.2.bb | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes-support/salt/salt_3000.2.bb b/recipes-support/salt/salt_3000.2.bb index 300b8326d..9bee997c7 100644 --- a/recipes-support/salt/salt_3000.2.bb +++ b/recipes-support/salt/salt_3000.2.bb @@ -142,6 +142,7 @@ DESCRIPTION:${PN}-common ="${DESCRIPTION_COMMON} This particular package provide salt-master, salt-minion, and salt-syndic require to function." RDEPENDS:${PN}-common = "\ python3-backports-ssl-match-hostname \ + python3-charset-normalizer \ python3-core \ python3-dateutil \ python3-fcntl \ From 2e53f49beddff692a447018f5c59e75d894fe43f Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 10 May 2024 16:45:57 -0400 Subject: [PATCH 093/147] Move Skyline RDEPENDS to packagegroup Some RDEPENDS declared in the salt recipe don't actually apply to any packages. But are rather intended to instigate the core packagefeed to include dependencies which will be needed by closed-source skyline packages at runtime. This is pretty confusing. So instead move those deps to an ni-skyline packagegroup, and make that a dependency of the core feed. Signed-off-by: Alex Stewart --- .../packagegroups/packagefeed-ni-core.bb | 1 + .../packagegroups/packagegroup-ni-skyline.bb | 37 +++++++++++++++++++ recipes-support/salt/salt_3000.2.bb | 26 ------------- 3 files changed, 38 insertions(+), 26 deletions(-) create mode 100644 recipes-core/packagegroups/packagegroup-ni-skyline.bb diff --git a/recipes-core/packagegroups/packagefeed-ni-core.bb b/recipes-core/packagegroups/packagefeed-ni-core.bb index ef4b9ba6c..499bd7194 100644 --- a/recipes-core/packagegroups/packagefeed-ni-core.bb +++ b/recipes-core/packagegroups/packagefeed-ni-core.bb @@ -16,6 +16,7 @@ RDEPENDS:${PN} = "\ packagegroup-ni-restoremode \ packagegroup-ni-runmode \ packagegroup-ni-safemode \ + packagegroup-ni-skyline \ packagegroup-ni-tzdata \ packagegroup-ni-wifi \ packagegroup-core-x11 \ diff --git a/recipes-core/packagegroups/packagegroup-ni-skyline.bb b/recipes-core/packagegroups/packagegroup-ni-skyline.bb new file mode 100644 index 000000000..eaf2f280f --- /dev/null +++ b/recipes-core/packagegroups/packagegroup-ni-skyline.bb @@ -0,0 +1,37 @@ +# (C) Copyright 2024, +# National Instruments Corporation. +# All rights reserved. + +SUMMARY = "Open Source package dependencies for the NI Skyline software stack." +LICENSE = "MIT" + +PACKAGE_ARCH = "${MACHINE_ARCH}" + +inherit packagegroup + +RDEPENDS:${PN} = "\ + salt-common \ + salt-minion \ +" + +# Miscellaneous python deps for closed-source Skyline components +RDEPENDS:${PN}:append = "\ + python3-aiodns \ + python3-aiohttp \ + python3-avahi \ + python3-configparser \ + python3-difflib \ + python3-misc \ + python3-mmap \ + python3-multiprocessing \ + python3-pika \ + python3-profile \ + python3-psutil \ + python3-pyiface \ + python3-pyinotify \ + python3-pyroute2 \ + python3-resource \ + python3-terminal \ + python3-unixadmin \ + python3-xmlrpc \ +" diff --git a/recipes-support/salt/salt_3000.2.bb b/recipes-support/salt/salt_3000.2.bb index 9bee997c7..bd606b6ea 100644 --- a/recipes-support/salt/salt_3000.2.bb +++ b/recipes-support/salt/salt_3000.2.bb @@ -56,32 +56,6 @@ PACKAGES += "\ ${PN}-bash-completion \ " -RDEPENDS:${PN}-minion += "\ - python3-aiodns \ - python3-aiohttp \ - python3-avahi \ - python3-mmap \ - python3-pyinotify \ - python3-pyroute2 \ - python3-pika \ - python3-psutil \ -" - -# NI Skyline dependencies only; not needed for base salt. -RDEPENDS:${PN}-common += " \ - python3-configparser \ - python3-dateutil \ - python3-difflib \ - python3-misc \ - python3-multiprocessing \ - python3-profile \ - python3-pyiface \ - python3-resource \ - python3-terminal \ - python3-unixadmin \ - python3-xmlrpc \ -" - INITSCRIPT_PARAMS:${PN}-minion = "defaults 93 7" do_install:append() { From 045a3f7e05a58f89bfd5ceb31aab5ceb2aef957c Mon Sep 17 00:00:00 2001 From: Rajendra Desai Date: Mon, 13 May 2024 12:04:59 +0530 Subject: [PATCH 094/147] packagefeed-ni-extra: temporarily disable unbuildable packages This change is to unblock building of packagefeed-ni-extra group. Signed-off-by: Rajendra Desai --- recipes-core/packagegroups/packagefeed-ni-extra.bb | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/recipes-core/packagegroups/packagefeed-ni-extra.bb b/recipes-core/packagegroups/packagefeed-ni-extra.bb index 445243f35..27cfbc275 100644 --- a/recipes-core/packagegroups/packagefeed-ni-extra.bb +++ b/recipes-core/packagegroups/packagefeed-ni-extra.bb @@ -14,7 +14,6 @@ RDEPENDS:${PN}:append:x64 = "\ geany \ gimp \ gnuplot \ - gnuradio \ gtk+3 \ iceauth \ fltk \ @@ -29,7 +28,6 @@ RDEPENDS:${PN}:append:x64 = "\ sessreg \ setxkbmap \ sysconfig-settings-ui \ - toscoterm \ tk \ twm \ upower \ @@ -193,7 +191,6 @@ RDEPENDS:${PN} += "\ nss-myhostname \ pinentry \ ptest-runner \ - sqlite \ " # openembedded-core/meta/recipes-kernel @@ -355,7 +352,6 @@ RDEPENDS:${PN}:append:x64 = "\ # meta-openembedded/meta-oe/recipes-multimedia RDEPENDS:${PN} += "\ - alsa-oss \ audiofile \ jack \ media-ctl \ @@ -497,19 +493,10 @@ RDEPENDS:${PN} += "\ tftp-hpa \ conntrack-tools \ ebtables \ - netkit-ftp \ - netkit-rpc \ - netkit-rsh-client \ - netkit-rsh-server \ - netkit-rwho-server \ - netkit-tftp-client \ - netkit-tftp-server \ - netkit-telnet \ net-snmp \ openflow \ openl2tp \ pptp-linux \ - quagga \ radiusclient-ng \ rp-pppoe \ xl2tpd \ From 766260960a53dafe5053449622164f2b73a700ec Mon Sep 17 00:00:00 2001 From: Rajendra Desai Date: Mon, 13 May 2024 12:10:16 +0530 Subject: [PATCH 095/147] passwd/group: add uid/gid for arpwatch extra packagefeed fails to build if we do not define the static ID for arpwatch package. Signed-off-by: Rajendra Desai --- files/group | 1 + files/passwd | 1 + 2 files changed, 2 insertions(+) diff --git a/files/group b/files/group index 40b4358cf..7fcb64e26 100644 --- a/files/group +++ b/files/group @@ -6,6 +6,7 @@ ni:x:500: openvpn:x:499: niwscerts:x:498: # free space +arpwatch:x:402: ptest:x:401: ossec:x:400: mysql:x:399: diff --git a/files/passwd b/files/passwd index 1e2f796a1..92630f824 100644 --- a/files/passwd +++ b/files/passwd @@ -6,6 +6,7 @@ webserv:x:501:::: lvuser:x:500:::: openvpn:x:499:::: # free space +arpwatch:x:402:::: ptest:x:401:::: ossec:x:400:::: mysql:x:399:::: From e0840fa56f149951558097991306f917544a7085 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Thu, 25 Apr 2024 18:01:47 -0500 Subject: [PATCH 096/147] nisystemreplication: Initialize grubenv Other than bootdelay and consoleoutputenable, we don't want any grubenv variables to be carried over from another target. So keep bootdelay and consoleoutputenable from the image and initialize others with default values. Also added some cleanup code from ni_provisioning.safemode. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 04679f518fd00fac64117cae0edd7cddc7956ea9) Signed-off-by: Chaitanya Vadrevu --- recipes-ni/ni-systemimage/files/nisystemimage | 2 -- .../files/nisystemreplication | 28 +++++++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/recipes-ni/ni-systemimage/files/nisystemimage b/recipes-ni/ni-systemimage/files/nisystemimage index ee86d22c2..ba7559cfd 100644 --- a/recipes-ni/ni-systemimage/files/nisystemimage +++ b/recipes-ni/ni-systemimage/files/nisystemimage @@ -58,8 +58,6 @@ BLACKLIST_PATHS=( BLACKLIST_PATHS_get_set=( # Safe mode /boot/.safe - # Grub variables - /boot/grub # Default bitfile /boot/.defbit # User bitfile diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index f610b2439..4204652a6 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -12,12 +12,34 @@ image_get () { image_set () { source /ni_provisioning.common early_setup + source /ni_provisioning.safemode.common + install_safemode + nisystemimage setall -x tgz -f getall.tgz -p reset -s reset - # save rootfs UUID for grub to reference - ROOTUUID=`lsblk ${TARGET_DISK}${PART_SEPARATOR}4 -n -o PARTUUID` - echo set rootuuid=$ROOTUUID >> $GRUB_MOUNTPOINT/grubvar_readonly + # Retain some grubenv settings + if grep --quiet "^consoleoutenable=" $BOOTFS_MOUNTPOINT/grub/grubenv; then + grubenv_consoleoutenable=$(grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv list |grep "^consoleoutenable=" | cut -f 2 -d '=') + fi + if grep --quiet "^bootdelay=" $BOOTFS_MOUNTPOINT/grub/grubenv; then + grubenv_bootdelay=$(grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv list |grep "^bootdelay=" | cut -f 2 -d '=') + fi + rm -rf mkdir $BOOTFS_MOUNTPOINT/grub + + install_grubenv + set_versions + + echo $LOG_LEVEL > /proc/sys/kernel/printk + sanity_check + + print_info "Re-enabling automount..." + enable_automount + print_done + + trap - ERR + exec 1>&3 + exec 2>&4 } image_${MODE} From d5cb7cf172a9517c8fb0985407adce4da7700dbe Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Tue, 30 Apr 2024 22:09:42 -0500 Subject: [PATCH 097/147] grub_old.cfg: Add grub menu options for usb replication Implement get/set grub menu options for usb replication. Increase grub timeout to 3s as 1s is too short to be able to select menu options. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit efb067a3435809b85a3d261d55cb41c3d9ac847d) Signed-off-by: Chaitanya Vadrevu --- .../initrdscripts/files/ni_provisioning | 15 ++++++++++++++- .../files/nisystemreplication | 18 ++++++++++++++---- scripts/lib/wic/canned-wks/grub_old.cfg | 12 +++++++++++- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/recipes-core/initrdscripts/files/ni_provisioning b/recipes-core/initrdscripts/files/ni_provisioning index 205b4abd6..bb0142b3c 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning +++ b/recipes-core/initrdscripts/files/ni_provisioning @@ -102,10 +102,23 @@ elif [[ $restore == "provision" ]]; then restore_runmode_image "$nilrt_path" "$uuid" "$PROVISION_PART_NILRT_LABEL" fi fi - elif [[ $restore == "migrate" ]]; then PROVISION_REBOOT_METHOD="reboot" provision_target onboard +elif [[ $restore == "get-image" ]]; then + echo "Get Image" + source /sbin/nisystemreplication get + if [[ "${FORCE_PROVISIONING}" -eq 1 ]]; then + PROVISION_REBOOT_METHOD=${RESTART_OPTION} + fi + ASK_BEFORE_REBOOT=1 +elif [[ $restore == "set-image" ]]; then + echo "Set Image" + source /sbin/nisystemreplication set + if [[ "${FORCE_PROVISIONING}" -eq 1 ]]; then + PROVISION_REBOOT_METHOD=${RESTART_OPTION} + fi + ASK_BEFORE_REBOOT=1 else die "Invalid restore mode." fi diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 4204652a6..5f4e545d2 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -5,18 +5,28 @@ set -e # get or set declare -r MODE="${1:-}" +NIRECOVERY_MOUNTPOINT=/mnt/NIRECOVERY + image_get () { - nisystemimage getall -x tgz -f getall.tgz + mount -o remount,rw $NIRECOVERY_MOUNTPOINT + mkdir -p $NIRECOVERY_MOUNTPOINT/replication_images + nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/replication_images/systemimage.tgz + mount -o remount,ro $NIRECOVERY_MOUNTPOINT + echo "Done getting image" } image_set () { - source /ni_provisioning.common - early_setup + if [[ $(type -t early_setup) != function ]]; then + # If we're here, the script was invoked manually i.e., not from init. + # So do the required setup. + source /ni_provisioning.common + early_setup + fi source /ni_provisioning.safemode.common install_safemode - nisystemimage setall -x tgz -f getall.tgz -p reset -s reset + nisystemimage setall -x tgz -f $NIRECOVERY_MOUNTPOINT/replication_images/systemimage.tgz -p reset -s reset # Retain some grubenv settings if grep --quiet "^consoleoutenable=" $BOOTFS_MOUNTPOINT/grub/grubenv; then diff --git a/scripts/lib/wic/canned-wks/grub_old.cfg b/scripts/lib/wic/canned-wks/grub_old.cfg index 1f5df762f..f44b7184c 100644 --- a/scripts/lib/wic/canned-wks/grub_old.cfg +++ b/scripts/lib/wic/canned-wks/grub_old.cfg @@ -1,6 +1,6 @@ serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1 default='Default Provisioning' -timeout=1 +timeout=3 search --set=root --label NIRECOVERY-CD @@ -16,4 +16,14 @@ menuentry 'Verbose Provisioning' { initrd /initrd } +menuentry 'Get Image' { + linux /bzImage rootwait restore=get-image quiet loglevel=1 + initrd /initrd +} + +menuentry 'Set Image' { + linux /bzImage rootwait restore=set-image quiet loglevel=1 + initrd /initrd +} + menu background splash.jpg From eedf11529c500dcea68d1ff7d1adcdb695b305d3 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Thu, 2 May 2024 16:09:20 -0500 Subject: [PATCH 098/147] ni-netcfgutil: Add required dependencies to RDEPENDS Added following packages which are required dependencies - ni-utils: Provides /etc/natinst/networking/functions.common - initscripts: Provides /etc/init.d/hostname.sh - initscripts-nilrt: Provides /etc/init.d/populateconfig Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 90c829dd776fcdb6c682d1d926851c415ad98a73) Signed-off-by: Chaitanya Vadrevu --- recipes-ni/ni-netcfgutil/ni-netcfgutil.bb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/recipes-ni/ni-netcfgutil/ni-netcfgutil.bb b/recipes-ni/ni-netcfgutil/ni-netcfgutil.bb index a1dc6605e..2b482640a 100644 --- a/recipes-ni/ni-netcfgutil/ni-netcfgutil.bb +++ b/recipes-ni/ni-netcfgutil/ni-netcfgutil.bb @@ -40,4 +40,9 @@ FILES:${PN} += "\ /etc/natinst/networking/ninetcfgutil_platdep.sh \ " -RDEPENDS:${PN} += "bash" +RDEPENDS:${PN} += "\ + bash \ + initscripts \ + initscripts-nilrt \ + ni-utils \ +" From 0abebf5defaf80a5d865c04a839b3848d804e122 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Tue, 7 May 2024 16:02:19 -0500 Subject: [PATCH 099/147] nilrt-recovery-media: Install nirtcfg nirtcfg is a dependency for ni-netcfgutil but since this is built outside of OE, we need to add it here. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 161fe67b2f99ad4c0892fae2756a34bc63ab8209) Signed-off-by: Chaitanya Vadrevu --- recipes-core/images/nilrt-recovery-media.bb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipes-core/images/nilrt-recovery-media.bb b/recipes-core/images/nilrt-recovery-media.bb index 7329427e8..8a5ceb2cc 100644 --- a/recipes-core/images/nilrt-recovery-media.bb +++ b/recipes-core/images/nilrt-recovery-media.bb @@ -8,6 +8,8 @@ PACKAGE_INSTALL = "${ROOTFS_BOOTSTRAP_INSTALL} \ IMAGE_FEATURES += "empty-root-password" +IMAGE_INSTALL_NODEPS += "nirtcfg" + DEPENDS += "init-restore-mode" INITRAMFS_MAXSIZE = "524288" From 241b8c0dbe8aa743f0528a3513191afc47e6c931 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Tue, 7 May 2024 16:03:08 -0500 Subject: [PATCH 100/147] nisystemimage: Handle netcfg operations in setall - Handle netcfg operations in setall - Refactor init_fs_restoremode for readability - Add cleanup_fs_restoremode to correctly handle cleanup in restoremode Signed-off-by: Chaitanya Vadrevu (cherry picked from commit e00d41768deb4a1e5c8245cac189185444530d3e) Signed-off-by: Chaitanya Vadrevu --- recipes-ni/ni-systemimage/files/nisystemimage | 71 ++++++++++++------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/recipes-ni/ni-systemimage/files/nisystemimage b/recipes-ni/ni-systemimage/files/nisystemimage index ba7559cfd..f19d3b76d 100644 --- a/recipes-ni/ni-systemimage/files/nisystemimage +++ b/recipes-ni/ni-systemimage/files/nisystemimage @@ -573,7 +573,14 @@ image_set () { } image_setall () { - image_set_tgz + netcfg_pre_apply_reset + netcfg_pre_preserve + # TODO: Fix wipe_fs to work in restoremode + # wipe_fs + image_set_${IMAGE_TYPE} + netcfg_apply + netcfg_preserve + netcfg_reset } init_fd_set () { @@ -634,9 +641,34 @@ init_fd () { init_fd_$MODE } +init_fs_restoremode () { + # If the util did not previously exit cleanly, it's possible that the + # working mount points didn't get unmounted. To avoid double-mounts, + # try to unmount them first + is_mounted $ORIG_BOOTFS && run_cmd_always umount -f $ORIG_BOOTFS || true + is_mounted $ORIG_CONFIGFS && run_cmd_always umount -f $ORIG_CONFIGFS || true + is_mounted $ORIG_USERFS && run_cmd_always umount -f $ORIG_USERFS || true + + run_cmd_always mkdir -p $ORIG_USERFS + run_cmd_always mount -L $USERFS_LABEL $ORIG_USERFS \ + || die FS_PREP_FAILED "Couldn't mount label $USERFS_LABEL to $ORIG_USERFS " + run_cmd_always mkdir -p $ORIG_BOOTFS + run_cmd_always mount -L $BOOTFS_LABEL $ORIG_BOOTFS \ + || die FS_PREP_FAILED "Couldn't mount label $BOOTFS_LABEL to $ORIG_BOOTFS " + run_cmd_always mkdir -p $ORIG_CONFIGFS + run_cmd_always mount -L $CONFIGFS_LABEL $ORIG_CONFIGFS \ + || die FS_PREP_FAILED "Couldn't mount label $CONFIGFS_LABEL to $ORIG_CONFIGFS" +} + init_fs () { + if in_restoremode; then + # Tools such as ninetcfg and nirtcfg expect same mountpoints as safemode and runmode. + # So mount userfs, bootfs, configfs at expected locations. + init_fs_restoremode + fi + # If the util did not previously exit cleanly, it's possible that the - # working mount points didn't get unmount them. To avoid double-mounts, + # working mount points didn't get unmounted. To avoid double-mounts, # try to unmount them first is_mounted $WORKING_BOOTFS && run_cmd_always umount -f $WORKING_BOOTFS || true is_mounted $WORKING_CONFIGFS && run_cmd_always umount -f $WORKING_CONFIGFS || true @@ -655,31 +687,20 @@ init_fs () { init_fs_$MODE } -init_fs_restoremode () { - # If the util did not previously exit cleanly, it's possible that the - # working mount points didn't get unmount them. To avoid double-mounts, - # try to unmount them first - is_mounted $WORKING_BOOTFS && run_cmd_always umount -f $WORKING_BOOTFS || true - is_mounted $WORKING_CONFIGFS && run_cmd_always umount -f $WORKING_CONFIGFS || true - is_mounted $WORKING_USERFS && run_cmd_always umount -f $WORKING_USERFS || true - - run_cmd_always mkdir -p $WORKING_USERFS - run_cmd_always mount -L $USERFS_LABEL $WORKING_USERFS \ - || die FS_PREP_FAILED "Couldn't mount label $USERFS_LABEL to $WORKING_USERFS" - run_cmd_always mkdir -p $WORKING_BOOTFS - run_cmd_always mount -L $BOOTFS_LABEL $WORKING_BOOTFS \ - || die FS_PREP_FAILED "Couldn't mount label $BOOTFS_LABEL to $WORKING_BOOTFS" - run_cmd_always mkdir -p $WORKING_CONFIGFS - run_cmd_always mount -L $CONFIGFS_LABEL $WORKING_CONFIGFS \ - || die FS_PREP_FAILED "Couldn't mount label CONFIGFS_LABEL to $WORKING_CONFIGFS" - - init_fs_$MODE +cleanup_fs_restoremode () { + is_mounted $ORIG_BOOTFS && run_cmd_always umount -f $ORIG_BOOTFS || true + is_mounted $ORIG_CONFIGFS && run_cmd_always umount -f $ORIG_CONFIGFS || true + is_mounted $ORIG_USERFS && run_cmd_always umount -f $ORIG_USERFS || true } ## ## Cleanup ## cleanup_fs () { + if in_restoremode; then + cleanup_fs_restoremode + fi + cd $ORIG_PWD || true echo_verbose 1 "starting fs cleanup: PWD=$PWD" @@ -834,7 +855,7 @@ check_settings () { # NOTE: "Set Image" under run mode is guaranteed to fail. "Get Image" is # merely unsupported and probably won't work. - if ! in_safemode; then ORIG_USERFS=/; fi + if ! in_safemode && ! in_restoremode ; then ORIG_USERFS=/; fi local -a escaped_excludes=() escaped_excludes+=("${BLACKLIST_PATHS[@]:+${BLACKLIST_PATHS[@]#/}}") @@ -878,11 +899,7 @@ dump_settings run_hook hook_settings_post # Initialize filesystem and file descriptor state -if in_restoremode; then - init_fs_restoremode -else - init_fs -fi +init_fs init_fd # Perform the image operation From c5fd7cb6cbeb8ee01d41cf4f0c693cb1aab07e74 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 8 May 2024 15:11:51 -0500 Subject: [PATCH 101/147] populateconfig: Check for directory presence before creating links populateconfig is going to be used in restoremode for usb replication feature. Since /var/local/natinst doesn't exist in restormode, check for it before creating /var/local/natinst/certstore symlink. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit daa49d1ec7c6d9981f6beac00300bc7587b6770c) Signed-off-by: Chaitanya Vadrevu --- recipes-ni/initscripts-nilrt/files/populateconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-ni/initscripts-nilrt/files/populateconfig b/recipes-ni/initscripts-nilrt/files/populateconfig index b11f481b2..a4c790c1e 100644 --- a/recipes-ni/initscripts-nilrt/files/populateconfig +++ b/recipes-ni/initscripts-nilrt/files/populateconfig @@ -54,7 +54,7 @@ then mkdir -p "/etc/natinst/share/certstore/server_certs" mkdir -p "/etc/natinst/share/certstore/temp" - if [ ! -x /var/local/natinst/certstore ]; then + if [ ! -x /var/local/natinst/certstore ] && [ -d /var/local/natinst ]; then ln -sf -T /etc/natinst/share/certstore /var/local/natinst/certstore fi From 51a77e380c5b1dd0f542118220a82d178ebc0d05 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Fri, 10 May 2024 14:23:55 -0500 Subject: [PATCH 102/147] grub.cfg: Replace grub.cfg with grub_old.cfg grub_old.cfg is used for safemode provisioning and grub.cfg was for rauc which we no longer use. So replace grub.cfg with grub_old.cfg. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit b66ec9e6fb2f9e8994fd4a7f46effc3abc98d7c8) Signed-off-by: Chaitanya Vadrevu --- scripts/lib/wic/canned-wks/grub.cfg | 18 ++++++++++-- scripts/lib/wic/canned-wks/grub_old.cfg | 29 ------------------- .../canned-wks/nilrt-recovery-media.x64.wks | 2 +- 3 files changed, 16 insertions(+), 33 deletions(-) delete mode 100644 scripts/lib/wic/canned-wks/grub_old.cfg diff --git a/scripts/lib/wic/canned-wks/grub.cfg b/scripts/lib/wic/canned-wks/grub.cfg index c75c2cf5d..f44b7184c 100644 --- a/scripts/lib/wic/canned-wks/grub.cfg +++ b/scripts/lib/wic/canned-wks/grub.cfg @@ -1,16 +1,28 @@ serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1 default='Default Provisioning' -timeout=1 +timeout=3 search --set=root --label NIRECOVERY-CD menuentry 'Default Provisioning' { - linux /bzImage rootwait restore=provision quiet loglevel=1 + linux /bzImage rootwait restore=provision-safe quiet loglevel=1 initrd /initrd } menuentry 'Verbose Provisioning' { - linux /bzImage rootwait restore=provision verbose_mode=1 + echo 'Loading Linux ...' + linux /bzImage rootwait restore=provision-safe verbose_mode=1 + echo 'Loading initial ramdisk ...' + initrd /initrd +} + +menuentry 'Get Image' { + linux /bzImage rootwait restore=get-image quiet loglevel=1 + initrd /initrd +} + +menuentry 'Set Image' { + linux /bzImage rootwait restore=set-image quiet loglevel=1 initrd /initrd } diff --git a/scripts/lib/wic/canned-wks/grub_old.cfg b/scripts/lib/wic/canned-wks/grub_old.cfg deleted file mode 100644 index f44b7184c..000000000 --- a/scripts/lib/wic/canned-wks/grub_old.cfg +++ /dev/null @@ -1,29 +0,0 @@ -serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1 -default='Default Provisioning' -timeout=3 - -search --set=root --label NIRECOVERY-CD - -menuentry 'Default Provisioning' { - linux /bzImage rootwait restore=provision-safe quiet loglevel=1 - initrd /initrd -} - -menuentry 'Verbose Provisioning' { - echo 'Loading Linux ...' - linux /bzImage rootwait restore=provision-safe verbose_mode=1 - echo 'Loading initial ramdisk ...' - initrd /initrd -} - -menuentry 'Get Image' { - linux /bzImage rootwait restore=get-image quiet loglevel=1 - initrd /initrd -} - -menuentry 'Set Image' { - linux /bzImage rootwait restore=set-image quiet loglevel=1 - initrd /initrd -} - -menu background splash.jpg diff --git a/scripts/lib/wic/canned-wks/nilrt-recovery-media.x64.wks b/scripts/lib/wic/canned-wks/nilrt-recovery-media.x64.wks index bc922e627..8453b7a3e 100644 --- a/scripts/lib/wic/canned-wks/nilrt-recovery-media.x64.wks +++ b/scripts/lib/wic/canned-wks/nilrt-recovery-media.x64.wks @@ -6,4 +6,4 @@ part /boot --source isoimage-isohybrid --sourceparams="esp_label=NIRECOVERY,esp_ # note that --append is ignored by grub when using a configfile # and only processed by syslinux as configured below -bootloader --configfile="grub_old.cfg" --append="restore=provision-safe console=ttyS0" +bootloader --configfile="grub.cfg" --append="restore=provision-safe console=ttyS0" From 88360d78f01f05cfbd9a4aca0c5b9352c43480c5 Mon Sep 17 00:00:00 2001 From: rajendra-desai-ni Date: Fri, 17 May 2024 19:05:39 +0530 Subject: [PATCH 103/147] cyclictest-container: install influxdb with apt Changed the installation of influxdb in the Dockerfile to install with apt install instead of pip install as the kernel-containerized-performance-tests seemed to be broken due to this. Signed-off-by: Rajendra Desai (cherry picked from commit 32ca39754673027b0eafa8b06410227ed0f75735) Signed-off-by: Chaitanya Vadrevu --- .../kernel-tests/files/cyclictest-container/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile b/recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile index ea2e71d14..6d67f8fcd 100644 --- a/recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile +++ b/recipes-kernel/kernel-tests/files/cyclictest-container/Dockerfile @@ -6,7 +6,5 @@ RUN DEBIAN_FRONTEND=noninteractive apt update && apt install -y \ python3 \ python3-pip \ dmidecode \ - grub-common - -# Need to send to InfluxDB -RUN pip install influxdb + grub-common \ + python3-influxdb From 496a07321952093354d310a05b2946d3b8264b16 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Thu, 16 May 2024 16:28:35 -0500 Subject: [PATCH 104/147] nisystemreplication: Improve error msg When "Get Image" menu is selected but the media is unwritable or has wrong label, show a more descriptive error. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 791dda0201a3285d773f64e1e0bc53a59373d419) Signed-off-by: Chaitanya Vadrevu --- recipes-ni/ni-systemreplication/files/nisystemreplication | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 5f4e545d2..3197cad8c 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -8,7 +8,10 @@ declare -r MODE="${1:-}" NIRECOVERY_MOUNTPOINT=/mnt/NIRECOVERY image_get () { - mount -o remount,rw $NIRECOVERY_MOUNTPOINT + if ! mount -o remount,rw $NIRECOVERY_MOUNTPOINT; then + echo "Make sure the media is labeled NIRECOVERY and is writable" && false + fi + mkdir -p $NIRECOVERY_MOUNTPOINT/replication_images nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/replication_images/systemimage.tgz mount -o remount,ro $NIRECOVERY_MOUNTPOINT From 44dd9b4f5d83507e2404d42dca0b47de67e6b978 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 22 May 2024 19:07:25 -0500 Subject: [PATCH 105/147] nisystemreplication: Custom image name for "Get Image" Add functionality to choose custom image name for "Get Image" Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 6463165718e5d4a1fe8fd483b4948f5bbb70fb2b) Signed-off-by: Chaitanya Vadrevu --- .../files/nisystemreplication | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 3197cad8c..f8620f2ae 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -7,13 +7,49 @@ declare -r MODE="${1:-}" NIRECOVERY_MOUNTPOINT=/mnt/NIRECOVERY +get_default_image_name () { + local DeviceDesc=$(fw_printenv -n DeviceDesc 2>/dev/null) + if [ -z "$DeviceDesc" ]; then + DeviceDesc="UnknownDevice" + fi + + local Date=$(date +%F-%H-%M-%S) + + echo $DeviceDesc-$Date +} + +get_image_name () { + local default_image_name=$(get_default_image_name) + read -e -p "Enter image name (no spaces, no special chars except '-'): " -i "$default_image_name" image_name + echo $image_name +} + +is_image_valid () { + local image_name=$1 + if [ -z "$image_name" ]; then + echo "No image name specified. Please specify an image name" && return 1 + fi + # If image already exists, confirm if it should be overwritten + if [ -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz ]; then + read -e -p "An image already exits with this name. Overwrite? (y/N): " -i "N" overwrite + [ $overwrite != "y" ] && return 1 + fi + return 0 +} + image_get () { if ! mount -o remount,rw $NIRECOVERY_MOUNTPOINT; then echo "Make sure the media is labeled NIRECOVERY and is writable" && false fi - mkdir -p $NIRECOVERY_MOUNTPOINT/replication_images - nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/replication_images/systemimage.tgz + # If no image name is provided, retry + local image_name=$(get_image_name) + while ! is_image_valid $image_name; do + image_name=$(get_image_name) + done + + mkdir -p $NIRECOVERY_MOUNTPOINT/Images/$image_name + nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz mount -o remount,ro $NIRECOVERY_MOUNTPOINT echo "Done getting image" } @@ -29,7 +65,7 @@ image_set () { source /ni_provisioning.safemode.common install_safemode - nisystemimage setall -x tgz -f $NIRECOVERY_MOUNTPOINT/replication_images/systemimage.tgz -p reset -s reset + nisystemimage setall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/systemimage.tgz -p reset -s reset # Retain some grubenv settings if grep --quiet "^consoleoutenable=" $BOOTFS_MOUNTPOINT/grub/grubenv; then From 56191de59e3bc6a83c70e68b0d990ac02da4af7b Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Thu, 23 May 2024 19:10:18 -0500 Subject: [PATCH 106/147] nisystemreplication: Custom image name for "Set Image" Add functionality to choose custom image name for "Set Image" Signed-off-by: Chaitanya Vadrevu (cherry picked from commit e061a1454beae3c5033c1f9e6a98128f83d860af) Signed-off-by: Chaitanya Vadrevu --- .../files/nisystemreplication | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index f8620f2ae..2c33c13b1 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -7,6 +7,8 @@ declare -r MODE="${1:-}" NIRECOVERY_MOUNTPOINT=/mnt/NIRECOVERY +# ---------------------- Get Image ---------------------- + get_default_image_name () { local DeviceDesc=$(fw_printenv -n DeviceDesc 2>/dev/null) if [ -z "$DeviceDesc" ]; then @@ -54,6 +56,31 @@ image_get () { echo "Done getting image" } +# ---------------------- Set Image ---------------------- + +# Presents a list of existing images and let's user select one +select_image_from_list () { + [ ! -d $NIRECOVERY_MOUNTPOINT/Images ] && return + + readarray -t images <<< "$(find $NIRECOVERY_MOUNTPOINT/Images -name systemimage.tgz | sort | sed "s,$NIRECOVERY_MOUNTPOINT/Images/,," | sed 's,/systemimage.tgz,,')" + [ -z $images ] && return + local no_of_images=${#images[@]} + + echo "System images found on media:" >&2 + + PS3="Select an image to use for provisioning: " + select opt in "${images[@]}"; do + case $opt in + *) + if [[ "$REPLY" =~ ^[0-9]+$ ]] && [ $REPLY -gt 0 ] && [ $REPLY -le $no_of_images ]; then + echo $opt + break + fi + ;; + esac + done +} + image_set () { if [[ $(type -t early_setup) != function ]]; then # If we're here, the script was invoked manually i.e., not from init. @@ -62,10 +89,13 @@ image_set () { early_setup fi + local image_name=$(select_image_from_list) + [ -z "$image_name" ] && die "No system images found on the media" + source /ni_provisioning.safemode.common install_safemode - nisystemimage setall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/systemimage.tgz -p reset -s reset + nisystemimage setall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz -p reset -s reset # Retain some grubenv settings if grep --quiet "^consoleoutenable=" $BOOTFS_MOUNTPOINT/grub/grubenv; then From 2ed060b7e5169a3c5fb9eaa784cd2076caff395b Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Mon, 27 May 2024 13:52:27 -0500 Subject: [PATCH 107/147] nisystemreplication: Ask user for confirmation Ask user for confirmation to provision during "Set Image" operation. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 6e52cf89e59f6a01d6a693de2f0e4a614d74ad22) Signed-off-by: Chaitanya Vadrevu --- .../files/ni_provisioning.common | 2 +- .../files/nisystemreplication | 42 +++++++++++-------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/recipes-core/initrdscripts/files/ni_provisioning.common b/recipes-core/initrdscripts/files/ni_provisioning.common index 26f6e3fe5..578655060 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning.common +++ b/recipes-core/initrdscripts/files/ni_provisioning.common @@ -53,7 +53,7 @@ ask_for_continue() variable=${2:-"PROVISION_REPARTITION_TARGET"} BUILD_IDENTIFIER=$(get_image_info BUILD_IDENTIFIER) - IMAGE_DISPLAY_NAME=$(get_image_info IMAGE_DISPLAY_NAME) + IMAGE_DISPLAY_NAME=${3:-$(get_image_info IMAGE_DISPLAY_NAME)} printf "\nNI Linux Real-Time $type $BUILD_IDENTIFIER. \n\n" printf "Continuing will partition the disk, format, and install $IMAGE_DISPLAY_NAME to the target.\n\n" diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 2c33c13b1..0d6500d6e 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -92,29 +92,35 @@ image_set () { local image_name=$(select_image_from_list) [ -z "$image_name" ] && die "No system images found on the media" - source /ni_provisioning.safemode.common - install_safemode + if [[ "$PROVISION_REPARTITION_TARGET" != "y" ]]; then + ask_for_continue "ReplicateImage" "PROVISION_REPARTITION_TARGET" "$image_name" + fi - nisystemimage setall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz -p reset -s reset + if [[ $PROVISION_REPARTITION_TARGET == "y" ]]; then + source /ni_provisioning.safemode.common + install_safemode - # Retain some grubenv settings - if grep --quiet "^consoleoutenable=" $BOOTFS_MOUNTPOINT/grub/grubenv; then - grubenv_consoleoutenable=$(grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv list |grep "^consoleoutenable=" | cut -f 2 -d '=') - fi - if grep --quiet "^bootdelay=" $BOOTFS_MOUNTPOINT/grub/grubenv; then - grubenv_bootdelay=$(grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv list |grep "^bootdelay=" | cut -f 2 -d '=') - fi - rm -rf mkdir $BOOTFS_MOUNTPOINT/grub + nisystemimage setall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz -p reset -s reset - install_grubenv - set_versions + # Retain some grubenv settings + if grep --quiet "^consoleoutenable=" $BOOTFS_MOUNTPOINT/grub/grubenv; then + grubenv_consoleoutenable=$(grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv list |grep "^consoleoutenable=" | cut -f 2 -d '=') + fi + if grep --quiet "^bootdelay=" $BOOTFS_MOUNTPOINT/grub/grubenv; then + grubenv_bootdelay=$(grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv list |grep "^bootdelay=" | cut -f 2 -d '=') + fi + rm -rf mkdir $BOOTFS_MOUNTPOINT/grub - echo $LOG_LEVEL > /proc/sys/kernel/printk - sanity_check + install_grubenv + set_versions - print_info "Re-enabling automount..." - enable_automount - print_done + echo $LOG_LEVEL > /proc/sys/kernel/printk + sanity_check + + print_info "Re-enabling automount..." + enable_automount + print_done + fi trap - ERR exec 1>&3 From 3f3f175f579d83112cdbdcffe438f17c7e56264d Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Mon, 27 May 2024 22:42:04 -0500 Subject: [PATCH 108/147] nisystemreplication: Check if image fits on media Estimate image size and ensure it fits on media for "Get Image". Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 0c9e049bace25a37ee1d8484ec52f1755405197f) Signed-off-by: Chaitanya Vadrevu --- .../files/nisystemreplication | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 0d6500d6e..342dd7c72 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -26,6 +26,40 @@ get_image_name () { echo $image_name } +get_filesystem_attribute () { + local label=$1 + local column=$2 + + local tempdir=$(mktemp -d) + + if ! mount -L $label $tempdir &>/dev/null; then + echo "Error mounting $label partition" >&2 && return + fi + + available_space=$(df $tempdir | tail -1 | awk -v column="$column" '{print $column}') + umount $tempdir + rmdir $tempdir + echo $available_space +} + +get_used_filesystem_size () { + get_filesystem_attribute $1 3 +} + +get_available_filesystem_size () { + get_filesystem_attribute $1 4 +} + +get_estimated_image_size () { + nigrub_size=$(get_used_filesystem_size "nigrub") + nibootfs_size=$(get_used_filesystem_size "nibootfs") + niconfig_size=$(get_used_filesystem_size "niconfig") + nirootfs_size=$(get_used_filesystem_size "nirootfs") + + total_size=$(($nigrub_size + $nibootfs_size + $niconfig_size + $nirootfs_size)) + echo $total_size +} + is_image_valid () { local image_name=$1 if [ -z "$image_name" ]; then @@ -41,9 +75,15 @@ is_image_valid () { image_get () { if ! mount -o remount,rw $NIRECOVERY_MOUNTPOINT; then - echo "Make sure the media is labeled NIRECOVERY and is writable" && false + die "ERROR: Mount failure. No writable media labelled NIRECOVERY." fi + local nirecovery_available_space=$(get_available_filesystem_size "NIRECOVERY") + local estimated_image_size=$(get_estimated_image_size) + + [ $estimated_image_size -eq "0" ] && die "Error estimating image size" + [ $nirecovery_available_space -lt $estimated_image_size ] && die "Image might not fit on media. Estimated image size: $estimated_image_size. Available space on media: $nirecovery_available_space" + # If no image name is provided, retry local image_name=$(get_image_name) while ! is_image_valid $image_name; do @@ -53,7 +93,7 @@ image_get () { mkdir -p $NIRECOVERY_MOUNTPOINT/Images/$image_name nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz mount -o remount,ro $NIRECOVERY_MOUNTPOINT - echo "Done getting image" + echo "Image $image_name saved." } # ---------------------- Set Image ---------------------- From 8eae7a7f6b093fc10835232cc799648d2ed62209 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Mon, 27 May 2024 22:53:06 -0500 Subject: [PATCH 109/147] nisystemreplication: Mount NIRECOVERY async NIRECOVERY is mounted with "sync" option initially but this causes "Get Image" to take very long. So remount NIRECOVERY as async. Also add an explicit sync call after storing image. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 4e5e28eeb85837e85dc11925d11a70fbe58b2c79) Signed-off-by: Chaitanya Vadrevu --- recipes-ni/ni-systemreplication/files/nisystemreplication | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 342dd7c72..dba085cf1 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -74,7 +74,7 @@ is_image_valid () { } image_get () { - if ! mount -o remount,rw $NIRECOVERY_MOUNTPOINT; then + if ! mount -o remount,rw,async $NIRECOVERY_MOUNTPOINT; then die "ERROR: Mount failure. No writable media labelled NIRECOVERY." fi @@ -92,6 +92,8 @@ image_get () { mkdir -p $NIRECOVERY_MOUNTPOINT/Images/$image_name nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz + + sync mount -o remount,ro $NIRECOVERY_MOUNTPOINT echo "Image $image_name saved." } From 8e418d634968b6ff181a095c057740b4da9fa35d Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Tue, 28 May 2024 15:33:41 -0500 Subject: [PATCH 110/147] nisystemreplication: Add compatibility checks Store DeviceDesc during "Get Image" and only list images with compatible DeviceDesc during "Set Image". Signed-off-by: Chaitanya Vadrevu (cherry picked from commit deac9fd6c5d66d25f9a912fcefdf69ccd2177e5e) Signed-off-by: Chaitanya Vadrevu --- .../files/nisystemreplication | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index dba085cf1..17f75a1dc 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -9,12 +9,17 @@ NIRECOVERY_MOUNTPOINT=/mnt/NIRECOVERY # ---------------------- Get Image ---------------------- -get_default_image_name () { +get_device_desc () { local DeviceDesc=$(fw_printenv -n DeviceDesc 2>/dev/null) if [ -z "$DeviceDesc" ]; then DeviceDesc="UnknownDevice" fi + echo $DeviceDesc +} + +get_default_image_name () { + local DeviceDesc=$(get_device_desc) local Date=$(date +%F-%H-%M-%S) echo $DeviceDesc-$Date @@ -92,6 +97,7 @@ image_get () { mkdir -p $NIRECOVERY_MOUNTPOINT/Images/$image_name nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz + echo "DeviceDesc=$(get_device_desc)" > $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.conf sync mount -o remount,ro $NIRECOVERY_MOUNTPOINT @@ -100,15 +106,29 @@ image_get () { # ---------------------- Set Image ---------------------- -# Presents a list of existing images and let's user select one +# Presents a list of existing compatible images and let's user select one select_image_from_list () { [ ! -d $NIRECOVERY_MOUNTPOINT/Images ] && return - readarray -t images <<< "$(find $NIRECOVERY_MOUNTPOINT/Images -name systemimage.tgz | sort | sed "s,$NIRECOVERY_MOUNTPOINT/Images/,," | sed 's,/systemimage.tgz,,')" + images=() + + for image in $(ls $NIRECOVERY_MOUNTPOINT/Images); do + if [ ! -f $NIRECOVERY_MOUNTPOINT/Images/$image/systemimage.tgz ] || [ ! -f $NIRECOVERY_MOUNTPOINT/Images/$image/systemimage.conf ]; then + echo "WARNING: Image \"$image\" is missing systemimage.tgz or systemimage.conf" >&2 + continue + fi + + local ImageDeviceDesc=$(awk -F= '{if ($1 == "DeviceDesc") print $2}' $NIRECOVERY_MOUNTPOINT/Images/$image/systemimage.conf) + if [ "$(get_device_desc)" = "$ImageDeviceDesc" ]; then + images+=($image) + fi + done + [ -z $images ] && return local no_of_images=${#images[@]} - echo "System images found on media:" >&2 + echo "" >&2 + echo "Compatible system images found on media:" >&2 PS3="Select an image to use for provisioning: " select opt in "${images[@]}"; do @@ -132,7 +152,7 @@ image_set () { fi local image_name=$(select_image_from_list) - [ -z "$image_name" ] && die "No system images found on the media" + [ -z "$image_name" ] && die "No compatible system images found on the media" if [[ "$PROVISION_REPARTITION_TARGET" != "y" ]]; then ask_for_continue "ReplicateImage" "PROVISION_REPARTITION_TARGET" "$image_name" From 024b8cce923154967a779b7470c25df20ff65d45 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Tue, 28 May 2024 20:54:48 -0500 Subject: [PATCH 111/147] nisystemreplication: Add support for answers file Add support for answers file to allow for non-interactive "Get Image" and "Set Image". Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 923cc654aa126336dbdb9f9943a447d72ec565e6) Signed-off-by: Chaitanya Vadrevu --- .../initrdscripts/files/ni_provisioning | 10 ++++++++++ .../files/ni_provisioning.answers.default | 4 ++++ .../files/nisystemreplication | 18 ++++++++++++------ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/recipes-core/initrdscripts/files/ni_provisioning b/recipes-core/initrdscripts/files/ni_provisioning index bb0142b3c..79a2835b8 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning +++ b/recipes-core/initrdscripts/files/ni_provisioning @@ -107,6 +107,11 @@ elif [[ $restore == "migrate" ]]; then provision_target onboard elif [[ $restore == "get-image" ]]; then echo "Get Image" + + # source user's answer file, if present + source_answer_file + echo + source /sbin/nisystemreplication get if [[ "${FORCE_PROVISIONING}" -eq 1 ]]; then PROVISION_REBOOT_METHOD=${RESTART_OPTION} @@ -114,6 +119,11 @@ elif [[ $restore == "get-image" ]]; then ASK_BEFORE_REBOOT=1 elif [[ $restore == "set-image" ]]; then echo "Set Image" + + # source user's answer file, if present + source_answer_file + echo + source /sbin/nisystemreplication set if [[ "${FORCE_PROVISIONING}" -eq 1 ]]; then PROVISION_REBOOT_METHOD=${RESTART_OPTION} diff --git a/recipes-core/initrdscripts/files/ni_provisioning.answers.default b/recipes-core/initrdscripts/files/ni_provisioning.answers.default index cf8af1be4..7aeeccd1d 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning.answers.default +++ b/recipes-core/initrdscripts/files/ni_provisioning.answers.default @@ -45,3 +45,7 @@ PROVISION_PART_NILRT_LABEL="nilrt" # A hostname to configure after provisioning. # An empty string means hostname is generated on first boot. PROVISION_HOSTNAME="" + +# System Image name used for "Get Image" and "Set Image" +# No spaces, no special characters except '-' +PROVISION_SYSTEMIMAGE_NAME="" diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 17f75a1dc..6220be377 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -89,11 +89,14 @@ image_get () { [ $estimated_image_size -eq "0" ] && die "Error estimating image size" [ $nirecovery_available_space -lt $estimated_image_size ] && die "Image might not fit on media. Estimated image size: $estimated_image_size. Available space on media: $nirecovery_available_space" - # If no image name is provided, retry - local image_name=$(get_image_name) - while ! is_image_valid $image_name; do + local image_name=$PROVISION_SYSTEMIMAGE_NAME + if [ -z $image_name ]; then image_name=$(get_image_name) - done + # If no image name is provided, retry + while ! is_image_valid $image_name; do + image_name=$(get_image_name) + done + fi mkdir -p $NIRECOVERY_MOUNTPOINT/Images/$image_name nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz @@ -151,8 +154,11 @@ image_set () { early_setup fi - local image_name=$(select_image_from_list) - [ -z "$image_name" ] && die "No compatible system images found on the media" + local image_name=$PROVISION_SYSTEMIMAGE_NAME + if [ -z $image_name ]; then + image_name=$(select_image_from_list) + [ -z "$image_name" ] && die "No compatible system images found on the media" + fi if [[ "$PROVISION_REPARTITION_TARGET" != "y" ]]; then ask_for_continue "ReplicateImage" "PROVISION_REPARTITION_TARGET" "$image_name" From 5a8f5ae7dcfcaa62246e49d197b26c5e343c3233 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Fri, 17 May 2024 18:48:29 -0500 Subject: [PATCH 112/147] nisystemimage: Show progress for get and set Use pv to show progress for get image and set image operations. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit f4749db2a76742082639691620b68a5f9149ebd8) Signed-off-by: Chaitanya Vadrevu --- recipes-ni/ni-systemimage/files/nisystemimage | 4 ++-- recipes-ni/ni-systemimage/ni-systemimage.bb | 1 + recipes-ni/ni-systemreplication/files/nisystemreplication | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/recipes-ni/ni-systemimage/files/nisystemimage b/recipes-ni/ni-systemimage/files/nisystemimage index f19d3b76d..5a3659af4 100644 --- a/recipes-ni/ni-systemimage/files/nisystemimage +++ b/recipes-ni/ni-systemimage/files/nisystemimage @@ -369,7 +369,7 @@ image_get_tgz () { cd $WORKING_USERFS CMDLINE+=(*) - run_cmd "${CMDLINE[@]}" 1>&"$FD_IMAGE" + run_cmd "${CMDLINE[@]}" | pv 1>&"$FD_IMAGE" echo_verbose 2 "finished" } @@ -551,7 +551,7 @@ image_set_tgz () { cd $WORKING_USERFS if [[ ! $NOEXEC ]]; then - "${CMDLINE[@]}" 0<&"$FD_IMAGE" + pv 0<&"$FD_IMAGE" | "${CMDLINE[@]}" else if (( $VERBOSE > 0 )); then echo_verbose 1 "skipping extraction; 'tar t' output:" diff --git a/recipes-ni/ni-systemimage/ni-systemimage.bb b/recipes-ni/ni-systemimage/ni-systemimage.bb index 0893a38eb..9a8eaba90 100644 --- a/recipes-ni/ni-systemimage/ni-systemimage.bb +++ b/recipes-ni/ni-systemimage/ni-systemimage.bb @@ -33,5 +33,6 @@ RDEPENDS:${PN} += "\ gzip \ ni-netcfgutil \ niacctbase \ + pv \ tar \ " diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 6220be377..07fb66bdd 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -99,6 +99,7 @@ image_get () { fi mkdir -p $NIRECOVERY_MOUNTPOINT/Images/$image_name + echo "Getting system image $image_name. This may take a while" >&2 nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz echo "DeviceDesc=$(get_device_desc)" > $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.conf @@ -168,6 +169,7 @@ image_set () { source /ni_provisioning.safemode.common install_safemode + echo "Applying system image $image_name. This may take a while" >&2 nisystemimage setall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz -p reset -s reset # Retain some grubenv settings From ec4f4254245ce05fda87628653cb6559c121a097 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Thu, 30 May 2024 08:42:34 -0500 Subject: [PATCH 113/147] nisystemimage: Disable progress bar by default Keep progress bar disabled by default for nisystemimage and add a cmdline argument so that nisystemreplication can enable it. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit cd4ecee5eb4b91a1c75bf244cf3a80602d20ce46) Signed-off-by: Chaitanya Vadrevu --- recipes-ni/ni-systemimage/files/nisystemimage | 15 ++++++++++----- .../files/nisystemreplication | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/recipes-ni/ni-systemimage/files/nisystemimage b/recipes-ni/ni-systemimage/files/nisystemimage index 5a3659af4..70acdcf7f 100644 --- a/recipes-ni/ni-systemimage/files/nisystemimage +++ b/recipes-ni/ni-systemimage/files/nisystemimage @@ -70,6 +70,10 @@ if [[ "${MODE}" == "get" || "${MODE}" == "set" ]]; then BLACKLIST_PATHS+=(${BLACKLIST_PATHS_get_set[*]}) fi +# Passed as argument to pv. The default "-q" shows no progress. +# Set this to empty string to display progress. +DISPLAY_PROGRESS_ARG="-q" + # All excludes to pass to `tar`. Includes BLACKLIST_PATHS and contents of all # files in BLACKLIST_LISTS. The contents of EXCLUDES are paths relative to # /mnt/userfs; the contents of BLACKLIST_PATHS and the contents of the files in @@ -145,9 +149,9 @@ print_help () { cat < [-hnv] [-b ] [-B ] [-f ] +$BASENAME get -x [-dhnv] [-b ] [-B ] [-f ] -$BASENAME set -x [-hnv] [-b ] [-B ] [-f ] +$BASENAME set -x [-dhnv] [-b ] [-B ] [-f ] -p reset|preserve|apply -s reset|preserve|apply EOF @@ -369,7 +373,7 @@ image_get_tgz () { cd $WORKING_USERFS CMDLINE+=(*) - run_cmd "${CMDLINE[@]}" | pv 1>&"$FD_IMAGE" + run_cmd "${CMDLINE[@]}" | pv $DISPLAY_PROGRESS_ARG 1>&"$FD_IMAGE" echo_verbose 2 "finished" } @@ -551,7 +555,7 @@ image_set_tgz () { cd $WORKING_USERFS if [[ ! $NOEXEC ]]; then - pv 0<&"$FD_IMAGE" | "${CMDLINE[@]}" + pv $DISPLAY_PROGRESS_ARG 0<&"$FD_IMAGE" | "${CMDLINE[@]}" else if (( $VERBOSE > 0 )); then echo_verbose 1 "skipping extraction; 'tar t' output:" @@ -755,10 +759,11 @@ dump_settings () { } parse_settings () { - while getopts :b:B:f:hnp:s:vx: opt ; do + while getopts :b:B:df:hnp:s:vx: opt ; do case $opt in b) BLACKLIST_LISTS+=("$OPTARG") ;; B) BLACKLIST_PATHS+=("$OPTARG") ;; + d) DISPLAY_PROGRESS_ARG="" ;; f) IMAGE_FILE="$OPTARG" ;; h) print_help; exit 0 ;; n) NOEXEC=1 ;; diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 07fb66bdd..74c27f036 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -100,7 +100,7 @@ image_get () { mkdir -p $NIRECOVERY_MOUNTPOINT/Images/$image_name echo "Getting system image $image_name. This may take a while" >&2 - nisystemimage getall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz + nisystemimage getall -d -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz echo "DeviceDesc=$(get_device_desc)" > $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.conf sync @@ -170,7 +170,7 @@ image_set () { install_safemode echo "Applying system image $image_name. This may take a while" >&2 - nisystemimage setall -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz -p reset -s reset + nisystemimage setall -d -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz -p reset -s reset # Retain some grubenv settings if grep --quiet "^consoleoutenable=" $BOOTFS_MOUNTPOINT/grub/grubenv; then From 95741aec0abd8c2a910046317219df1195560db6 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Thu, 27 Jun 2024 12:18:42 -0500 Subject: [PATCH 114/147] nisystemreplication: Replicate ext4 features When "Set Image" operation is done from a recovery media containing newer OS version, mkfs.ext4 may create ext4 partitions with newer features enabled which may not be compatible with the grub version from the System Image being applied. So save ext4 features used for creating nibootfs during "Get Image" operation and only enable them during "Set Image" operation. Signed-off-by: Chaitanya Vadrevu --- .../ni-systemreplication/files/nisystemreplication | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 74c27f036..13aab9e31 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -31,6 +31,11 @@ get_image_name () { echo $image_name } +get_ext4_features () { + local bootfs_device=$(blkid -L nibootfs) + echo $(debugfs -R features $bootfs_device 2>/dev/null | awk -F: '{if ($1 == "Filesystem features") print $2}' | xargs | sed 's/ /,/g') +} + get_filesystem_attribute () { local label=$1 local column=$2 @@ -100,8 +105,11 @@ image_get () { mkdir -p $NIRECOVERY_MOUNTPOINT/Images/$image_name echo "Getting system image $image_name. This may take a while" >&2 + local ext4_features=$(get_ext4_features) nisystemimage getall -d -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz + echo "DeviceDesc=$(get_device_desc)" > $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.conf + echo "Ext4Features=$ext4_features" >> $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.conf sync mount -o remount,ro $NIRECOVERY_MOUNTPOINT @@ -166,6 +174,9 @@ image_set () { fi if [[ $PROVISION_REPARTITION_TARGET == "y" ]]; then + ext4_features=$(awk -F= '{if ($1 == "Ext4Features") print $2}' $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.conf) + # Start with "none" so that default features are not enabled if not listed in $ext4_features + MKFS_ARGS="${MKFS_ARGS} -O none,$ext4_features" source /ni_provisioning.safemode.common install_safemode From 3e903c7284a6ecaa0868c8dbff5f0d8abb94cd30 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Thu, 27 Jun 2024 15:52:45 -0500 Subject: [PATCH 115/147] nisystemreplication: Detect unsupported fs features during "Get Image" When older recovery media is used to "Get Image" on a system provisioned with newer recovery media, some file system features may not be supported by older recovery media in which case error out. Signed-off-by: Chaitanya Vadrevu --- .../ni-systemreplication/files/nisystemreplication | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 13aab9e31..2ff92686a 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -36,6 +36,10 @@ get_ext4_features () { echo $(debugfs -R features $bootfs_device 2>/dev/null | awk -F: '{if ($1 == "Filesystem features") print $2}' | xargs | sed 's/ /,/g') } +ext4_has_unsupported_features () { + e2fsck -n LABEL=nibootfs 2>&1 | grep "unsupported feature" +} + get_filesystem_attribute () { local label=$1 local column=$2 @@ -85,7 +89,11 @@ is_image_valid () { image_get () { if ! mount -o remount,rw,async $NIRECOVERY_MOUNTPOINT; then - die "ERROR: Mount failure. No writable media labelled NIRECOVERY." + die "Mount failure. No writable media labelled NIRECOVERY." + fi + + if ext4_has_unsupported_features; then + die "Unsupported filesystem features detected. Please use a newer version of recovery media." fi local nirecovery_available_space=$(get_available_filesystem_size "NIRECOVERY") From 2ed568394922f28075862da2b91687fe11dff02d Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Mon, 5 Aug 2024 14:56:58 -0500 Subject: [PATCH 116/147] rtgroups: set priority for ktimers/* Set the per-CPU 'ktimers' threads priority to FIFO 10. This places them above 'ksoftirqd' threads which run at FIFO 8 and matches the intent of the upstream stable-rt kernel commit c1d68f0da449 ("softirq: Use a dedicated thread for timer wake-ups.") which states: "Using a dedicated variable to store the pending softirq bits values ensure that the timer are not accidentally picked up by ksoftirqd and other threaded interrupts. It shouldn't be picked up by ksoftirqd since it runs at lower priority." On our systems, up until this commit, 'ksoftirqd' was running at FIFO/8 whereas 'ktimers' were running at the default FIFO/1 which is not the intent of the upstream change. Also, add the corresponding test that goes with it. Signed-off-by: Gratian Crisan (cherry picked from commit 50f28dfab4046af57e31b22f48e887e184f0fe02) Signed-off-by: Chaitanya Vadrevu --- recipes-rt/rt-tests/files/kthread_test_priority.sh | 4 ++++ recipes-rt/rtctl/files/rtgroups | 1 + 2 files changed, 5 insertions(+) diff --git a/recipes-rt/rt-tests/files/kthread_test_priority.sh b/recipes-rt/rt-tests/files/kthread_test_priority.sh index 60f04477e..e62e04a68 100755 --- a/recipes-rt/rt-tests/files/kthread_test_priority.sh +++ b/recipes-rt/rt-tests/files/kthread_test_priority.sh @@ -50,4 +50,8 @@ ptest_change_subtest 4 irq_work check_prio_for_task irq_work/ FIFO 12 ptest_report +ptest_change_subtest 5 ktimers +check_prio_for_task ktimers/ FIFO 10 +ptest_report + exit $rc_first diff --git a/recipes-rt/rtctl/files/rtgroups b/recipes-rt/rtctl/files/rtgroups index 87da030da..4731e83c6 100644 --- a/recipes-rt/rtctl/files/rtgroups +++ b/recipes-rt/rtctl/files/rtgroups @@ -41,5 +41,6 @@ kthreadd:f:25:*:\[kthreadd\]$ irqthread:f:15:*:\[irq\/.+\]$ irq_work:f:12:*:\[irq_work\/.+\]$ +ktimers:f:10:*:\[ktimers\/.+\]$ ksoftirqd:f:8:*:\[ksoftirqd\/.+\]$ From 86dc2a1376ea0ee8e164172bec479e7fd4366be6 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Mon, 5 Aug 2024 18:49:03 -0500 Subject: [PATCH 117/147] nilrt.conf: Bump NILRT_FEED_NAME Signed-off-by: Chaitanya Vadrevu --- conf/distro/nilrt.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/distro/nilrt.conf b/conf/distro/nilrt.conf index dec02d255..abe8438af 100644 --- a/conf/distro/nilrt.conf +++ b/conf/distro/nilrt.conf @@ -4,7 +4,7 @@ DISTRO_VERSION = "11.0" DISTRO_CODENAME = "${LAYERSERIES_COMPAT_meta-nilrt}" -NILRT_FEED_NAME ?= "2024Q3" +NILRT_FEED_NAME ?= "2024Q4" DISTRO_FEATURES:append:x64 = "\ x11 \ From 1c98ae39686847fa84fd7e3b4e966301c981bd45 Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Mon, 5 Aug 2024 15:14:44 -0500 Subject: [PATCH 118/147] kernel-test-nohz: use 'tsc=reliable' Use the 'tsc=reliable' kernel parameter instead of 'tsc=nowatchdog' when running the nohz test. This option is a super-set of the 'nowatchdog' option - in addition to disabling the TSC watchdog it also disables a repeating 10 minute timer which gets scheduled on every CPU (including isolated ones) to check for TSC synchronization. This timer results in additional latency spikes on the isolated CPUs (which we are trying to avoid during this test). Signed-off-by: Gratian Crisan (cherry picked from commit f67f78a119d29d3273ffafd355db3d0f2e256bad) Signed-off-by: Chaitanya Vadrevu --- recipes-kernel/kernel-tests/kernel-test-nohz.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/kernel-tests/kernel-test-nohz.bb b/recipes-kernel/kernel-tests/kernel-test-nohz.bb index 350ffc5ee..b592c6345 100644 --- a/recipes-kernel/kernel-tests/kernel-test-nohz.bb +++ b/recipes-kernel/kernel-tests/kernel-test-nohz.bb @@ -40,7 +40,7 @@ pkg_postinst_ontarget:${PN}-ptest:append() { ISOLATED_CPU=$((CPUS - 1)) if [ $ISOLATED_CPU -gt 0 ]; then - echo 'set otherbootargs="${otherbootargs} isolcpus=nohz,domain,managed_irq,'$ISOLATED_CPU' nohz_full='$ISOLATED_CPU' mitigations=off intel_pstate=disable intel_idle.max_cstate=0 processor.max_cstate=0 nosoftlockup mce=ignore_ce audit=0 tsc=nowatchdog"' > /boot/runmode/no-hz-full-params.cfg + echo 'set otherbootargs="${otherbootargs} isolcpus=nohz,domain,managed_irq,'$ISOLATED_CPU' nohz_full='$ISOLATED_CPU' mitigations=off intel_pstate=disable intel_idle.max_cstate=0 processor.max_cstate=0 nosoftlockup mce=ignore_ce audit=0 tsc=reliable"' > /boot/runmode/no-hz-full-params.cfg grep -qsxF 'source /runmode/no-hz-full-params.cfg' /boot/runmode/bootimage.cfg || echo 'source /runmode/no-hz-full-params.cfg' >> /boot/runmode/bootimage.cfg else echo "[kernel-test-nohz:error] This test requires a system with 2 or more CPUs" From adf02cd954f152566e4e39717d17fc70dfee1f76 Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Mon, 5 Aug 2024 15:33:47 -0500 Subject: [PATCH 119/147] kernel-test-nohz: run test at FIFO/1 Run nohz ptest at priority FIFO/1 instead of FIFO/98. Somewhat counter intuitively, this results in better performance because it allows the other kernel threads which are pinned to the isolated CPUs to run and get out of the way of our test. Otherwise those threads, being lower priority than our test, never get a chance to run but remain runnable. This in turn prevents the scheduler tick from being disabled and negates the whole reason for running a nohz kernel. Signed-off-by: Gratian Crisan (cherry picked from commit 4dfdb7a9f37b6eb4c5c88eed958a0dc542f795d7) Signed-off-by: Chaitanya Vadrevu --- recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c b/recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c index 61b18269e..f73cc1ad0 100644 --- a/recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c +++ b/recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c @@ -20,7 +20,7 @@ #include #include -#define TEST_PRIO 98 +#define TEST_PRIO 1 #define NSEC_PER_USEC 1000ULL #define NSEC_PER_SEC 1000000000ULL From fb8be7dda1a7fb3690025643acf08a3489b17c3d Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Wed, 7 Aug 2024 17:46:19 -0500 Subject: [PATCH 120/147] kernel-test-nohz: fix max latency check in validate_results When running the nohz test we exit the test loop early if the measured time delta between two consecutive clock_gettime() calls is equal or greater to the set maximum latency threshold. However when validating the test results in order to decide if the test has failed a 'greater'-only comparison was used missing the cases where the recorded max latency is exactly equal to the max latency failure threshold. This can lead to a false 'pass' result being reported. Fix the comparison to account for this. Signed-off-by: Gratian Crisan (cherry picked from commit 4cedbb777c566d8c04b4f0c700b2a8812ae79509) Signed-off-by: Chaitanya Vadrevu --- recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c b/recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c index f73cc1ad0..c89c4b259 100644 --- a/recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c +++ b/recipes-kernel/kernel-tests/kernel-test-nohz-files/nohz_test.c @@ -420,7 +420,7 @@ static void validate_results(struct histogram_data *h) p_99_999 = get_percentile(99.999, h); p_99_9999 = get_percentile(99.9999, h); - if (h->max > max_latency) + if (h->max >= max_latency) error_exit("Maximum latency exceeded", stats_summary, h->cnt, h->max, p_99_999, p_99_9999); if (p_99_999 > percentile_99_999) error_exit("99.999%% threshold exceeded", stats_summary, h->cnt, h->max, p_99_999, p_99_9999); From 8b84d2d3db1bcde08cb45265fa811ab3dd223615 Mon Sep 17 00:00:00 2001 From: erickshepherdNI Date: Mon, 12 Aug 2024 11:58:19 -0500 Subject: [PATCH 121/147] kernel_dmesg_diff.py: Update the software IO TLB filter rule. The IO TLB filter message changed format and is no longer caught by the rule. This change updates the rule to catch the new format. Signed-off-by: Erick Shepherd (cherry picked from commit e49ea772f83f0023c8e0ce7f123d2f8ce37d5b4a) Signed-off-by: Chaitanya Vadrevu --- .../kernel-tests/kernel-tests-files/kernel_dmesg_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py index c77e1b1d6..c9ce412a5 100644 --- a/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py +++ b/recipes-kernel/kernel-tests/kernel-tests-files/kernel_dmesg_diff.py @@ -214,7 +214,7 @@ def strip_timestamps(log): [r'NODE_DATA\(0\) allocated \[mem .*\]', 'NODE_DATA(0) allocated [mem ]'], [r'ACPI: SSDT 0x.*', 'ACPI: SSDT 0x'], [r'tsc: Refined TSC clocksource calibration: \d+.\d+ MHz', 'tsc: Refined TSC clocksource calibration: MHz'], - [r'software IO TLB: mapped mem .*', 'software IO TLB: mapped mem'], + [r'software IO TLB: mapped .*', 'software IO TLB: mapped mem'], [r'eth\d:', 'ethX:'], [r'renamed from eth\d', 'renamed from ethX'], [r'mounted filesystem [0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}', 'mounted filesystem xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'] From c683180cd7314c6150365e8d66f7ceee2c55da6c Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 13 Aug 2024 15:48:59 -0400 Subject: [PATCH 122/147] packagegroups: add pkggrp-ni-snac Add a packagegroup to track package dependencies of the NILRT Secured, Network-Attached Controller (SNAC) configuration. Since the SNAC configuration is officially supported, add it to the core package feed. This packagegroup SHOULD NOT be installed to the NILRT runmode or safemode images. Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagefeed-ni-core.bb | 7 ++++--- recipes-core/packagegroups/packagegroup-ni-snac.bb | 11 +++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 recipes-core/packagegroups/packagegroup-ni-snac.bb diff --git a/recipes-core/packagegroups/packagefeed-ni-core.bb b/recipes-core/packagegroups/packagefeed-ni-core.bb index 499bd7194..4c669ebc2 100644 --- a/recipes-core/packagegroups/packagefeed-ni-core.bb +++ b/recipes-core/packagegroups/packagefeed-ni-core.bb @@ -6,6 +6,9 @@ inherit packagegroup RDEPENDS:${PN} = "\ packagegroup-base \ packagegroup-core-boot \ + packagegroup-core-standalone-sdk-target \ + packagegroup-core-x11 \ + packagegroup-kernel-module-build \ packagegroup-ni-base \ packagegroup-ni-contributors \ packagegroup-ni-crio \ @@ -17,11 +20,9 @@ RDEPENDS:${PN} = "\ packagegroup-ni-runmode \ packagegroup-ni-safemode \ packagegroup-ni-skyline \ + packagegroup-ni-snac \ packagegroup-ni-tzdata \ packagegroup-ni-wifi \ - packagegroup-core-x11 \ - packagegroup-core-standalone-sdk-target \ - packagegroup-kernel-module-build \ dkms \ bolt \ " diff --git a/recipes-core/packagegroups/packagegroup-ni-snac.bb b/recipes-core/packagegroups/packagegroup-ni-snac.bb new file mode 100644 index 000000000..bfc9c86a7 --- /dev/null +++ b/recipes-core/packagegroups/packagegroup-ni-snac.bb @@ -0,0 +1,11 @@ +SUMMARY = "Open source package dependencies for the NILRT SNAC configuration." +LICENSE = "MIT" + + +inherit packagegroup + + +RDEPENDS:${PN} = "" + +RDEPENDS:${PN}:append = "\ +" From 2b025f3ac13c78c9de97a99863b21943026efdcc Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Tue, 13 Aug 2024 15:53:04 -0400 Subject: [PATCH 123/147] pkggrp-ni-snac: add initial top level deps Add some initial packages that we know are going to be used in the SNAC v1.0 design. Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagegroup-ni-snac.bb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/recipes-core/packagegroups/packagegroup-ni-snac.bb b/recipes-core/packagegroups/packagegroup-ni-snac.bb index bfc9c86a7..e791f70cd 100644 --- a/recipes-core/packagegroups/packagegroup-ni-snac.bb +++ b/recipes-core/packagegroups/packagegroup-ni-snac.bb @@ -8,4 +8,7 @@ inherit packagegroup RDEPENDS:${PN} = "" RDEPENDS:${PN}:append = "\ + cryptsetup \ + ntp \ + tmux \ " From fe72374f54d429305a388c25d98f3a1c5af0586a Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Wed, 14 Aug 2024 10:05:04 -0400 Subject: [PATCH 124/147] pkggrp-ni-snac: remove unneeded append assignment The snac packagegroup is only expected to contain one logical subsection of packages, so there is no need to split up the RDEPENDS assignments. Use a single section. Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagegroup-ni-snac.bb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/recipes-core/packagegroups/packagegroup-ni-snac.bb b/recipes-core/packagegroups/packagegroup-ni-snac.bb index e791f70cd..aa2a0fb66 100644 --- a/recipes-core/packagegroups/packagegroup-ni-snac.bb +++ b/recipes-core/packagegroups/packagegroup-ni-snac.bb @@ -5,9 +5,7 @@ LICENSE = "MIT" inherit packagegroup -RDEPENDS:${PN} = "" - -RDEPENDS:${PN}:append = "\ +RDEPENDS:${PN} = "\ cryptsetup \ ntp \ tmux \ From 6da2d19c6a90648db377297eb0aaa69477fb8e53 Mon Sep 17 00:00:00 2001 From: Can Wong Date: Thu, 15 Aug 2024 12:37:41 -0500 Subject: [PATCH 125/147] pkggrp-ni-internal-deps: Add PAtools dependencies Signed-off-by: Can Wong --- .../packagegroups/packagegroup-ni-internal-deps.bb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb index 9b0c49695..72f04240f 100644 --- a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb +++ b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb @@ -60,7 +60,7 @@ RDEPENDS:${PN} += "\ libxml-parser-perl \ " -# Required for a mobilize step that installs a specific Python version +# Required for a mobilize step that installs a specific Python version # and requires building Python on the test system # Contact: ulf.glaeser@ni.com # Team: DAQ.SW.Ops@ni.com @@ -109,3 +109,13 @@ RDEPENDS:${PN} += "\ RDEPENDS:${PN} += "\ libglu \ " +# Required by PAtools Runtimesystem +# Team: Transportation BU - Modern Battery Lab +# Contact: Deborah Bryant +RDEPENDS:${PN} += "\ + libxslt \ + samba \ + ntp \ + ntp-tickadj \ + ntp-utils \ +" From 689d16e7d415ffa222054a1d79af47871a3eaed6 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Mon, 19 Aug 2024 18:06:42 -0400 Subject: [PATCH 126/147] .github: Add pull request template Add a Github PR template to help contributors write high-quality PRs correctly the first time. Signed-off-by: Alex Stewart --- .github/PULL_REQUEST_TEMPLATE.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..4d1b7b433 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,20 @@ +### Summary of Changes + +TODO: Include a high-level description of the changes in this pull request. + + +### Justification + +TODO: Justify why this contribution should be part of the project. Link to an AZDO work item with `AB#${AZDO ID}`. + + +### Testing + +TODO: Detail what testing has been done to ensure this submission meets requirements. + +* [ ] I have built the core package feed with this PR in place. (`bitbake packagefeed-ni-core`) + + +### Procedure + +* [ ] I certify that the contents of this pull request complies with the [Developer Certificate of Origin](https://github.com/ni/nilrt/blob/HEAD/docs/CONTRIBUTING.md#developer-certificate-of-origin-dco). From 8dc36c69fb1337a60f4a913a44da79e626f85724 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Thu, 15 Aug 2024 15:59:21 -0400 Subject: [PATCH 127/147] pkggrp-ni-internal-deps: change nissl contact Haris Okanovic no longer works at NI. Change NISSL dependency ownership to Rich Tollerton. Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagegroup-ni-internal-deps.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb index 72f04240f..a39c20044 100644 --- a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb +++ b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb @@ -16,7 +16,7 @@ RDEPENDS:${PN} += "\ " # nissl and nissleay -# Contact: Haris Okanovic +# Contact: Rich Tollerton RDEPENDS:${PN} += "\ apache-websocket \ apache2 \ From 2fb42226c839e4835ce1788ea498cf4f5be450bb Mon Sep 17 00:00:00 2001 From: Mark Silva Date: Tue, 20 Aug 2024 10:27:53 -0400 Subject: [PATCH 128/147] libpwquality: Build recipe Signed-off-by: Mark Silva --- recipes-core/packagegroups/packagegroup-ni-snac.bb | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes-core/packagegroups/packagegroup-ni-snac.bb b/recipes-core/packagegroups/packagegroup-ni-snac.bb index aa2a0fb66..f0d862b40 100644 --- a/recipes-core/packagegroups/packagegroup-ni-snac.bb +++ b/recipes-core/packagegroups/packagegroup-ni-snac.bb @@ -9,4 +9,5 @@ RDEPENDS:${PN} = "\ cryptsetup \ ntp \ tmux \ + libpwquality \ " From 94ffc6e845609209f14b28f279c7344867aa094b Mon Sep 17 00:00:00 2001 From: Mark Silva Date: Tue, 20 Aug 2024 13:03:17 -0400 Subject: [PATCH 129/147] libpwquality: Add config file to recipe * Add pwquality.conf that has the values that are required for SNAC mode * These values are more restrictive than the default values * Add bbappend file to install the config file to the correct location Signed-off-by: Mark Silva --- .../libpwquality/files/pwquality.conf | 78 +++++++++++++++++++ .../libpwquality/libpwquality_1.%.bbappend | 13 ++++ 2 files changed, 91 insertions(+) create mode 100644 recipes-extended/libpwquality/files/pwquality.conf create mode 100644 recipes-extended/libpwquality/libpwquality_1.%.bbappend diff --git a/recipes-extended/libpwquality/files/pwquality.conf b/recipes-extended/libpwquality/files/pwquality.conf new file mode 100644 index 000000000..9a652d5cd --- /dev/null +++ b/recipes-extended/libpwquality/files/pwquality.conf @@ -0,0 +1,78 @@ +# Configuration for systemwide password quality limits +# +# Number of characters in the new password that must not be present in the +# old password. +difok = 8 +# +# Minimum acceptable size for the new password (plus one if +# credits are not disabled which is the default). (See pam_cracklib manual.) +# Cannot be set to lower value than 6. +minlen = 15 +# +# The maximum credit for having digits in the new password. If less than 0 +# it is the minimum number of digits in the new password. +dcredit = -1 +# +# The maximum credit for having uppercase characters in the new password. +# If less than 0 it is the minimum number of uppercase characters in the new +# password. +ucredit = -1 +# +# The maximum credit for having lowercase characters in the new password. +# If less than 0 it is the minimum number of lowercase characters in the new +# password. +lcredit = -1 +# +# The maximum credit for having other characters in the new password. +# If less than 0 it is the minimum number of other characters in the new +# password. +ocredit = -1 +# +# The minimum number of required classes of characters for the new +# password (digits, uppercase, lowercase, others). +minclass = 4 +# +# The maximum number of allowed consecutive same characters in the new password. +# The check is disabled if the value is 0. +maxrepeat = 3 +# +# The maximum number of allowed consecutive characters of the same class in the +# new password. +# The check is disabled if the value is 0. +maxclassrepeat = 4 +# +# Whether to check for the words from the passwd entry GECOS string of the user. +# The check is enabled if the value is not 0. +# gecoscheck = 0 +# +# Whether to check for the words from the cracklib dictionary. +# The check is enabled if the value is not 0. +dictcheck = 1 +# +# Whether to check if it contains the user name in some form. +# The check is enabled if the value is not 0. +# usercheck = 1 +# +# Length of substrings from the username to check for in the password +# The check is enabled if the value is greater than 0 and usercheck is enabled. +# usersubstr = 0 +# +# Whether the check is enforced by the PAM module and possibly other +# applications. +# The new password is rejected if it fails the check and the value is not 0. +# enforcing = 1 +# +# Path to the cracklib dictionaries. Default is to use the cracklib default. +# dictpath = +# +# Prompt user at most N times before returning with error. The default is 1. +retry = 3 +# +# Enforces pwquality checks on the root user password. +# Enabled if the option is present. +# enforce_for_root +# +# Skip testing the password quality for users that are not present in the +# /etc/passwd file. +# Enabled if the option is present. +# local_users_only \ No newline at end of file diff --git a/recipes-extended/libpwquality/libpwquality_1.%.bbappend b/recipes-extended/libpwquality/libpwquality_1.%.bbappend new file mode 100644 index 000000000..c7ecb86c9 --- /dev/null +++ b/recipes-extended/libpwquality/libpwquality_1.%.bbappend @@ -0,0 +1,13 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +SRC_URI:append = "\ + file://pwquality.conf \ +" + +do_install:append() { + install -d ${D}${sysconfdir}/security + install -m 644 ${WORKDIR}/pwquality.conf ${D}${sysconfdir}/security/pwquality.conf +} + +FILES:${PN} += "${sysconfdir}/security/pwquality.conf" +CONFFILES:${PN} += "${sysconfdir}/security/pwquality.conf" From a3fbdd40adbaccea4975fd032e0b82225d4111aa Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Tue, 20 Aug 2024 21:24:56 -0500 Subject: [PATCH 130/147] nisystemreplication: Split image files to fit FAT32 FAT32 max file size is limited to 2^32 - 1. So split the image files at this boundary. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 47e9f1f481d396363214458ea48004c5cbcfbc12) Signed-off-by: Chaitanya Vadrevu --- .../ni-systemreplication/files/nisystemreplication | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index 2ff92686a..a6fc41ba9 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -80,7 +80,7 @@ is_image_valid () { echo "No image name specified. Please specify an image name" && return 1 fi # If image already exists, confirm if it should be overwritten - if [ -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz ]; then + if [ -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.conf ]; then read -e -p "An image already exits with this name. Overwrite? (y/N): " -i "N" overwrite [ $overwrite != "y" ] && return 1 fi @@ -111,10 +111,13 @@ image_get () { done fi + # Cleanup any existing image files with same name first + rm -rf $NIRECOVERY_MOUNTPOINT/Images/$image_name mkdir -p $NIRECOVERY_MOUNTPOINT/Images/$image_name echo "Getting system image $image_name. This may take a while" >&2 local ext4_features=$(get_ext4_features) - nisystemimage getall -d -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz + # Split files at FAT32's max file size i.e., 2^32 - 1 + nisystemimage getall -d -x tgz | split --bytes=$((2**32 - 1)) -d -a4 - $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz echo "DeviceDesc=$(get_device_desc)" > $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.conf echo "Ext4Features=$ext4_features" >> $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.conf @@ -133,8 +136,8 @@ select_image_from_list () { images=() for image in $(ls $NIRECOVERY_MOUNTPOINT/Images); do - if [ ! -f $NIRECOVERY_MOUNTPOINT/Images/$image/systemimage.tgz ] || [ ! -f $NIRECOVERY_MOUNTPOINT/Images/$image/systemimage.conf ]; then - echo "WARNING: Image \"$image\" is missing systemimage.tgz or systemimage.conf" >&2 + if [ ! -f $NIRECOVERY_MOUNTPOINT/Images/$image/systemimage.tgz0000 ] || [ ! -f $NIRECOVERY_MOUNTPOINT/Images/$image/systemimage.conf ]; then + echo "WARNING: Image \"$image\" is missing systemimage.tgz* or systemimage.conf" >&2 continue fi @@ -189,7 +192,7 @@ image_set () { install_safemode echo "Applying system image $image_name. This may take a while" >&2 - nisystemimage setall -d -x tgz -f $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz -p reset -s reset + cat $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz* | nisystemimage setall -d -x tgz -p reset -s reset # Retain some grubenv settings if grep --quiet "^consoleoutenable=" $BOOTFS_MOUNTPOINT/grub/grubenv; then From b09f844d9fc00ffd9be6bf5efed4714e646c6073 Mon Sep 17 00:00:00 2001 From: Mark Silva Date: Mon, 19 Aug 2024 18:07:20 -0400 Subject: [PATCH 131/147] tmux-ipk: Add tmux configuration file to tmux IPK * Create a directory for tmux configuration files in /usr/share/tmux/conf.d * Add a /etc/tmux.conf file with default lock command * Add a line to source all files in /usr/share/tmux/conf.d in /etc/tmux.conf * -q is to ignore errors if the directory is empty * This allows snac mode to add a conf file to this location without modifying the main tmux.conf file Signed-off-by: Mark Silva tmux-bbappend: Add conf.d directory --- recipes-extended/tmux/files/tmux.conf | 3 +++ recipes-extended/tmux/tmux_3.%.bbappend | 15 +++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 recipes-extended/tmux/files/tmux.conf create mode 100644 recipes-extended/tmux/tmux_3.%.bbappend diff --git a/recipes-extended/tmux/files/tmux.conf b/recipes-extended/tmux/files/tmux.conf new file mode 100644 index 000000000..76fa63182 --- /dev/null +++ b/recipes-extended/tmux/files/tmux.conf @@ -0,0 +1,3 @@ +set -g lock-command vlock +bind X lock-session +source-file -q /usr/share/tmux/conf.d/* diff --git a/recipes-extended/tmux/tmux_3.%.bbappend b/recipes-extended/tmux/tmux_3.%.bbappend new file mode 100644 index 000000000..bb808ea1a --- /dev/null +++ b/recipes-extended/tmux/tmux_3.%.bbappend @@ -0,0 +1,15 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +SRC_URI += "\ + file://tmux.conf \ +" + +do_install:append() { + install -d ${D}${sysconfdir} + install -m 644 ${WORKDIR}/tmux.conf ${D}${sysconfdir}/tmux.conf + install -d ${D}/usr/share/tmux/conf.d +} + +FILES:${PN} += "${sysconfdir}/tmux.conf" +CONFFILES:${PN} += "${sysconfdir}/tmux.conf" +RDEPENDS:${PN}:append = " vlock" From b877114b20e587beb4bef59557b2076ad55820fa Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Wed, 21 Aug 2024 16:10:16 -0500 Subject: [PATCH 132/147] rtgroups: adjust ksoftirqd and rcu* priorities The default upstream kernel configuration uses SCHED_OTHER scheduling policy for per-CPU 'ksoftirqd/x' kernel threads. Early in NI Linux RT we have bumped their priority to SCHED_FIFO/8 with the assumption this will help deferred I/O work in drivers. In the intervening time several RCU kernel threads were added upstream: rcub, rcuc, rcuog, and rcu_preempt. These by default run as SCHED_FIFO/1 i.e. they were intended to have higher priority than softirq threads. In our configuration this priority is inverted. Also, new 'irq_work/x' threads were added by linux-stable-rt kernel commit b4c6f86ec2f6 ("irq_work: Handle some irq_work in a per-CPU thread on PREEMPT_RT"). These use 'rcuwait' for scheduling and run at SCHED_FIFO/12 on NILRT. When combined with the priority inversion described above this can lead to a deadlock in certain rare cases. Set the 'ksoftirqd/x' priority to the lowest RT priority SCHED_FIFO/1. This preserves the previous behavior where they are scheduled before other normal or background tasks but is closer to the upstream default. Set the priority of RCU threads which require a RT scheduling policy to SCHED_FIFO/2 to align with upstream assumptions where the priority of: irq_work > rcu* > ksoftirqd Additionally, modify the rt-tests ptest to account for the new priorities. AB#2795435 Signed-off-by: Gratian Crisan (cherry picked from commit 5da2ed4f72aa13e6376b093c2291273d9fc26f2d) Signed-off-by: Chaitanya Vadrevu --- .../rt-tests/files/kthread_test_priority.sh | 18 +++++++++++++++++- recipes-rt/rtctl/files/rtgroups | 5 +++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/recipes-rt/rt-tests/files/kthread_test_priority.sh b/recipes-rt/rt-tests/files/kthread_test_priority.sh index e62e04a68..577adfd34 100755 --- a/recipes-rt/rt-tests/files/kthread_test_priority.sh +++ b/recipes-rt/rt-tests/files/kthread_test_priority.sh @@ -39,7 +39,7 @@ ptest_report rc_first=$ptest_rc ptest_change_subtest 2 ksoftirqd -check_prio_for_task ksoftirqd/0 FIFO 8 +check_prio_for_task ksoftirqd/0 FIFO 1 ptest_report ptest_change_subtest 3 irq @@ -54,4 +54,20 @@ ptest_change_subtest 5 ktimers check_prio_for_task ktimers/ FIFO 10 ptest_report +ptest_change_subtest 6 rcub +check_prio_for_task rcub/ FIFO 2 +ptest_report + +ptest_change_subtest 7 rcuc +check_prio_for_task rcuc/ FIFO 2 +ptest_report + +ptest_change_subtest 8 rcuog +check_prio_for_task rcuog/ FIFO 2 +ptest_report + +ptest_change_subtest 9 rcu_preempt +check_prio_for_task rcu_preempt FIFO 2 +ptest_report + exit $rc_first diff --git a/recipes-rt/rtctl/files/rtgroups b/recipes-rt/rtctl/files/rtgroups index 4731e83c6..71bb7b8d2 100644 --- a/recipes-rt/rtctl/files/rtgroups +++ b/recipes-rt/rtctl/files/rtgroups @@ -42,5 +42,6 @@ kthreadd:f:25:*:\[kthreadd\]$ irqthread:f:15:*:\[irq\/.+\]$ irq_work:f:12:*:\[irq_work\/.+\]$ ktimers:f:10:*:\[ktimers\/.+\]$ -ksoftirqd:f:8:*:\[ksoftirqd\/.+\]$ - +rcu:f:2:*:\[rcu(b|c|og)\/[0-9]+\]$ +rcu_preempt:f:2:\[rcu_preempt\]$ +ksoftirqd:f:1:*:\[ksoftirqd\/.+\]$ From 33df8964d1721e13c881a158b21d53ee789788eb Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Wed, 21 Aug 2024 17:37:41 -0500 Subject: [PATCH 133/147] rt_tests: add ptest to catch default FIFO/1 tasks Add a ptest to catch any new (kernel) threads which get added with the default SCHED_FIFO/1 priority. On NI Linux RT we expect this priority level to be used by 'ksoftirqd/x' tasks. Signed-off-by: Gratian Crisan (cherry picked from commit 4588ac7ab9ca2f001c284e6c32a495e8748fc456) Signed-off-by: Chaitanya Vadrevu --- .../rt-tests/files/kthread_test_priority.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/recipes-rt/rt-tests/files/kthread_test_priority.sh b/recipes-rt/rt-tests/files/kthread_test_priority.sh index 577adfd34..2f7850443 100755 --- a/recipes-rt/rt-tests/files/kthread_test_priority.sh +++ b/recipes-rt/rt-tests/files/kthread_test_priority.sh @@ -33,6 +33,21 @@ function check_prio_for_task() { fi } +# Check if there are any tasks running as SCHED_FIFO priority 1. On NILRT +# systems we expect this RT policy/priority level to be used only by the +# ksoftirqd/x tasks. +function check_tasks_fifo_1() { + ptest_pass + + fifo_1_tasks=$(ps -e -x -o policy,rtprio,comm | awk '$1 == "FF" && $2 == 1 && $3 !~ /^ksoftirqd\/[0-9]+/ {print $3}') + + if [ ! -z "$fifo_1_tasks" ]; then + echo "unexpected tasks found running at FIFO/1 priority:" + echo "$fifo_1_tasks" + ptest_fail + fi +} + ptest_change_subtest 1 kthreadd check_prio_for_task kthreadd FIFO 25 ptest_report @@ -70,4 +85,8 @@ ptest_change_subtest 9 rcu_preempt check_prio_for_task rcu_preempt FIFO 2 ptest_report +ptest_change_subtest 10 fifo_1 +check_tasks_fifo_1 +ptest_report + exit $rc_first From 6edcf25da719451f80b8560bf42abdb83b54e906 Mon Sep 17 00:00:00 2001 From: Gratian Crisan Date: Wed, 21 Aug 2024 17:13:15 -0500 Subject: [PATCH 134/147] ni-utils: remove unused wl12xx workaround The wl12xx WiFi driver is not enabled in our x86_64 configurations. Additionally it is a workaround introduced for AB#735824 and was never revisited in AB#779061. On top of it the assumptions underlying the workaround - that RCU callbacks are processed in softirqs - no longer apply. Remove this dead code. Signed-off-by: Gratian Crisan (cherry picked from commit 805579d6d77a08630f25273274cc50a03c017523) Signed-off-by: Chaitanya Vadrevu --- recipes-core/busybox/files/ifplugd.action | 3 --- recipes-ni/ni-utils/files/functions.common | 13 ------------- 2 files changed, 16 deletions(-) diff --git a/recipes-core/busybox/files/ifplugd.action b/recipes-core/busybox/files/ifplugd.action index a192b0b03..bd3211d7a 100755 --- a/recipes-core/busybox/files/ifplugd.action +++ b/recipes-core/busybox/files/ifplugd.action @@ -10,9 +10,6 @@ export INTERFACE=$1 -# If there are drivers that need priorities changed, do it -update_driver_priorities - case "$2" in up) # On link-up, we're not guaranteed to be on the same network as diff --git a/recipes-ni/ni-utils/files/functions.common b/recipes-ni/ni-utils/files/functions.common index aa152cbc5..a127885b3 100644 --- a/recipes-ni/ni-utils/files/functions.common +++ b/recipes-ni/ni-utils/files/functions.common @@ -695,19 +695,6 @@ update_all_interface_info () { done } -update_driver_priorities () { - for d in /proc/irq/*; do - # Adjust wl12xx priority to something - # that allows reclamation of RCU objects - # through ksoftirqd under prolonged heavy - # wireless traffic conditions - if [ -d "$d"/wl12xx ]; then - echo 7 > "$d"/priority - break - fi - done -} - # Usage: enable_net_hotplug # Sets the flag so that enables hotplug.script to run enable_net_hotplug () { From 51b5ef2d1b45a7ff3fd5c7ed7d66927ce32f9746 Mon Sep 17 00:00:00 2001 From: Chaitanya Vadrevu Date: Wed, 21 Aug 2024 19:14:57 -0500 Subject: [PATCH 135/147] nisystemreplication: Fix unexpected differences Existing "Set Image" workflow installed grub and safemode afresh from provisioning media while setting some variables to ensure settings from system image get carried over but this left undesired changes in some files. This commit fixes this by copying nigrub and nibootfs from system image and later fixing the few files in them that are required to be different. Signed-off-by: Chaitanya Vadrevu (cherry picked from commit 1d4eb3260f4b8b042e7bbcfcb9abf4fb1e345b1a) Signed-off-by: Chaitanya Vadrevu --- .../files/ni_provisioning.safemode | 16 ++++ .../files/ni_provisioning.safemode.common | 96 ++++++++++++------- .../files/nisystemreplication | 30 +++--- 3 files changed, 96 insertions(+), 46 deletions(-) diff --git a/recipes-core/initrdscripts/files/ni_provisioning.safemode b/recipes-core/initrdscripts/files/ni_provisioning.safemode index a79185ae9..c7778bac7 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning.safemode +++ b/recipes-core/initrdscripts/files/ni_provisioning.safemode @@ -3,6 +3,22 @@ set -e source /ni_provisioning.safemode.common +check_all_used_binaries + +echo "Installing safemode to: $TARGET_DISK." +echo 6 > /proc/sys/kernel/printk + +print_info "Disabling automount..." +disable_automount +print_done + +partitioning_disk +wait_for_partitions $TARGET_DISK +create_filesystems + +prune_efi_crash_vars +install_grub + install_safemode install_bootmode_file install_grubenv diff --git a/recipes-core/initrdscripts/files/ni_provisioning.safemode.common b/recipes-core/initrdscripts/files/ni_provisioning.safemode.common index cbbd53556..bfb873f1f 100755 --- a/recipes-core/initrdscripts/files/ni_provisioning.safemode.common +++ b/recipes-core/initrdscripts/files/ni_provisioning.safemode.common @@ -110,6 +110,7 @@ check_all_used_binaries() override_primaryport_grubenv() { + local grubenv_file=$1 # Add target-specific override of primary ethernet port if the lowest ifIndex is not to be used local device_code=$(get_target_id) # CVS-1458RT @@ -117,7 +118,7 @@ override_primaryport_grubenv() # Set eth0 (note: renamed to eth0 via udev) as the primary port. Do not let the primary # port selection logic decide since it selects the 'eth' port with the lowest ifIndex # value. (Renaming the port via udev does not alter ifIndex, so eth0 is not the lowest) - grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "ethaddr=`cat /sys/class/net/eth0/address`" + grub-editenv $grubenv_file set "ethaddr=`cat /sys/class/net/eth0/address`" fi } @@ -199,17 +200,8 @@ create_filesystems() print_done } -install_grub() +set_efiboot_entry() { - GRUB_MOUNTPOINT=/var/volatile/grub - mkdir $GRUB_MOUNTPOINT -p - MOUNT_ERROR=`mount -L $PART1_LABEL $GRUB_MOUNTPOINT 2>&1` || die "$MOUNT_ERROR" - - print_info "Configuring EFI grub2..." - GRUB_TARGET_DIR=$GRUB_MOUNTPOINT/efi/boot - mkdir -p $GRUB_TARGET_DIR - GRUB_TARGET=$(uname -m) - cp /boot/EFI/BOOT/bootx64.efi $GRUB_TARGET_DIR # Delete existing NILRT entries with "-B" option for ENTRY in $(efibootmgr | grep -Ei '(LabVIEW RT)|(niboota)|(nibootb)' | grep -Eo 'Boot[0-9A-Fa-f]{4}' || true); do @@ -219,6 +211,26 @@ install_grub() done efibootmgr $VERBOSE_ARGS -c -d ${TARGET_DISK} -p 1 -L 'LabVIEW RT' -l '\efi\boot\bootx64.efi' print_done +} + +mount_grub_partition() +{ + GRUB_MOUNTPOINT=/var/volatile/grub + mkdir $GRUB_MOUNTPOINT -p + MOUNT_ERROR=`mount -L $PART1_LABEL $GRUB_MOUNTPOINT 2>&1` || die "$MOUNT_ERROR" +} + +install_grub() +{ + mount_grub_partition + + print_info "Configuring EFI grub2..." + GRUB_TARGET_DIR=$GRUB_MOUNTPOINT/efi/boot + mkdir -p $GRUB_TARGET_DIR + GRUB_TARGET=$(uname -m) + cp /boot/EFI/BOOT/bootx64.efi $GRUB_TARGET_DIR + + set_efiboot_entry print_info "Installing grub.cfg..." @@ -228,16 +240,21 @@ install_grub() print_done } -install_safemode() +mount_bootfs_partition() { - print_info "Installing safemode kernel and ramdisk..." - BOOTFS_MOUNTPOINT=/var/volatile/bootfs mkdir $BOOTFS_MOUNTPOINT -p BOOTFS_ERROR=`mount -L $PART2_LABEL $BOOTFS_MOUNTPOINT 2>&1` || die "$BOOTFS_ERROR" +} + +install_safemode() +{ + print_info "Installing safemode kernel and ramdisk..." + + mount_bootfs_partition mkdir -p $BOOTFS_MOUNTPOINT/.safe - cp "$SOURCE_DIR"/*Image "$BOOTFS_MOUNTPOINT/.safe/" + cp "$SOURCE_DIR"/*Image "$BOOTFS_MOUNTPOINT/.safe/" cp "$SOURCE_DIR"/ramdisk.* "$BOOTFS_MOUNTPOINT/.safe/" cp "$SOURCE_DIR"/bootimage.* "$BOOTFS_MOUNTPOINT/.safe/" @@ -252,6 +269,13 @@ install_bootmode_file() print_done } +add_rootuuid_to_grubenv() +{ + # save rootfs UUID for grub to reference + ROOTUUID=`lsblk ${TARGET_DISK}${PART_SEPARATOR}4 -n -o PARTUUID` + echo set rootuuid=$ROOTUUID >> $GRUB_MOUNTPOINT/grubvar_readonly +} + install_grubenv() { NI_TARGET="false" @@ -267,7 +291,7 @@ install_grubenv() cp $SOURCE_DIR/SMBIOS_NI_vars $BOOTFS_MOUNTPOINT/.safe cp $SOURCE_DIR/EFI_NI_vars $BOOTFS_MOUNTPOINT/.safe add_USB_gadget_args_to_grubenv - override_primaryport_grubenv + override_primaryport_grubenv $BOOTFS_MOUNTPOINT/grub/grubenv else cp $SOURCE_DIR/grubenv_non_ni_target $BOOTFS_MOUNTPOINT/grub/grubenv fi @@ -282,9 +306,7 @@ install_grubenv() echo "$varname" >> "$BOOTFS_MOUNTPOINT/.safe/GRUB_NI_readonly_vars" done - # save rootfs UUID for grub to reference - ROOTUUID=`lsblk ${TARGET_DISK}${PART_SEPARATOR}4 -n -o PARTUUID` - echo set rootuuid=$ROOTUUID >> $GRUB_MOUNTPOINT/grubvar_readonly + add_rootuuid_to_grubenv # set proper permissions on and backup firmware variable files chown 0:500 $BOOTFS_MOUNTPOINT/grub/grubenv @@ -325,6 +347,26 @@ install_grubenv() fi } +fix_grubenv_after_usbreplication() +{ + # Fix files in nigrub filesystem + rm $GRUB_MOUNTPOINT/grubvar_readonly + if is_ni_device ; then + add_USB_gadget_args_to_grubenv + fi + set_serial_port + add_rootuuid_to_grubenv + + # Fix files in nibootfs filesystem + if is_ni_device ; then + override_primaryport_grubenv $BOOTFS_MOUNTPOINT/grub/grubenv + override_primaryport_grubenv $BOOTFS_MOUNTPOINT/grub/grubenv.bak + else + grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv set "serial#=ABCDEFG" + fi + grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv unset primaryethaddr +} + set_versions() { BUILD_IDENTIFIER=$(get_image_info BUILD_IDENTIFIER) @@ -404,20 +446,4 @@ sanity_check() print_done } -check_all_used_binaries - -echo "Installing safemode to: $TARGET_DISK." -echo 6 > /proc/sys/kernel/printk - SOURCE_DIR=/payload - -print_info "Disabling automount..." -disable_automount -print_done - -partitioning_disk -wait_for_partitions $TARGET_DISK -create_filesystems - -prune_efi_crash_vars -install_grub diff --git a/recipes-ni/ni-systemreplication/files/nisystemreplication b/recipes-ni/ni-systemreplication/files/nisystemreplication index a6fc41ba9..fd9414c7b 100644 --- a/recipes-ni/ni-systemreplication/files/nisystemreplication +++ b/recipes-ni/ni-systemreplication/files/nisystemreplication @@ -189,22 +189,30 @@ image_set () { # Start with "none" so that default features are not enabled if not listed in $ext4_features MKFS_ARGS="${MKFS_ARGS} -O none,$ext4_features" source /ni_provisioning.safemode.common - install_safemode + + check_all_used_binaries + + echo "Installing to: $TARGET_DISK." + echo 6 > /proc/sys/kernel/printk + + print_info "Disabling automount..." + disable_automount + print_done + + partitioning_disk + wait_for_partitions $TARGET_DISK + create_filesystems + + prune_efi_crash_vars echo "Applying system image $image_name. This may take a while" >&2 cat $NIRECOVERY_MOUNTPOINT/Images/$image_name/systemimage.tgz* | nisystemimage setall -d -x tgz -p reset -s reset - # Retain some grubenv settings - if grep --quiet "^consoleoutenable=" $BOOTFS_MOUNTPOINT/grub/grubenv; then - grubenv_consoleoutenable=$(grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv list |grep "^consoleoutenable=" | cut -f 2 -d '=') - fi - if grep --quiet "^bootdelay=" $BOOTFS_MOUNTPOINT/grub/grubenv; then - grubenv_bootdelay=$(grub-editenv $BOOTFS_MOUNTPOINT/grub/grubenv list |grep "^bootdelay=" | cut -f 2 -d '=') - fi - rm -rf mkdir $BOOTFS_MOUNTPOINT/grub + set_efiboot_entry - install_grubenv - set_versions + mount_grub_partition + mount_bootfs_partition + fix_grubenv_after_usbreplication echo $LOG_LEVEL > /proc/sys/kernel/printk sanity_check From 3718278554e32d6ae125012c4c2d0e212765954a Mon Sep 17 00:00:00 2001 From: Richard Tollerton Date: Thu, 29 Aug 2024 14:33:24 -0500 Subject: [PATCH 136/147] packagegroup-ni-runmode: add nftables nftables is the modern packet filtering solution on Linux. The nftables userspace can be installed alongside iptables, but in general, only one can be enabled at once. This commit adds the userspace but does not do anything with the present iptables-based firewall configuration. At present, there are no plans to introduce nftables into safemode, so add it to the runmode packagegroup, not base. Signed-off-by: Rich Tollerton --- recipes-core/packagegroups/packagegroup-ni-runmode.bb | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes-core/packagegroups/packagegroup-ni-runmode.bb b/recipes-core/packagegroups/packagegroup-ni-runmode.bb index c23940a88..a0fc86a24 100644 --- a/recipes-core/packagegroups/packagegroup-ni-runmode.bb +++ b/recipes-core/packagegroups/packagegroup-ni-runmode.bb @@ -23,6 +23,7 @@ RDEPENDS:${PN} = "\ librtpi \ linux-firmware-radeon \ lldpd \ + nftables \ ni-configpersistentlogs \ ni-locale-alias \ ni-modules-autoload \ From b03a9498821262a2015b4cbae0dc0ded0ec5553b Mon Sep 17 00:00:00 2001 From: Richard Tollerton Date: Thu, 29 Aug 2024 15:15:29 -0500 Subject: [PATCH 137/147] packagegroup-ni-snac: add firewalld firewalld is the best-maintained high-level firewall administration tool on Linux. We ultimately wish to replace our present direct use of iptables (via initscript) with firewalld; but at present the existing configuration is unchanged. Signed-off-by: Rich Tollerton --- recipes-core/packagegroups/packagegroup-ni-snac.bb | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes-core/packagegroups/packagegroup-ni-snac.bb b/recipes-core/packagegroups/packagegroup-ni-snac.bb index f0d862b40..f6ca21c54 100644 --- a/recipes-core/packagegroups/packagegroup-ni-snac.bb +++ b/recipes-core/packagegroups/packagegroup-ni-snac.bb @@ -7,6 +7,7 @@ inherit packagegroup RDEPENDS:${PN} = "\ cryptsetup \ + firewalld \ ntp \ tmux \ libpwquality \ From eeb7b911a79c4ef474e9de590fed11cde42749a0 Mon Sep 17 00:00:00 2001 From: Alex Hearn Date: Tue, 27 Aug 2024 12:10:11 -0500 Subject: [PATCH 138/147] libpam: enable pam-plugin-faillock when it's installed and customize settings - update the pam-plugin-faillock package so that the plugin gets enabled when it's installed - modify some faillock configuration settings - prevent pam-plugin-faillock from being installed when ni-auth is installed This change simplifies Secured, Network-Attached Controller (SNAC) configuration. faillock is required to be enabled on a SNAC. The faillock settings were chosen to comply with SNAC requirements. The conflict with ni-auth was added because from testing it appears that the faillock plugin is incompatible with the ni-auth plugin. Signed-off-by: Alex Hearn --- .../pam/libpam/security/faillock.conf | 62 +++++++++++++++++++ recipes-extended/pam/libpam_1.%.bbappend | 22 +++++++ 2 files changed, 84 insertions(+) create mode 100644 recipes-extended/pam/libpam/security/faillock.conf diff --git a/recipes-extended/pam/libpam/security/faillock.conf b/recipes-extended/pam/libpam/security/faillock.conf new file mode 100644 index 000000000..abec69ac7 --- /dev/null +++ b/recipes-extended/pam/libpam/security/faillock.conf @@ -0,0 +1,62 @@ +# Configuration for locking the user after multiple failed +# authentication attempts. +# +# The directory where the user files with the failure records are kept. +# The default is /var/run/faillock. +# dir = /var/run/faillock +# +# Will log the user name into the system log if the user is not found. +# Enabled if option is present. +audit +# +# Don't print informative messages. +# Enabled if option is present. +silent +# +# Don't log informative messages via syslog. +# Enabled if option is present. +# no_log_info +# +# Only track failed user authentications attempts for local users +# in /etc/passwd and ignore centralized (AD, IdM, LDAP, etc.) users. +# The `faillock` command will also no longer track user failed +# authentication attempts. Enabling this option will prevent a +# double-lockout scenario where a user is locked out locally and +# in the centralized mechanism. +# Enabled if option is present. +# local_users_only +# +# Deny access if the number of consecutive authentication failures +# for this user during the recent interval exceeds n tries. +# The default is 3. +deny = 3 +# +# The length of the interval during which the consecutive +# authentication failures must happen for the user account +# lock out is n seconds. +# The default is 900 (15 minutes). +fail_interval = 900 +# +# The access will be re-enabled after n seconds after the lock out. +# The value 0 has the same meaning as value `never` - the access +# will not be re-enabled without resetting the faillock +# entries by the `faillock` command. +# The default is 600 (10 minutes). +unlock_time = 0 +# +# Root account can become locked as well as regular accounts. +# Enabled if option is present. +# even_deny_root +# +# This option implies the `even_deny_root` option. +# Allow access after n seconds to root account after the +# account is locked. In case the option is not specified +# the value is the same as of the `unlock_time` option. +# root_unlock_time = 900 +# +# If a group name is specified with this option, members +# of the group will be handled by this module the same as +# the root account (the options `even_deny_root>` and +# `root_unlock_time` will apply to them. +# By default, the option is not set. +# admin_group = diff --git a/recipes-extended/pam/libpam_1.%.bbappend b/recipes-extended/pam/libpam_1.%.bbappend index 9accc57ef..1a582a480 100644 --- a/recipes-extended/pam/libpam_1.%.bbappend +++ b/recipes-extended/pam/libpam_1.%.bbappend @@ -1 +1,23 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/${BPN}:" + +SRC_URI += "\ + file://security/faillock.conf \ +" + +do_install:append() { + install -m 644 ${WORKDIR}/security/faillock.conf ${D}${sysconfdir}/security/faillock.conf +} + +pkg_postinst:pam-plugin-faillock:append() { + # enable faillock + sed -E -i 's/^(.+)success=1(.+)$/auth requisite pam_faillock.so preauth\n\1success=2\2\nauth [default=die] pam_faillock.so authfail/' "${sysconfdir}/pam.d/common-auth" + echo "auth sufficient pam_faillock.so authsucc" >> "${sysconfdir}/pam.d/common-auth" +} + +pkg_prerm:pam-plugin-faillock:append() { + # disable faillock + sed -E -i '/pam_faillock.so/d' "${sysconfdir}/pam.d/common-auth" + sed -E -i 's/^(.+)success=2(.+)$/\1success=1\2/' "${sysconfdir}/pam.d/common-auth" +} + +RCONFLICTS:pam-plugin-faillock:append = " ni-auth" From 6633023ce98e03edfb9b474d44d4278eef08b0b4 Mon Sep 17 00:00:00 2001 From: Alex Hearn Date: Thu, 5 Sep 2024 12:25:33 -0500 Subject: [PATCH 139/147] sudo: give the sudo group permission to run the sudo command Signed-off-by: Alex Hearn --- recipes-extended/sudo/sudo_1.%.bbappend | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 recipes-extended/sudo/sudo_1.%.bbappend diff --git a/recipes-extended/sudo/sudo_1.%.bbappend b/recipes-extended/sudo/sudo_1.%.bbappend new file mode 100644 index 000000000..a094dca7b --- /dev/null +++ b/recipes-extended/sudo/sudo_1.%.bbappend @@ -0,0 +1,3 @@ +do_install:append() { + sed -i 's/^# \(%sudo ALL=(ALL:ALL) ALL\)$/\1/' ${D}${sysconfdir}/sudoers +} From 50e89ba291c369f85be2a1cb9620c4d9653f610c Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 6 Sep 2024 13:09:26 -0400 Subject: [PATCH 140/147] recipes-ni: add nilrt-snac recipe nilrt-snac is an NI configuration tool for NILRT that allows a system administrator to semi-automate the process of configuring the system into the Secured, Network-Attached Controller (SNAC) configuration. Add a recipe for it. This recipe is NILRT-specific. Signed-off-by: Alex Stewart --- recipes-ni/nilrt-snac/nilrt-snac_git.bb | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 recipes-ni/nilrt-snac/nilrt-snac_git.bb diff --git a/recipes-ni/nilrt-snac/nilrt-snac_git.bb b/recipes-ni/nilrt-snac/nilrt-snac_git.bb new file mode 100644 index 000000000..c03a4d35e --- /dev/null +++ b/recipes-ni/nilrt-snac/nilrt-snac_git.bb @@ -0,0 +1,32 @@ +SUMMARY = "NILRT SNAC Configuration Tool" +DESCRIPTION = "\ +A utility for admins to put a NILRT system into the SNAC configuration.\ +" +HOMEPAGE = "https://github.com/ni/nilrt-snac" +SECTION = "base" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://LICENSE;md5=380df876633ca23587b9851600778cc0" + + +SRC_URI = "\ + git://github.com/ni/nilrt-snac;branch=master;protocol=https \ +" + + +SRCREV = "${AUTOREV}" +PV = "0.1.1+git${SRCPV}" + +S = "${WORKDIR}/git" + + +do_install() { + oe_runmake install \ + DESTDIR=${D} +} + + +RDEPENDS:${PN} = "\ + bash \ + opkg \ + python3-core \ +" From dcc67efb8042dce97e786365db59327f2ee58814 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 6 Sep 2024 13:11:47 -0400 Subject: [PATCH 141/147] pkggrp-ni-snac: add nilrt-snac Add the nilrt-snac configuration tool to the SNAC packagegroup to ensure that it is always built into the core packagefeed. It should not be installed to the base system image by default. Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagegroup-ni-snac.bb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes-core/packagegroups/packagegroup-ni-snac.bb b/recipes-core/packagegroups/packagegroup-ni-snac.bb index f6ca21c54..0e7b9e784 100644 --- a/recipes-core/packagegroups/packagegroup-ni-snac.bb +++ b/recipes-core/packagegroups/packagegroup-ni-snac.bb @@ -8,7 +8,8 @@ inherit packagegroup RDEPENDS:${PN} = "\ cryptsetup \ firewalld \ + libpwquality \ + nilrt-snac \ ntp \ tmux \ - libpwquality \ " From 8e6ecc22fc42d1eb4189340af964fd8197fdf9b9 Mon Sep 17 00:00:00 2001 From: Rajendra Desai Date: Fri, 13 Sep 2024 12:03:59 +0530 Subject: [PATCH 142/147] pkggrp-ni-internal-deps: enable ni-grpc-device ni-grpc-device has been updated to compile with grpc >= 1.60. This commit enables the recipe back into the package feeds. Signed-off-by: Rajendra Desai --- .../packagegroup-ni-internal-deps.bb | 6 +- ...grpc-device-buildable-on-NILRT-11-10.patch | 448 ++++++++++++++++++ .../0001-CMakeLists-find-gpr-library.patch | 35 -- ...akeLists.txt-remove-local-protobuf-i.patch | 35 -- ...MakeLists-fixup-utf8cpp-library-link.patch | 29 -- ...akeLists-use-native-python3-binaries.patch | 54 --- .../ni-grpc-device/ni-grpc-device_git.bb | 22 +- 7 files changed, 461 insertions(+), 168 deletions(-) create mode 100644 recipes-ni/ni-grpc-device/files/0001-CMakeLists-Make-grpc-device-buildable-on-NILRT-11-10.patch delete mode 100644 recipes-ni/ni-grpc-device/files/0001-CMakeLists-find-gpr-library.patch delete mode 100644 recipes-ni/ni-grpc-device/files/0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch delete mode 100644 recipes-ni/ni-grpc-device/files/0002-CMakeLists-fixup-utf8cpp-library-link.patch delete mode 100644 recipes-ni/ni-grpc-device/files/0003-CMakeLists-use-native-python3-binaries.patch diff --git a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb index a39c20044..09f6e1419 100644 --- a/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb +++ b/recipes-core/packagegroups/packagegroup-ni-internal-deps.bb @@ -24,9 +24,9 @@ RDEPENDS:${PN} += "\ " # ni-sync -#RDEPENDS:${PN} += "\ -# ni-grpc-device \ -#" +RDEPENDS:${PN} += "\ + ni-grpc-device \ +" # Required components for Veristand. # Engineering contact: Marcelo Izaguirre diff --git a/recipes-ni/ni-grpc-device/files/0001-CMakeLists-Make-grpc-device-buildable-on-NILRT-11-10.patch b/recipes-ni/ni-grpc-device/files/0001-CMakeLists-Make-grpc-device-buildable-on-NILRT-11-10.patch new file mode 100644 index 000000000..f5552c422 --- /dev/null +++ b/recipes-ni/ni-grpc-device/files/0001-CMakeLists-Make-grpc-device-buildable-on-NILRT-11-10.patch @@ -0,0 +1,448 @@ +From b64a69d7b4a6d8335013fb7bea16b83a8f167c7e Mon Sep 17 00:00:00 2001 +From: Rajendra Desai <111044286+rajendra-desai-ni@users.noreply.github.com> +Date: Wed, 11 Sep 2024 21:07:38 +0530 +Subject: [PATCH] CMakeLists: Make grpc-device buildable on NILRT 11 (#1082) + +* CMakeLists: Make grpc-device buildable on NILRT 11 + +NILRT 11 will ship with grpc >1.60 and protobuf >v25.2. +grpc-device doesn't compile with these toolchain versions, +throwing errors about undefined symbols. Downgrading grpc +back to 1.51 (with python3-grpcio and protobuf recipes) +successfully works around these issues. But for security and +general currency reasons, we cannot afford to ship NILRT 11 +with these downgrades. +Current CMakeLists.txt is limited to some of the bitbake +functionalities which makes it difficult to build with new changes. + +Changes in this commit will make sure NILRT 11 compiles grpc-device +with the latest/upgraded grpc version without affecting the existing +build process. + +Changes: +1. refactor toolchain link logic + - Deprecate the CMAKE_CROSSCOMPILING variable, in favor of + USE_SUBMODULE_LIBS cmake option. Refactor the linking logic to + be a consolidation of all the linking actions from across the + file, and to better support builds in generic linux environments. +2. fixup utf8cpp library link + - The utf8cpp cmake library namespace is incorrectly identified + as 'utf8cpp', instead of the proper 'utf8cpp:utf8cpp'. As a + result, cmake does not link the utf8.h header and compilation + fails. +3. parameterize python3 venv + - Create a USE_PYTHON_VIRTUALENV cmake option. When asserted, it + will add the bespoke venv to the toolchain. Otherwise, the + cmake config will use the system python environment. +4. link the device server to grpc_gpr + - ni_grpc_device_server target depends on symbols from grpc + gpr.so, namely gpr_log. Add grpc_gpr to link libraries for + ni_grpc_device_server. +5. add abseil_sync dep to server target + - ni_grpc_device_server uses symbology from libabsl_synchronization + library. Add a library dependency to reflect that relationship. +6. add utf8cpp dep to IntegrationTestsRunner + - The IntegrationTestsRunner depends on utf8.h header indirectly, + via its access to the device server source. +7. fill out target lib deps + - Shove ni_grpc_device_server library dependencies into a variable, + so that it can be easily passed along to the test targets. +8. suppress protobuf installation in SM + - Set protobuf_INSTALL=OFF, which suppresses the protobuf + installation codepaths - that we don't want to use anyway and + which cause the failure. +9. add necessary gRPC dep to ni_grpc_device_server + - ni_grpc_device_server must be linked against libgrpc, as well as + the grpccpp libs. +10.fixup venv codegen deps + - Give the codegen targets a dependency on the python virtualenv + via the all_codegen_dependencies variable. + +Signed-off-by: Rajendra Desai + +* Updated variable name of NILRT_LEGACY_TOOLCHAIN in build artifacts and fixed a small typo + +Signed-off-by: Rajendra Desai + +* build_nilrt.yml: Fixed a typo + +* Enforce consistent runtime library settings in case of MSVC + +Signed-off-by: Rajendra Desai + +* Update linker condition + +* Add a condition to check for POLICY CMP0091 (enabling the use of +MSVC_RUNTIME_LIBRARY property) which is supported on cmake version +>3.15 in case of MSVC compiler + +Signed-off-by: Rajendra Desai + +* CMakeLists: Code cleanup + +* CMakeLists: use static runtime library + +* CMakeLists: change gtest_force_shared_crt to default + +* CMakeLists: add compile options if MSVC + +* Workaround: Do a clean build + +* Revert "CMakeLists: add compile options if MSVC" + +This reverts commit 2ea12e9fbde1c969c8a57eddc375346a7f6900b5. + +* Revert "CMakeLists: change gtest_force_shared_crt to default" + +This reverts commit 36890c9fff9f6ccdc1ee322a100a07770c4ef1d9. + +* Revert "CMakeLists: use static runtime library" + +This reverts commit 26a3c63d05e64b326c18cbe778f0d14e886433ab. + +* Workaround to fix Linker issues + +* Remove duplicate line + +* build_cmake.yml: Added a comment + +* build_cmake.yml: Move comment to avoid syntax error + +--------- +[rajendra: Discarded upstream changes in .github/workflows/build_cmake.yml] +[rajendra: Discarded upstream changes in .github/workflows/build_nilrt.yml] + +Upstream-Status: Backport [https://github.com/ni/grpc-device/commit/0d1658ea666d25a8577f348d78645b591eb45cd8] + +Signed-off-by: Rajendra Desai +--- + CMakeLists.txt | 188 ++++++++++++++++++++++++--------------- + cmake/nilrt-x86_64.cmake | 1 - + 2 files changed, 115 insertions(+), 74 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index b0384680..3980bffb 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -5,62 +5,100 @@ project(ni_grpc_device_server + VERSION 2.6.0) + + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +-include(CreateVirtualEnvironment) ++ + + # Workaround for: https://bugs.chromium.org/p/boringssl/issues/detail?id=423 + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + set(CMAKE_SYSTEM_PROCESSOR "amd64") + endif() + ++# Enforce consistent runtime library settings in case of MSVC ++if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") ++ if (POLICY CMP0091) ++ cmake_policy(SET CMP0091 NEW) ++ set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") ++ message("Setting policy CMP0091 and runtime library") ++ else() ++ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MD") ++ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd") ++ message("Using dynamic runtime library") ++ endif() ++endif() ++ ++#--- ++# Configuration options ++#--- ++option(USE_NILRT_LEGACY_TOOLCHAIN "Enable to use tools and libraries from a NILRT compile toolchain." OFF) ++option(USE_PYTHON_VIRTUALENV "Enable to use the automatically-generated python venv, local to this project source." ON) ++option(USE_SUBMODULE_LIBS "Enable to link against the submodule libraries, rather than their native-OS equivalents." ON) ++ + #---------------------------------------------------------------------- +-# Use the grpc targets directly from this build, only when not cross-compiling. ++# Setup build dependencies, according to the toolchain options. + #---------------------------------------------------------------------- +-if(CMAKE_CROSSCOMPILING) ++ ++if(USE_SUBMODULE_LIBS) ++ # The archetypical WIN32 build case ++ # protobuf_INSTALL must be turned OFF whenever building it as a cmake subdir. ++ set(protobuf_INSTALL OFF) ++ ++ add_subdirectory(third_party/grpc ${CMAKE_CURRENT_BINARY_DIR}/grpc) ++ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) ++ add_subdirectory(third_party/gtest ${CMAKE_CURRENT_BINARY_DIR}/gtest EXCLUDE_FROM_ALL) ++ add_subdirectory(third_party/json ${CMAKE_CURRENT_BINARY_DIR}/json) ++ add_subdirectory(third_party/utfcpp ${CMAKE_CURRENT_BINARY_DIR}/utfcpp) ++ ++ set(_PROTOBUF_PROTOC $) ++ set(_REFLECTION grpc++_reflection) ++ set(_GRPC grpc) ++ set(_GRPC_CPP_PLUGIN_EXECUTABLE $) ++ set(_GRPC_GRPCPP grpc++) ++ set(_PROTOBUF_LIBPROTOBUF libprotobuf) ++ set(_UTF8CPP utf8cpp) ++else() + find_program(_PROTOBUF_PROTOC protoc) + find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin) ++ find_library(_GRPC_GPR gpr) ++ find_library(_ABSEIL_SYNC absl_synchronization REQUIRED) + +- if(NOT _GRPC_DEVICE_NILRT_LEGACY_TOOLCHAIN) +- find_package(gRPC REQUIRED) +- find_library(_REFLECTION grpc++_reflection) +- find_library(_GRPC_GRPCPP grpc++) +- find_library(_PROTOBUF_LIBPROTOBUF protobuf) +- else() ++ if(USE_NILRT_LEGACY_TOOLCHAIN) ++ # The archetypical NILRT SDK toolchain build case + add_subdirectory(third_party/grpc ${CMAKE_CURRENT_BINARY_DIR}/grpc EXCLUDE_FROM_ALL) + set(_REFLECTION grpc++_reflection) + set(_GRPC_GRPCPP grpc++) + set(_PROTOBUF_LIBPROTOBUF libprotobuf) +- endif() ++ else() ++ # The archetypical linux build case (including OpenEmbedded) ++ find_library(_GRPC_GRPCPP grpc++) ++ find_library(_PROTOBUF_LIBPROTOBUF protobuf) ++ find_library(_REFLECTION grpc++_reflection) + +-else() +- add_subdirectory(third_party/grpc ${CMAKE_CURRENT_BINARY_DIR}/grpc EXCLUDE_FROM_ALL) +- set(_PROTOBUF_PROTOC $) +- set(_REFLECTION grpc++_reflection) +- set(_GRPC_CPP_PLUGIN_EXECUTABLE $) +- set(_GRPC_GRPCPP grpc++) +- set(_PROTOBUF_LIBPROTOBUF libprotobuf) +-endif() ++ find_package(gRPC REQUIRED) ++ find_package(GTest REQUIRED) ++ find_package(nlohmann_json REQUIRED) ++ find_package(utf8cpp REQUIRED) + +-#---------------------------------------------------------------------- +-CreateVirtualEnvironment(virtual_environment +- REQUIREMENTS_TXT +- ${CMAKE_SOURCE_DIR}/python_build_requirements.txt +- ENV_NAME +- venv +- OUT_PYTHON_EXE +- PYTHON_EXE +-) ++ set(_GRPC gRPC::grpc) ++ set(_UTF8CPP utf8cpp::utf8cpp) ++ endif() ++endif() + +-#---------------------------------------------------------------------- +-# Use the utfcpp targets directly from this build, only when not cross-compiling. +-#---------------------------------------------------------------------- +-if(CMAKE_CROSSCOMPILING AND NOT _GRPC_DEVICE_NILRT_LEGACY_TOOLCHAIN) +- find_package(utf8cpp REQUIRED) ++# Python3 Virtual Environment ++if(USE_PYTHON_VIRTUALENV) ++ include(CreateVirtualEnvironment) ++ CreateVirtualEnvironment(virtual_environment ++ REQUIREMENTS_TXT ++ ${CMAKE_SOURCE_DIR}/python_build_requirements.txt ++ ENV_NAME ++ venv ++ OUT_PYTHON_EXE ++ PYTHON_EXE ++ ) + else() +- add_subdirectory(third_party/utfcpp ${CMAKE_CURRENT_BINARY_DIR}/utfcpp EXCLUDE_FROM_ALL) ++ find_package(Python3 REQUIRED) ++ set(PYTHON_EXE ${Python3_EXECUTABLE}) + endif() +-include_directories( +- "./third_party/utfcpp/source" +-) ++ ++enable_testing() + + #---------------------------------------------------------------------- + # Use C++17 (needed for shared_mutex support on Linux) +@@ -141,6 +179,10 @@ endif() + + set(all_codegen_dependencies "") + ++if(USE_PYTHON_VIRTUALENV) ++ list(APPEND all_codegen_dependencies virtual_environment) ++endif() ++ + foreach(api ${nidrivers}) + set(codegen_dependencies + "${metadata_dir}/${api}/attributes.py" +@@ -191,7 +233,10 @@ foreach(api ${nidrivers}) + ${nidriver_client_srcs} + "${service_output_dir}/${api}/${api}_client.cpp") + endif() +- set(proto_dependencies ${codegen_dependencies} ${codegen_scripts} virtual_environment) ++ set(proto_dependencies ${codegen_dependencies} ${codegen_scripts}) ++ if (USE_PYTHON_VIRTUALENV) ++ list(APPEND proto_dependencies virtual_environment) ++ endif() + add_custom_command(OUTPUT ${output_files} + ${gen_command} + COMMENT "Generating proto file and service for ${api}" +@@ -211,7 +256,6 @@ add_custom_command( + DEPENDS + ${all_codegen_dependencies} + ${codegen_scripts} +- virtual_environment + ) + + set(nidriver_service_srcs +@@ -240,6 +284,9 @@ function(GenerateGrpcSources) + cmake_parse_arguments(GENERATE_ARGS "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(output_files "${GENERATE_ARGS_OUTPUT}") + set(proto_file "${GENERATE_ARGS_PROTO}") ++ if(USE_SUBMODULE_LIBS) ++ set(protobuf_includes_arg -I ${CMAKE_SOURCE_DIR}/third_party/grpc/third_party/protobuf/src/) ++ endif() + get_filename_component(proto_path "${proto_file}" PATH) + # Asssumption: all outputs are in the same directory: use the zero-th. + list(GET output_files 0 proto_srcs) +@@ -250,7 +297,7 @@ function(GenerateGrpcSources) + ARGS --grpc_out "${proto_out_path}" + --cpp_out "${proto_out_path}" + -I "${proto_path}" +- -I ${CMAKE_SOURCE_DIR}/third_party/grpc/third_party/protobuf/src/ ++ ${protobuf_includes_arg} + -I ${CMAKE_SOURCE_DIR}/imports/protobuf + --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" + "${proto_file}" +@@ -472,14 +519,21 @@ if(CMAKE_SYSTEM_NAME STREQUAL Windows) + PRIVATE "source/server/windows/console_ctrl_handler.cpp") + endif() + ++set(server_lib_deps ++ ${_ABSEIL_SYNC} ++ ${_GRPC_GPR} ++ ${_GRPC_GRPCPP} ++ ${_GRPC} ++ ${_PROTOBUF_LIBPROTOBUF} ++ ${_REFLECTION} ++ ${_UTF8CPP} ++ ${CMAKE_DL_LIBS} ++ nlohmann_json::nlohmann_json ++) ++ + target_link_libraries(ni_grpc_device_server +- ${_REFLECTION} +- ${_GRPC_GRPCPP} +- ${_PROTOBUF_LIBPROTOBUF} +- ${CMAKE_DL_LIBS} +- nlohmann_json::nlohmann_json +- utf8cpp +- ) ++ ${server_lib_deps} ++) + + set_target_properties(ni_grpc_device_server PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE) + +@@ -500,20 +554,6 @@ add_custom_command( + COMMAND ${PYTHON_EXE} ${codegen_dir}/generate_server_capabilities.py ${metadata_dir}/ + -o $/) + +- +-#---------------------------------------------------------------------- +-# Add JSON parser and configure google tests +-#---------------------------------------------------------------------- +-if(CMAKE_CROSSCOMPILING AND NOT _GRPC_DEVICE_NILRT_LEGACY_TOOLCHAIN) +- find_package(nlohmann_json REQUIRED) +- find_package(GTest REQUIRED) +-else() +- add_subdirectory(third_party/json ${CMAKE_CURRENT_BINARY_DIR}/json EXCLUDE_FROM_ALL) +- enable_testing() +- set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) +- add_subdirectory(third_party/gtest ${CMAKE_CURRENT_BINARY_DIR}/gtest EXCLUDE_FROM_ALL) +-endif() +- + # Link test executable against gtest + add_executable(IntegrationTestsRunner + "imports/include/nierr_Status.cpp" +@@ -568,12 +608,12 @@ add_executable(IntegrationTestsRunner + find_package(Threads REQUIRED) + + target_link_libraries(IntegrationTestsRunner +- gtest ++ ${server_lib_deps} + gmock +- ${_GRPC_GRPCPP} +- ${CMAKE_DL_LIBS} ++ grpc ++ gtest + Threads::Threads +- nlohmann_json::nlohmann_json) ++) + + # Ignore the use of deprecated functions in test code + target_compile_definitions(IntegrationTestsRunner +@@ -673,12 +713,13 @@ target_include_directories(UnitTestsRunner + PRIVATE "source/server") + + target_link_libraries(UnitTestsRunner +- gtest +- gmock +- ${_GRPC_GRPCPP} + ${CMAKE_DL_LIBS} ++ ${server_lib_deps} ++ gmock ++ grpc ++ gtest + Threads::Threads +- nlohmann_json::nlohmann_json) ++) + + #---------------------------------------------------------------------- + # Copy test asset certificates to binary output certs sub-directory +@@ -695,8 +736,10 @@ foreach(api ${nidrivers_to_build}) + "${service_output_dir}/${api}/${api}_compilation_test.cpp") + endforeach() + add_library(CompilationTests STATIC ${compilation_test_sources}) ++ + target_link_libraries(CompilationTests + ${_GRPC_GRPCPP} ++ ${_UTF8CPP} + ) + add_custom_target(generated_nidriver_service_library_hdrs DEPENDS ${nidriver_service_library_hdrs}) + add_dependencies(CompilationTests generated_nidriver_service_library_hdrs) +@@ -802,11 +845,10 @@ endif() + add_executable(SystemTestsRunner ${system_test_runner_sources}) + + target_link_libraries(SystemTestsRunner +- gtest +- gmock +- ${_GRPC_GRPCPP} + ${CMAKE_DL_LIBS} +- nlohmann_json::nlohmann_json ++ ${server_lib_deps} ++ gmock ++ gtest + ) + + # Ignore the use of deprecated functions in test code +@@ -819,7 +861,7 @@ add_custom_command( + ${CMAKE_SOURCE_DIR}/source/tests/assets/data/ + $/) + +-if(_GRPC_DEVICE_NILRT_LEGACY_TOOLCHAIN) ++if(USE_NILRT_LEGACY_TOOLCHAIN) + target_link_libraries(SystemTestsRunner stdc++fs) + target_compile_definitions(SystemTestsRunner PRIVATE FS_EXPERIMENTAL) + endif() +diff --git a/cmake/nilrt-x86_64.cmake b/cmake/nilrt-x86_64.cmake +index c007f696..7f00969f 100644 +--- a/cmake/nilrt-x86_64.cmake ++++ b/cmake/nilrt-x86_64.cmake +@@ -3,7 +3,6 @@ + #---------------------------------------------------------------------- + set(CMAKE_SYSTEM_NAME Linux) + set(CMAKE_SYSTEM_PROCESSOR x86_64) +-set(_GRPC_DEVICE_NILRT_LEGACY_TOOLCHAIN TRUE) + + #---------------------------------------------------------------------- + # Path variables for toolchains +-- +2.34.1 + diff --git a/recipes-ni/ni-grpc-device/files/0001-CMakeLists-find-gpr-library.patch b/recipes-ni/ni-grpc-device/files/0001-CMakeLists-find-gpr-library.patch deleted file mode 100644 index 99f8f3402..000000000 --- a/recipes-ni/ni-grpc-device/files/0001-CMakeLists-find-gpr-library.patch +++ /dev/null @@ -1,35 +0,0 @@ -From e74df62e09595d32ef8c52a359fcfb29f432d7e2 Mon Sep 17 00:00:00 2001 -From: Alex Stewart -Date: Tue, 23 Jan 2024 15:57:22 -0500 -Subject: [PATCH] CMakeLists: find gpr library - -The ni_grpc_device_server must be linked to the grpc-provided gpr.so -library during compilation. - -Modify the CMakeLists logic to find and link gpr.so. - -Signed-off-by: Alex Stewart ---- - CMakeLists.txt | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 6c966a2b..67074c65 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -18,6 +18,7 @@ endif() - if(CMAKE_CROSSCOMPILING) - find_program(_PROTOBUF_PROTOC protoc) - find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin) -+ find_library(_GRPC_GPR gpr) - - if(NOT _GRPC_DEVICE_NILRT_LEGACY_TOOLCHAIN) - find_package(gRPC REQUIRED) -@@ -441,6 +442,7 @@ endif() - target_link_libraries(ni_grpc_device_server - ${_REFLECTION} - ${_GRPC_GRPCPP} -+ ${_GRPC_GPR} - ${_PROTOBUF_LIBPROTOBUF} - ${CMAKE_DL_LIBS} - nlohmann_json::nlohmann_json diff --git a/recipes-ni/ni-grpc-device/files/0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch b/recipes-ni/ni-grpc-device/files/0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch deleted file mode 100644 index 9dd86b091..000000000 --- a/recipes-ni/ni-grpc-device/files/0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch +++ /dev/null @@ -1,35 +0,0 @@ -From aca928177133eea00462fa720034bc203a959add Mon Sep 17 00:00:00 2001 -From: Alex Stewart -Date: Thu, 18 Jan 2024 14:37:36 -0500 -Subject: [PATCH] Subject: [PATCH] CMakeLists.txt: remove local protobuf - includes - -When building grpc from within OE, protobuf is supplied by the native -sysroot. Do not add an includes path to the un-checked-out submodule. - -Upstream-Status: Inappropriate [Would break non-cross-compile builds] - -Signed-off-by: Alex Stewart ---- - CMakeLists.txt | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index d52498d7..5471d6cd 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -250,7 +250,6 @@ function(GenerateGrpcSources) - ARGS --grpc_out "${proto_out_path}" - --cpp_out "${proto_out_path}" - -I "${proto_path}" -- -I ${CMAKE_SOURCE_DIR}/third_party/grpc/third_party/protobuf/src/ - -I ${CMAKE_SOURCE_DIR}/imports/protobuf - --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" - "${proto_file}" -@@ -787,4 +786,4 @@ endif() - # add_test( NAME UnitTests COMMAND UnitTestsRunner ) - add_test( NAME UnitTests COMMAND UnitTestsRunner ) - add_test( NAME IntegrationTests COMMAND IntegrationTestsRunner ) --add_test( NAME SystemTests COMMAND SystemTestsRunner ) -\ No newline at end of file -+add_test( NAME SystemTests COMMAND SystemTestsRunner ) diff --git a/recipes-ni/ni-grpc-device/files/0002-CMakeLists-fixup-utf8cpp-library-link.patch b/recipes-ni/ni-grpc-device/files/0002-CMakeLists-fixup-utf8cpp-library-link.patch deleted file mode 100644 index 71c2e2110..000000000 --- a/recipes-ni/ni-grpc-device/files/0002-CMakeLists-fixup-utf8cpp-library-link.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 7de2f0751b126851a003392f2cc08901e84bfd5e Mon Sep 17 00:00:00 2001 -From: Alex Stewart -Date: Fri, 19 Jan 2024 14:45:14 -0500 -Subject: [PATCH] CMakeLists: fixup utf8cpp library link - -The utf8cpp cmake library namespace is incorrectly identified as -'utf8cpp', instead of the proper 'utf8cpp:utf8cpp'. As a result, cmake -does not link the utf8.h header and compilation fails. - -Correct the library namespace. - -Signed-off-by: Alex Stewart ---- - CMakeLists.txt | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 5471d6cd..48b5c07e 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -452,7 +452,7 @@ target_link_libraries(ni_grpc_device_server - ${_PROTOBUF_LIBPROTOBUF} - ${CMAKE_DL_LIBS} - nlohmann_json::nlohmann_json -- utf8cpp -+ utf8cpp::utf8cpp - ) - - set_target_properties(ni_grpc_device_server PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE) diff --git a/recipes-ni/ni-grpc-device/files/0003-CMakeLists-use-native-python3-binaries.patch b/recipes-ni/ni-grpc-device/files/0003-CMakeLists-use-native-python3-binaries.patch deleted file mode 100644 index 45ca6b279..000000000 --- a/recipes-ni/ni-grpc-device/files/0003-CMakeLists-use-native-python3-binaries.patch +++ /dev/null @@ -1,54 +0,0 @@ -From 6536cafe3d6e753fdc4de70b3cfc3360e39cdfb5 Mon Sep 17 00:00:00 2001 -From: Alex Stewart -Date: Mon, 22 Jan 2024 10:37:19 -0500 -Subject: [PATCH] CMakeLists: use native python3 binaries - -The existing CMakeLists implementation assumes that you want to setup a -python virtual environment, instead of using what is provided by the -local system. For OE, this isn't sensible since part of the missions is -to build the whole native toolchain. - -So remove this functionality. - -Signed-off-by: Alex Stewart ---- - CMakeLists.txt | 12 ++---------- - 1 file changed, 2 insertions(+), 10 deletions(-) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 48b5c07e..6c966a2b 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -41,14 +41,7 @@ else() - endif() - - #---------------------------------------------------------------------- --CreateVirtualEnvironment(virtual_environment -- REQUIREMENTS_TXT -- ${CMAKE_SOURCE_DIR}/python_build_requirements.txt -- ENV_NAME -- venv -- OUT_PYTHON_EXE -- PYTHON_EXE --) -+set(PYTHON_EXE ${Python3_EXECUTABLE}) - - #---------------------------------------------------------------------- - # Use the utfcpp targets directly from this build, only when not cross-compiling. -@@ -191,7 +184,7 @@ foreach(api ${nidrivers}) - ${nidriver_client_srcs} - "${service_output_dir}/${api}/${api}_client.cpp") - endif() -- set(proto_dependencies ${codegen_dependencies} ${codegen_scripts} virtual_environment) -+ set(proto_dependencies ${codegen_dependencies} ${codegen_scripts}) - add_custom_command(OUTPUT ${output_files} - ${gen_command} - COMMENT "Generating proto file and service for ${api}" -@@ -211,7 +204,6 @@ add_custom_command( - DEPENDS - ${all_codegen_dependencies} - ${codegen_scripts} -- virtual_environment - ) - - set(nidriver_service_srcs diff --git a/recipes-ni/ni-grpc-device/ni-grpc-device_git.bb b/recipes-ni/ni-grpc-device/ni-grpc-device_git.bb index 1be102878..cf91e39cc 100644 --- a/recipes-ni/ni-grpc-device/ni-grpc-device_git.bb +++ b/recipes-ni/ni-grpc-device/ni-grpc-device_git.bb @@ -10,37 +10,36 @@ LIC_FILES_CHKSUM = "\ DEPENDS += "\ - grpc-native \ googletest-native \ + grpc \ + grpc-native \ nlohmann-json-native \ + protobuf \ protobuf-native \ python3-grpcio-tools-native \ python3-mako-native \ python3-native \ - utf8cpp-native \ python3-schema-native \ + utf8cpp-native \ " -PV = "2.4.0" +PV = "2.6.0" SRC_URI = "\ git://github.com/ni/grpc-device.git;name=grpc-device;branch=main;protocol=https \ file://ptest \ - file://0001-Subject-PATCH-CMakeLists.txt-remove-local-protobuf-i.patch \ - file://0002-CMakeLists-fixup-utf8cpp-library-link.patch \ - file://0003-CMakeLists-use-native-python3-binaries.patch \ - file://0001-CMakeLists-find-gpr-library.patch \ + file://0001-CMakeLists-Make-grpc-device-buildable-on-NILRT-11-10.patch \ " -SRCREV_grpc-device = "aeef9995eb634ed5dc4d87dc5adbfd7c66d9fa64" + +SRCREV_grpc-device = "609fdf8c7ec99597373cf35f2b9608422b1955c9" SRCREV_FORMAT = "grpc-device" S = "${WORKDIR}/git" - inherit cmake python3native -EXTRA_OECMAKE += "-DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release" +EXTRA_OECMAKE += "-DCMAKE_CROSSCOMPILING=True -DCMAKE_BUILD_TYPE=Release -DUSE_SUBMODULE_LIBS=OFF -DUSE_PYTHON_VIRTUALENV=OFF" OECMAKE_TARGET_COMPILE = "ni_grpc_device_server" OECMAKE_GENERATOR = "Unix Makefiles" @@ -95,7 +94,6 @@ do_install () { cp -r ${S}/examples ${D}${datadir}/${BPN} } - PACKAGE_BEFORE_PN = "${PN}-examples" # ${PN} @@ -114,4 +112,4 @@ RDEPENDS:${PN}-dev:append = "\ FILES:${PN}-examples = "${datadir}/${BPN}/examples" RDEPENDS:${PN}-examples = "\ python3 \ -" +" \ No newline at end of file From 889156fca57e02c42cb2dd7c64c169527c44e3a9 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 13 Sep 2024 11:32:56 -0400 Subject: [PATCH 143/147] nilrt-snac: add integration ptests The nilrt-snac project now has an integration test suite. Add a run-ptest entrypoint and pytest subpackage to run it. Signed-off-by: Alex Stewart --- recipes-ni/nilrt-snac/files/run-ptest | 8 ++++++++ recipes-ni/nilrt-snac/nilrt-snac_git.bb | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 recipes-ni/nilrt-snac/files/run-ptest diff --git a/recipes-ni/nilrt-snac/files/run-ptest b/recipes-ni/nilrt-snac/files/run-ptest new file mode 100644 index 000000000..053978fee --- /dev/null +++ b/recipes-ni/nilrt-snac/files/run-ptest @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +import pytest + +pytest.main([ + "-v", + "/usr/lib/nilrt-snac/tests/integration", +]) diff --git a/recipes-ni/nilrt-snac/nilrt-snac_git.bb b/recipes-ni/nilrt-snac/nilrt-snac_git.bb index c03a4d35e..4e00ccffb 100644 --- a/recipes-ni/nilrt-snac/nilrt-snac_git.bb +++ b/recipes-ni/nilrt-snac/nilrt-snac_git.bb @@ -10,23 +10,38 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=380df876633ca23587b9851600778cc0" SRC_URI = "\ git://github.com/ni/nilrt-snac;branch=master;protocol=https \ + file://run-ptest \ " - SRCREV = "${AUTOREV}" PV = "0.1.1+git${SRCPV}" S = "${WORKDIR}/git" +inherit ptest + + do_install() { oe_runmake install \ DESTDIR=${D} } +do_install_ptest() { + install -m 0755 ${WORKDIR}/run-ptest ${D}${PTEST_PATH} +} + RDEPENDS:${PN} = "\ bash \ opkg \ python3-core \ " + +FILES:${PN}-ptest += "${libdir}/${PN}/tests/integration" +RDEPENDS:${PN}-ptest = "\ + bash \ + nilrt-snac \ + python3-core \ + python3-pytest \ +" From 4713fbdaf6e8efd79402496ce061075fba4c6a60 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 13 Sep 2024 11:34:27 -0400 Subject: [PATCH 144/147] pkggrp-ni-smoke: use tabs for indentation Signed-off-by: Alex Stewart --- .../packagegroup-ni-ptest-smoke.bb | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/recipes-core/packagegroups/packagegroup-ni-ptest-smoke.bb b/recipes-core/packagegroups/packagegroup-ni-ptest-smoke.bb index b0222d41a..c00e074d7 100644 --- a/recipes-core/packagegroups/packagegroup-ni-ptest-smoke.bb +++ b/recipes-core/packagegroups/packagegroup-ni-ptest-smoke.bb @@ -13,47 +13,47 @@ RDEPENDS:${PN} = "ptest-runner" # ptest packages RDEPENDS:${PN}:append = "\ - busybox-ptest \ - bzip2-ptest \ - coreutils-ptest \ - docker-functional-tests-ptest \ - e2fsprogs-ptest \ - elfutils-ptest \ - ethtool-ptest \ - flex-ptest \ - gettext-ptest \ - glibc-locale-tests-ptest \ - glibc-tests-ptest \ - kernel-tests-ptest \ - liberror-perl-ptest \ - libxml2-ptest \ - mdadm-ptest \ - nettle-ptest \ - ni-hw-scripts-ptest \ - ni-test-boot-time-ptest \ - opkg-ptest \ - pango-ptest \ - parted-ptest \ - perl-ptest \ - pstore-save-ptest \ - python3-appdirs-ptest \ - python3-atomicwrites-ptest \ - python3-bcrypt-ptest \ - python3-cryptography-ptest \ - python3-markupsafe-ptest \ - python3-more-itertools-ptest \ - python3-msgpack-ptest \ - python3-multidict-ptest \ - python3-pluggy-ptest \ - python3-pyasn1-ptest \ - python3-pyroute2-ptest \ - python3-pyserial-ptest \ - python3-pytz-ptest \ - python3-wcwidth-ptest \ - rt-tests-ptest \ - run-postinsts-ptest \ - sed-ptest \ - util-linux-ptest \ - xorg-fonts-100dpi-ptest \ - zeromq-ptest \ + busybox-ptest \ + bzip2-ptest \ + coreutils-ptest \ + docker-functional-tests-ptest \ + e2fsprogs-ptest \ + elfutils-ptest \ + ethtool-ptest \ + flex-ptest \ + gettext-ptest \ + glibc-locale-tests-ptest \ + glibc-tests-ptest \ + kernel-tests-ptest \ + liberror-perl-ptest \ + libxml2-ptest \ + mdadm-ptest \ + nettle-ptest \ + ni-hw-scripts-ptest \ + ni-test-boot-time-ptest \ + opkg-ptest \ + pango-ptest \ + parted-ptest \ + perl-ptest \ + pstore-save-ptest \ + python3-appdirs-ptest \ + python3-atomicwrites-ptest \ + python3-bcrypt-ptest \ + python3-cryptography-ptest \ + python3-markupsafe-ptest \ + python3-more-itertools-ptest \ + python3-msgpack-ptest \ + python3-multidict-ptest \ + python3-pluggy-ptest \ + python3-pyasn1-ptest \ + python3-pyroute2-ptest \ + python3-pyserial-ptest \ + python3-pytz-ptest \ + python3-wcwidth-ptest \ + rt-tests-ptest \ + run-postinsts-ptest \ + sed-ptest \ + util-linux-ptest \ + xorg-fonts-100dpi-ptest \ + zeromq-ptest \ " From 80eff37f7df9e4d7a158bca5841ae0222cc981a6 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 13 Sep 2024 11:35:19 -0400 Subject: [PATCH 145/147] pkggrp-ni-ptest-smoke: add nilrt-snac-ptest Signed-off-by: Alex Stewart --- recipes-core/packagegroups/packagegroup-ni-ptest-smoke.bb | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes-core/packagegroups/packagegroup-ni-ptest-smoke.bb b/recipes-core/packagegroups/packagegroup-ni-ptest-smoke.bb index c00e074d7..b4a328a8f 100644 --- a/recipes-core/packagegroups/packagegroup-ni-ptest-smoke.bb +++ b/recipes-core/packagegroups/packagegroup-ni-ptest-smoke.bb @@ -31,6 +31,7 @@ RDEPENDS:${PN}:append = "\ nettle-ptest \ ni-hw-scripts-ptest \ ni-test-boot-time-ptest \ + nilrt-snac-ptest \ opkg-ptest \ pango-ptest \ parted-ptest \ From 052582bfa03709f1b9d365e3ca03c9049291a170 Mon Sep 17 00:00:00 2001 From: Alex Stewart Date: Fri, 13 Sep 2024 13:36:06 -0400 Subject: [PATCH 146/147] nilrt_snac: remove unneeded RDEPENDS The ptest.bbclass already adds an RDEPENDS on `nilrt-snac`, but it is overriden by the nilrt-snac-ptest RDEPENDS bb assignment. Instead, append the ptest RDEPENDS. Signed-off-by: Alex Stewart --- recipes-ni/nilrt-snac/nilrt-snac_git.bb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipes-ni/nilrt-snac/nilrt-snac_git.bb b/recipes-ni/nilrt-snac/nilrt-snac_git.bb index 4e00ccffb..9e559e798 100644 --- a/recipes-ni/nilrt-snac/nilrt-snac_git.bb +++ b/recipes-ni/nilrt-snac/nilrt-snac_git.bb @@ -39,9 +39,8 @@ RDEPENDS:${PN} = "\ " FILES:${PN}-ptest += "${libdir}/${PN}/tests/integration" -RDEPENDS:${PN}-ptest = "\ +RDEPENDS:${PN}-ptest += "\ bash \ - nilrt-snac \ python3-core \ python3-pytest \ " From 6c2a2de7d13dafd4cc66c71edccbdf0ef7f24df6 Mon Sep 17 00:00:00 2001 From: pratheekshasn Date: Thu, 19 Sep 2024 10:23:53 +0530 Subject: [PATCH 147/147] onboard: add onboard to next branch (scarthgap) Signed-off by: Pratheeksha S N --- ...001-add-xfce-to-autostart-onlyshowin.patch | 10 +++ .../onboard/files/01-gnome-accessibility | 3 + recipes-support/onboard/files/NI.colors | 62 +++++++++++++++++++ recipes-support/onboard/files/NI.theme | 16 +++++ .../onboard/files/onboard-defaults.conf | 13 ++++ .../files/onboard_hover_seg_fault.patch | 37 +++++++++++ recipes-support/onboard/onboard_%.bbappend | 41 ++++++++++++ 7 files changed, 182 insertions(+) create mode 100644 recipes-support/onboard/files/0001-add-xfce-to-autostart-onlyshowin.patch create mode 100644 recipes-support/onboard/files/01-gnome-accessibility create mode 100644 recipes-support/onboard/files/NI.colors create mode 100644 recipes-support/onboard/files/NI.theme create mode 100644 recipes-support/onboard/files/onboard-defaults.conf create mode 100644 recipes-support/onboard/files/onboard_hover_seg_fault.patch create mode 100644 recipes-support/onboard/onboard_%.bbappend diff --git a/recipes-support/onboard/files/0001-add-xfce-to-autostart-onlyshowin.patch b/recipes-support/onboard/files/0001-add-xfce-to-autostart-onlyshowin.patch new file mode 100644 index 000000000..28f4dff62 --- /dev/null +++ b/recipes-support/onboard/files/0001-add-xfce-to-autostart-onlyshowin.patch @@ -0,0 +1,10 @@ +diff --git a/data/onboard-autostart.desktop.in b/data/onboard-autostart.desktop.in +index 8fb55ac..cf022a0 100644 +--- a/data/onboard-autostart.desktop.in ++++ b/data/onboard-autostart.desktop.in +@@ -9,4 +9,4 @@ NoDisplay=true + X-Ubuntu-Gettext-Domain=onboard + AutostartCondition=GSettings org.gnome.desktop.a11y.applications screen-keyboard-enabled + X-GNOME-AutoRestart=true +-OnlyShowIn=GNOME;Unity;MATE; ++OnlyShowIn=GNOME;Unity;MATE;XFCE; diff --git a/recipes-support/onboard/files/01-gnome-accessibility b/recipes-support/onboard/files/01-gnome-accessibility new file mode 100644 index 000000000..a92f5648d --- /dev/null +++ b/recipes-support/onboard/files/01-gnome-accessibility @@ -0,0 +1,3 @@ +# We need accessibility on in order to use Onboard's auto-show. +[org/gnome/desktop/interface] +toolkit-accessibility=true diff --git a/recipes-support/onboard/files/NI.colors b/recipes-support/onboard/files/NI.colors new file mode 100644 index 000000000..7f8217b92 --- /dev/null +++ b/recipes-support/onboard/files/NI.colors @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + icon0 + + + + icon1, icon2 + + + + + + + icon3, + RCTL, LCTL, RALT, LALT, LWIN, CAPS, + LFSH, RTSH, NMLK, + MENU, RWIN, BKSP, TAB, RTRN, + KPDL, KPEN, KPSU, KPDV, KPAD, KPMU, + LEFT, RGHT, UP, DOWN, INS, DELE, HOME, END, PGUP, PGDN, + F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11, F12, + Prnt, Pause, ESC, Scroll, + secondaryclick, middleclick, doubleclick, dragclick, hoverclick, + hide, showclick, move, layer, + quit + + + + + wordlist, prediction, pause-learning.wordlist, language.wordlist, hide.wordlist + + + + + + + m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14, m15 + + + + + + settings + + + diff --git a/recipes-support/onboard/files/NI.theme b/recipes-support/onboard/files/NI.theme new file mode 100644 index 000000000..c611cd9db --- /dev/null +++ b/recipes-support/onboard/files/NI.theme @@ -0,0 +1,16 @@ + + + + NI + flat + 20 + 30 + 70 + -3 + + + diff --git a/recipes-support/onboard/files/onboard-defaults.conf b/recipes-support/onboard/files/onboard-defaults.conf new file mode 100644 index 000000000..daaa5b9ff --- /dev/null +++ b/recipes-support/onboard/files/onboard-defaults.conf @@ -0,0 +1,13 @@ +[main] +layout=Compact +theme=NI +key-label-font=DejaVu Sans + +[window] +force-to-top=True + +[auto-show] +# Enable autoshow when there's no keyboard detected. +enabled=True +keyboard-device-detection-enabled=True +keyboard-device-detection-exceptions=['::noserial'] diff --git a/recipes-support/onboard/files/onboard_hover_seg_fault.patch b/recipes-support/onboard/files/onboard_hover_seg_fault.patch new file mode 100644 index 000000000..7a59d251c --- /dev/null +++ b/recipes-support/onboard/files/onboard_hover_seg_fault.patch @@ -0,0 +1,37 @@ +--- a/Onboard/osk/osk_devices.c ++++ b/Onboard/osk/osk_devices.c +@@ -97,13 +97,15 @@ osk_device_event_dealloc (OskDeviceEvent + static OskDeviceEvent* + new_device_event (void) + { +- OskDeviceEvent *ev = PyObject_New(OskDeviceEvent, &osk_device_event_type); ++ OskDeviceEvent *ev; ++ PyGILState_STATE gstate = PyGILState_Ensure(); ++ ev = PyObject_New(OskDeviceEvent, &osk_device_event_type); + if (ev) + { + osk_device_event_type.tp_init((PyObject*) ev, NULL, NULL); +- return ev; + } +- return NULL; ++ PyGILState_Release(gstate); ++ return ev; + } + + static PyObject * +@@ -334,6 +336,7 @@ osk_devices_dealloc (OskDevices *dev) + static void + queue_event (OskDevices* dev, OskDeviceEvent* event, Bool discard_pending) + { ++ PyGILState_STATE state = PyGILState_Ensure (); + GQueue* queue = dev->event_queue; + if (queue) + { +@@ -364,6 +367,7 @@ queue_event (OskDevices* dev, OskDeviceE + Py_INCREF(event); + g_queue_push_head(queue, event); + } ++ PyGILState_Release (state); + } + + static gboolean idle_process_event_queue (OskDevices* dev) diff --git a/recipes-support/onboard/onboard_%.bbappend b/recipes-support/onboard/onboard_%.bbappend new file mode 100644 index 000000000..40b6dfc5a --- /dev/null +++ b/recipes-support/onboard/onboard_%.bbappend @@ -0,0 +1,41 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/files:" + +# file://0001-add-xfce-to-autostart-onlyshowin.patch + +SRC_URI += " \ + file://0001-add-xfce-to-autostart-onlyshowin.patch \ + file://onboard_hover_seg_fault.patch \ + file://01-gnome-accessibility \ + file://NI.colors \ + file://NI.theme \ + file://onboard-defaults.conf \ +" + +CFLAGS += "-Werror=declaration-after-statement" + +do_install:append () { + install -d ${D}${sysconfdir}/dconf/db/local.d + install -d ${D}${sysconfdir}/onboard + install -d ${D}${datadir}/onboard/themes + + install -m 644 ${WORKDIR}/01-gnome-accessibility ${D}${sysconfdir}/dconf/db/local.d/ + install -m 644 ${WORKDIR}/onboard-defaults.conf ${D}${sysconfdir}/onboard/ + + install -m 644 ${WORKDIR}/NI.colors ${D}${datadir}/onboard/themes/ + install -m 644 ${WORKDIR}/NI.theme ${D}${datadir}/onboard/themes/ +} + +pkg_postinst:${PN} () { + dconf update +} + + +CONFFILES:${PN}:append := " \ + ${sysconfdir}/onboard/onboard-defaults.conf \ + ${sysconfdir}/dconf/db/local.d/01-gnome-accessibility \ +" + +RDEPENDS:${PN}:append = " dconf" +# Onboard uses unicode glyphs in its key_defs.xml file, which means +# we need a font that has those glyphs present. +RDEPENDS:${PN}:append = " ttf-dejavu-sans"