Up2023.10

.github/workflows/basic.yml

@@ -86,15 +86,33 @@
       - name: stop docker
         run: cd output/local && docker-compose down
 
+#      - name: Clean old install
+#        run: rm -r output
+#      - name: Copy backup
+#        run: cp -v backup-latest/* lava-master/backup/
+#      - name: Run lavalab-gen
+#        run: ./lavalab-gen.py boards-ci-2023.06.yaml
+#      - name: Build lava-docker 2023.06
+#        run: cd output/local && docker-compose build
+#      - name: Launch lava-docker 2023.06
+#        run: cd output/local && docker-compose up -d
+#      - name: Wait for LAVA 2023.06 to be started
+#        run: sh .github/workflows/wait-for-docker.sh
+#
+#      - name: Run backup of 2023.06
+#        run: ./backup.sh
+#      - name: stop docker
+#        run: cd output/local && docker-compose down
+
       - name: Clean old install
         run: rm -r output
       - name: Copy backup
         run: cp -v backup-latest/* lava-master/backup/
       - name: Run lavalab-gen
-        run: ./lavalab-gen.py boards-ci-2023.06.yaml
-      - name: Build lava-docker 2023.06
+        run: ./lavalab-gen.py boards-ci-2023.10.yaml
+      - name: Build lava-docker 2023.10
         run: cd output/local && docker-compose build
-      - name: Launch lava-docker 2023.06
+      - name: Launch lava-docker 2023.10
         run: cd output/local && docker-compose up -d
-      - name: Wait for LAVA 2023.06 to be started
+      - name: Wait for LAVA 2023.10 to be started
         run: sh .github/workflows/wait-for-docker.sh

README.md

@@ -217,8 +217,9 @@ masters:
  - name:  lava-master	name of the master
     host: name		name of the host running lava-master (default to "local")
     version: "202x.xx"	LAVA version for master
-    webadmin_https:	Does the LAVA webadmin is accessed via https
-    webinterface_port: Port number to use for the LAVA web interface (default to "10080")
+    webadmin_https:	    Does the LAVA webadmin is accessed via https
+    listen_address:     Address where webinterface_port will listen (default to 0.0.0.0)
+    webinterface_port:  Port number to use for the LAVA web interface (default to "10080")
     lava-coordinator:		Does the master should ran a lava-coordinator and export its port
     persistent_db: true/false	(default false) Is the postgresql DB is persistent over reboot.
                                 WARNING: this is working accross the same LAVA version, this do not work when there is a postgresql major update
@@ -344,6 +345,7 @@ boards:
       devpath: the UDEV devpath to this uart for UART without serial number
       interfacenum:	(optional) The interfacenumber of the serial. (Used with two serial in one device)
       use_ser2net: 	True/False (Deprecated, ser2net is the default uart handler)
+      worker:          (optional) an host/IP where ser2net is running
       ser2net_keepopen:	True/False (optional) Use the recent ser2net keepopen
       ser2net_options:	(optional) A list of ser2net options to add
         - option1

boards-ci-2023.10.yaml

@@ -0,0 +1,29 @@
+---
+masters:
+  - name: masterci1
+    version: "2023.10"
+    host: local
+    healthcheck_url: http://healthcheck
+    users:
+      - name: admin
+        token: tokenforci
+        password: passwordforci
+        superuser: true
+        staff: true
+    tokens:
+      - username: admin
+        token: dfjdfkfkdjfkdsjfslforci
+        description: no description
+slaves:
+  - name: lab-ci-0
+    version: "2023.10"
+    host: local
+    remote_master: masterci1
+    remote_user: admin
+    use_overlay_server: false
+    use_tftp: false
+    host_healthcheck: true
+
+boards:
+  - name: qemu-01
+    type: qemu

lava-master/Dockerfile

@@ -1,4 +1,4 @@
-FROM lavasoftware/lava-server:2023.08
+FROM lavasoftware/lava-server:2023.10
 
 RUN apt-get update && apt-get -y install sudo git
 

lava-master/entrypoint.d/01_setup.sh

@@ -72,7 +72,7 @@ fi
 # if we came from 2023.01 to 2023.05, we need to handle the migration bug
 grep -q '2023.0[1-5]' /tmp/workerversions
 if [ $? -eq 0 ];then
-	grep -q 2023.06 /usr/lib/python3/dist-packages/lava_common/VERSION
+	grep -qE '2023.0[68]|2023.10' /usr/lib/python3/dist-packages/lava_common/VERSION
 	if [ $? -eq 0 ];then
 		echo "============================="
 		echo "DEBUG: handle DB migration BUG"
@@ -222,6 +222,12 @@ if [ -e /root/lava-callback-tokens ];then
 	done
 fi
 
+lava-server manage device-types --no-color list > /tmp/device-types.list
+if [ $? -ne 0 ];then
+	echo "ERROR: fail to get device-types"
+	exit 1
+fi
+
 # This directory is used for storing device-types already added
 mkdir -p /root/.lavadocker/
 if [ -e /root/device-types ];then
@@ -234,7 +240,7 @@ if [ -e /root/device-types ];then
 		cp $i /etc/lava-server/dispatcher-config/device-types/
 		chown lavaserver:lavaserver /etc/lava-server/dispatcher-config/device-types/$(basename $i)
 		devicetype=$(basename $i |sed 's,.jinja2,,')
-		lava-server manage device-types list | grep -q "[[:space:]]$devicetype[[:space:]]"
+		grep -q "[[:space:]]$devicetype[[:space:]]" /tmp/device-types.list
 		if [ $? -eq 0 ];then
 			echo "Skip already known $devicetype"
 		else
@@ -266,6 +272,12 @@ do
 	done
 done
 
+if [ ! -z "$GUNICORN_WORKERS" ];then
+	echo "DEBUG: set gunicorn workers to $GUNICORN_WORKERS"
+	grep -ri workers /etc
+	sed -i "s,.*WORKERS=.*,WORKERS=$GUNICORN_WORKERS," /etc/lava-server/lava-server-gunicorn || exit $?
+fi
+
 echo "DEBUG: fix owning rights on /etc/lava-server/dispatcher-config"
 chown -Rc lavaserver:lavaserver /etc/lava-server/dispatcher-config
 exit 0

lava-slave/Dockerfile

@@ -1,4 +1,4 @@
-FROM lavasoftware/lava-dispatcher:2023.08
+FROM lavasoftware/lava-dispatcher:2023.10
 
 RUN apt-get update
 

lavalab-gen.py

@@ -31,7 +31,7 @@
 """)
 
 template_device_ser2net = string.Template("""
-{% set connection_command = 'telnet 127.0.0.1 ${port}' %}
+{% set connection_command = 'telnet ${telnet_host} ${port}' %}
 """)
 
 ser2net_dict = {}
@@ -114,8 +114,9 @@ def main():
             "allowed_hosts",
             "build_args",
             "event_notifications",
-            "groups",
+            "groups", "gunicorn_workers",
             "healthcheck_url", "host", "http_fqdn",
+            "listen_address",
             "loglevel", "lava-coordinator",
             "name",
             "persistent_db", "pg_lava_password",
@@ -141,13 +142,17 @@ def main():
             webinterface_port = "10080"
         else:
             webinterface_port = master["webinterface_port"]
+        if "listen_address" in master:
+            listen_address = master["listen_address"]
+        else:
+            listen_address = '0.0.0.0'
         dockcomp = {}
         dockcomp["version"] = "2.0"
         dockcomp["services"] = {}
         dockcomposeymlpath = "output/%s/docker-compose.yml" % host
         dockcomp["services"][name] = {}
         dockcomp["services"][name]["hostname"] = name
-        dockcomp["services"][name]["ports"] = [ str(webinterface_port) + ":80", "5555:5555", "5556:5556", "5500:5500" ]
+        dockcomp["services"][name]["ports"] = [ listen_address + ":" + str(webinterface_port) + ":80"]
         dockcomp["services"][name]["volumes"] = [ "/boot:/boot", "/lib/modules:/lib/modules" ]
         dockcomp["services"][name]["build"] = {}
         dockcomp["services"][name]["build"]["context"] = name
@@ -199,6 +204,10 @@ def main():
             f_entrypoint.write("exit $?\n")
             f_entrypoint.close()
             os.chmod("%s/entrypoint.d/02_lava-coordinator.sh" % workerdir, 0o755)
+        if "gunicorn_workers" in worker:
+            dockcomp["services"][name]["environment"] = {}
+            dockcomp["services"][name]["environment"]["GUNICORN_WORKERS"] = worker["gunicorn_workers"]
+
         with open(dockcomposeymlpath, 'w') as f:
             yaml.dump(dockcomp, f)
         if "healthcheck_url" in master:
@@ -388,7 +397,7 @@ def main():
                 fenv.close()
         if "loglevel" in worker:
             for component in worker["loglevel"]:
-                if component != "lava-master" and component != "lava-logs" and component != 'lava-server-gunicorn':
+                if component != "lava-master" and component != "lava-logs" and component != 'lava-server-gunicorn' and component != "lava-scheduler":
                     print("ERROR: invalid loglevel component %s" % component)
                     sys.exit(1)
                 loglevel = worker["loglevel"][component]
@@ -569,8 +578,8 @@ def main():
             dockcomp["services"]["healthcheck"]["ports"] = ["8080:8080"]
             dockcomp["services"]["healthcheck"]["build"] = {}
             dockcomp["services"]["healthcheck"]["build"]["context"] = "healthcheck"
-            if "build_args" in master:
-                dockcomp["services"]["healthcheck"]["build"]["args"] = master['build_args']
+            if remote_master in worker and "build_args" in worker[remote_master]:
+                dockcomp["services"]["healthcheck"]["build"]["args"] = worker[remote_master]['build_args']
             shutil.copytree("healthcheck", "output/%s/healthcheck" % host)
         if "extra_actions" in worker:
             fp = open("%s/scripts/extra_actions" % workerdir, "w")
@@ -736,16 +745,41 @@ def main():
             if not use_ser2net and not "connection_command" in board:
                 use_ser2net = True
             if use_ser2net:
-                if not worker_name in ser2net_ports:
-                    ser2net_ports[worker_name] = ser2net_port_start
-                    fp = open("%s/ser2net.yaml" % workerdir, "a")
+                if "worker" in uart:
+                    worker_ser2net = uart["worker"]
+                    telnet_host = worker_ser2net
+                else:
+                    worker_ser2net = worker_name
+                    telnet_host = "127.0.0.1"
+                ser2netdir = "output/%s/%s" % (host, worker_ser2net)
+                if not os.path.isdir(ser2netdir):
+                    os.mkdir(ser2netdir)
+                if (not "bind_dev" in slave or not slave["bind_dev"]) and worker_ser2net == worker_name:
+                    dockcomp_add_device(dockcomp, worker_name, "/dev/%s:/dev/%s" % (board_name, board_name))
+                udev_line = 'SUBSYSTEM=="tty", ATTRS{idVendor}=="%04x", ATTRS{idProduct}=="%04x",' % (idvendor, idproduct)
+                if "serial" in uart:
+                    udev_line += 'ATTRS{serial}=="%s", ' % board["uart"]["serial"]
+                if "devpath" in uart:
+                    udev_line += 'ATTRS{devpath}=="%s", ' % board["uart"]["devpath"]
+                if "interfacenum" in uart:
+                    udev_line += 'ENV{ID_USB_INTERFACE_NUM}=="%s", ' % board["uart"]["interfacenum"]
+                udev_line += 'MODE="0664", OWNER="uucp", SYMLINK+="%s"\n' % board_name
+                udevdir = "output/%s/%s/udev" % (host, worker_ser2net)
+                if not os.path.isdir(udevdir):
+                    os.mkdir(udevdir)
+                fp = open("%s/99-lavaworker-udev.rules" % udevdir, "a")
+                fp.write(udev_line)
+                fp.close()
+                if not worker_ser2net in ser2net_ports:
+                    ser2net_ports[worker_ser2net] = ser2net_port_start
+                    fp = open("%s/ser2net.yaml" % ser2netdir, "a")
                     fp.write("%YAML 1.1\n---\n")
                     fp.close()
-                device_line += template_device_ser2net.substitute(port=ser2net_ports[worker_name])
+                device_line += template_device_ser2net.substitute(port=ser2net_ports[worker_ser2net], telnet_host=telnet_host)
                 # YAML version
-                fp = open("%s/ser2net.yaml" % workerdir, "a")
-                fp.write("connection: &con%d\n" % ser2net_ports[worker_name])
-                fp.write("  accepter: telnet(rfc2217),tcp,%d\n" % ser2net_ports[worker_name])
+                fp = open("%s/ser2net.yaml" % ser2netdir, "a")
+                fp.write("connection: &con%d\n" % ser2net_ports[worker_ser2net])
+                fp.write("  accepter: telnet(rfc2217),tcp,%d\n" % ser2net_ports[worker_ser2net])
                 fp.write("  enable: on\n")
                 if ser2net_keepopen:
                     ser2net_yaml_line= "  connector: keepopen(retry-time=2000,discard-badwrites),serialdev,/dev/%s,%dn81,local" % (board_name, baud)
@@ -758,7 +792,7 @@ def main():
                 fp.write(ser2net_yaml_line)
                 fp.write("  options:\n")
                 fp.write("    max-connections: 10\n")
-                ser2net_ports[worker_name] += 1
+                ser2net_ports[worker_ser2net] += 1
                 fp.close()
         if "connection_command" in board:
             connection_command = board["connection_command"]