linux: add linux rump kernel build

build-rr.sh adds new -r option to switch what rump kernel is used:
NetBSD is the default one and add Linux one for the alternative in this
patch.

Although there are several limitations (listed below) in this stage,
this would be a first step toward supporting alternate rump kernel
across different ones.

Limitations:
- no xen execution support (build should be fine)
- no i486 build support
- CXX is always false: CXX=false should be added to build linux rump
  kernel
- no libz support
- no cookfs (/rootfs) support
- rumprun-packages needs more work: current test on travis is only build
  test and the executions might have various issues.

.gitmodules

@@ -1,6 +1,12 @@
 [submodule "buildrump.sh"]
 	path = buildrump.sh
-	url = https://github.com/rumpkernel/buildrump.sh
+	url = https://github.com/libos-nuse/buildrump.sh
 [submodule "src-netbsd"]
 	path = src-netbsd
 	url = https://github.com/rumpkernel/src-netbsd
+[submodule "linux"]
+	path = linux
+	url = https://github.com/libos-nuse/lkl-linux.git
+[submodule "musl"]
+	path = musl
+	url = https://github.com/libos-nuse/musl.git

.travis.yml

@@ -11,17 +11,33 @@ before_script:
   - sudo apt-get install --only-upgrade binutils gcc -y
 
 env:
-  - PLATFORM=hw MACHINE=x86_64 TESTS=qemu EXTRAFLAGS=
-  - PLATFORM=hw MACHINE=i486 ELF=elf TESTS=qemu EXTRAFLAGS='-- -F ACLFLAGS=-m32 -F ACLFLAGS=-march=i686'
-  - PLATFORM=xen MACHINE=x86_64 TESTS=none EXTRAFLAGS=
-  - PLATFORM=xen MACHINE=i486 ELF=elf TESTS=none EXTRAFLAGS='-- -F ACLFLAGS=-m32'
-  - PLATFORM=hw MACHINE=x86_64 TESTS=qemu EXTRAFLAGS= CXX='false'
-  - PLATFORM=hw MACHINE=x86_64 TESTS=none KERNONLY=-k EXTRAFLAGS= 
-  - PLATFORM=xen MACHINE=x86_64 TESTS=none KERNONLY=-k EXTRAFLAGS=
+  - PLATFORM=hw MACHINE=x86_64 TESTS=qemu RUMPKERNEL="-r netbsd" EXTRAFLAGS=
+  - PLATFORM=hw MACHINE=i486 ELF=elf TESTS=qemu RUMPKERNEL="-r netbsd" EXTRAFLAGS='-- -F ACLFLAGS=-m32 -F ACLFLAGS=-march=i686'
+  - PLATFORM=xen MACHINE=x86_64 TESTS=none RUMPKERNEL="-r netbsd" EXTRAFLAGS=
+  - PLATFORM=xen MACHINE=i486 ELF=elf TESTS=none RUMPKERNEL="-r netbsd" EXTRAFLAGS='-- -F ACLFLAGS=-m32'
+  - PLATFORM=hw MACHINE=x86_64 TESTS=qemu RUMPKERNEL="-r netbsd" EXTRAFLAGS= CXX='false'
+  - PLATFORM=hw MACHINE=x86_64 TESTS=none RUMPKERNEL="-r netbsd" KERNONLY=-k EXTRAFLAGS=
+  - PLATFORM=xen MACHINE=x86_64 TESTS=none RUMPKERNEL="-r netbsd" KERNONLY=-k EXTRAFLAGS=
+  - PLATFORM=hw MACHINE=x86_64 TESTS=qemu RUMPKERNEL="-r linux" EXTRAFLAGS=
+  - PLATFORM=hw MACHINE=i486 ELF=elf TESTS=qemu RUMPKERNEL="-r linux" EXTRAFLAGS='-- -F ACLFLAGS=-m32 -F ACLFLAGS=-march=i686'
+  - PLATFORM=xen MACHINE=x86_64 TESTS=none RUMPKERNEL="-r linux" EXTRAFLAGS=
+  - PLATFORM=xen MACHINE=i486 ELF=elf TESTS=none RUMPKERNEL="-r linux" EXTRAFLAGS='-- -F ACLFLAGS=-m32'
+  - PLATFORM=hw MACHINE=x86_64 TESTS=qemu RUMPKERNEL="-r linux" EXTRAFLAGS= CXX='false'
+  - PLATFORM=hw MACHINE=x86_64 TESTS=none RUMPKERNEL="-r linux" KERNONLY=-k EXTRAFLAGS=
+  - PLATFORM=xen MACHINE=x86_64 TESTS=none RUMPKERNEL="-r linux" KERNONLY=-k EXTRAFLAGS=
+
+matrix:
+   allow_failures:
+      # no CXX
+      - env: PLATFORM=hw MACHINE=x86_64 TESTS=qemu RUMPKERNEL="-r linux" EXTRAFLAGS=
+      - env: PLATFORM=xen MACHINE=x86_64 TESTS=none RUMPKERNEL="-r linux" EXTRAFLAGS=
+      # no i486 build yet
+      - env: PLATFORM=hw MACHINE=i486 ELF=elf TESTS=qemu RUMPKERNEL="-r linux" EXTRAFLAGS='-- -F ACLFLAGS=-m32 -F ACLFLAGS=-march=i686'
+      - env: PLATFORM=xen MACHINE=i486 ELF=elf TESTS=none RUMPKERNEL="-r linux" EXTRAFLAGS='-- -F ACLFLAGS=-m32'
 
 script:
   - git submodule update --init
-  - ./build-rr.sh -o myobj -j16 -qq ${KERNONLY} ${PLATFORM} ${EXTRAFLAGS}
+  - ./build-rr.sh -o myobj -j16 -qq ${RUMPKERNEL} ${KERNONLY} ${PLATFORM} ${EXTRAFLAGS}
   - . ./myobj/config
   - ./tests/buildtests.sh ${KERNONLY}
   - ./tests/runtests.sh ${TESTS}

app-tools/rumprun

@@ -691,7 +691,7 @@ run_qemu ()
 	if [ -n "$opt_interactive" ]; then
 		${qemucmd} -append "${json_coma}"
 	else
-		qemucmd="${qemucmd} -display none"
+		qemucmd="${qemucmd} -display none -vga none"
 		${qemucmd} -append "${json_coma}" 1>/dev/null 2>&1 &
 		echo qemu:$!
 	fi
@@ -703,7 +703,13 @@ bake_iso ()
 	[ -z "${DUMPCMD}" ] || die -D not supported by iso.  Use -T.
 
 	type xorriso >/dev/null || die bake_iso needs xorriso
-	type grub-mkrescue >/dev/null || die bake_iso needs grub-mkrescue
+	if type grub-mkrescue 2>/dev/null >/dev/null; then
+		MKRESCUE=grub-mkrescue
+	elif type grub2-mkrescue 2>/dev/null >/dev/null; then
+		MKRESCUE=grub2-mkrescue
+	else
+		die bake_iso needs grub-mkrescue or grub2-mkrescue
+	fi
 
 	store_blkspec=json_store_iso_blkspec
 	store_netspec=json_store_netspec
@@ -777,7 +783,7 @@ bake_iso ()
 	    $(basename $1) >> "${ISODIR}/boot/grub/grub.cfg"
 	cp ${bootimage} "${ISODIR}/boot"
 	cp "${TMPDIR}/json.cfg" "${ISODIR}"
-	grub-mkrescue -o ${opt_name} "${ISODIR}"
+	${MKRESCUE} -o ${opt_name} "${ISODIR}"
 }
 
 make_ec2 ()

app-tools/rumprun-bake.conf

@@ -149,3 +149,17 @@ conf hw_generic
 			_pciether		\
 			_usb
 fnoc
+
+conf hw_lkl
+	create		"LKL generic (a.k.a all) targets"
+	add		-llkl
+	add		-lrumprun_base
+	add		-lrumpdev_linux_pci
+fnoc
+
+conf xen_pv_lkl
+	create		"Xen with paravirtualized I/O drivers with LKL"
+	add		-llkl
+	add		-lrumprun_base
+	add		-lrumpdev_linux_pci
+fnoc

build-rr.sh

@@ -49,6 +49,7 @@ helpme ()
 	printf "\t-o: use non-default object directory\n"
 	printf "\t-k: build kernel only, without libc or tools\n"
 	printf "\t-s: specify alternative src-netbsd location\n\n"
+	printf "\t-r: type of rump kernel [netbsd|linux]. default netbsd\n\n\n"
 	printf "\tbuildrump.sh opts are passed to buildrump.sh\n"
 	printf "\n"
 	printf "The toolchain is picked up from the environment.  See the\n"
@@ -96,14 +97,16 @@ parseargs ()
 	KERNONLY=false
 	RROBJ=
 	RUMPSRC=src-netbsd
+	LKLSRC=linux
+	RUMPKERNEL=netbsd
 	STDJ=-j4
 	EXTSRC=
 
 	DObuild=false
 	DOinstall=false
 
 	orignargs=$#
-	while getopts '?d:hj:ko:qs:' opt; do
+	while getopts '?d:hj:kr:o:qs:' opt; do
 		case "$opt" in
 		'j')
 			[ -z "$(echo ${OPTARG} | tr -d '[0-9]')" ] \
@@ -116,6 +119,14 @@ parseargs ()
 		'k')
 			KERNONLY=true
 			;;
+		'r')
+			RUMPKERNEL="${OPTARG}"
+			if [ ${RUMPKERNEL} != "netbsd" -a ${RUMPKERNEL} != "linux" ]; then
+				echo '>> ERROR:'
+				echo '>> -r option (RUMPKERNEL) must be netbsd or linux'
+				exit 1
+			fi
+			;;
 		'o')
 			RROBJ="${OPTARG}"
 			;;
@@ -180,6 +191,8 @@ parseargs ()
 		DOinstall=true
 	fi
 
+	. rumpkernel/${RUMPKERNEL}.sh
+
 	case ${RUMPSRC} in
 	/*)
 		;;
@@ -190,6 +203,7 @@ parseargs ()
 
 	export RUMPSRC
 	export BUILD_QUIET
+	export RUMPKERNEL
 
 	ARGSSHIFT=$((${orignargs} - $#))
 }
@@ -297,6 +311,9 @@ setvars ()
 	abspath RRDEST
 	abspath RROBJ
 	abspath RUMPSRC
+	abspath LKLSRC
+	export RROBJ
+	export LKLSRC
 }
 
 checktools ()
@@ -357,7 +374,7 @@ buildrump ()
 	    -s ${RUMPSRC} -T ${RUMPTOOLS} -o ${BROBJ} -d ${STAGING}	\
 	    -V MKPIC=no -V RUMP_CURLWP=__thread				\
 	    -V RUMP_KERNEL_IS_LIBC=1 -V BUILDRUMP_SYSROOT=yes		\
-	    ${extracflags} "$@" tools
+	    ${extracflags} -l ${RUMPKERNEL} "$@" tools
 
 	echo '>>'
 	echo '>> Now that we have the appropriate tools, performing'
@@ -367,7 +384,7 @@ buildrump ()
 	RUMPMAKE=$(pwd)/${RUMPTOOLS}/rumpmake
 
 	TOOLTUPLE=$(${RUMPMAKE} -f bsd.own.mk \
-	    -V '${MACHINE_GNU_PLATFORM:S/--netbsd/-rumprun-netbsd/}')
+	    -V '${MACHINE_GNU_PLATFORM:S/--netbsd/-rumprun-${RUMPKERNEL}/}')
 
 	[ $(${RUMPMAKE} -f bsd.own.mk -V '${_BUILDRUMP_CXX}') != 'yes' ] \
 	    || HAVECXX=true
@@ -406,7 +423,7 @@ EOF
 	# build rump kernel
 	${BUILDRUMP}/buildrump.sh ${BUILD_QUIET} ${STDJ} -k		\
 	    -s ${RUMPSRC} -T ${RUMPTOOLS} -o ${BROBJ} -d ${STAGING}	\
-	    "$@" build kernelheaders install
+	    -l ${RUMPKERNEL} "$@" build kernelheaders install
 
 	echo '>>'
 	echo '>> Rump kernel components built.  Proceeding to build'
@@ -421,33 +438,6 @@ buildapptools ()
 	${MAKE} -C app-tools BUILDRR=true install
 }
 
-builduserspace ()
-{
-
-	usermtree ${STAGING}
-
-	LIBS="$(stdlibs ${RUMPSRC})"
-	! ${HAVECXX} || LIBS="${LIBS} $(stdlibsxx ${RUMPSRC})"
-
-	userincludes ${RUMPSRC} ${LIBS} $(pwd)/lib/librumprun_tester
-	for lib in ${LIBS}; do
-		makeuserlib ${lib}
-	done
-}
-
-buildpci ()
-{
-
-	if eval ${PLATFORM_PCI_P}; then
-		(
-			cd ${PLATFORMDIR}/pci
-			${RUMPMAKE} ${STDJ} obj
-			${RUMPMAKE} ${STDJ} dependall
-			${RUMPMAKE} ${STDJ} install
-		)
-	fi
-}
-
 wraponetool ()
 {
 
@@ -476,6 +466,7 @@ makeconfig ()
 	echo "TOOLTUPLE=${quote}${TOOLTUPLE}${quote}" >> ${1}
 	echo "KERNONLY=${quote}${KERNONLY}${quote}" >> ${1}
 	echo "PLATFORM=${quote}${PLATFORM}${quote}" >> ${1}
+	echo "RUMPKERNEL=${quote}${RUMPKERNEL}${quote}" >> ${1}
 
 	echo "RRDEST=${quote}${RRDEST}${quote}" >> ${1}
 	echo "RROBJ=${quote}${RROBJ}${quote}" >> ${1}
@@ -517,7 +508,7 @@ dobuild ()
 
 	# do final build of the platform bits
 	( cd ${PLATFORMDIR} \
-	    && ${MAKE} BUILDRR=true \
+	    && ${MAKE} BUILDRR=true RUMPKERNEL=${RUMPKERNEL} \
 	    && ${MAKE} BUILDRR=true install || exit 1)
 	[ $? -eq 0 ] || die platform make failed!
 }
@@ -543,11 +534,19 @@ doinstall ()
 		# first, move things to where we want them to be
 		cd ${STAGING}
 		rm -rf lib/pkgconfig
-		find lib -maxdepth 1 -name librump\*.a \
-		    -exec mv -f '{}' rumprun-${MACHINE_GNU_ARCH}/lib/rumprun-${PLATFORM}/ \;
-		find lib -maxdepth 1 -name \*.a \
-		    -exec mv -f '{}' rumprun-${MACHINE_GNU_ARCH}/lib/ \;
-
+		if [ ${RUMPKERNEL} = "netbsd" ] ; then
+			find lib -maxdepth 1 -name librump\*.a \
+			    -exec mv -f '{}' rumprun-${MACHINE_GNU_ARCH}/lib/rumprun-${PLATFORM}/ \;
+			find lib -maxdepth 1 -name \*.a \
+			    -exec mv -f '{}' rumprun-${MACHINE_GNU_ARCH}/lib/ \;
+		elif [ ${RUMPKERNEL} = "linux" ] ; then
+			find lib -maxdepth 1 -name \*.a \
+			    -exec cp -f '{}' rumprun-${MACHINE_GNU_ARCH}/lib/rumprun-${PLATFORM}/ \;
+			find lib -maxdepth 1 -name \*.a \
+			    -exec mv -f '{}' rumprun-${MACHINE_GNU_ARCH}/lib/ \;
+			# FIXME: need to create empty librump.a for linux
+			ar rc rumprun-${MACHINE_GNU_ARCH}/lib/rumprun-${PLATFORM}/librump.a
+		fi
 		# make sure special cases are visible everywhere
 		for x in c pthread ; do
 			rm -f rumprun-${MACHINE_GNU_ARCH}/lib/rumprun-${PLATFORM}/lib${x}.a

lib/libbmk_core/sched.c

@@ -719,6 +719,14 @@ bmk_sched_init_mainlwp(void *cookie)
 	return bmk_current;
 }
 
+void *bmk_sched_get_cookie(void);
+void *
+bmk_sched_get_cookie(void)
+{
+
+	return bmk_current->bt_cookie;
+}
+
 const char *
 bmk_sched_threadname(struct bmk_thread *thread)
 {

lib/libbmk_rumpuser/rumpuser_synch.c

@@ -123,6 +123,14 @@ rumpuser_thread_join(void *p)
 	return 0;
 }
 
+/* FIXME */
+void *rumpuser_thread_self(void);
+void *
+rumpuser_thread_self(void)
+{
+	return bmk_current;
+}
+
 struct rumpuser_mtx {
 	struct waithead waiters;
 	int v;
@@ -481,3 +489,19 @@ rumpuser_curlwp(void)
 
 	return current_lwp;
 }
+
+
+void *rumpuser_getcookie(void);
+void *bmk_sched_get_cookie(void);
+void *
+rumpuser_getcookie(void)
+{
+	return bmk_sched_get_cookie();
+}
+
+void rumpuser_setcookie(void*);
+void
+rumpuser_setcookie(void* cookie)
+{
+	bmk_sched_init_mainlwp(cookie);
+}

lib/librumprun_base/Makefile

@@ -2,18 +2,27 @@ LIB=		rumprun_base
 
 SRCS=		main.c rumprun.c
 SRCS+=		parseargs.c config.c
-SRCS+=		malloc.c netbsd_initfini.c signals.c
-SRCS+=		syscall_mman.c syscall_misc.c
-SRCS+=		__errno.c _lwp.c libc_stubs.c
-SRCS+=		daemon.c
+SRCS+=		malloc.c
+SRCS+=		__errno.c libc_stubs.c
+SRCS+=		daemon.c syscall_misc.c
 SRCS+=		sysproxy.c
 
+.if ${RUMPKERNEL} == "netbsd"
+SRCS+=		netbsd_initfini.c signals.c
+SRCS+=		_lwp.c
+SRCS+=		syscall_mman.c
+
 # doesn't really belong here, but at the moment we don't have
 # a rumpkernel-only "userspace" lib
 SRCS+=		platefs.c
 
 INCS=		platefs.h
 INCSDIR=	/usr/include/rumprun
+.else
+SRCS+=		linux_initfini.c
+.include "${.CURDIR}/pthread/Makefile.inc"
+CFLAGS+=	-DCONFIG_LKL -DMUSL_LIBC -include ${.CURDIR}/../../rumpkernel/linux.h
+.endif
 
 WARNS=		5