Merge remote-tracking branch 'canonical/master' into bundled-raft-only

.github/workflows/build-and-test.yml

@@ -28,6 +28,8 @@ jobs:
       run: |
           sudo apt update
           sudo apt install -y lcov libsqlite3-dev liblz4-dev libuv1-dev
+          # TODO: remove once the mysterious hang is fixed
+          sudo apt install -y gdb
 
     - name: Build dqlite
       env:
@@ -36,19 +38,23 @@ jobs:
           autoreconf -i
           ./configure --enable-debug --enable-code-coverage --enable-sanitize \
                       --enable-build-raft --enable-dqlite-next=${{ matrix.dqlite-next }}
-          make -j4 unit-test integration-test \
-                   raft-core-fuzzy-test \
-                   raft-core-integration-test \
-                   raft-core-unit-test \
-                   raft-uv-integration-test \
-                   raft-uv-unit-test
+          make -j$(nproc) check-norun
 
     - name: Test
       env:
         CC: ${{ matrix.compiler }}
         LIBDQLITE_TRACE: 1
       run: |
-           make check || (cat ./test-suite.log && false)
+          # TODO: return to just `make check` once the mysterious hang is fixed
+          tests="$(make print-test-programs | tr ' ' '\n' | grep -v '^unit-test$')"
+          make check TESTS="$tests"
+
+          # Grab backtraces when the unit-test binary hangs (this is a heisenbug
+          # that we've only been able to trigger in GHA jobs)
+          ./unit-test --no-fork &
+          pid=$!
+          bash -c "sleep 10m; sudo gdb -p $pid -batch -ex 'thread apply all bt' -ex quit; false" &
+          wait -n
 
     - name: Coverage
       env:

.github/workflows/nolz4.yaml

@@ -0,0 +1,39 @@
+name: CI Tests (no liblz4)
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y libsqlite3-dev libuv1-dev
+          sudo apt remove -y liblz4-dev
+
+    - name: Build dqlite (liblz4 not present)
+      run: |
+          autoreconf -i
+          ./configure --enable-build-raft
+          make -j$(nproc)
+          make clean
+
+    - name: Build dqlite (liblz4 requested and not present)
+      run: |
+          autoreconf -i
+          ! ./configure --enable-build-raft --with-lz4
+    - name: Install liblz4
+      run: |
+          sudo apt install liblz4-dev
+
+    - name: Build dqlite (liblz4 present but ignored)
+      run: |
+          ./configure --enable-build-raft --without-lz4
+          make -j$(nproc)
+          ! ldd .libs/libdqlite.so | grep lz4

.github/workflows/packages.yml

@@ -12,8 +12,8 @@ jobs:
         target:
           - focal
           - jammy
-          - mantic
           - noble
+          - oracular
     runs-on: ubuntu-20.04
     environment:
       name: ppa
@@ -28,6 +28,11 @@ jobs:
           sudo apt-get update -qq
           sudo apt-get install -qq debhelper devscripts gnupg
 
+    - name: Work around GPG issue
+      run: |
+          mkdir -p ~/.gnupg
+          echo "pinentry-mode loopback" >~/.gnupg/gpg.conf
+
     - name: Setup GPG signing key
       env:
           PPA_SECRET_KEY: ${{ secrets.PPA_SECRET_KEY }}

.github/workflows/static.yml

@@ -0,0 +1,22 @@
+name: CI Tests (musl build)
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install dependencies
+      run: |
+           sudo apt install -y build-essential automake libtool gettext autopoint tclsh tcl libsqlite3-dev pkg-config git
+
+    - name: Build and test
+      env:
+        LIBDQLITE_TRACE: 1
+      run: |
+           contrib/build-static.sh || (cat ./test-suite.log && false)

Makefile.am

@@ -2,7 +2,16 @@ ACLOCAL_AMFLAGS = -I m4
 
 AM_CFLAGS += $(CODE_COVERAGE_CFLAGS)
 AM_CFLAGS += $(SQLITE_CFLAGS) $(UV_CFLAGS) $(PTHREAD_CFLAGS)
-AM_LDFLAGS = $(UV_LIBS) $(PTHREAD_LIBS)
+
+if WITH_STATIC_DEPS
+AM_CFLAGS += -DDQLITE_STATIC_LIBC
+static = -all-static
+else
+static =
+endif
+
+AM_LDFLAGS = $(static)
+AM_LDFLAGS += $(UV_LIBS) $(PTHREAD_LIBS)
 
 if DQLITE_NEXT_ENABLED
 AM_CFLAGS += -DDQLITE_NEXT
@@ -206,7 +215,7 @@ libtest_la_SOURCES += \
   test/raft/lib/loop.c
 
 libraft_la_CFLAGS = $(AM_CFLAGS)
-libraft_la_LDFLAGS = $(UV_LIBS)
+libraft_la_LDFLAGS = $(static) $(UV_LIBS)
 
 raft_core_unit_test_SOURCES = \
   $(libraft_la_SOURCES) \
@@ -250,7 +259,7 @@ raft_core_integration_test_SOURCES = \
   test/raft/integration/test_transfer.c \
   test/raft/integration/test_voter_contacts.c
 raft_core_integration_test_CFLAGS = $(AM_CFLAGS) -Wno-conversion
-raft_core_integration_test_LDFLAGS = -no-install
+raft_core_integration_test_LDFLAGS = $(static) -no-install
 raft_core_integration_test_LDADD = libtest.la libraft.la
 
 raft_core_fuzzy_test_SOURCES = \
@@ -263,7 +272,7 @@ raft_core_fuzzy_test_SOURCES = \
   test/raft/fuzzy/test_membership.c \
   test/raft/fuzzy/test_replication.c
 raft_core_fuzzy_test_CFLAGS = $(AM_CFLAGS) -Wno-conversion
-raft_core_fuzzy_test_LDFLAGS = -no-install
+raft_core_fuzzy_test_LDFLAGS = $(static) -no-install
 raft_core_fuzzy_test_LDADD = libtest.la libraft.la
 
 raft_uv_unit_test_SOURCES = \
@@ -305,14 +314,14 @@ raft_uv_integration_test_SOURCES = \
   test/raft/integration/test_uv_truncate_snapshot.c \
   test/raft/integration/test_uv_work.c
 raft_uv_integration_test_CFLAGS = $(AM_CFLAGS) -Wno-type-limits -Wno-conversion
-raft_uv_integration_test_LDFLAGS = -no-install
+raft_uv_integration_test_LDFLAGS = $(static) -no-install
 raft_uv_integration_test_LDADD = libtest.la $(UV_LIBS)
 
 if LZ4_AVAILABLE
 libdqlite_la_CFLAGS += -DLZ4_AVAILABLE $(LZ4_CFLAGS)
 libdqlite_la_LDFLAGS += $(LZ4_LIBS)
 raft_core_unit_test_CFLAGS += -DLZ4_AVAILABLE $(LZ4_CFLAGS)
-raft_core_unit_test_LDFLAGS = $(LZ4_LIBS)
+raft_core_unit_test_LDFLAGS = $(static) $(LZ4_LIBS)
 libraft_la_CFLAGS += -DLZ4_AVAILABLE $(LZ4_CFLAGS)
 libraft_la_LDFLAGS += $(LZ4_LIBS)
 raft_uv_integration_test_CFLAGS += -DLZ4_AVAILABLE
@@ -339,6 +348,14 @@ endif
 
 TESTS = $(check_PROGRAMS)
 
+check-norun: $(TESTS)
+
+# Sometimes we want to run the test binaries manually. This target allows us to
+# get the list of binaries out of the build system for that use-case,
+# maintaining a single source of truth.
+print-test-programs:
+	@echo '$(TESTS)'
+
 if CODE_COVERAGE_ENABLED
 
 include $(top_srcdir)/aminclude_static.am

README.md

@@ -119,6 +119,28 @@ The `--enable-build-raft` option causes dqlite to use its bundled Raft
 implementation instead of linking to an external libraft; the latter is a
 legacy configuration that should not be used for new development.
 
+Building for static linking
+---------------------------
+
+If you're building dqlite for eventual use in a statically-linked
+binary, there are some additional considerations. You should pass
+`--with-static-deps` to the configure script; this disables code that
+relies on dependencies being dynamically linked. (Currently it only
+affects the test suite, but you should use it even when building
+`libdqlite.a` only for future compatibility.)
+
+When linking libdqlite with musl libc, it's recommended to increase
+the default stack size, which is otherwise too low for dqlite's
+needs:
+
+```
+LDFLAGS="-Wl,-z,stack-size=1048576"
+```
+
+The `contrib/build-static.sh` script demonstrates building and
+testing dqlite with all dependencies (including libc) statically
+linked.
+
 Usage Notes
 -----------
 

configure.ac

@@ -6,8 +6,7 @@ AC_CONFIG_AUX_DIR([ac])
 AM_INIT_AUTOMAKE([subdir-objects -Wall -Werror -Wno-portability foreign])
 AM_SILENT_RULES([yes])
 
-# Without this line, AC_PROG_CC boneheadedly adds `-g -O2` to our CFLAGS.
-AC_SUBST(CFLAGS, "")
+AC_SUBST(AM_CFLAGS)
 AC_PROG_CC
 AC_USE_SYSTEM_EXTENSIONS
 
@@ -48,6 +47,11 @@ AS_IF([test "x$enable_build_raft" != "xyes"],
 AC_ARG_ENABLE(dqlite-next, AS_HELP_STRING([--enable-dqlite-next[=ARG]], [build with the experimental dqlite backend [default=no]]))
 AM_CONDITIONAL(DQLITE_NEXT_ENABLED, test "x$enable_dqlite_next" = "xyes")
 
+AC_ARG_WITH(static-deps,
+	    AS_HELP_STRING([--with-static-deps[=ARG]],
+			   [skip building a shared library and link test binaries statically]))
+AM_CONDITIONAL(WITH_STATIC_DEPS, test "x$with_static_deps" = "xyes")
+
 # Whether to enable code coverage.
 AX_CODE_COVERAGE
 
@@ -71,7 +75,7 @@ AC_ARG_WITH([lz4], AS_HELP_STRING([--without-lz4], [never link to liblz4]))
 AS_IF([test "x$with_lz4" != "xno"],
      [PKG_CHECK_MODULES(LZ4, [liblz4 >= 1.7.1], [have_lz4=yes], [have_lz4=no])],
      [have_lz4=no])
-AS_IF([test "x$with_lz4" != "xno" -a "x$have_lz4" = "xno"],
+AS_IF([test "x$with_lz4" = "xyes" -a "x$have_lz4" = "xno"],
      [AC_MSG_ERROR([liblz4 required but not found])],
      [])
 
@@ -92,10 +96,8 @@ CC_CHECK_FLAGS_APPEND([AM_CFLAGS],[CFLAGS],[ \
   -pipe \
   -fno-strict-aliasing \
   -fdiagnostics-color \
-  -fexceptions \
   -fstack-clash-protection \
   -fstack-protector-strong \
-  -fasynchronous-unwind-tables \
   -fdiagnostics-show-option \
   -Wall \
   -Wextra \
@@ -114,6 +116,7 @@ CC_CHECK_FLAGS_APPEND([AM_CFLAGS],[CFLAGS],[ \
   -Wdate-time \
   -Wnested-externs \
   -Wconversion \
+  -Wno-format-nonliteral \
   -Werror \
 ])
 # To enable: