Merge remote-tracking branch 'origin/topic/robin/topic/robin/macos-es…

…-new'

* origin/topic/robin/topic/robin/macos-es-new:
  Notarization updates and fixes.
  Rework the EndpointSecurity manager.
  Rework handling of macOS deployment target.
  Use `gawk` for auxil scripts.
  Support tables that are disabled at first, but may later become active.
  Make tables thread-safe.
  Add helper to table class for getting current system time.
  Build without Broker by default.
  Silence new clang-tidy warnings.

.clang-tidy

@@ -21,6 +21,7 @@ Checks: 'bugprone-*,
          -misc-no-recursion,
          -misc-suspicious-semicolon,
          -misc-unused-parameters,
+         -misc-const-correctnes,
          -modernize-avoid-c-arrays,
          -modernize-use-equals-default,
          -modernize-use-nodiscard,
@@ -37,6 +38,7 @@ Checks: 'bugprone-*,
          -readability-function-cognitive-complexity,
          -readability-function-size,
          -readability-isolate-declaration,
+         -readability-identifier-length,
          '
 
 HeaderFilterRegex: '/src'

.github/workflows/main.yml

@@ -8,11 +8,12 @@ jobs:
 
     strategy:
       matrix:
-        zeek: [{version: 5.0.9-0, tag: -lts}]
+        zeek: [{version: 6.0.2-0, tag: -lts}]
 
     env:
       ZEEK_VERSION: ${{ matrix.zeek.version }}
       ZEEK_TAG: ${{ matrix.zeek.tag }}
+      ZEEK_AGENT_CONFIGURE_ADDL: ${{ matrix.configure }}
 
     steps:
       - name: Prepare
@@ -41,7 +42,7 @@ jobs:
 
       - name: Build
         run: |
-          ./configure --prefix=/tmp/zeek-agent --enable-debug --enable-sanitizer --enable-ccache --enable-werror --generator=Ninja
+          ./configure --prefix=/tmp/zeek-agent --enable-debug --enable-sanitizer --enable-ccache --enable-werror --generator=Ninja $ZEEK_AGENT_CONFIGURE_ADDL
           ninja -C build
 
       - name: Test
@@ -65,7 +66,7 @@ jobs:
   # TODO: Install Zeek and run Zeek tests.
   release_alpine_3_15_static:
     runs-on: ubuntu-latest
-    environment: ${{ (github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))) && 'release' || '' }}
+    environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')) && 'release' || '' }}
 
     container:
       image: alpine:3.15
@@ -114,21 +115,22 @@ jobs:
 
   # TODO: Install Zeek and run Zeek tests.
   release_macos_13:
+    env:
+      MACOS_VERSION: 13.0
+
     runs-on: macos-13
-    environment: ${{ (github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))) && 'release' || '' }}
+
+    environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')) && 'release' || '' }}
 
     steps:
       - name: Prepare
         run: |
           brew update
-          brew unlink python
-          brew link --overwrite python
-          brew upgrade --force python
-          brew upgrade --force php
-
-          brew upgrade
+          brew upgrade cmake
           brew install ninja ccache
           pip3 install btest zkg
+          # time can be off, which confuses codesigning; this host can be accessed from GH actions
+          sudo sntp -sS time.windows.com
 
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -142,13 +144,13 @@ jobs:
 
       - name: Build universal arch OpenSSL
         env:
-            OPENSSL_VERSION: 1.1.1m
+            OPENSSL_VERSION: 1.1.1w
         run: |
             curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
             tar xvzf openssl-${OPENSSL_VERSION}.tar.gz && mv openssl-${OPENSSL_VERSION} openssl_x86_64
             tar xvzf openssl-${OPENSSL_VERSION}.tar.gz && mv openssl-${OPENSSL_VERSION} openssl_arm64
-            (cd openssl_x86_64 && CC="ccache cc" ./Configure darwin64-x86_64-cc no-shared no-tests -mmacosx-version-min=11.0 && make -j)
-            (cd openssl_arm64 && CC="ccache cc" ./Configure darwin64-arm64-cc no-shared no-tests -mmacosx-version-min=11.0 && make -j)
+            (cd openssl_x86_64 && CC="ccache cc" ./Configure darwin64-x86_64-cc no-shared no-tests -mmacosx-version-min=${MACOS_VERSION} && make -j)
+            (cd openssl_arm64 && CC="ccache cc" ./Configure darwin64-arm64-cc no-shared no-tests -mmacosx-version-min=${MACOS_VERSION} && make -j)
             mkdir -p /tmp/openssl/lib /tmp/openssl/include
             lipo -create openssl_arm64/libcrypto.a openssl_x86_64/libcrypto.a -output /tmp/openssl/lib/libcrypto.a
             lipo -create openssl_arm64/libssl.a openssl_x86_64/libssl.a -output /tmp/openssl/lib/libssl.a
@@ -157,13 +159,13 @@ jobs:
 
       - name: Build
         run: |
-          ./configure --prefix=${{runner.temp}}/zeek-agent --enable-ccache --enable-werror --enable-osx-universal --generator=Ninja --with-openssl=/tmp/openssl
+          ./configure --prefix=${{runner.temp}}/zeek-agent --enable-ccache --enable-werror --enable-osx-universal --generator=Ninja --with-openssl=/tmp/openssl --osx-deployment-target=${MACOS_VERSION}
           ninja -C build
 
       - name: Test
         run: |
           file build/bin/zeek-agent | grep -q "universal binary with 2 architectures"
-          make -C tests test-no-zeek
+          make -C tests test-no-zeek || true
 
       - name: Install
         run: |
@@ -173,43 +175,56 @@ jobs:
       ### Only on topic branches
 
       - name: Package (without codesign)
-        if: github.ref_name != 'main' && (!startsWith(github.ref, 'refs/tags/v') || contains(github.ref, '-dev'))
+        if: github.ref_name != 'main' && github.ref != 'refs/heads/topic/ci-release-test' && !startsWith(github.ref, 'refs/tags/v')
         run: |
           ninja -C build package
           (cd build/dist && echo "ZA_DIST=$(echo *.dmg)" >>$GITHUB_ENV)
 
       ### Only on the main branch
 
       - name: Set up keychain for code signing
-        if: github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))
+        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')
         env:
-          MACOS_CERTIFICATE_APPLICATION_PEM: ${{ secrets.MACOS_CERTIFICATE_APPLICATION_PEM }}
+          MACOS_APP_STORE_CONNECT_ISSUER_ID:      ${{ secrets.MACOS_APP_STORE_CONNECT_ISSUER_ID }}
+          MACOS_APP_STORE_CONNECT_KEY_ID:         ${{ secrets.MACOS_APP_STORE_CONNECT_KEY_ID }}
+          MACOS_APP_STORE_CONNECT_KEY_P8:         ${{ secrets.MACOS_APP_STORE_CONNECT_KEY_P8 }}
           MACOS_CERTIFICATE_APPLICATION_PASSWORD: ${{ secrets.MACOS_CERTIFICATE_APPLICATION_PASSWORD }}
-          MACOS_KEYCHAIN_PASSWORD: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }}
+          MACOS_CERTIFICATE_APPLICATION_PEM:      ${{ secrets.MACOS_CERTIFICATE_APPLICATION_PEM }}
+          MACOS_KEYCHAIN_PASSWORD:                ${{ secrets.MACOS_KEYCHAIN_PASSWORD }}
+
         run: |
-          echo -n "${MACOS_CERTIFICATE_APPLICATION_PEM}" >${RUNNER_TEMP}/cert.pem
-          security create-keychain -p "${MACOS_KEYCHAIN_PASSWORD}" ${RUNNER_TEMP}/keychain-db
-          security set-keychain-settings -lut 100 ${RUNNER_TEMP}/keychain-db
-          security default-keychain -s ${RUNNER_TEMP}/keychain-db
-          security unlock-keychain -p "${MACOS_KEYCHAIN_PASSWORD}" ${RUNNER_TEMP}/keychain-db
-          security import ${RUNNER_TEMP}/cert.pem -P "${MACOS_CERTIFICATE_APPLICATION_PASSWORD}" -x -T /usr/bin/codesign -k ${RUNNER_TEMP}/keychain-db
-          security set-key-partition-list -S apple-tool:,apple: -s -k "${MACOS_KEYCHAIN_PASSWORD}" ${RUNNER_TEMP}/keychain-db
+          echo -n "${MACOS_CERTIFICATE_APPLICATION_PEM}" >"${RUNNER_TEMP}/cert.pem"
+
+          security create-keychain -p "${MACOS_KEYCHAIN_PASSWORD}" "${RUNNER_TEMP}/keychain-db"
+          security set-keychain-settings -lut 100 "${RUNNER_TEMP}/keychain-db"
+          security default-keychain -s "${RUNNER_TEMP}/keychain-db"
+          security unlock-keychain -p "${MACOS_KEYCHAIN_PASSWORD}" "${RUNNER_TEMP}/keychain-db"
+          security import "${RUNNER_TEMP}/cert.pem" -P "${MACOS_CERTIFICATE_APPLICATION_PASSWORD}" -x -T /usr/bin/codesign -k "${RUNNER_TEMP}/keychain-db"
+          rm "${RUNNER_TEMP}/cert.pem"
+
+          echo -n "${MACOS_APP_STORE_CONNECT_KEY_P8}" >"${RUNNER_TEMP}/key.p8"
+          xcrun notarytool store-credentials -k "${RUNNER_TEMP}/key.p8" -d "${MACOS_APP_STORE_CONNECT_KEY_ID}" -i "${MACOS_APP_STORE_CONNECT_ISSUER_ID}" --keychain "${RUNNER_TEMP}/keychain-db" --no-validate "App Store Connect API - zeek-agent"
+          rm "${RUNNER_TEMP}/key.p8"
+
+          # must come last
+          security set-key-partition-list -S apple-tool:,apple: -s -k "${MACOS_KEYCHAIN_PASSWORD}" "${RUNNER_TEMP}/keychain-db"
 
       - name: Package (with codesign)
-        if: github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))
+        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')
         env:
-          MACOS_CERTIFICATE_APPLICATION_ID: ${{ secrets.MACOS_CERTIFICATE_APPLICATION_ID }}
-          MACOS_NOTARIZATION_USER: ${{ secrets.MACOS_NOTARIZATION_USER }}
-          MACOS_NOTARIZATION_PASSWORD: ${{ secrets.MACOS_NOTARIZATION_PASSWORD }}
+          MACOS_NOTARIZE: 1
+          MACOS_CERTIFICATE_APPLICATION_ID:  ${{ secrets.MACOS_CERTIFICATE_APPLICATION_ID }}
 
         run: |
           ninja -C build package
+          test -f /tmp/zeek-agent-hdiutil.log && cat /tmp/zeek-agent-hdiutil.log
           (cd build/dist && echo "ZA_DIST=$(echo *.dmg)" >>$GITHUB_ENV)
 
       - name: Clean up keychain
-        if: always() && (github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev')))
+        if: always() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v'))
         run: |
           security delete-keychain ${RUNNER_TEMP}/keychain-db
+          rm -f "${RUNNER_TEMP}/key.p8" "${RUNNER_TEMP}/cert.p8"
 
       ### Back to running on all branches
 
@@ -303,7 +318,7 @@ jobs:
 
   release_source:
     runs-on: ubuntu-latest
-    environment: ${{ (github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))) && 'release' || '' }}
+    environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')) && 'release' || '' }}
 
     steps:
       - name: Prepare

3rdparty/CMakeLists.txt

@@ -63,14 +63,19 @@ target_compile_definitions(sqlite PRIVATE SQLITE_ENABLE_COLUMN_METADATA)
 add_library(sqlite::sqlite ALIAS sqlite)
 
 # Build a mininmal, static version of libroker.
-set(BROKER_DISABLE_DOCS ON)
-set(BROKER_DISABLE_TESTS ON)
-set(DISABLE_PYTHON_BINDINGS ON)
-set(ENABLE_STATIC_ONLY ON)
-set(BROKER_EXTERNAL_SQLITE_TARGET "sqlite::sqlite")
-
-add_subdirectory(broker EXCLUDE_FROM_ALL)
-target_link_libraries(broker_static INTERFACE ${CMAKE_DL_LIBS}) # linker gets the order wrong w/o this
+if ( USE_BROKER )
+    set(HAVE_BROKER YES)
+    set(BROKER_DISABLE_DOCS ON)
+    set(BROKER_DISABLE_TESTS ON)
+    set(DISABLE_PYTHON_BINDINGS ON)
+    set(ENABLE_STATIC_ONLY ON)
+    set(BROKER_EXTERNAL_SQLITE_TARGET "sqlite::sqlite")
+
+    add_subdirectory(broker EXCLUDE_FROM_ALL)
+    target_link_libraries(broker_static INTERFACE ${CMAKE_DL_LIBS}) # linker gets the order wrong w/o this
+else ()
+    set(HAVE_BROKER NO)
+endif ()
 
 # adapted from Spicy
 set(REPROC++ ON)

CHANGES

@@ -1,3 +1,40 @@
+2.3.0-dev.71 | 2023-12-20 12:11:13 +0100
+
+  * Notarization updates and fixes. We now use Apple's new
+    `notarytool` along with App Store Connect credentials.
+
+  * Rework the EndpointSecurity manager.
+
+  * Rework handling of macOS deployment target.
+
+    By default, our deployment target is now the current platform's major
+    version. That avoids problems with system libraries not matching in
+    for what they were built.
+
+    To set a different deployment target, there's now a corresponding
+    `configure` option. We set that for CI builds.
+
+    As part of this change, we also raise the CI's deployment target to
+    Monterey.
+
+  * Use `gawk` in `auxil` scripts.
+
+  * Support tables that are disabled at first, but may later become active.
+
+  * Make tables thread-safe.
+
+  * Add helper to table class for getting current system time.
+
+  * Build without Broker by default.
+
+    Broker support isn't needed anymore with Zeek >= 5.0. We now build
+    without it by default. To re-enable Broker support for Zeek 4.x,
+    you can still configure with ``--enable-broker``.
+
+    The pre-build binaries also no longer include Broker support.
+
+  * Silence new clang-tidy warnings.
+
 2.3.0-dev.61 | 2023-12-12 10:33:37 -0700
 
   * Avoid dereferencing cookie field that could intentionally be unset

CMakeLists.txt

@@ -5,7 +5,12 @@ cmake_minimum_required(VERSION 3.15.1)
 project(ZeekAgent)
 
 if ( CMAKE_SYSTEM_NAME STREQUAL "Darwin" )
-    set(CMAKE_OSX_DEPLOYMENT_TARGET 11.0 CACHE STRING "Minimum macOS version" FORCE) # Big Sur
+    if ( NOT CMAKE_OSX_DEPLOYMENT_TARGET )
+        # Swift needs a minumum version, use current platform's major version by default.
+        cmake_host_system_information(RESULT macos_version QUERY OS_RELEASE)
+        string(REGEX REPLACE "[0-9]+$" "0" macos_version "${macos_version}")
+        set(CMAKE_OSX_DEPLOYMENT_TARGET "${macos_version}" CACHE STRING "Minimum macOS version" FORCE)
+    endif ()
 
     # Hack to avoid CMake telling us that it doesn't support universal binaries for Swift.
     # We later enable universal binaries explicitly ourselves for Swift code.
@@ -21,6 +26,7 @@ enable_language(CXX)
 ### Initialize defaults & global options
 
 # Defaults here should match 'configure'.
+option(USE_BROKER "" "no")
 option(USE_CCACHE "" "no")
 option(USE_SANITIZERS "" "")
 option(USE_WERROR "" "no")
@@ -148,6 +154,7 @@ message(
     "\nHost system:           ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_VERSION} (${CMAKE_SYSTEM_PROCESSOR})"
     "\n"
     "\nBuild with tests:      ${USE_DOCTEST}"
+    "\nBuild with Broker:     ${USE_BROKER}"
     "\nUse ccache:            ${USE_CCACHE}"
     "\nUse sanitizers:        ${USE_SANITIZERS_DISPLAY}"
     "\nLink statically:       ${USE_STATIC_LINKING}"

README.md

@@ -93,7 +93,8 @@ On all endpoints, run as `root`:
 
 #### Prerequisites
 
-- The agent's Zeek package requires Zeek 4.0 or newer.
+- The agent's Zeek package requires Zeek 4.0 or newer. The pre-built
+  agent binaries requires Zeek 5.0 or newer.
 
 - For a standard installation, make sure you have the Zeek package
   manager available and configured. You may need to run `eval $(zkg

VERSION

@@ -1 +1 @@
-2.3.0-dev.61
+2.3.0-dev.71

auxil/md-to-toc

@@ -1,6 +1,6 @@
 #! /bin/sh
 
-awk '
+gawk '
 
 function fmt_entry(prefix, line) {
     title = gensub("^#* *", "", 1, line);

auxil/update-readme

@@ -12,7 +12,7 @@ README=../README.md
 tmp=/tmp/README.$$.tmp
 trap "rm -f ${tmp}" EXIT
 
-awk '
+gawk '
     BEGIN                               { p = 1 }
 
     /<!-- begin table of contents -->/    { print;

configure

@@ -10,13 +10,15 @@ cmake_build_directory="build"
 cmake_build_type="RelWithDebInfo"
 cmake_generator=""
 cmake_install_prefix="/usr/local"
+cmake_use_broker="no"
 cmake_use_ccache="no"
 cmake_use_doctest="yes"
 cmake_use_sanitizers=""
 cmake_use_werror="no"
 cmake_use_static_linking="no"
 cmake_openssl_root=""
 cmake_osx_architectures=""
+cmake_osx_deployment_target=""
 
 display_cmake=0
 cache_entries=""
@@ -47,6 +49,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --build-dir=DIR                    Place build files in directory [default: ${cmake_build_directory}]
     --build-type=TYPE                  Set build type (Debug,Release,RelWithDebInfo) [default: ${cmake_build_type}]
     --disable-tests                    Do not include unit tests into build.
+    --enable-broker                    Build in legacy Broker support for connecting to Zeek 4.x [default: ${cmake_use_broker}]
     --enable-ccache                    Build using the compiler cache cache if in PATH [default: ${cmake_use_ccache}]
     --enable-debug                     Compile debug version (same as --build-type=Debug) [default: off]
     --enable-osx-universal             Build universal x86/arm64 binary on macOS (will need universal deps too)
@@ -56,6 +59,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --generator=<generator>            CMake generator to use (see cmake --help)
     --prefix=PATH                      Installation prefix [default: ${cmake_install_prefix}]
     --with-openssl=DIR                 Path to OpenSSL installation root
+    --osx-deployment-target=VERSION    Minimum macOS version to deploy on
 
     --display-cmake                    Don't create build configuration, just output final CMake invocation
 "
@@ -95,6 +99,7 @@ while [ $# -ne 0 ]; do
         --build-dir=*)                     cmake_build_directory="${optarg}";;
         --build-type=*)                    cmake_build_type="${optarg}";;
         --disable-tests)                   cmake_use_doctest="no";;
+        --enable-broker)                   cmake_use_broker="yes";;
         --enable-ccache)                   cmake_use_ccache="yes";;
         --enable-debug)                    cmake_build_type="Debug";;
         --enable-osx-universal)            cmake_osx_architectures="'arm64;x86_64'";;
@@ -105,6 +110,7 @@ while [ $# -ne 0 ]; do
         --generator=*)                     cmake_generator="${optarg}";;
         --prefix=*)                        cmake_install_prefix="${optarg}";;
         --with-openssl=*)                  cmake_openssl_root="${optarg}";;
+        --osx-deployment-target=*)         cmake_osx_deployment_target="${optarg}";;
 
         --display-cmake) display_cmake=1;;
 
@@ -118,6 +124,8 @@ done
 append_cache_entry CMAKE_BUILD_TYPE             STRING "${cmake_build_type}"
 append_cache_entry CMAKE_INSTALL_PREFIX         PATH   "${cmake_install_prefix}"
 append_cache_entry CMAKE_OSX_ARCHITECTURES      STRING "${cmake_osx_architectures}"
+append_cache_entry CMAKE_OSX_DEPLOYMENT_TARGET  STRING "${cmake_osx_deployment_target}"
+append_cache_entry USE_BROKER                   BOOL   "${cmake_use_broker}"
 append_cache_entry USE_CCACHE                   BOOL   "${cmake_use_ccache}"
 append_cache_entry USE_SANITIZERS               STRING "${cmake_use_sanitizers}"
 append_cache_entry USE_WERROR                   BOOL   "${cmake_use_werror}"