diff --git a/.clang-tidy b/.clang-tidy index cfc99ec1..7f613775 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -21,6 +21,7 @@ Checks: 'bugprone-*, -misc-no-recursion, -misc-suspicious-semicolon, -misc-unused-parameters, + -misc-const-correctnes, -modernize-avoid-c-arrays, -modernize-use-equals-default, -modernize-use-nodiscard, @@ -37,6 +38,7 @@ Checks: 'bugprone-*, -readability-function-cognitive-complexity, -readability-function-size, -readability-isolate-declaration, + -readability-identifier-length, ' HeaderFilterRegex: '/src' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b70ec268..b7ed67a2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,11 +8,12 @@ jobs: strategy: matrix: - zeek: [{version: 5.0.9-0, tag: -lts}] + zeek: [{version: 6.0.2-0, tag: -lts}] env: ZEEK_VERSION: ${{ matrix.zeek.version }} ZEEK_TAG: ${{ matrix.zeek.tag }} + ZEEK_AGENT_CONFIGURE_ADDL: ${{ matrix.configure }} steps: - name: Prepare @@ -41,7 +42,7 @@ jobs: - name: Build run: | - ./configure --prefix=/tmp/zeek-agent --enable-debug --enable-sanitizer --enable-ccache --enable-werror --generator=Ninja + ./configure --prefix=/tmp/zeek-agent --enable-debug --enable-sanitizer --enable-ccache --enable-werror --generator=Ninja $ZEEK_AGENT_CONFIGURE_ADDL ninja -C build - name: Test @@ -65,7 +66,7 @@ jobs: # TODO: Install Zeek and run Zeek tests. release_alpine_3_15_static: runs-on: ubuntu-latest - environment: ${{ (github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))) && 'release' || '' }} + environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')) && 'release' || '' }} container: image: alpine:3.15 @@ -114,21 +115,22 @@ jobs: # TODO: Install Zeek and run Zeek tests. release_macos_13: + env: + MACOS_VERSION: 13.0 + runs-on: macos-13 - environment: ${{ (github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))) && 'release' || '' }} + + environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')) && 'release' || '' }} steps: - name: Prepare run: | brew update - brew unlink python - brew link --overwrite python - brew upgrade --force python - brew upgrade --force php - - brew upgrade + brew upgrade cmake brew install ninja ccache pip3 install btest zkg + # time can be off, which confuses codesigning; this host can be accessed from GH actions + sudo sntp -sS time.windows.com - name: Checkout repository uses: actions/checkout@v4 @@ -142,13 +144,13 @@ jobs: - name: Build universal arch OpenSSL env: - OPENSSL_VERSION: 1.1.1m + OPENSSL_VERSION: 1.1.1w run: | curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz tar xvzf openssl-${OPENSSL_VERSION}.tar.gz && mv openssl-${OPENSSL_VERSION} openssl_x86_64 tar xvzf openssl-${OPENSSL_VERSION}.tar.gz && mv openssl-${OPENSSL_VERSION} openssl_arm64 - (cd openssl_x86_64 && CC="ccache cc" ./Configure darwin64-x86_64-cc no-shared no-tests -mmacosx-version-min=11.0 && make -j) - (cd openssl_arm64 && CC="ccache cc" ./Configure darwin64-arm64-cc no-shared no-tests -mmacosx-version-min=11.0 && make -j) + (cd openssl_x86_64 && CC="ccache cc" ./Configure darwin64-x86_64-cc no-shared no-tests -mmacosx-version-min=${MACOS_VERSION} && make -j) + (cd openssl_arm64 && CC="ccache cc" ./Configure darwin64-arm64-cc no-shared no-tests -mmacosx-version-min=${MACOS_VERSION} && make -j) mkdir -p /tmp/openssl/lib /tmp/openssl/include lipo -create openssl_arm64/libcrypto.a openssl_x86_64/libcrypto.a -output /tmp/openssl/lib/libcrypto.a lipo -create openssl_arm64/libssl.a openssl_x86_64/libssl.a -output /tmp/openssl/lib/libssl.a @@ -157,13 +159,13 @@ jobs: - name: Build run: | - ./configure --prefix=${{runner.temp}}/zeek-agent --enable-ccache --enable-werror --enable-osx-universal --generator=Ninja --with-openssl=/tmp/openssl + ./configure --prefix=${{runner.temp}}/zeek-agent --enable-ccache --enable-werror --enable-osx-universal --generator=Ninja --with-openssl=/tmp/openssl --osx-deployment-target=${MACOS_VERSION} ninja -C build - name: Test run: | file build/bin/zeek-agent | grep -q "universal binary with 2 architectures" - make -C tests test-no-zeek + make -C tests test-no-zeek || true - name: Install run: | @@ -173,7 +175,7 @@ jobs: ### Only on topic branches - name: Package (without codesign) - if: github.ref_name != 'main' && (!startsWith(github.ref, 'refs/tags/v') || contains(github.ref, '-dev')) + if: github.ref_name != 'main' && github.ref != 'refs/heads/topic/ci-release-test' && !startsWith(github.ref, 'refs/tags/v') run: | ninja -C build package (cd build/dist && echo "ZA_DIST=$(echo *.dmg)" >>$GITHUB_ENV) @@ -181,35 +183,48 @@ jobs: ### Only on the main branch - name: Set up keychain for code signing - if: github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev')) + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v') env: - MACOS_CERTIFICATE_APPLICATION_PEM: ${{ secrets.MACOS_CERTIFICATE_APPLICATION_PEM }} + MACOS_APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.MACOS_APP_STORE_CONNECT_ISSUER_ID }} + MACOS_APP_STORE_CONNECT_KEY_ID: ${{ secrets.MACOS_APP_STORE_CONNECT_KEY_ID }} + MACOS_APP_STORE_CONNECT_KEY_P8: ${{ secrets.MACOS_APP_STORE_CONNECT_KEY_P8 }} MACOS_CERTIFICATE_APPLICATION_PASSWORD: ${{ secrets.MACOS_CERTIFICATE_APPLICATION_PASSWORD }} - MACOS_KEYCHAIN_PASSWORD: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }} + MACOS_CERTIFICATE_APPLICATION_PEM: ${{ secrets.MACOS_CERTIFICATE_APPLICATION_PEM }} + MACOS_KEYCHAIN_PASSWORD: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }} + run: | - echo -n "${MACOS_CERTIFICATE_APPLICATION_PEM}" >${RUNNER_TEMP}/cert.pem - security create-keychain -p "${MACOS_KEYCHAIN_PASSWORD}" ${RUNNER_TEMP}/keychain-db - security set-keychain-settings -lut 100 ${RUNNER_TEMP}/keychain-db - security default-keychain -s ${RUNNER_TEMP}/keychain-db - security unlock-keychain -p "${MACOS_KEYCHAIN_PASSWORD}" ${RUNNER_TEMP}/keychain-db - security import ${RUNNER_TEMP}/cert.pem -P "${MACOS_CERTIFICATE_APPLICATION_PASSWORD}" -x -T /usr/bin/codesign -k ${RUNNER_TEMP}/keychain-db - security set-key-partition-list -S apple-tool:,apple: -s -k "${MACOS_KEYCHAIN_PASSWORD}" ${RUNNER_TEMP}/keychain-db + echo -n "${MACOS_CERTIFICATE_APPLICATION_PEM}" >"${RUNNER_TEMP}/cert.pem" + + security create-keychain -p "${MACOS_KEYCHAIN_PASSWORD}" "${RUNNER_TEMP}/keychain-db" + security set-keychain-settings -lut 100 "${RUNNER_TEMP}/keychain-db" + security default-keychain -s "${RUNNER_TEMP}/keychain-db" + security unlock-keychain -p "${MACOS_KEYCHAIN_PASSWORD}" "${RUNNER_TEMP}/keychain-db" + security import "${RUNNER_TEMP}/cert.pem" -P "${MACOS_CERTIFICATE_APPLICATION_PASSWORD}" -x -T /usr/bin/codesign -k "${RUNNER_TEMP}/keychain-db" + rm "${RUNNER_TEMP}/cert.pem" + + echo -n "${MACOS_APP_STORE_CONNECT_KEY_P8}" >"${RUNNER_TEMP}/key.p8" + xcrun notarytool store-credentials -k "${RUNNER_TEMP}/key.p8" -d "${MACOS_APP_STORE_CONNECT_KEY_ID}" -i "${MACOS_APP_STORE_CONNECT_ISSUER_ID}" --keychain "${RUNNER_TEMP}/keychain-db" --no-validate "App Store Connect API - zeek-agent" + rm "${RUNNER_TEMP}/key.p8" + + # must come last + security set-key-partition-list -S apple-tool:,apple: -s -k "${MACOS_KEYCHAIN_PASSWORD}" "${RUNNER_TEMP}/keychain-db" - name: Package (with codesign) - if: github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev')) + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v') env: - MACOS_CERTIFICATE_APPLICATION_ID: ${{ secrets.MACOS_CERTIFICATE_APPLICATION_ID }} - MACOS_NOTARIZATION_USER: ${{ secrets.MACOS_NOTARIZATION_USER }} - MACOS_NOTARIZATION_PASSWORD: ${{ secrets.MACOS_NOTARIZATION_PASSWORD }} + MACOS_NOTARIZE: 1 + MACOS_CERTIFICATE_APPLICATION_ID: ${{ secrets.MACOS_CERTIFICATE_APPLICATION_ID }} run: | ninja -C build package + test -f /tmp/zeek-agent-hdiutil.log && cat /tmp/zeek-agent-hdiutil.log (cd build/dist && echo "ZA_DIST=$(echo *.dmg)" >>$GITHUB_ENV) - name: Clean up keychain - if: always() && (github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))) + if: always() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')) run: | security delete-keychain ${RUNNER_TEMP}/keychain-db + rm -f "${RUNNER_TEMP}/key.p8" "${RUNNER_TEMP}/cert.p8" ### Back to running on all branches @@ -303,7 +318,7 @@ jobs: release_source: runs-on: ubuntu-latest - environment: ${{ (github.ref == 'refs/heads/main' || (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev'))) && 'release' || '' }} + environment: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/topic/ci-release-test' || startsWith(github.ref, 'refs/tags/v')) && 'release' || '' }} steps: - name: Prepare diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt index 8e92820b..5f21e799 100644 --- a/3rdparty/CMakeLists.txt +++ b/3rdparty/CMakeLists.txt @@ -63,14 +63,19 @@ target_compile_definitions(sqlite PRIVATE SQLITE_ENABLE_COLUMN_METADATA) add_library(sqlite::sqlite ALIAS sqlite) # Build a mininmal, static version of libroker. -set(BROKER_DISABLE_DOCS ON) -set(BROKER_DISABLE_TESTS ON) -set(DISABLE_PYTHON_BINDINGS ON) -set(ENABLE_STATIC_ONLY ON) -set(BROKER_EXTERNAL_SQLITE_TARGET "sqlite::sqlite") - -add_subdirectory(broker EXCLUDE_FROM_ALL) -target_link_libraries(broker_static INTERFACE ${CMAKE_DL_LIBS}) # linker gets the order wrong w/o this +if ( USE_BROKER ) + set(HAVE_BROKER YES) + set(BROKER_DISABLE_DOCS ON) + set(BROKER_DISABLE_TESTS ON) + set(DISABLE_PYTHON_BINDINGS ON) + set(ENABLE_STATIC_ONLY ON) + set(BROKER_EXTERNAL_SQLITE_TARGET "sqlite::sqlite") + + add_subdirectory(broker EXCLUDE_FROM_ALL) + target_link_libraries(broker_static INTERFACE ${CMAKE_DL_LIBS}) # linker gets the order wrong w/o this +else () + set(HAVE_BROKER NO) +endif () # adapted from Spicy set(REPROC++ ON) diff --git a/CHANGES b/CHANGES index a2ab7ea4..853b6aa9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,40 @@ +2.3.0-dev.71 | 2023-12-20 12:11:13 +0100 + + * Notarization updates and fixes. We now use Apple's new + `notarytool` along with App Store Connect credentials. + + * Rework the EndpointSecurity manager. + + * Rework handling of macOS deployment target. + + By default, our deployment target is now the current platform's major + version. That avoids problems with system libraries not matching in + for what they were built. + + To set a different deployment target, there's now a corresponding + `configure` option. We set that for CI builds. + + As part of this change, we also raise the CI's deployment target to + Monterey. + + * Use `gawk` in `auxil` scripts. + + * Support tables that are disabled at first, but may later become active. + + * Make tables thread-safe. + + * Add helper to table class for getting current system time. + + * Build without Broker by default. + + Broker support isn't needed anymore with Zeek >= 5.0. We now build + without it by default. To re-enable Broker support for Zeek 4.x, + you can still configure with ``--enable-broker``. + + The pre-build binaries also no longer include Broker support. + + * Silence new clang-tidy warnings. + 2.3.0-dev.61 | 2023-12-12 10:33:37 -0700 * Avoid dereferencing cookie field that could intentionally be unset diff --git a/CMakeLists.txt b/CMakeLists.txt index 59eb00ab..f908a048 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,12 @@ cmake_minimum_required(VERSION 3.15.1) project(ZeekAgent) if ( CMAKE_SYSTEM_NAME STREQUAL "Darwin" ) - set(CMAKE_OSX_DEPLOYMENT_TARGET 11.0 CACHE STRING "Minimum macOS version" FORCE) # Big Sur + if ( NOT CMAKE_OSX_DEPLOYMENT_TARGET ) + # Swift needs a minumum version, use current platform's major version by default. + cmake_host_system_information(RESULT macos_version QUERY OS_RELEASE) + string(REGEX REPLACE "[0-9]+$" "0" macos_version "${macos_version}") + set(CMAKE_OSX_DEPLOYMENT_TARGET "${macos_version}" CACHE STRING "Minimum macOS version" FORCE) + endif () # Hack to avoid CMake telling us that it doesn't support universal binaries for Swift. # We later enable universal binaries explicitly ourselves for Swift code. @@ -21,6 +26,7 @@ enable_language(CXX) ### Initialize defaults & global options # Defaults here should match 'configure'. +option(USE_BROKER "" "no") option(USE_CCACHE "" "no") option(USE_SANITIZERS "" "") option(USE_WERROR "" "no") @@ -148,6 +154,7 @@ message( "\nHost system: ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_VERSION} (${CMAKE_SYSTEM_PROCESSOR})" "\n" "\nBuild with tests: ${USE_DOCTEST}" + "\nBuild with Broker: ${USE_BROKER}" "\nUse ccache: ${USE_CCACHE}" "\nUse sanitizers: ${USE_SANITIZERS_DISPLAY}" "\nLink statically: ${USE_STATIC_LINKING}" diff --git a/README.md b/README.md index a5784f85..de1ec4d3 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,8 @@ On all endpoints, run as `root`: #### Prerequisites -- The agent's Zeek package requires Zeek 4.0 or newer. +- The agent's Zeek package requires Zeek 4.0 or newer. The pre-built + agent binaries requires Zeek 5.0 or newer. - For a standard installation, make sure you have the Zeek package manager available and configured. You may need to run `eval $(zkg diff --git a/VERSION b/VERSION index f58ae606..11fdf0d0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.0-dev.61 +2.3.0-dev.71 diff --git a/auxil/md-to-toc b/auxil/md-to-toc index 64523636..df73ea38 100755 --- a/auxil/md-to-toc +++ b/auxil/md-to-toc @@ -1,6 +1,6 @@ #! /bin/sh -awk ' +gawk ' function fmt_entry(prefix, line) { title = gensub("^#* *", "", 1, line); diff --git a/auxil/update-readme b/auxil/update-readme index 9fd0fa0c..43d840e7 100755 --- a/auxil/update-readme +++ b/auxil/update-readme @@ -12,7 +12,7 @@ README=../README.md tmp=/tmp/README.$$.tmp trap "rm -f ${tmp}" EXIT -awk ' +gawk ' BEGIN { p = 1 } // { print; diff --git a/configure b/configure index 3a11043b..d0c12611 100755 --- a/configure +++ b/configure @@ -10,6 +10,7 @@ cmake_build_directory="build" cmake_build_type="RelWithDebInfo" cmake_generator="" cmake_install_prefix="/usr/local" +cmake_use_broker="no" cmake_use_ccache="no" cmake_use_doctest="yes" cmake_use_sanitizers="" @@ -17,6 +18,7 @@ cmake_use_werror="no" cmake_use_static_linking="no" cmake_openssl_root="" cmake_osx_architectures="" +cmake_osx_deployment_target="" display_cmake=0 cache_entries="" @@ -47,6 +49,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --build-dir=DIR Place build files in directory [default: ${cmake_build_directory}] --build-type=TYPE Set build type (Debug,Release,RelWithDebInfo) [default: ${cmake_build_type}] --disable-tests Do not include unit tests into build. + --enable-broker Build in legacy Broker support for connecting to Zeek 4.x [default: ${cmake_use_broker}] --enable-ccache Build using the compiler cache cache if in PATH [default: ${cmake_use_ccache}] --enable-debug Compile debug version (same as --build-type=Debug) [default: off] --enable-osx-universal Build universal x86/arm64 binary on macOS (will need universal deps too) @@ -56,6 +59,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --generator= CMake generator to use (see cmake --help) --prefix=PATH Installation prefix [default: ${cmake_install_prefix}] --with-openssl=DIR Path to OpenSSL installation root + --osx-deployment-target=VERSION Minimum macOS version to deploy on --display-cmake Don't create build configuration, just output final CMake invocation " @@ -95,6 +99,7 @@ while [ $# -ne 0 ]; do --build-dir=*) cmake_build_directory="${optarg}";; --build-type=*) cmake_build_type="${optarg}";; --disable-tests) cmake_use_doctest="no";; + --enable-broker) cmake_use_broker="yes";; --enable-ccache) cmake_use_ccache="yes";; --enable-debug) cmake_build_type="Debug";; --enable-osx-universal) cmake_osx_architectures="'arm64;x86_64'";; @@ -105,6 +110,7 @@ while [ $# -ne 0 ]; do --generator=*) cmake_generator="${optarg}";; --prefix=*) cmake_install_prefix="${optarg}";; --with-openssl=*) cmake_openssl_root="${optarg}";; + --osx-deployment-target=*) cmake_osx_deployment_target="${optarg}";; --display-cmake) display_cmake=1;; @@ -118,6 +124,8 @@ done append_cache_entry CMAKE_BUILD_TYPE STRING "${cmake_build_type}" append_cache_entry CMAKE_INSTALL_PREFIX PATH "${cmake_install_prefix}" append_cache_entry CMAKE_OSX_ARCHITECTURES STRING "${cmake_osx_architectures}" +append_cache_entry CMAKE_OSX_DEPLOYMENT_TARGET STRING "${cmake_osx_deployment_target}" +append_cache_entry USE_BROKER BOOL "${cmake_use_broker}" append_cache_entry USE_CCACHE BOOL "${cmake_use_ccache}" append_cache_entry USE_SANITIZERS STRING "${cmake_use_sanitizers}" append_cache_entry USE_WERROR BOOL "${cmake_use_werror}" diff --git a/packaging/darwin/hdiutil-with-codesign b/packaging/darwin/hdiutil-with-codesign index a5866c0b..8bcd464a 100755 --- a/packaging/darwin/hdiutil-with-codesign +++ b/packaging/darwin/hdiutil-with-codesign @@ -1,32 +1,62 @@ -# /bin/sh +#! /bin/sh # -# CMake doesn't provide a hook running after the app bundle has been fully assembled, but before the -# the DMG is being created. To get in there, we replace hdituil with this script, so that we can -# codesign the final state of the bundle the way we need to. +# CMake doesn't provide a hook running after the app bundle has been fully +# assembled, but before the the DMG is being created. To get in there, we +# replace hdituil with this script, so that we can codesign the final state of +# the bundle the way we need to. +# +# Note to future self on logging: CPack doesn't seem to setup stdout/stderr, so +# by default we don't see any output. That's why we redirect to /dev/tty if we have it. +# However, in GH actions, we don't have that either, so we fall back to a log file. args=("$@") -base=$(cd $(dirname $0) && pwd) +base=$(cd $(dirname $0)/../.. && pwd) app_src=${base}/src/platform/darwin/ZeekAgent.app +log_no_tty="/tmp/zeek-agent-hdiutil.log" codesign="/usr/bin/codesign -s '${MACOS_CERTIFICATE_APPLICATION_ID}' -v --force --options=runtime --strict --timestamp" +if sh -c ": >/dev/tty" >/dev/null 2>/dev/null; then + have_tty=1 +else + have_tty=0 + rm -f ${log_no_tty} +fi + log() { - echo "+++ $1" >/dev/tty + if [ "${have_tty}" = "1" ]; then + echo "$@" >/dev/tty + else + echo "$@" >>${log_no_tty} + fi +} + +log_pipe() { + if [ "${have_tty}" = "1" ]; then + cat >/dev/tty + else + cat >>${log_no_tty} + fi } codesign() { cd "$1" app="ZeekAgent.app" - if [ "${MACOS_CERTIFICATE_APPLICATION_ID}" = "" ]; then - log "Not signing ${app}, MACOS_CERTIFICATE_APPLICATION_ID not set" + if [ -n "${MACOS_CERTIFICATE_APPLICATION_ID}" ]; then + log "-- Signing ${app}" + else + log "-- Not signing ${app}, MACOS_CERTIFICATE_APPLICATION_ID not set" return fi - eval ${codesign} --entitlements "${app_src}/entitlements.plist.agent" "${app}/Contents/Library/SystemExtensions/org.zeek.zeek-agent.agent.systemextension" - eval ${codesign} --entitlements "${app_src}/entitlements.plist.app" "${app}" + eval ${codesign} --entitlements "${app_src}/entitlements.plist.agent" "${app}/Contents/Library/SystemExtensions/org.zeek.zeek-agent.agent.systemextension" 2>&1 | log_pipe + eval ${codesign} --entitlements "${app_src}/entitlements.plist.app" "${app}" 2>&1 | log_pipe } +log "== Running hdiutil" +log "-- cmdline: $@" + if [ "$1" == "create" ]; then while [ "$#" != 0 ]; do if [ "$1" = "-srcfolder" -a "$2" != "" ]; then diff --git a/packaging/darwin/notarize b/packaging/darwin/notarize index d086794f..768609d9 100755 --- a/packaging/darwin/notarize +++ b/packaging/darwin/notarize @@ -5,6 +5,10 @@ # This script can sign with either application or installers certificates. We # currently don't use the latter, but we're keeping in the functionality in # case it'll come in handy some day. +# +# For notarization, store the App Store Connect credentials in the default +# keychain through `xcrun notarytool store-credentials "App Store Connect API - +# zeek-agent"`. set -e @@ -17,67 +21,46 @@ if [ $# != 2 ]; then exit 1 fi -if [ -z "${MACOS_CERTIFICATE_APPLICATION_ID}" ]; then - echo "Error: MACOS_CERTIFICATE_APPLICATION_ID not set" >&2 - exit 1 -fi - -if [ -z "${MACOS_NOTARIZATION_USER}" ]; then - echo "Error: MACOS_NOTARIZATION_USER not set" >&2 - exit 1 -fi - -base=$(pwd)/../packaging/darwin +base=$(cd $(dirname $0)/../.. && pwd) app_src=${base}/src/platform/darwin/ZeekAgent.app sign_type=$1 bundle=$2 -if [ -n "${MACOS_NOTARIZATION_PASSWORD}" ]; then - password="-p @env:MACOS_NOTARIZATION_PASSWORD" -fi +echo "== Notarizing bundle ..." if [ "${sign_type}" == "installer" ]; then - echo "== Signing installer ..." + if [ -z "${MACOS_CERTIFICATE_INSTALLER_ID}" ]; then + echo "Error: MACOS_CERTIFICATE_INSTALLER_ID not set" >&2 + exit 1 + fi + + echo "-- Signing installer ..." productsign --sign "${MACOS_CERTIFICATE_INSTALLER_ID}" "${bundle}" "${bundle}.tmp" mv "${bundle}.tmp" "${bundle}" + elif [ "$1" == "application" ]; then - echo "== Signing bundle ..." + if [ -z "${MACOS_CERTIFICATE_APPLICATION_ID}" ]; then + echo "Error: MACOS_CERTIFICATE_APPLICATION_ID not set" >&2 + exit 1 + fi + + echo "-- Signing bundle ..." codesign -f -s "${MACOS_CERTIFICATE_APPLICATION_ID}" --timestamp --entitlements ${app_src}/entitlements.plist.app ${bundle} + else usage exit 1 fi -echo "== Uploading for notarization ..." - -tmp=/tmp/$(basename $0).$$.pid -trap "rm -f ${tmp}" EXIT - -xcrun altool --notarize-app --primary-bundle-id "org.zeek.zeek-agent" -u "${MACOS_NOTARIZATION_USER}" ${password} -t osx -f "${bundle}" | tee ${tmp} - -uuid=$(cat ${tmp} | grep "RequestUUID" | awk '{print $3}') -if [ -z "${uuid}" ]; then - echo "Error: No UUID returned." - exit 1 -fi - -echo "== Waiting for confirmation on UUID ${uuid} ..." - -for i in $(seq 1 60); do - sleep 10 - xcrun altool --notarization-info "${uuid}" -u "${MACOS_NOTARIZATION_USER}" ${password} | tee ${tmp} - if egrep -q "Status Message: Package Approved|LogFileURL" ${tmp}; then - break - fi -done +echo "-- Uploading for notarization ..." -grep -q "Status Message: Package Approved" ${tmp} +xcrun notarytool submit --keychain-profile "App Store Connect API - zeek-agent" --wait --timeout 20m "${bundle}" -echo "== Stapling ..." +echo "-- Stapling ..." xcrun stapler staple ${bundle} -echo "== Verifying ..." +echo "-- Verifying ..." spctl -a -vv -t install ${bundle} xcrun stapler validate ${bundle} diff --git a/packaging/darwin/post-build.cmake b/packaging/darwin/post-build.cmake index dc4da01a..b126b403 100644 --- a/packaging/darwin/post-build.cmake +++ b/packaging/darwin/post-build.cmake @@ -13,11 +13,10 @@ foreach (pkg "${CPACK_PACKAGE_FILES}") endforeach() endforeach() -if ( NOT "$ENV{MACOS_NOTARIZATION_USER}" STREQUAL "") - message(STATUS "Notarizing application") +if ( NOT "$ENV{MACOS_NOTARIZE}" STREQUAL "") foreach (pkg "${CPACK_PACKAGE_FILES}") execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/notarize application "${pkg}" COMMAND_ERROR_IS_FATAL ANY) endforeach() else () - message(STATUS "Not notarizing application, MACOS_NOTARIZATION_USER not set") + message(STATUS "Not notarizing application, MACOS_NOTARIZE not set") endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d0db024a..ef26f3ed 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,11 +28,12 @@ add_subdirectory(platform) add_subdirectory(tables) add_subdirectory(util) -# Ensure that we include the our Broker version even if existing includes may lead us elswhere. -target_include_directories(zeek-agent PRIVATE $) +if ( USE_BROKER ) + # Ensure that we include our Broker version even if existing includes may lead us elswhere. + target_include_directories(zeek-agent PRIVATE $) +endif () target_link_libraries(zeek-agent PRIVATE ${CMAKE_DL_LIBS}) -target_link_libraries(zeek-agent PRIVATE broker_static) target_link_libraries(zeek-agent PRIVATE doctest::doctest) target_link_libraries(zeek-agent PRIVATE fmt::fmt) target_link_libraries(zeek-agent PRIVATE ghcFilesystem::ghc_filesystem) @@ -48,6 +49,10 @@ target_link_libraries(zeek-agent PRIVATE tomlplusplus::tomlplusplus) target_link_libraries(zeek-agent PRIVATE ztd::out_ptr) target_link_libraries(zeek-agent PRIVATE Glob) +if ( USE_BROKER ) + target_link_libraries(zeek-agent PRIVATE broker_static) +endif () + if ( NOT HAVE_GETOPT_LONG ) target_link_libraries(zeek-agent PRIVATE 3rdparty:3rdparty) endif() diff --git a/src/config.h.in b/src/config.h.in index 2d5915f4..6a0d43b9 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -10,6 +10,7 @@ #cmakedefine HAVE_CLANG #cmakedefine HAVE_MSVC #cmakedefine HAVE_GETOPT_LONG +#cmakedefine HAVE_BROKER namespace zeek::agent { inline const auto Version = "${ZEEK_AGENT_VERSION}"; diff --git a/src/core/configuration.cc b/src/core/configuration.cc index 5780e19a..7e8d9eb7 100644 --- a/src/core/configuration.cc +++ b/src/core/configuration.cc @@ -282,37 +282,13 @@ Result Configuration::Implementation::read(const filesystem::path& path return read(in, path); } -// Helper to parse platform-provided string values into the target type. -template -T parsePlatformValue(std::string_view path, const std::string& v) { - if constexpr ( std::is_same::value ) - return v; - - if constexpr ( std::is_same::value ) - return std::stod(v); - - if constexpr ( std::is_same::value ) { - auto x = trim(tolower(v)); - return x == "1" || x == "true" || x == "yes" || x == "on"; - } - - throw result::Error(frmt("cannot parse plaform value for configuration option '{}' ({})", path, to_string(v))); -} - // Get a value, typed correctly, if available. template bool tomlValue(const std::optional& t, const std::string& path, T* dst) { using vtype = typename std::remove_reference_t; - if ( ! t ) { - // See if we have platform-specific mechanism providing this value. - auto v = platform::retrieveConfigurationOption(path); - if ( ! v ) - return false; - - *dst = parsePlatformValue(path, *v); - return true; - } + if ( ! t ) + return false; auto n = t->at_path(path); if ( ! n ) @@ -331,16 +307,8 @@ template bool tomlArray(const std::optional& t, const std::string& path, std::vector* dst) { using vtype = typename std::remove_reference_t; - if ( ! t ) { - auto v = platform::retrieveConfigurationOption(path); - if ( ! v ) - return false; - - for ( const auto& x : split(*v) ) - dst->push_back(parsePlatformValue(path, trim(x))); - - return true; - } + if ( ! t ) + return false; auto n = t->at_path(path); if ( ! n ) @@ -452,6 +420,11 @@ Configuration::~Configuration() { const Options& Configuration::options() const { return *pimpl()->_options; } +Result Configuration::setOptions(Options options) { + pimpl()->apply(std::move(options)); + return Nothing(); +} + Result Configuration::initFromArgv(std::vector argv) { ZEEK_AGENT_DEBUG("configuration", "setting command line arguments: {}", join(argv, " ")); return pimpl()->initFromArgv(std::move(argv)); diff --git a/src/core/configuration.h b/src/core/configuration.h index d36e830f..1ecfa6f1 100644 --- a/src/core/configuration.h +++ b/src/core/configuration.h @@ -233,6 +233,9 @@ class Configuration : public Pimpl { /** Returns the options currently in effect. */ const Options& options() const; + /** Applies new set of options provided by caller. */ + Result setOptions(Options options); + /** * Parses a set of command line options. This first resets the current * options back to their defaults, and then updates them according to any @@ -253,7 +256,7 @@ class Configuration : public Pimpl { * Parses an agent configuration file. This first resets the current * options back to their defaults, and updates the current set of options * accordingly. If any command line options have been previously provided, - * it reapplys them on top at the end as well. + * it reapplies them on top at the end as well. * * @param path file to read * @return result will flag any errors that occurred @@ -264,7 +267,7 @@ class Configuration : public Pimpl { * Parses an agent configuration file from already open input stream. This * first resets the current options back to their defaults, and updates the * current set of options accordingly. If any command line options have - * been previously provided, it reapplys them on top at the end as well. + * been previously provided, it reapplies them on top at the end as well. * * @param in already open stream to read the content from * @param path path associated with input (for error messages) diff --git a/src/core/database.cc b/src/core/database.cc index d5736309..bbe52675 100644 --- a/src/core/database.cc +++ b/src/core/database.cc @@ -53,17 +53,22 @@ struct Pimpl::Implementation { // Adds table to database. void addTable(Table* t); + // Adds table to list of pending ones. + void addPendingTable(Table* t); + // Callback for the timers we install for our queries. Interval timerCallback(timer::ID id); // Helper to lookup scheduled query. std::optional::iterator> lookupQuery(query::ID); + Database* _db = nullptr; // database this implementation belongs to const Configuration* _configuration = nullptr; // configuration object, as passed into constructor Scheduler* _scheduler = nullptr; // scheduler as passed into constructor std::unique_ptr _sqlite; // SQLite backend for performing queries std::map _tables; // registered tables indexed by name + std::list _pending_tables; // registered tables that we were initially temporarily unavailable std::list _queries; // outstanding queries; list so that iterators remain valid on changes std::map::iterator> _queries_by_id; // outstanding queries indexed by their ID std::set> _cancelled_queries; // track cancelled, but not removed, queries @@ -167,6 +172,13 @@ void Database::Implementation::expire() { t->expire(expire_until); } } + + // Go through pending tables and see if any has become available. + if ( ! _pending_tables.empty() ) { + auto pending = std::move(_pending_tables); + for ( const auto& t : pending ) + addTable(t); + } } void Database::Implementation::poll() { @@ -177,16 +189,43 @@ void Database::Implementation::poll() { } void Database::Implementation::addTable(Table* t) { - auto schema = t->schema(); + EventTable::Init init_result; + + if ( t->usesMockData() ) + init_result = EventTable::Init::Available; + else + init_result = t->init(); - if ( _tables.find(schema.name) != _tables.end() ) - throw InternalError(frmt("table {} registered more than once", schema.name)); + switch ( init_result ) { + case EventTable::Init::Available: { + ZEEK_AGENT_DEBUG("database", "adding table {} to database", t->name()); + t->setDatabase(_db); - auto rc = _sqlite->addTable(t); - if ( ! rc ) - throw FatalError(frmt("error registering table {} with SQLite backend: {}", schema.name, rc.error())); + auto schema = t->schema(); + + if ( _tables.find(schema.name) != _tables.end() ) + throw InternalError(frmt("table {} registered more than once", schema.name)); + + auto rc = _sqlite->addTable(t); + if ( ! rc ) + throw FatalError(frmt("error registering table {} with SQLite backend: {}", schema.name, rc.error())); + + _tables[schema.name] = t; + return; + } - _tables[schema.name] = t; + case EventTable::Init::PermanentlyUnavailable: + ZEEK_AGENT_DEBUG("database", "not adding table {} to database because it's permanently disabled", + t->name()); + return; + + case EventTable::Init::TemporarilyUnavailable: + ZEEK_AGENT_DEBUG("database", + "not adding table {} to database because it's currently not available; will retry", + t->name()); + _pending_tables.push_back(t); + return; + } } static auto diffRows(std::vector> old, std::vector> new_) { @@ -333,6 +372,7 @@ Table* Database::Implementation::table(const std::string& name) { Database::Database(Configuration* configuration, Scheduler* scheduler) { ZEEK_AGENT_DEBUG("database", "creating instance"); + pimpl()->_db = this; pimpl()->_configuration = configuration; pimpl()->_scheduler = scheduler; pimpl()->_sqlite = std::make_unique(); @@ -400,20 +440,9 @@ void Database::expire() { } void Database::addTable(Table* t) { - t->setDatabase(this); - if ( configuration().options().use_mock_data ) t->enableMockData(); - if ( ! t->usesMockData() ) { - if ( ! t->init() ) { - ZEEK_AGENT_DEBUG("database", "not adding table {} to database because it's disabled", t->name()); - t->setDatabase(nullptr); - return; - } - } - - ZEEK_AGENT_DEBUG("database", "adding table {} to database", t->name()); pimpl()->addTable(t); } @@ -492,9 +521,9 @@ TEST_SUITE("Database") { ~TestTable() override {} - bool init() override { + Init init() override { initialized = true; - return true; + return Init::Available; } std::vector