Update Splice from CCI (#25)

Signed-off-by: CircleCI <[email protected]>
Co-authored-by: CircleCI <[email protected]>

.circleci/canton-scripts/check-logs.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+###############################################################################
+# This script was copied from the Canton repo and then heavily adapted to use
+# ripgrep instead of lnav for filtering the logs. To say that it's faster this
+# way is an understatement.
+###############################################################################
+
+
+
+set -eou pipefail
+
+LOGFILE="$1"
+shift
+FILES_WITH_IGNORE_PATTERNS=("$@")
+
+# Succeed if logfile does not exist
+if [[ ! -f "$LOGFILE" ]]
+then
+  echo "$LOGFILE does not exist."
+  echo "Skipping log file check."
+  echo ""
+  exit 0
+else
+  echo "Checking $LOGFILE while ignoring: ${FILES_WITH_IGNORE_PATTERNS[*]}"
+fi
+
+# Get the full path to the .circleci directory
+SRCDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
+
+# Load utility functions
+source "$SRCDIR"/io-utils.sh
+
+# Load ignore patterns for later use
+IGNORE_PATTERNS="$(cat "${FILES_WITH_IGNORE_PATTERNS[@]}" | remove_comment_and_blank_lines)"
+
+# Keywords are based on `canton-json.lnav.json`
+MIN_LOG_LEVEL_WARNING="\"level\":\"WARN\"|\"level\":\"ERROR\""
+
+### Output ignored log lines
+
+# Reads ignore patterns from stdin (one pattern per line).
+# Outputs log entries matching an ignore pattern.
+read_ignored_entries() {
+  rg -f - <<< "$IGNORE_PATTERNS" "$LOGFILE" |
+    rg -e "$MIN_LOG_LEVEL_WARNING" || true
+}
+
+# Read ignored entries
+read_ignored_entries |
+  # We print the ignored entries as part of our sbt run, and we don't want them to trigger the sbt output checker.
+  # We thus add this suffix which is ignored by default in the sbt output checker.
+  sed 's/$/ (ignore this line in check-sbt-output.sh)/' |
+  # Output ignored entries, succeeding in any case
+  output_problems "ignored entries" "$LOGFILE" "0"
+
+### Check for errors and warnings
+
+# this differs from `read_ignored_entries` by one `-v`
+find_errors() {
+  # rg returns 1 if there were not matches so we add the || true
+  rg -v -f - <<< "$IGNORE_PATTERNS" "$LOGFILE" |
+    rg -e "$MIN_LOG_LEVEL_WARNING" || true
+}
+
+# Find the errors
+find_errors |
+  # Output errors, failing if there are some
+  output_problems "problems" "$LOGFILE"
+
+### Output exceptions
+
+find_exceptions() {
+  set +o pipefail # rg returns 1 if there were not matches
+  rg -v -f - <<< "$IGNORE_PATTERNS" "$LOGFILE" |
+    rg -e "(?-i:Exception|Error)(?::\s.*)?$" || true
+    # not replacing anything here on purpose as these log lines
+    # might have different structure; better to avoid removing
+    # anything we might need
+}
+
+# Find exceptions
+find_exceptions |
+  # Output total number of exceptions
+  echo "Total: $(wc -l) lines with stack traces."; echo
+
+### Look for leaked secrets
+
+find_secrets() {
+  set +o pipefail # rg returns 1 if there were not matches
+  rg -o -e "(secret|token|private-key|password)=[^,[:space:]]*" "$LOGFILE" |
+    # we mask secrets as "****" in our logs and testcontainers obfuscates secrets as "hidden non-blank value"
+    # (https://github.com/testcontainers/testcontainers-java/blob/bf5605a2031d7f29f86a85430e3509a198c6e125/core/src/main/java/org/testcontainers/utility/AuthConfigUtil.java#L33)
+    rg -v -e "=\\\\\"\*\*\*\*\\\\\"" -e "=hidden" || true
+}
+
+# Find leaked secrets
+find_secrets |
+  # Output unmasked secrets, failing if there are some
+  output_problems "unmasked secrets" "$LOGFILE"
+
+find_deprecated_configs() {
+  set +o pipefail # rg returns 1 if there were not matches
+  rg -v -f - <<< "$IGNORE_PATTERNS" "$LOGFILE" |
+    rg -e 'Config field at (\S+) is deprecated' || true
+}
+
+find_deprecated_configs |
+  output_problems "deprecated config paths" "$LOGFILE"
+
+# Give the next command some space
+echo ""

.circleci/canton-scripts/check-sbt-output.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -eo pipefail
+
+###############################################################################
+# This script was copied from the Canton repo and slightly adapted for the CN repo.
+###############################################################################
+
+
+SBT_OUTPUT_FILE="$1"
+# Note the files with ignore patterns are hardcoded, as we have only one call-site
+# and our preflight checks can lead to mismatches in arguments on the call-site
+# and the actual files available in the checkout as of which the preflight check
+# runs; see #3528.
+FILES_WITH_IGNORE_PATTERNS=("project/ignore-patterns/sbt-output.ignore.txt")
+
+# Get the full path to the .circleci directory
+SRCDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
+
+# Load utility functions
+source "$SRCDIR"/io-utils.sh
+
+read_sbt_output() {
+  grep -v -E "$@" "$SBT_OUTPUT_FILE"
+}
+
+filter_errors() {
+  grep -i -w -e error -e severe -e warn -e warning -e exception -e critical -e fatal || true
+}
+
+# Read ignore patterns
+cat "${FILES_WITH_IGNORE_PATTERNS[@]}" |
+  remove_comment_and_blank_lines |
+  # Prepend each pattern with '-e'
+  sed 'i -e' |
+  # Read sbt output and remove ignored lines
+  with_input_as_params read_sbt_output |
+  # Detect the errors
+  filter_errors |
+  # Report errors and fail on error
+  output_problems "problems" "the sbt output"

.circleci/canton-scripts/io-utils.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+###############################################################################
+# This script was copied from the Canton repo.
+###############################################################################
+
+
+
+# Copy stdin to stdout, while removing all lines that start with '#' or consist of blanks
+remove_comment_and_blank_lines() {
+  while IFS= read -r # Read a file line by line; IFS= ensures that no separators other than newline are used
+  do
+    if [[ -n "${REPLY// }" ]] && [[ "$REPLY" != "#"* ]] # Filter out comment lines and blank lines
+    then
+      cat <<< "$REPLY"
+    fi
+  done
+}
+
+# Read lines from stdin and invoke "$@" with input lines as extra arguments
+with_input_as_params() {
+  local -a PARAMS
+  while IFS= read -r; do
+    PARAMS+=("$REPLY")
+  done
+  "$@" "${PARAMS[@]}"
+}
+
+# Output a problem report.
+# Problems are read from stdin.
+output_problems() {
+  local PROBLEMS_NAME="${1:?PROBLEMS_NAME undefined}"
+  local SOURCE="${2:?SOURCE undefined}"
+  local RETURN_CODE_ON_PROBLEM="${3:-1}"
+  local PROBLEMS
+  PROBLEMS=$(cat)
+  if [[ -z "$PROBLEMS" ]]
+  then
+    echo "No $PROBLEMS_NAME found in $SOURCE."
+    echo
+  else
+    echo "Found $PROBLEMS_NAME in $SOURCE:"
+    cat <<< "$PROBLEMS"
+    if [[ $RETURN_CODE_ON_PROBLEM != "0" ]]
+    then cat <<< "$PROBLEMS" >> found_problems.txt
+    fi
+    echo "Total: $(wc -l <<< "$PROBLEMS") lines with $PROBLEMS_NAME."
+    echo
+    return "$RETURN_CODE_ON_PROBLEM"
+  fi
+}

.circleci/canton-scripts/split-canton-logs.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+###############################################################################
+# Split Canton logs into logs before and after shutdown
+###############################################################################
+
+
+
+set -eou pipefail
+
+LOGFILE="$1"
+LOGFILE_BEFORE="$2"
+LOGFILE_AFTER="$3"
+
+# We're using this rather specific pattern to avoid skipping over logs.
+SHUTDOWN_MESSAGE_PATTERN='"message":"Shutting down\.\.\.","logger_name":"c\.d\..*\.Canton.*App$"'
+
+# Succeed if logfile does not exist
+if [[ ! -f "$LOGFILE" ]]
+then
+  echo "$LOGFILE does not exist."
+  echo "Skipping splitting of log file."
+  exit 0
+fi
+
+# Fail if the shutdown pattern is present more than once
+NUM_SHUTDOWN_PATTERNS=$(grep --count --max-count=2 -E "$SHUTDOWN_MESSAGE_PATTERN" "$LOGFILE" || true)
+if [[ "2" == "$NUM_SHUTDOWN_PATTERNS" ]]
+then
+  # This error will be picked up by the sbt output checker
+  echo "ERROR - not splitting the log-files, as there are two or more shutdown messages. See:"
+  grep "$SHUTDOWN_MESSAGE_PATTERN" "$LOGFILE"
+  exit 2
+fi
+
+# Using range addresses to split the file: https://www.gnu.org/software/sed/manual/sed.html#Range-Addresses
+# This will print everything up to and including the shutdown log line.
+sed -n "0,/$SHUTDOWN_MESSAGE_PATTERN/p" "$LOGFILE" >> "$LOGFILE_BEFORE"
+
+# This will print the shutdown log line and all lines until the end of the file.
+sed -n "/$SHUTDOWN_MESSAGE_PATTERN/,\$p" "$LOGFILE" >> "$LOGFILE_AFTER"
+
+# Delete original logfile to ensure every log-line is present in at most one file
+rm -f "$LOGFILE"

.envrc

@@ -0,0 +1,30 @@
+use_flake_subdir() {
+  # Newer versions of direnv have watch_dir but the ancient version in
+  # Ubuntu does not so we just manually watch all (git-tracked) files.
+  for file in $(git ls-files nix); do
+    watch_file $file
+  done
+  mkdir -p "$(direnv_layout_dir)"
+  eval "$(nix print-dev-env --profile "$(direnv_layout_dir)/flake-profile" "path:nix" "$@")"
+}
+
+# TODO(#3876) work around for $TMPDIR is removed. #3876 to investigate more
+OLD_TMPDIR=$TMPDIR
+
+use flake_subdir
+
+# TODO(#3876) work around for $TMPDIR is removed. #3876 to investigate more
+export TMPDIR=$OLD_TMPDIR
+
+source_env .envrc.vars
+
+# Private .envrc
+[[ -f .envrc.private ]] && [[ -z "$IGNORE_PRIVATE_ENVRC" ]] && source_env .envrc.private || true
+
+if stat --printf='' .envrc.include.* 2>/dev/null; then
+  for file in .envrc.include.*; do
+    source_env $file
+  done
+fi
+
+

.envrc.vars

@@ -0,0 +1,29 @@
+# -*- shell-script -*-
+
+### Set UTC timezone, as CometBft always logs in UTC while java tools log using the local timezone
+export TZ=UTC
+
+### Separate (out of direnv) the env vars that are sourced both locally *and* in CircleCI
+export REPO_ROOT="${PWD}"
+export TOOLS_LIB=$REPO_ROOT/build-tools/lib
+
+# Inrease code heap sizes to avoid issues
+# Defaults NonNMethodCodeHeapSize=7M,NonProfiledCodeHeapSize=122M,ProfiledCodeHeapSize=122M
+export SBT_OPTS="-Xmx6G -Xms2G -Xss2M -XX:+UseG1GC -XX:NonNMethodCodeHeapSize=32M -XX:NonProfiledCodeHeapSize=256M -XX:ProfiledCodeHeapSize=256M -XX:ReservedCodeCacheSize=544M"
+
+# Shortcut var to refer to release bundle contents
+export BUNDLE=${REPO_ROOT}/apps/app/target/release/splice-node
+
+export PATH=${PATH}:${REPO_ROOT}/build-tools
+export PATH=${PATH}:${BUNDLE}/bin
+
+export POSTGRES_HOST="localhost"
+export POSTGRES_USER=postgres
+export POSTGRES_PASSWORD=postgres
+
+# Include all organization specific `.envrc.vars.*` files
+if stat --printf='' .envrc.vars.* 2>/dev/null; then
+  for file in .envrc.vars.*; do
+    source_env $file || . $file
+  done
+fi

.gitignore

@@ -1 +1,68 @@
+.daml
+*.dar
+!daml/dars/*
+*.log
+*.clog
+.DS_Store
+node_modules/
+build
+daml.js
+log/
+logfile
+testlog
+_build/
+.idea/*
+.envrc.private
+.bloop
+.metals
+.ammonite
+**/metals.sbt
+
+# We want to share run configurations
+!.idea/runConfigurations
+.vscode/*
+!.vscode/settings.json
+target
+.bsp
+!canton/community/participant/src/test/resources/daml/illformed.dar
+!canton/community/participant/src/main/resources/dar/AdminWorkflows.dar
 *~
+*#
+*.pid
+canton.ports
+canton-simtime.ports
+canton-release
+start-frontends-http-ledger-api-port
+.disable-unused-warnings
+todo-out
+temp/
+.protobufs-*.zip
+# Exclude generated ts protobuf sources
+**/com/daml/ledger/api/**/*.ts
+**/com/daml/ledger/api/**/*.js
+**/com/daml/network/**/*.ts
+**/com/daml/network/**/*.js
+**/com/digitalasset/canton/**/*.ts
+**/com/digitalasset/canton/**/*.js
+**/google/**/*.js
+**/google/**/*.ts
+**/scalapb/**/*.js
+**/scalapb/**/*.ts
+canton.patch
+# files generated by start-canton.sh
+*.tokens
+*.participants
+canton-classpath
+openapi-ts-client
+external-openapi-ts-client
+/cluster/manifest/test-results/actual.json
+.direnv
+rust_mozprofile*
+openapi-cache-key.txt
+apps/splitwell/src/test/resources/splitwell-bundle*.tar.gz
+.tmp_ERRFILE
+.k9s/*
+!.k9s/plugins.yaml
+.ignore
+
+**/SingletonCookie

LATEST_RELEASE

@@ -0,0 +1 @@
+0.2.1