Generic project improvements (#12)

.github/workflows/release.yml

@@ -98,6 +98,7 @@ jobs:
           pdm sync -p ./docs
 
 
+
       - name: Build docs
         run: |
           ./dev_scripts/docs.sh build

.gitignore

@@ -6,6 +6,7 @@ private/
 !*private*.*.zetch
 
 process_data/
+logs/
 
 # Tempate files
 .cop.*.yml

.pre-commit-config.yaml

@@ -48,7 +48,7 @@ repos:
 
   # Ruff: linting and formatting for python code:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.5
+    rev: v0.2.1
     hooks:
       # Run the formatter.
       - id: ruff-format
@@ -73,13 +73,21 @@ repos:
         types: [rust]
         args: [--manifest-path=./py_rust/Cargo.toml, --]
       - id: cargo-check
-        name: cargo-check
+        name: cargo-check-all-features
         description: Check the package for errors.
         entry: cargo +nightly check
         language: rust
         types: [rust]
         args: [--manifest-path=./py_rust/Cargo.toml, --all-features]
         pass_filenames: false
+      - id: cargo-check
+        name: cargo-check-default-features
+        description: Check the package for errors.
+        entry: cargo +nightly check
+        language: rust
+        types: [rust]
+        args: [--manifest-path=./py_rust/Cargo.toml]
+        pass_filenames: false
       - id: cargo-clippy
         name: cargo-clippy
         description: Lint rust sources

dev_scripts/_internal/match_substring.py

@@ -0,0 +1,23 @@
+"""Used internally by utils.sh match_substring."""
+
+import re
+import sys
+
+r = sys.argv[1]
+s = sys.argv[2]
+
+
+def err_inf():
+    """Helper for error message."""
+    return "\n\nString:\n{}\n\nRegex: '{}'\n\n".format(s, r)
+
+
+res = re.findall(r, s)
+if len(res) != 1:
+    raise ValueError("{}Expected 1 match, got: {}.".format(err_inf(), len(res)))
+
+match = res[0]
+if match == "":
+    raise ValueError("{}Matched empty string.".format(err_inf()))
+
+print(match.strip())

dev_scripts/initial_setup.sh

@@ -20,48 +20,66 @@ _install_yaml_fmt () {
     echo "yamlfmt version $1 installed!"
 }
 
+
+
 _install_biome () {
     echo "Installing biome version $1..."
 
     # os lowercase:
     OS=$(uname -s | tr '[:upper:]' '[:lower:]')
     ARCH=$(uname -m)
 
-    curl -L https://github.com/biomejs/biome/releases/download/cli%2Fv$1/biome-$OS-$ARCH -o biome
+    echo "Downloading biome version $1 for ${OS}-${ARCH}..."
+    curl -L https://github.com/biomejs/biome/releases/download/cli%2Fv$1/biome-${OS}-${ARCH} -o biome -f
     chmod +x biome
     sudo mv biome /usr/local/bin
 }
 
-initial_setup () {
-    # Make sure zetch is installed and up to date:
-    if command -v zetch > /dev/null 2>&1; then
-        echo "zetch already installed"
-    else
-        echo "zetch could not be found, installing..."
-        pipx install zetch
+_ensure_biome() {
+    req_ver="$1"
+
+    if [[ -z "$req_ver" ]]; then
+        echo "biome version not provided!"
+        exit 1
     fi
 
-    # Install biome if missing:
-    biome_req_ver="1.5.3"
     if version=$(biome --version 2>/dev/null); then
         # Will be "Version: $ver", make sure starts with "Version: " and remove that:
         if [[ ! "$version" =~ ^Version:\  ]]; then
-            echo "biome version not found in expected format, expected 'Version: x.x.x', got $version!"
+            echo "Biome version not found in expected format, expected 'Version: x.x.x', got '$version'!"
             exit 1
         fi
 
         # Strip prefix:
         version=${version#Version: }
 
-        if [[ "$version" == "$biome_req_ver" ]]; then
+        if [[ "$version" == "$req_ver" ]]; then
             echo "biome already installed with correct version $version!"
         else
-            echo "biome incorrect version, upgrading..."
-            _install_biome $biome_req_ver
+            echo "biome incorrect version, upgrading to $version..."
+            _install_biome $req_ver
         fi
     else
-        _install_biome $biome_req_ver
+        _install_biome $req_ver
     fi
+}
+
+initial_setup () {
+    # Install useful local directories (might be unused):
+    mkdir -p ./process_data
+    mkdir -p ./logs
+
+    # Make sure zetch is installed and up to date:
+    if command -v zetch > /dev/null 2>&1; then
+        echo "zetch already installed"
+    else
+        echo "zetch could not be found, installing..."
+        pipx install zetch
+    fi
+
+
+    # Make sure biome is installed for linting and formatting various files:
+    _ensure_biome "1.5.3"
 
     # Make sure bun installed:
     if command -v bun > /dev/null 2>&1; then

dev_scripts/process.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# Stop on error:
+set -e
+
+PROCESS_PREFIX="dev_script_process"
+
+# Start a process with this system. $1: process_name, $2: stringified command to run
+# These processes will be tracked and are listable and stopable.
+# All processes should stop when terminal is shut
+start() {
+    if [ "$#" -ne 2 ]; then
+        echo "Usage: $0 <process_name> <stringified command to run>"
+        return 1
+    fi
+
+    # If process_name is empty, error
+    if [ -z "$1" ]; then
+        echo "Process name cannot be empty!"
+        return 1
+    fi
+
+    local process_name="$1"
+    local process_command="$2"
+    local process_id_file="/tmp/${PROCESS_PREFIX}_${process_name}.pid"
+
+    # Check if the process is already running
+    if [ -e "$process_id_file" ]; then
+        local existing_pid=$(<"$process_id_file")
+        if ps -p "$existing_pid" > /dev/null; then
+            echo "Process '$process_name' is already running with PID $existing_pid"
+            return 1
+        else
+            # Remove stale PID file
+            rm "$process_id_file"
+        fi
+    fi
+
+    # Start the process and write the processes output to $(pwd)/process_data/processes/$process_name.log
+    local log_file="$(pwd)/logs/proc_$process_name.log"
+    mkdir -p "$(dirname "$log_file")"
+    # Clear the logfile to start from scratch:
+    > "$log_file"
+
+    # Start the process in its own process group, rerouting all output to the logfile:
+    eval "$process_command" > "$log_file" 2>&1 &
+
+    # Capture the PID of the process and write it to a file
+    local new_pid=$!
+
+    # Wait for 0.2 seconds, if the process has exited already with a non-zero exit code, then print the log file
+    sleep 0.2
+    if ! ps -p "$new_pid" > /dev/null; then
+        cat "$log_file"
+        echo "Process '$process_name' failed (wasn't running after 0.2 seconds), output in logfile printed above."
+        return 1
+    fi
+
+    echo "$new_pid" > "$process_id_file"
+    echo "Process '$process_name' started with PID $new_pid, output will write to '$log_file'"
+}
+
+# Stop processes with a given namespace started with this system. $1: process_name
+# E.g. scr process.sh stop my_process would stop my_process and any processes with that as a prefix.
+stop() {
+    if [ "$#" -ne 1 ]; then
+        echo "Usage: $0 <process_name>"
+        return 1
+    fi
+
+    local process_name="$1"
+    local process_id_files=(/tmp/${PROCESS_PREFIX}_${process_name}*.pid)
+
+    echo "Stopping processes matching or prefixed by '$process_name'..."
+    for process_id_file in "${process_id_files[@]}"; do
+        if [ -e "$process_id_file" ]; then
+            while IFS= read -r pid_to_kill; do
+                echo "Stopping process with PID: $pid_to_kill"
+                terminate "$pid_to_kill"
+            done < "$process_id_file"
+            rm "$process_id_file"
+        fi
+    done
+    echo "Stopped ${#process_id_files[@]} process."
+}
+
+# List all processes actively running processes that were started with start()
+list() {
+    local pid_files=$(ls /tmp/${PROCESS_PREFIX}_*.pid 2>/dev/null)
+
+    if [ -z "$pid_files" ]; then
+        echo "No processes running!"
+        return
+    fi
+
+    echo "Processes:"
+    for pid_file in $pid_files; do
+        local process_name=$(basename "$pid_file" | sed "s/${PROCESS_PREFIX}_//g" | sed "s/.pid//g")
+        local pid=$(<"$pid_file")
+        echo "  Process: $process_name, PID: $pid"
+    done
+}
+
+# Terminate a process and all of its child processes
+terminate() {
+    local parent_pid="$1"
+    local IS_CHILD=$2
+
+    # Terminate the child processes of the parent PID
+    local child_pids=$(pgrep -P "$parent_pid")
+    for pid in $child_pids; do
+        terminate "$pid" "true"
+    done
+
+    # Terminate the parent PID
+    if ps -p "$parent_pid" > /dev/null; then
+        if [ "$IS_CHILD" = "true" ]; then
+            echo "Terminating child: $parent_pid"
+        else
+            echo "Terminating root: $parent_pid"
+        fi
+        # Or true to not error if the process is already dead:
+        kill -9 "$parent_pid" > /dev/null 2>&1 || true
+    fi
+}
+
+# Has to come at the end of these files:
+source ./dev_scripts/_scr_setup/setup.sh "$@"

dev_scripts/run.sh

@@ -3,6 +3,16 @@
 # Stop on error:
 set -e
 
+# Prep for running top-level services
+_prep () {
+    # A custom env version may have been used before, reset zetch to make sure not the case.
+    zetch
+
+
+
+}
+
+
 
 # Has to come at the end of these files:
 source ./dev_scripts/_scr_setup/setup.sh "$@"

dev_scripts/utils.sh

@@ -5,7 +5,8 @@ set -e
 
 # Run commands in parallel. E.g. run_parallel "sleep 1" "sleep 1" "sleep 1"
 run_parallel () {
-    parallel --ungroup -j 0 ::: "$@"
+    # --halt now,fail=1 stops all processes if any of the error
+    parallel --ungroup -j 0 --halt now,fail=1 ::: "$@"
 }
 
 py_install_if_missing () {
@@ -26,5 +27,41 @@ replace_text () {
 
 
 
+# Returns "true" if looks like in_ci, "false" otherwise:
+in_ci () {
+    # Check if any of the CI/CD environment variables are set
+    if [ -n "$GITHUB_ACTIONS" ] || [ -n "$TRAVIS" ] || [ -n "$CIRCLECI" ] || [ -n "$GITLAB_CI" ]; then
+        echo "true"
+    else
+        echo "false"
+    fi
+}
+
+# If python exists and is a 3.x version, runs the command. Otherwise, runs with python3.12/3.11/3, whichever found first.
+anypython () {
+    # Use python by default (e.g. virtualenv) as long as its a 3.x version:
+    if command -v python &> /dev/null && [[ $(python -c 'import sys; print(sys.version_info[0])') == "3" ]]; then
+        python "$@"
+    elif command -v python3.12 &> /dev/null; then
+        python3.12 "$@"
+    elif command -v python3.11 &> /dev/null; then
+        python3.11 "$@"
+    elif command -v python3 &> /dev/null; then
+        python3 "$@"
+    else
+        echo "No python found."
+        exit 1
+    fi
+}
+
+# Uses python re.findall(), if more than one match or no matches, errors. Otherwise returns the matched substring.
+# Args:
+# $1: regex string, e.g. 'foo_(.*?)_ree' (make sure to use single quotes to escape the special chars)
+# $2: string to search in e.g. "foo_bar_ree"
+# Returns: the matched substring, e.g. "bar"
+match_substring () {
+    anypython ./dev_scripts/_internal/match_substring.py "$1" "$2"
+}
+
 # Has to come at the end of these files:
 source ./dev_scripts/_scr_setup/setup.sh "$@"