diff --git a/.dockerignore b/.dockerignore index 0da9ab921..b63f0cdfa 100644 --- a/.dockerignore +++ b/.dockerignore @@ -29,6 +29,7 @@ arkime-raw kubernetes malcolm-iso sensor-iso +sensor-raspi nginx/nginx_ldap*.conf pcap _site diff --git a/.github/workflows/api-build-and-push-ghcr.yml b/.github/workflows/api-build-and-push-ghcr.yml index dacbb7a4f..2b0113e20 100644 --- a/.github/workflows/api-build-and-push-ghcr.yml +++ b/.github/workflows/api-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/arkime-build-and-push-ghcr.yml b/.github/workflows/arkime-build-and-push-ghcr.yml index c847c9995..c8d174d16 100644 --- a/.github/workflows/arkime-build-and-push-ghcr.yml +++ b/.github/workflows/arkime-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index b893f6782..000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: "CodeQL" - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: "5 17 * * 5" - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ javascript, python ] - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - queries: +security-and-quality - - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - if: ${{ matrix.language == 'javascript' || matrix.language == 'python' }} - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/dashboards-build-and-push-ghcr.yml b/.github/workflows/dashboards-build-and-push-ghcr.yml index ada149096..f9f1e39a6 100644 --- a/.github/workflows/dashboards-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml index 1ec639dcb..4ca3c2d8a 100644 --- a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/dirinit-build-and-push-ghcr.yml b/.github/workflows/dirinit-build-and-push-ghcr.yml index db9b8bdd3..9abe1a7f4 100644 --- a/.github/workflows/dirinit-build-and-push-ghcr.yml +++ b/.github/workflows/dirinit-build-and-push-ghcr.yml @@ -49,7 +49,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/file-monitor-build-and-push-ghcr.yml b/.github/workflows/file-monitor-build-and-push-ghcr.yml index 10c6885da..c5fecc8a5 100644 --- a/.github/workflows/file-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/file-monitor-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/file-upload-build-and-push-ghcr.yml b/.github/workflows/file-upload-build-and-push-ghcr.yml index 6228bfa20..be51b63ef 100644 --- a/.github/workflows/file-upload-build-and-push-ghcr.yml +++ b/.github/workflows/file-upload-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/filebeat-build-and-push-ghcr.yml b/.github/workflows/filebeat-build-and-push-ghcr.yml index 5d67fd099..884b91364 100644 --- a/.github/workflows/filebeat-build-and-push-ghcr.yml +++ b/.github/workflows/filebeat-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/freq-build-and-push-ghcr.yml b/.github/workflows/freq-build-and-push-ghcr.yml index a6411ca25..314182cde 100644 --- a/.github/workflows/freq-build-and-push-ghcr.yml +++ b/.github/workflows/freq-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/htadmin-build-and-push-ghcr.yml b/.github/workflows/htadmin-build-and-push-ghcr.yml index 954d245aa..0b9db5e1f 100644 --- a/.github/workflows/htadmin-build-and-push-ghcr.yml +++ b/.github/workflows/htadmin-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/logstash-build-and-push-ghcr.yml b/.github/workflows/logstash-build-and-push-ghcr.yml index 19ae8e43e..030dbd394 100644 --- a/.github/workflows/logstash-build-and-push-ghcr.yml +++ b/.github/workflows/logstash-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml index 488575fe8..cf6486a42 100644 --- a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -92,12 +92,12 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Build image run: | - IMAGES=( $(grep image: docker-compose.yml | awk '{print $2}' | sort -u) ) + IMAGES=( $(grep image: docker-compose-dev.yml | awk '{print $2}' | sort -u) ) for IMAGE in "${IMAGES[@]}"; do REPO_IMAGE="$(echo "$IMAGE" | sed "s@^.*\(malcolm\)@ghcr.io/${{ github.repository_owner }}/\1@" | sed "s/:.*/:${{ steps.extract_branch.outputs.branch }}/")" docker pull "$REPO_IMAGE" && \ diff --git a/.github/workflows/netbox-build-and-push-ghcr.yml b/.github/workflows/netbox-build-and-push-ghcr.yml index c59567995..aa2ca78d1 100644 --- a/.github/workflows/netbox-build-and-push-ghcr.yml +++ b/.github/workflows/netbox-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/nginx-build-and-push-ghcr.yml b/.github/workflows/nginx-build-and-push-ghcr.yml index 06f028768..2343e6f31 100644 --- a/.github/workflows/nginx-build-and-push-ghcr.yml +++ b/.github/workflows/nginx-build-and-push-ghcr.yml @@ -64,7 +64,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/opensearch-build-and-push-ghcr.yml b/.github/workflows/opensearch-build-and-push-ghcr.yml index 8a0083bad..c12913a79 100644 --- a/.github/workflows/opensearch-build-and-push-ghcr.yml +++ b/.github/workflows/opensearch-build-and-push-ghcr.yml @@ -56,7 +56,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/pcap-capture-build-and-push-ghcr.yml b/.github/workflows/pcap-capture-build-and-push-ghcr.yml index 332d0b560..e0cfe4d7d 100644 --- a/.github/workflows/pcap-capture-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-capture-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml index 385ec4fec..f8bdc2c33 100644 --- a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/postgresql-build-and-push-ghcr.yml b/.github/workflows/postgresql-build-and-push-ghcr.yml index c90fd8b64..dd3908422 100644 --- a/.github/workflows/postgresql-build-and-push-ghcr.yml +++ b/.github/workflows/postgresql-build-and-push-ghcr.yml @@ -56,7 +56,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/redis-build-and-push-ghcr.yml b/.github/workflows/redis-build-and-push-ghcr.yml index 921e11c91..881531576 100644 --- a/.github/workflows/redis-build-and-push-ghcr.yml +++ b/.github/workflows/redis-build-and-push-ghcr.yml @@ -56,7 +56,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml index aade513c9..14fe12261 100644 --- a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml @@ -88,7 +88,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Build image diff --git a/.github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml b/.github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml new file mode 100644 index 000000000..0d5faee04 --- /dev/null +++ b/.github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml @@ -0,0 +1,102 @@ +name: sensor-raspi-build-docker-wrap-push-ghcr + +on: + # push: + # branches: + # - main + # - development + # paths: + # - '.trigger_raspi_workflow_build' + workflow_dispatch: + # repository_dispatch: + +jobs: + build: + runs-on: ubuntu-22.04 + permissions: + actions: write + packages: write + contents: read + security-events: write + defaults: + run: + shell: bash + steps: + - + name: Cancel previous run in progress + uses: styfle/cancel-workflow-action@0.12.0 + with: + ignore_sha: true + all_but_latest: true + access_token: ${{ secrets.GITHUB_TOKEN }} + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver-opts: | + image=moby/buildkit:master + - + name: Build environment setup + run: | + sudo apt-get -q update + sudo env DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y -q \ + binfmt-support \ + bmap-tools \ + ca-certificates \ + debootstrap \ + dosfstools \ + kpartx \ + python3 \ + qemu-user-static \ + qemu-utils \ + time \ + vmdb2 \ + zerofree + - + name: Checkout + uses: actions/checkout@v4 + - + name: Extract branch name + shell: bash + run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT + id: extract_branch + - + name: Extract commit SHA + shell: bash + run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + id: extract_commit_sha + - + name: Extract Malcolm version + shell: bash + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + id: extract_malcolm_version + - + name: Build image + run: | + pushd ./sensor-raspi + mkdir -p ./shared + echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt + echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt + echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot + echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot + echo "BUILD_JOBS=2" > ./shared/environment.chroot + sudo make raspi_4_bookworm.img + sudo chmod 644 ./raspi_4_bookworm*.* + popd + - + name: ghcr.io login + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Build and push IMG image + uses: docker/build-push-action@v5 + with: + context: ./sensor-raspi + push: true + tags: ghcr.io/${{ github.repository_owner }}/malcolm/hedgehog-raspi:${{ steps.extract_branch.outputs.branch }} diff --git a/.github/workflows/suricata-build-and-push-ghcr.yml b/.github/workflows/suricata-build-and-push-ghcr.yml index 2d9da5d1e..2b467e0db 100644 --- a/.github/workflows/suricata-build-and-push-ghcr.yml +++ b/.github/workflows/suricata-build-and-push-ghcr.yml @@ -57,7 +57,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.github/workflows/zeek-build-and-push-ghcr.yml b/.github/workflows/zeek-build-and-push-ghcr.yml index 27f6e05db..dd6c44935 100644 --- a/.github/workflows/zeek-build-and-push-ghcr.yml +++ b/.github/workflows/zeek-build-and-push-ghcr.yml @@ -56,7 +56,7 @@ jobs: - name: Extract Malcolm version shell: bash - run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT + run: echo "mversion=$(grep -P "^\s+image:.*/malcolm/" docker-compose-dev.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" >> $GITHUB_OUTPUT id: extract_malcolm_version - name: Set up QEMU diff --git a/.gitignore b/.gitignore index c47bd8a97..eadc2e2ff 100644 --- a/.gitignore +++ b/.gitignore @@ -28,10 +28,12 @@ config.*/ .envrc .direnv .vagrant +.fuse_* malcolm_*images.tar.gz malcolm_*images.tar.xz malcolm_netbox_backup_*.gz *.iso +*.img *-build.log Gemfile.lock _site diff --git a/.trigger_iso_workflow_build b/.trigger_iso_workflow_build index bf8dee067..7d921ebb8 100644 --- a/.trigger_iso_workflow_build +++ b/.trigger_iso_workflow_build @@ -1,2 +1,2 @@ # this file exists solely for the purpose of being updated and seen by github to trigger a commit build action -2 \ No newline at end of file +3 \ No newline at end of file diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index b43c5310c..1c11505b1 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -7,7 +7,7 @@ ENV TERM xterm ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 -ENV ARKIME_VERSION "v4.6.0" +ENV ARKIME_VERSION "v5.0.0" ENV ARKIME_DIR "/opt/arkime" ENV ARKIME_URL "https://github.com/arkime/arkime.git" ENV ARKIME_LOCALELASTICSEARCH no @@ -49,6 +49,7 @@ RUN apt-get -q update && \ python3-pip \ python3-setuptools \ python3-wheel \ + re2c \ sudo \ swig \ wget \ @@ -57,7 +58,7 @@ RUN apt-get -q update && \ cd /opt && \ git clone --recurse-submodules --branch="$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION && \ cd "./arkime-"$ARKIME_VERSION && \ - bash -c 'for i in /opt/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ + bash -c 'for i in /opt/patches/*.patch; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ export PATH="$ARKIME_DIR/bin:${PATH}" && \ ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \ ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \ @@ -69,7 +70,6 @@ RUN apt-get -q update && \ sed -i "s/^\(ARKIME_LOCALELASTICSEARCH=\).*/\1"$ARKIME_LOCALELASTICSEARCH"/" ./release/Configure && \ sed -i "s/^\(ARKIME_INET=\).*/\1"$ARKIME_INET"/" ./release/Configure && \ ./easybutton-build.sh && \ - npm -g config set user root && \ make install && \ npm cache clean --force && \ rm -f ${ARKIME_DIR}/wiseService/source.* ${ARKIME_DIR}/etc/*.systemd.service && \ diff --git a/Dockerfiles/dirinit.Dockerfile b/Dockerfiles/dirinit.Dockerfile index 1b3cfd1d1..978ad6565 100644 --- a/Dockerfiles/dirinit.Dockerfile +++ b/Dockerfiles/dirinit.Dockerfile @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm' LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md' LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm' -LABEL org.opencontainers.image.vendor='Idaho National Laboratory' +LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency' LABEL org.opencontainers.image.title='ghcr.io/cisagov/malcolm/dirinit' LABEL org.opencontainers.image.description='Sidecar container that ensures the creation of some volume subdirectories and does nothing else' diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index dad767400..f3992d895 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -80,14 +80,15 @@ ENV EXTRACTED_FILE_ENABLE_CAPA $EXTRACTED_FILE_ENABLE_CAPA ENV EXTRACTED_FILE_CAPA_VERBOSE $EXTRACTED_FILE_CAPA_VERBOSE ENV SRC_BASE_DIR "/usr/local/src" ENV CLAMAV_RULES_DIR "/var/lib/clamav" -ENV YARA_VERSION "4.3.2" +ENV YARA_VERSION "4.5.0" ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz" ENV YARA_RULES_SRC_DIR "/yara-rules-src" ENV YARA_RULES_DIR "/yara-rules" -ENV CAPA_VERSION "6.1.0" +ENV CAPA_VERSION "7.0.1" ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip" ENV CAPA_DIR "/opt/capa" ENV CAPA_BIN "${CAPA_DIR}/capa" +ENV EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR "/opt/assets" ENV EXTRACTED_FILE_HTTP_SERVER_DEBUG $EXTRACTED_FILE_HTTP_SERVER_DEBUG ENV EXTRACTED_FILE_HTTP_SERVER_ENABLE $EXTRACTED_FILE_HTTP_SERVER_ENABLE ENV EXTRACTED_FILE_HTTP_SERVER_ZIP $EXTRACTED_FILE_HTTP_SERVER_ZIP @@ -141,6 +142,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour rsync && \ python3 -m pip install --break-system-packages --no-compile --no-cache-dir \ clamd \ + dominate \ psutil \ pycryptodome \ python-magic \ @@ -153,7 +155,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour chmod +x "$SUPERCRONIC" && \ mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \ ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \ - mkdir -p "${SRC_BASE_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" && \ + mkdir -p "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}" "${SRC_BASE_DIR}" "${YARA_RULES_DIR}" "${YARA_RULES_SRC_DIR}" && \ cd "${SRC_BASE_DIR}" && \ curl -sSL "${YARA_URL}" | tar xzf - -C "${SRC_BASE_DIR}" && \ cd "./yara-${YARA_VERSION}" && \ @@ -214,10 +216,29 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour ln -r -s /usr/local/bin/zeek_carve_scanner.py /usr/local/bin/capa_scan.py && \ echo "0 */6 * * * /bin/bash /usr/local/bin/capa-update.sh\n0 */6 * * * /usr/local/bin/yara_rules_setup.sh -r \"${YARA_RULES_SRC_DIR}\" -y \"${YARA_RULES_DIR}\"" > ${SUPERCRONIC_CRONTAB} +USER ${PUSER} + +RUN /usr/bin/freshclam freshclam --config-file=/etc/clamav/freshclam.conf + +USER root + +ADD nginx/landingpage/css "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css" +ADD nginx/landingpage/js "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/js" +ADD --chmod=644 docs/images/logo/Malcolm_background.png "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/assets/img/bg-masthead.png" +ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u_w4BMUTPHjxsI9w2_Gwfo.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" +ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u8w4BMUTPHjxsAXC-v.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" +ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u_w4BMUTPHjxsI5wq_Gwfo.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" +ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u9w4BMUTPHh7USSwiPHA.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" +ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6uyw4BMUTPHjx4wWw.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" +ADD --chmod=644 https://fonts.gstatic.com/s/lato/v24/S6u9w4BMUTPHh6UVSwiPHA.ttf "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/" +ADD --chmod=644 'https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/fonts/bootstrap-icons.woff2?856008caa5eb66df68595e734e59580d' "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/bootstrap-icons.woff2" +ADD --chmod=644 'https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/fonts/bootstrap-icons.woff?856008caa5eb66df68595e734e59580d' "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/css/bootstrap-icons.woff" + +COPY --chmod=644 docs/images/icon/favicon.ico "${EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR}/favicon.ico" COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ COPY --chmod=755 shared/bin/zeek_carve*.py /usr/local/bin/ -COPY --chmod=755 shared/bin/extracted_files_http_server.py /usr/local/bin/ +COPY --chmod=755 file-monitor/scripts/*.py /usr/local/bin/ COPY --chmod=644 shared/bin/watch_common.py /usr/local/bin/ COPY --chmod=644 scripts/malcolm_utils.py /usr/local/bin/ COPY --chmod=644 file-monitor/supervisord.conf /etc/supervisord.conf @@ -225,12 +246,6 @@ COPY --chmod=755 file-monitor/docker-entrypoint.sh /docker-entrypoint.sh COPY --chmod=755 file-monitor/*update.sh /usr/local/bin/ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic -USER ${PUSER} - -RUN /usr/bin/freshclam freshclam --config-file=/etc/clamav/freshclam.conf - -USER root - WORKDIR /zeek/extract_files ENV PATH "${CAPA_DIR}:${PATH}" diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 06c8b3a7d..46452ed8c 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.11.4 +FROM docker.elastic.co/beats/filebeat-oss:8.12.1 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index 6fcdc1512..845821b02 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash-oss:8.11.4 +FROM docker.elastic.co/logstash/logstash-oss:8.12.1 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile index d854a768c..a5dff5bad 100644 --- a/Dockerfiles/nginx.Dockerfile +++ b/Dockerfiles/nginx.Dockerfile @@ -4,7 +4,6 @@ # thanks to: nginx - https://github.com/nginxinc/docker-nginx/blob/master/mainline/alpine/Dockerfile # kvspb/nginx-auth-ldap - https://github.com/kvspb/nginx-auth-ldap # tiredofit/docker-nginx-ldap - https://github.com/tiredofit/docker-nginx-ldap/blob/master/Dockerfile -# jwilder/nginx-proxy - https://github.com/jwilder/nginx-proxy/blob/master/Dockerfile.alpine #################################################################################### @@ -101,8 +100,6 @@ ADD https://opensearch.org/assets/brand/SVG/Logo/opensearch_logo_default.svg /us ADD https://opensearch.org/assets/brand/SVG/Logo/opensearch_logo_darkmode.svg /usr/share/nginx/html/assets/img/ ADD https://opensearch.org/assets/brand/SVG/Mark/opensearch_mark_default.svg /usr/share/nginx/html/assets/img/ ADD https://opensearch.org/assets/brand/SVG/Mark/opensearch_mark_darkmode.svg /usr/share/nginx/html/assets/img/ -ADD https://raw.githubusercontent.com/arkime/arkime/main/assets/Arkime_Logo_FullGradientBlack.svg /usr/share/nginx/html/assets/img/ -ADD https://raw.githubusercontent.com/arkime/arkime/main/assets/Arkime_Logo_FullGradientWhite.svg /usr/share/nginx/html/assets/img/ ADD https://raw.githubusercontent.com/gchq/CyberChef/master/src/web/static/images/logo/cyberchef.svg /usr/share/nginx/html/assets/img/ ADD https://raw.githubusercontent.com/netbox-community/netbox/develop/netbox/project-static/img/netbox_icon.svg /usr/share/nginx/html/assets/img/ ADD https://fonts.gstatic.com/s/lato/v24/S6u_w4BMUTPHjxsI9w2_Gwfo.ttf /usr/share/nginx/html/css/ @@ -201,7 +198,7 @@ RUN set -x ; \ make -j$(getconf _NPROCESSORS_ONLN) ; \ make install ; \ rm -rf /etc/nginx/html/ ; \ - mkdir -p /etc/nginx/conf.d/ /etc/nginx/auth/ /usr/share/nginx/html/ ; \ + mkdir -p /etc/nginx/conf.d/ /etc/nginx/templates/ /etc/nginx/auth/ /usr/share/nginx/html/ ; \ install -m644 html/50x.html /usr/share/nginx/html/ ; \ install -m755 objs/nginx-debug /usr/sbin/nginx-debug ; \ install -m755 objs/ngx_http_xslt_filter_module-debug.so /usr/lib/nginx/modules/ngx_http_xslt_filter_module-debug.so ; \ @@ -227,7 +224,7 @@ RUN set -x ; \ | xargs -r apk info --installed \ | sort -u \ )" ; \ - apk add --no-cache --virtual .nginx-rundeps $runDeps ca-certificates bash wget openssl apache2-utils openldap shadow stunnel supervisor tini tzdata; \ + apk add --no-cache --virtual .nginx-rundeps $runDeps ca-certificates bash jq wget openssl apache2-utils openldap shadow stunnel supervisor tini tzdata; \ update-ca-certificates; \ apk del .nginx-build-deps ; \ apk del .gettext ; \ @@ -237,17 +234,16 @@ RUN set -x ; \ find /usr/share/nginx/html/ -type d -exec chmod 755 "{}" \; && \ find /usr/share/nginx/html/ -type f -exec chmod 644 "{}" \; -COPY --from=jwilder/nginx-proxy:alpine /app/nginx.tmpl /etc/nginx/ -COPY --from=jwilder/nginx-proxy:alpine /etc/nginx/network_internal.conf /etc/nginx/ -COPY --from=jwilder/nginx-proxy:alpine /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/ COPY --from=docbuild /site/_site /usr/share/nginx/html/readme ADD nginx/landingpage /usr/share/nginx/html COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ ADD nginx/scripts /usr/local/bin/ ADD nginx/*.conf /etc/nginx/ +ADD nginx/templates /etc/nginx/templates/ ADD nginx/supervisord.conf /etc/ COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/assets/favicon.ico +COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/favicon.ico COPY --chmod=644 docs/images/logo/Malcolm_background.png /usr/share/nginx/html/assets/img/bg-masthead.png VOLUME ["/etc/nginx/certs", "/etc/nginx/dhparam"] diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 90dbbcd22..d31fc814d 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -1,65 +1,3 @@ -FROM debian:12-slim as build - -ENV DEBIAN_FRONTEND noninteractive -ENV TERM xterm - -# for build -ARG ZEEK_VERSION=6.1.0 -ENV ZEEK_VERSION $ZEEK_VERSION -ARG ZEEK_DBG=0 -ENV ZEEK_DBG $ZEEK_DBG -ARG BUILD_JOBS=4 -ENV BUILD_JOBS $BUILD_JOBS -ENV CCACHE_DIR "/var/spool/ccache" -ENV CCACHE_COMPRESS 1 -ENV CMAKE_C_COMPILER clang-14 -ENV CMAKE_CXX_COMPILER clang++-14 -ENV CXXFLAGS "-stdlib=libc++ -lc++abi" -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 - -RUN apt-get -q update && \ - apt-get -y -q --no-install-recommends upgrade && \ - apt-get install -q -y --no-install-recommends \ - bison \ - ca-certificates \ - ccache \ - clang \ - cmake \ - curl \ - flex \ - git \ - libc++-dev \ - libc++abi-dev \ - libfl-dev \ - libgoogle-perftools-dev \ - libgoogle-perftools4 \ - libkrb5-3 \ - libkrb5-dev \ - libmaxminddb-dev \ - libpcap-dev \ - libssl-dev \ - libtcmalloc-minimal4 \ - make \ - ninja-build \ - python3 \ - python3-dev \ - python3-git \ - python3-semantic-version \ - sudo \ - swig \ - zlib1g-dev && \ - mkdir -p /usr/share/src/zeek "${CCACHE_DIR}" && \ - cd /usr/share/src && \ - ( curl -sSL "https://download.zeek.org/zeek-${ZEEK_VERSION}.tar.gz" | tar xzf - -C ./zeek --strip-components 1 ) && \ - cd /usr/share/src/zeek && \ - [ "$ZEEK_DBG" = "1" ] && \ - ./configure --prefix=/opt/zeek --generator=Ninja --ccache --enable-perftools --enable-debug || \ - ./configure --prefix=/opt/zeek --generator=Ninja --ccache --enable-perftools && \ - ninja -C build -j "${BUILD_JOBS}" && \ - cd ./build && \ - cpack -G DEB - FROM debian:12-slim # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. @@ -100,7 +38,7 @@ ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" # for download and install -ARG ZEEK_VERSION=6.1.0 +ARG ZEEK_VERSION=6.1.1-0 ENV ZEEK_VERSION $ZEEK_VERSION # put Zeek and Spicy in PATH @@ -110,13 +48,9 @@ ENV PATH "${ZEEK_DIR}/bin:${PATH}" # for build ENV CCACHE_DIR "/var/spool/ccache" ENV CCACHE_COMPRESS 1 -ENV CMAKE_C_COMPILER clang-14 -ENV CMAKE_CXX_COMPILER clang++-14 -ENV CXXFLAGS "-stdlib=libc++ -lc++abi" - -COPY --from=build /usr/share/src/zeek/build/*.deb /tmp/zeekdebs/ -# add script for building 3rd-party plugins +# add script for downloading zeek and building 3rd-party plugins +ADD shared/bin/zeek-deb-download.sh /usr/local/bin/ ADD shared/bin/zeek_install_plugins.sh /usr/local/bin/ # build and install system packages, zeek, spicy and plugins @@ -129,19 +63,18 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ bison \ ca-certificates \ ccache \ - clang \ cmake \ curl \ file \ flex \ + g++ \ + gcc \ git \ gnupg2 \ iproute2 \ jq \ less \ libatomic1 \ - libc++-dev \ - libc++abi-dev \ libcap2-bin \ libfl-dev \ libfl2 \ @@ -179,8 +112,10 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ vim-tiny \ xxd \ zlib1g-dev && \ - dpkg -i /tmp/zeekdebs/*.deb && \ python3 -m pip install --break-system-packages --no-cache-dir pymisp stix2 taxii2-client dateparser && \ + mkdir -p /tmp/zeek-packages && \ + bash /usr/local/bin/zeek-deb-download.sh -o /tmp/zeek-packages -z "${ZEEK_VERSION}" && \ + dpkg -i /tmp/zeek-packages/*.deb && \ curl -fsSLO "$SUPERCRONIC_URL" && \ echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \ chmod +x "$SUPERCRONIC" && \ @@ -230,15 +165,6 @@ ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SP ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 25 ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" -RUN mkdir -p /tmp/logs && \ - cd /tmp/logs && \ - "$ZEEK_DIR"/bin/zeek -NN local >zeeknn.log 2>/dev/null && \ - bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_PLUGINS_GREP" zeeknn.log) >= $ZEEK_THIRD_PARTY_PLUGINS_COUNT)) && echo 'Zeek plugins loaded correctly' || (echo 'One or more Zeek plugins did not load correctly' && cat zeeknn.log && exit 1)" && \ - "$ZEEK_DIR"/bin/zeek -C -r /tmp/pcaps/udp.pcap local policy/misc/loaded-scripts 2>/dev/null && \ - bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_SCRIPTS_GREP" loaded_scripts.log) == $ZEEK_THIRD_PARTY_SCRIPTS_COUNT)) && echo 'Zeek scripts loaded correctly' || (echo 'One or more Zeek scripts did not load correctly' && cat loaded_scripts.log && exit 1)" && \ - cd /tmp && \ - rm -rf /tmp/logs /tmp/pcaps - RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \ usermod -a -G tty ${PUSER} && \ @@ -251,6 +177,15 @@ RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \ ln -sfr /usr/local/bin/pcap_processor.py /usr/local/bin/pcap_zeek_processor.py && \ ln -sfr /usr/local/bin/malcolm_utils.py "${ZEEK_DIR}"/bin/malcolm_utils.py +RUN mkdir -p /tmp/logs && \ + cd /tmp/logs && \ + "$ZEEK_DIR"/bin/zeek-offline -NN local >zeeknn.log 2>/dev/null && \ + bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_PLUGINS_GREP" zeeknn.log) >= $ZEEK_THIRD_PARTY_PLUGINS_COUNT)) && echo 'Zeek plugins loaded correctly' || (echo 'One or more Zeek plugins did not load correctly' && cat zeeknn.log && exit 1)" && \ + "$ZEEK_DIR"/bin/zeek-offline -C -r /tmp/pcaps/udp.pcap local policy/misc/loaded-scripts 2>/dev/null && \ + bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_SCRIPTS_GREP" loaded_scripts.log) == $ZEEK_THIRD_PARTY_SCRIPTS_COUNT)) && echo 'Zeek scripts loaded correctly' || (echo 'One or more Zeek scripts did not load correctly' && cat loaded_scripts.log && exit 1)" && \ + cd /tmp && \ + rm -rf /tmp/logs /tmp/pcaps + #Whether or not to auto-tag logs based on filename ARG AUTO_TAG=true #Whether or not to start up the pcap_processor script to monitor pcaps @@ -301,6 +236,7 @@ ENV PCAP_FILTER $PCAP_FILTER ENV PCAP_NODE_NAME $PCAP_NODE_NAME # environment variables for zeek runtime tweaks (used in local.zeek) +ARG ZEEK_DISABLE_STATS=true ARG ZEEK_DISABLE_HASH_ALL_FILES= ARG ZEEK_DISABLE_LOG_PASSWORDS= ARG ZEEK_DISABLE_SSL_VALIDATE_CERTS= @@ -321,6 +257,7 @@ ARG ZEEK_DISABLE_SPICY_TFTP= ARG ZEEK_DISABLE_SPICY_WIREGUARD= ARG ZEEK_SYNCHROPHASOR_DETAILED= +ENV ZEEK_DISABLE_STATS $ZEEK_DISABLE_STATS ENV ZEEK_DISABLE_HASH_ALL_FILES $ZEEK_DISABLE_HASH_ALL_FILES ENV ZEEK_DISABLE_LOG_PASSWORDS $ZEEK_DISABLE_LOG_PASSWORDS ENV ZEEK_DISABLE_SSL_VALIDATE_CERTS $ZEEK_DISABLE_SSL_VALIDATE_CERTS diff --git a/NOTICE.txt b/NOTICE.txt index a27e8c07b..25a298a20 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,4 +1,4 @@ -This project contains code from the Cybersecurity and Infrastructure Security Agency's Malcolm Project +This project contains code from CISA's Malcolm Project https://github.com/cisagov/Malcolm diff --git a/_config.yml b/_config.yml index 12d935ffd..68cdb8369 100644 --- a/_config.yml +++ b/_config.yml @@ -57,7 +57,7 @@ exclude: - arkime-raw - config - dashboards - - docker-compose-standalone.yml + - docker-compose-dev.yml - docker-compose.yml - Dockerfiles - docs/images/font diff --git a/api/project/__init__.py b/api/project/__init__.py index 9e5106b34..96adbff9f 100644 --- a/api/project/__init__.py +++ b/api/project/__init__.py @@ -11,7 +11,7 @@ import urllib3 import warnings -from collections import defaultdict +from collections import defaultdict, OrderedDict from collections.abc import Iterable from datetime import datetime from flask import Flask, jsonify, request @@ -152,6 +152,15 @@ field_type_map['time'] = 'date' field_type_map['timestamp'] = 'date' +# field type maps to various supported "missing" values +# TODO: do I need to handle weird ones like "date" and "geo"? +missing_field_map = defaultdict(lambda: '-') +missing_field_map['double'] = 0.0 +missing_field_map['float'] = 0.0 +missing_field_map['integer'] = 0 +missing_field_map['ip'] = '0.0.0.0' +missing_field_map['long'] = 0 + urllib3.disable_warnings() warnings.filterwarnings( "ignore", @@ -212,32 +221,16 @@ ) -def deep_get(d, keys, default=None): - assert type(keys) is list - if d is None: - return default - if not keys: - return d - return deep_get(d.get(keys[0]), keys[1:], default) - - -def get_iterable(x): - if isinstance(x, Iterable) and not isinstance(x, str): - return x - else: - return (x,) - - def random_id(length=20): return ''.join(random.choices(string.ascii_letters + string.digits, k=length)) def get_request_arguments(req): arguments = {} - if 'POST' in get_iterable(req.method): + if 'POST' in malcolm_utils.get_iterable(req.method): if (data := req.get_json() if req.is_json else None) and isinstance(data, dict): arguments.update(data) - if 'GET' in get_iterable(req.method): + if 'GET' in malcolm_utils.get_iterable(req.method): arguments.update(request.args) if debugApi: print(f"{req.method} {req.path} arguments: {json.dumps(arguments)}") @@ -342,7 +335,7 @@ def urls_for_field(fieldname, start_time=None, end_time=None): translated = [] if databaseMode != malcolm_utils.DatabaseMode.ElasticsearchRemote: - for field in get_iterable(fieldname): + for field in malcolm_utils.get_iterable(fieldname): for url_regex_pair in fields_to_urls: if (len(url_regex_pair) == 2) and re.search(url_regex_pair[0], field, flags=re.IGNORECASE): for url in url_regex_pair[1]: @@ -370,7 +363,7 @@ def doctype_from_args(args): return doctype network|host """ - return deep_get(args, ["doctype"], app.config["DOCTYPE_DEFAULT"]) + return malcolm_utils.deep_get(args, ["doctype"], app.config["DOCTYPE_DEFAULT"]) def index_from_args(args): @@ -502,7 +495,7 @@ def filtervalues(search, args): # field != value s = s.exclude( "terms", - **{fieldname[1:]: get_iterable(filtervalue)}, + **{fieldname[1:]: malcolm_utils.get_iterable(filtervalue)}, ) else: # field exists ("is not null") @@ -513,7 +506,7 @@ def filtervalues(search, args): # field == value s = s.filter( "terms", - **{fieldname: get_iterable(filtervalue)}, + **{fieldname: malcolm_utils.get_iterable(filtervalue)}, ) else: # field does not exist ("is null") @@ -524,7 +517,7 @@ def filtervalues(search, args): return (filters, s) -def bucketfield(fieldname, current_request, urls=None): +def aggfields(fieldnames, current_request, urls=None): """Returns a bucket aggregation for a particular field over a given time range Parameters @@ -550,38 +543,60 @@ def bucketfield(fieldname, current_request, urls=None): global SearchClass args = get_request_arguments(current_request) + idx = index_from_args(args) s = SearchClass( using=databaseClient, - index=index_from_args(args), + index=idx, ).extra(size=0) start_time_ms, end_time_ms, s = filtertime(s, args) filters, s = filtervalues(s, args) - bucket_limit = int(deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"])) + bucket_limit = int(malcolm_utils.deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"])) last_bucket = s.aggs - for fname in get_iterable(fieldname): + + for fname in malcolm_utils.get_iterable(fieldnames): + # Get the field mapping type for this field, and map it to a good default "missing" + # (empty bucket) label for the bucket missing= parameter below + mapping = databaseClient.indices.get_field_mapping( + fname, + index=idx, + ) + missing_val = ( + missing_field_map[ + next( + iter( + malcolm_utils.dictsearch( + mapping[next(iter(OrderedDict(sorted(mapping.items(), reverse=True))))], 'type' + ) + ), + None, + ) + ] + if (mapping and isinstance(mapping, dict)) + else missing_field_map[None] + ) + + # chain on the aggregation for the next field last_bucket = last_bucket.bucket( - "values", + fname, "terms", field=fname, size=bucket_limit, + missing=missing_val, ) response = s.execute() + + top_bucket_name = next(iter(malcolm_utils.get_iterable(fieldnames))) + result_dict = { + top_bucket_name: response.aggregations.to_dict().get(top_bucket_name, {}), + 'range': (start_time_ms // 1000, end_time_ms // 1000), + 'filter': filters, + 'fields': malcolm_utils.get_iterable(fieldnames), + } if (urls is not None) and (len(urls) > 0): - return jsonify( - values=response.aggregations.to_dict().get("values", {}), - range=(start_time_ms // 1000, end_time_ms // 1000), - filter=filters, - fields=get_iterable(fieldname), - urls=urls, - ) - else: - return jsonify( - values=response.aggregations.to_dict().get("values", {}), - range=(start_time_ms // 1000, end_time_ms // 1000), - filter=filters, - fields=get_iterable(fieldname), - ) + result_dict['urls'] = urls + + return jsonify(result_dict) @app.route( @@ -594,14 +609,14 @@ def bucketfield(fieldname, current_request, urls=None): methods=['GET', 'POST'], ) def aggregate(fieldname): - """Returns the aggregated values and counts for a given field name, see bucketfield + """Returns the aggregated values and counts for a given field name, see aggfields Parameters ---------- fieldname : string the name of the field(s) to be bucketed (comma-separated if multiple fields) request : Request - see bucketfield + see aggfields Returns ------- @@ -612,7 +627,7 @@ def aggregate(fieldname): """ start_time, end_time = gettimes(get_request_arguments(request)) fields = fieldname.split(",") - return bucketfield( + return aggfields( fields, request, urls=urls_for_field(fields, start_time=start_time, end_time=end_time), @@ -645,7 +660,7 @@ def document(): s = SearchClass( using=databaseClient, index=index_from_args(args), - ).extra(size=int(deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"]))) + ).extra(size=int(malcolm_utils.deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"]))) start_time_ms, end_time_ms, s = filtertime(s, args, default_from="1970-1-1", default_to="now") filters, s = filtervalues(s, args) return jsonify( @@ -707,7 +722,7 @@ def fields(): args = get_request_arguments(request) - templateName = deep_get(args, ["template"], app.config["MALCOLM_TEMPLATE"]) + templateName = malcolm_utils.deep_get(args, ["template"], app.config["MALCOLM_TEMPLATE"]) arkimeFields = (templateName == app.config["MALCOLM_TEMPLATE"]) and (doctype_from_args(args) == 'network') fields = defaultdict(dict) @@ -720,12 +735,12 @@ def fields(): index=index_from_args(args), ).extra(size=5000) for hit in [x['_source'] for x in s.execute().to_dict().get('hits', {}).get('hits', [])]: - if (fieldname := deep_get(hit, ['dbField2'])) and (fieldname not in fields): + if (fieldname := malcolm_utils.deep_get(hit, ['dbField2'])) and (fieldname not in fields): if debugApi: hit['source'] = 'arkime' fields[fieldname] = { - 'description': deep_get(hit, ['help']), - 'type': field_type_map[deep_get(hit, ['type'])], + 'description': malcolm_utils.deep_get(hit, ['help']), + 'type': field_type_map[malcolm_utils.deep_get(hit, ['type'])], } if debugApi: fields[fieldname]['original'] = [hit] @@ -741,35 +756,39 @@ def fields(): verify=opensearchSslVerify, ).json() - for template in deep_get(getTemplateResponseJson, ["index_templates"]): + for template in malcolm_utils.deep_get(getTemplateResponseJson, ["index_templates"]): # top-level fields - for fieldname, fieldinfo in deep_get( + for fieldname, fieldinfo in malcolm_utils.deep_get( template, ["index_template", "template", "mappings", "properties"], ).items(): if debugApi: fieldinfo['source'] = f'opensearch.{templateName}' if 'type' in fieldinfo: - fields[fieldname]['type'] = field_type_map[deep_get(fieldinfo, ['type'])] + fields[fieldname]['type'] = field_type_map[malcolm_utils.deep_get(fieldinfo, ['type'])] if debugApi: fields[fieldname]['original'] = fields[fieldname].get('original', []) + [fieldinfo] # descendant component fields - for componentName in get_iterable(deep_get(template, ["index_template", "composed_of"])): + for componentName in malcolm_utils.get_iterable( + malcolm_utils.deep_get(template, ["index_template", "composed_of"]) + ): getComponentResponseJson = requests.get( f'{opensearchUrl}/_component_template/{componentName}', auth=opensearchReqHttpAuth, verify=opensearchSslVerify, ).json() - for component in get_iterable(deep_get(getComponentResponseJson, ["component_templates"])): - for fieldname, fieldinfo in deep_get( + for component in malcolm_utils.get_iterable( + malcolm_utils.deep_get(getComponentResponseJson, ["component_templates"]) + ): + for fieldname, fieldinfo in malcolm_utils.deep_get( component, ["component_template", "template", "mappings", "properties"], ).items(): if debugApi: fieldinfo['source'] = f'opensearch.{templateName}.{componentName}' if 'type' in fieldinfo: - fields[fieldname]['type'] = field_type_map[deep_get(fieldinfo, ['type'])] + fields[fieldname]['type'] = field_type_map[malcolm_utils.deep_get(fieldinfo, ['type'])] if debugApi: fields[fieldname]['original'] = fields[fieldname].get('original', []) + [fieldinfo] @@ -788,12 +807,12 @@ def fields(): auth=opensearchReqHttpAuth, verify=opensearchSslVerify, ).json()['fields']: - if fieldname := deep_get(field, ['name']): + if fieldname := malcolm_utils.deep_get(field, ['name']): if debugApi: field['source'] = 'dashboards' - field_types = deep_get(field, ['esTypes'], []) + field_types = malcolm_utils.deep_get(field, ['esTypes'], []) fields[fieldname]['type'] = field_type_map[ - field_types[0] if len(field_types) > 0 else deep_get(fields[fieldname], ['type']) + field_types[0] if len(field_types) > 0 else malcolm_utils.deep_get(fields[fieldname], ['type']) ] if debugApi: fields[fieldname]['original'] = fields[fieldname].get('original', []) + [field] @@ -934,7 +953,7 @@ def event(): data = get_request_arguments(request) nowTimeStr = datetime.now().astimezone(pytz.utc).isoformat().replace('+00:00', 'Z') if 'alert' in data: - alert[app.config["MALCOLM_NETWORK_INDEX_TIME_FIELD"]] = deep_get( + alert[app.config["MALCOLM_NETWORK_INDEX_TIME_FIELD"]] = malcolm_utils.deep_get( data, [ 'alert', @@ -944,7 +963,7 @@ def event(): nowTimeStr, ) alert['firstPacket'] = alert[app.config["MALCOLM_NETWORK_INDEX_TIME_FIELD"]] - alert['lastPacket'] = deep_get( + alert['lastPacket'] = malcolm_utils.deep_get( data, [ 'alert', @@ -964,7 +983,7 @@ def event(): alert['event']['dataset'] = 'alerting' alert['event']['module'] = 'alerting' alert['event']['url'] = '/dashboards/app/alerting#/dashboard' - alertId = deep_get( + alertId = malcolm_utils.deep_get( data, [ 'alert', @@ -972,7 +991,7 @@ def event(): ], ) alert['event']['id'] = alertId if alertId else random_id() - if alertBody := deep_get( + if alertBody := malcolm_utils.deep_get( data, [ 'alert', @@ -980,7 +999,7 @@ def event(): ], ): alert['event']['original'] = alertBody - if triggerName := deep_get( + if triggerName := malcolm_utils.deep_get( data, [ 'alert', @@ -989,7 +1008,7 @@ def event(): ], ): alert['event']['reason'] = triggerName - if monitorName := deep_get( + if monitorName := malcolm_utils.deep_get( data, [ 'alert', @@ -1000,7 +1019,7 @@ def event(): alert['rule'] = {} alert['rule']['name'] = monitorName if alertSeverity := str( - deep_get( + malcolm_utils.deep_get( data, [ 'alert', @@ -1014,7 +1033,7 @@ def event(): alert['event']['risk_score_norm'] = sevnum alert['event']['severity'] = sevnum alert['event']['severity_tags'] = 'Alert' - if alertResults := deep_get( + if alertResults := malcolm_utils.deep_get( data, [ 'alert', @@ -1022,7 +1041,7 @@ def event(): ], ): if len(alertResults) > 0: - if hitCount := deep_get(alertResults[0], ['hits', 'total', 'value'], 0): + if hitCount := malcolm_utils.deep_get(alertResults[0], ['hits', 'total', 'value'], 0): alert['event']['hits'] = hitCount docDateStr = dateparser.parse(alert[app.config["MALCOLM_NETWORK_INDEX_TIME_FIELD"]]).strftime('%y%m%d') diff --git a/api/requirements.txt b/api/requirements.txt index f76b2a87f..3f95576d4 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -5,5 +5,5 @@ opensearch-py==2.4.2 requests==2.31.0 regex==2022.3.2 dateparser==1.1.1 -elasticsearch==8.11.1 -elasticsearch-dsl==8.11.0 \ No newline at end of file +elasticsearch==8.12.0 +elasticsearch-dsl==8.12.0 \ No newline at end of file diff --git a/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile b/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile index 890787325..5039422bd 100644 --- a/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile +++ b/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile @@ -55,7 +55,7 @@ USER $ARKIME_USER RUN git clone --recursive --depth=1 --single-branch -b "$GITHUB_BRANCH" "$GITHUB_URL" "$ARKIME_DIR" && \ cd "$ARKIME_DIR" && \ - bash -c 'for i in /opt/patches/*; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ + bash -c 'for i in /opt/patches/*.patch; do patch -p 1 -r - --no-backup-if-mismatch < $i || true; done' && \ export PATH="$ARKIME_DIR/bin:$ARKIME_DIR/node-v10.21.0-linux-x64/bin:${PATH}" && \ sudo ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \ sudo ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \ diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 3a71c6291..a34b194b3 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -14,6 +14,7 @@ cronQueries=true dropGroup=arkime dropUser=arkime elasticsearch=http://opensearch:9200 +footerTemplate=_version_ | arkime.com 🦉 | Malc⦿lm 📄 | Dashboards 📊 | NetBox 💻 | _responseTime_ms ⏱️ freeSpaceG=10% geoLite2ASN=/opt/arkime/etc/GeoLite2-ASN.mmdb geoLite2Country=/opt/arkime/etc/GeoLite2-Country.mmdb @@ -51,6 +52,7 @@ rotateIndex=daily rulesFiles= smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: spiDataMaxIndices=7 +spiViewCategoryOrder=malcolm supportSha256=false tcpSaveTimeout=720 tcpTimeout=600 @@ -77,7 +79,7 @@ tpacketv3NumThreads=2 tpacketv3BlockSize=8388608 pcapWriteMethod=simple pcapWriteSize=2560000 -simpleCompression=none +simpleCompression=zstd simpleZstdLevel=3 simpleGzipLevel=3 packetThreads=2 @@ -133,6 +135,7 @@ network.protocol=db:network.protocol;group:malcolm;kind:termfield;viewerOnly:tru network.protocol_version=db:network.protocol_version;group:malcolm;kind:termfield;viewerOnly:true;friendly:Service Version;help:Service Version network.transport=db:network.transport;group:malcolm;kind:lotermfield;viewerOnly:true;friendly:Protocol;help:Protocol network.vlan.id=db:network.vlan.id;group:malcolm;kind:integer;viewerOnly:false;friendly:VLAN ID;help:VLAN ID +related.device_id=db:related.device_id;group:malcolm;kind:integer;viewerOnly:true;friendly:Related Device ID;help:Related Device ID related.device_name=db:related.device_name;group:malcolm;kind:termfield;viewerOnly:true;friendly:Related Device Name;help:Related Device Name related.device_type=db:related.device_type;group:malcolm;kind:termfield;viewerOnly:true;friendly:Related Device Type;help:Related Device Type related.hash=db:related.hash;group:malcolm;kind:termfield;viewerOnly:true;friendly:Related Hash;help:Related Hash @@ -2581,7 +2584,7 @@ n_netbox_device_originator=require:source.device.id;title:NetBox Device (Origina n_netbox_device_responder=require:destination.device.id;title:NetBox Device (Responder);fields:destination.device.cluster,destination.device.device_type,destination.device.id,destination.device.manufacturer,destination.device.name,destination.device.role,destination.device.service,destination.device.site,destination.device.url n_netbox_segment_originator=require:source.segment.id;title:NetBox Segment (Originator);fields:source.segment.id,source.segment.name,source.segment.site,source.segment.tenant,source.segment.url n_netbox_segment_responder=require:destination.segment.id;title:NetBox Segment (Responder);fields:destination.segment.id,destination.segment.name,destination.segment.site,destination.segment.tenant,destination.segment.url -n_netbox_z_related=require:related.site;title:NetBox Related;fields:related.site,network.name,related.manufacturer,related.device_type,related.role,related.device_name,related.service +n_netbox_z_related=require:related.site;title:NetBox Related;fields:related.site,network.name,related.manufacturer,related.device_type,related.role,related.device_id,related.device_name,related.service o_zeek_bacnet=require:zeek.bacnet;title:Zeek bacnet.log;fields:zeek.bacnet.bvlc_function,zeek.bacnet.pdu_type,zeek.bacnet.pdu_service,zeek.bacnet.invoke_id,zeek.bacnet.instance_number,zeek.bacnet.result_code o_zeek_bacnet_discovery=require:zeek.bacnet_discovery;title:Zeek bacnet_discovery.log;fields:zeek.bacnet.pdu_service,zeek.bacnet_discovery.object_type,zeek.bacnet_discovery.vendor,zeek.bacnet_discovery.range,zeek.bacnet_discovery.range_low,zeek.bacnet_discovery.range_high,zeek.bacnet_discovery.object_name o_zeek_bacnet_device_control=require:zeek.bacnet_device_control;title:Zeek bacnet_device_control.log;fields:zeek.bacnet.pdu_service,zeek.bacnet_device_control.time_duration,zeek.bacnet_device_control.device_state,zeek.bacnet_device_control.result,zeek.bacnet_device_control.result_code diff --git a/arkime/patch/db_pl_quiet_backup_warning.patch b/arkime/patch/db_pl_quiet_backup_warning.patch deleted file mode 100644 index aa2fb9e67..000000000 --- a/arkime/patch/db_pl_quiet_backup_warning.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/db/db.pl b/db/db.pl -index 9aa04d2d..44145db0 100755 ---- a/db/db.pl -+++ b/db/db.pl -@@ -7418,7 +7418,7 @@ my $health = dbCheckHealth(); - - my $nodes = esGet("/_nodes"); - $main::numberOfNodes = dataNodes($nodes->{nodes}); --logmsg "It is STRONGLY recommended that you stop ALL Arkime captures and viewers before proceeding. Use 'db.pl ${main::elasticsearch} backup' to backup db first.\n\n"; -+# logmsg "It is STRONGLY recommended that you stop ALL Arkime captures and viewers before proceeding. Use 'db.pl ${main::elasticsearch} backup' to backup db first.\n\n"; - if ($main::numberOfNodes == 1) { - logmsg "There is $main::numberOfNodes elastic search data node, if you expect more please fix first before proceeding.\n\n"; - } else { diff --git a/arkime/patch/empty.patch b/arkime/patch/empty.patch new file mode 100644 index 000000000..e69de29bb diff --git a/arkime/patch/footer_links.patch b/arkime/patch/footer_links.patch deleted file mode 100644 index c2ab1d837..000000000 --- a/arkime/patch/footer_links.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/viewer/vueapp/src/components/utils/Footer.vue b/viewer/vueapp/src/components/utils/Footer.vue -index 84b28a45..7a414ca2 100644 ---- a/viewer/vueapp/src/components/utils/Footer.vue -+++ b/viewer/vueapp/src/components/utils/Footer.vue -@@ -4,9 +4,12 @@ -

- - Arkime v{{ version }} | -- arkime.com -+ arkime.com 🦉 -+ | Malc⦿lm 📄 -+ | Dashboards 📊 -+ | NetBox 💻 - -- | {{ responseTime | commaString }}ms -+ | {{ responseTime | commaString }}ms ⏱️ - - - | diff --git a/arkime/patch/spi_sort_malcolm.patch b/arkime/patch/spi_sort_malcolm.patch deleted file mode 100644 index b1a60a225..000000000 --- a/arkime/patch/spi_sort_malcolm.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/viewer/vueapp/src/components/spiview/Spiview.vue b/viewer/vueapp/src/components/spiview/Spiview.vue -index c178fe13..0f99b8b7 100644 ---- a/viewer/vueapp/src/components/spiview/Spiview.vue -+++ b/viewer/vueapp/src/components/spiview/Spiview.vue -@@ -953,6 +953,8 @@ export default { - - // sorted list of categories for the view - this.categoryList = Object.keys(this.categoryObjects).sort(); -+ this.categoryList.splice(this.categoryList.indexOf('malcolm'), 1); -+ this.categoryList.unshift('malcolm'); - this.categoryList.splice(this.categoryList.indexOf('general'), 1); - this.categoryList.unshift('general'); - diff --git a/arkime/patch/viewer_wider_field_detail.patch b/arkime/patch/viewer_wider_field_detail.patch deleted file mode 100644 index 9cdd10b47..000000000 --- a/arkime/patch/viewer_wider_field_detail.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/viewer/vueapp/src/components/sessions/SessionDetail.vue b/viewer/vueapp/src/components/sessions/SessionDetail.vue -index b2f32a02..f0459680 100644 ---- a/viewer/vueapp/src/components/sessions/SessionDetail.vue -+++ b/viewer/vueapp/src/components/sessions/SessionDetail.vue -@@ -846,7 +846,7 @@ export default { - .session-detail dt { - float: left; - clear: left; -- width: 160px; -+ width: 320px; - text-align: right; - margin-right: 6px; - line-height: 1.7; -@@ -854,7 +854,7 @@ export default { - } - - .session-detail dd { -- margin-left: 165px; -+ margin-left: 325px; - } - - /* more items link */ diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 8f94bfa94..5494d65f8 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -109,8 +109,8 @@ if [[ ! -f "${ARKIME_CONFIG_FILE}" ]] && [[ -r "${ARKIME_DIR}"/etc/config.orig.i fi # pcap compression - COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}" - COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}" + COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-zstd}" + COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-3}" sed -r -i "s/(simpleCompression)\s*=\s*.*/\1=$COMPRESSION_TYPE/" "${ARKIME_CONFIG_FILE}" if [[ "$COMPRESSION_TYPE" == "zstd" ]]; then sed -r -i "s/(simpleZstdLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "${ARKIME_CONFIG_FILE}" diff --git a/arkime/scripts/initarkime.sh b/arkime/scripts/initarkime.sh index a47caab03..3de1cbf53 100755 --- a/arkime/scripts/initarkime.sh +++ b/arkime/scripts/initarkime.sh @@ -100,6 +100,14 @@ if [[ "$MALCOLM_PROFILE" == "malcolm" ]]; then fi # if/else OpenSearch database initialized + if [[ "${INDEX_MANAGEMENT_ENABLED:-false}" == "true" ]]; then + [[ "${INDEX_MANAGEMENT_HOT_WARM_ENABLED:-false}" == "true" ]] && HOT_WARM_FLAG=--hotwarm || HOT_WARM_FLAG= + [[ "${OPENSEARCH_PRIMARY}" == "elasticsearch-remote" ]] && LIFECYCLE_POLCY=ilm || LIFECYCLE_POLCY=ism + $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" ${LIFECYCLE_POLCY} "${INDEX_MANAGEMENT_OPTIMIZATION_PERIOD}" "${INDEX_MANAGEMENT_RETENTION_TIME}" ${HOT_WARM_FLAG} --segments "${INDEX_MANAGEMENT_SEGMENTS}" --replicas "${INDEX_MANAGEMENT_OLDER_SESSION_REPLICAS}" --history "${INDEX_MANAGEMENT_HISTORY_RETENTION_WEEKS}" + $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" upgradenoprompt --ifneeded --${LIFECYCLE_POLCY} + echo "${LIFECYCLE_POLCY} created" + fi + # increase OpenSearch max shards per node from default if desired if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then # see https://github.com/elastic/elasticsearch/issues/40803 diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index a2c919ba9..5e38bca78 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -142,6 +142,7 @@ class MalcolmSource extends WISESource { "oui.dst", "oui.src", "protocols", + "related.device_id", "related.device_name", "related.device_type", "related.hash", @@ -2221,18 +2222,23 @@ class MalcolmSource extends WISESource { // add rick-click for opening malcolm agg api var apiLabel = "Aggregate %DBFIELD%"; - var apiURL = "mapi/agg/%DBFIELD%?from=%ISOSTART%&to=%ISOSTOP%"; + var apiURL = "/mapi/agg/%DBFIELD%?from=%ISOSTART%&to=%ISOSTOP%"; this.api.addFieldAction("malcolm_mapi_fields_zeek", { name: apiLabel, url: apiURL, all: true }); + // add rick-click for extracted-files + var extractedFilesLabel = "Browse Extracted Files"; + var extractedFilesURL = "/extracted-files/"; + this.api.addFieldAction("malcolm_mapi_field_extracted_files", { name: extractedFilesLabel, url: extractedFilesURL, fields: carvedFieldsStr }); + // add right-click for viewing original JSON document - this.api.addValueAction("malcolm_json_source", { name: "%DBFIELD% Document(s) JSON", url: "mapi/document?filter={\"%DBFIELD%\":\"%TEXT%\"}", fields: "communityId,event.id,id,network.community_id,rootId,zeek.fuid,zeek.uid" }); + this.api.addValueAction("malcolm_json_source", { name: "%DBFIELD% Document(s) JSON", url: "/mapi/document?filter={\"%DBFIELD%\":\"%TEXT%\"}", fields: "communityId,event.id,id,network.community_id,rootId,zeek.fuid,zeek.uid" }); this.api.addView("malcolm_common", "if (session.event.hash)\n" + - // id information + // id and basic connection information " div.sessionDetailMeta.bold Malcolm Common Fields\n" + - " dl.sessionDetailMeta(suffix=\"IDs\")\n" + + " dl.sessionDetailMeta(suffix=\"IDs and Basic Connection Info\")\n" + " +arrayList(session.event, 'id', 'Log ID', 'event.id')\n" + " +arrayList(session.event, 'hash', 'Log Hash', 'event.hash')\n" + " +arrayList(session.network, 'community_id', 'Connection Community ID', 'network.community_id')\n" + @@ -2240,9 +2246,6 @@ class MalcolmSource extends WISESource { " +arrayList(session.event, 'dataset', 'Log Type', 'event.dataset')\n" + " +arrayList(session.event, 'module', 'Data Source Module', 'event.module')\n" + " +arrayList(session.host, 'name', 'Malcolm Node', 'host.name')\n" + - - // basic connection information - " dl.sessionDetailMeta(suffix=\"Basic Connection Info\")\n" + " +arrayList(session.network, 'transport', 'Protocol', 'network.transport')\n" + " +arrayList(session.network, 'protocol', 'Service', 'network.protocol')\n" + " +arrayList(session.network, 'protocol_version', 'Service Version', 'network.protocol_version')\n" + @@ -2285,10 +2288,7 @@ class MalcolmSource extends WISESource { " +arrayList(session.file, 'path', 'File Path', 'file.path')\n" + " +arrayList(session.file, 'mime_type', 'File Magic', 'file.mime_type')\n" + " +arrayList(session.file, 'source', 'File Transport', 'file.source')\n" + - " +arrayList(session.related, 'hash', 'Related Hash', 'related.hash')\n" + - - // #################################################################### - " br\n"); + " +arrayList(session.related, 'hash', 'Related Hash', 'related.hash')\n"); } } diff --git a/config/arkime.env.example b/config/arkime.env.example index c3afc8271..c80aa2402 100644 --- a/config/arkime.env.example +++ b/config/arkime.env.example @@ -7,4 +7,20 @@ ARKIME_FREESPACEG=10% # https://arkime.com/settings#rotateIndex ARKIME_ROTATE_INDEX=daily +# These variables manage setting for Arkime's ILM/ISM features (https://arkime.com/faq#ilm) +# Whether or not Arkime should perform index management +INDEX_MANAGEMENT_ENABLED=false +# Time in hours/days before moving to warm and force merge (number followed by h or d) +INDEX_MANAGEMENT_OPTIMIZATION_PERIOD=30d +# Time in hours/days before deleting index (number followed by h or d) +INDEX_MANAGEMENT_RETENTION_TIME=90d +# Number of replicas for older sessions indices +INDEX_MANAGEMENT_OLDER_SESSION_REPLICAS=0 +# Number of weeks of history to retain +INDEX_MANAGEMENT_HISTORY_RETENTION_WEEKS=13 +# Number of segments to optimize sessions for +INDEX_MANAGEMENT_SEGMENTS=1 +# Whether or not Arkime should use a hot/warm design (storing non-session data in a warm index) +INDEX_MANAGEMENT_HOT_WARM_ENABLED=false + OPENSEARCH_MAX_SHARDS_PER_NODE=2500 \ No newline at end of file diff --git a/config/dashboards.env.example b/config/dashboards.env.example index b636dc379..29a395ad8 100644 --- a/config/dashboards.env.example +++ b/config/dashboards.env.example @@ -4,3 +4,8 @@ # 'http://dashboards:5601/dashboards', otherwise specify the Dashboards URL # in the format 'protocol://host:port/uri'. DASHBOARDS_URL=http://dashboards:5601/dashboards +# These values are used to handle the Arkime value actions to pivot from Arkime +# to Dashboards. The nginx-proxy container's entrypoint will try to formulate +# them automatically, but they may be specified explicitly here. +NGINX_DASHBOARDS_PREFIX= +NGINX_DASHBOARDS_PROXY_PASS= \ No newline at end of file diff --git a/config/logstash.env.example b/config/logstash.env.example index b5e6f7e56..a9436ad51 100644 --- a/config/logstash.env.example +++ b/config/logstash.env.example @@ -11,10 +11,14 @@ LOGSTASH_SEVERITY_SCORING=true LOGSTASH_REVERSE_DNS=false # Whether or not Logstash will enrich network traffic metadata via NetBox API calls LOGSTASH_NETBOX_ENRICHMENT=false +# Which types of logs will be enriched via NetBox (comma-separated list of provider.dataset, or the string all to enrich all logs) +LOGSTASH_NETBOX_ENRICHMENT_DATASETS=suricata.alert,zeek.conn,zeek.known_hosts,zeek.known_services,zeek.notice,zeek.signatures,zeek.software,zeek.weird # Whether or not unobserved network entities in Logstash data will be used to populate NetBox LOGSTASH_NETBOX_AUTO_POPULATE=false # Caching parameters for NetBox's LogStash lookups LOGSTASH_NETBOX_CACHE_SIZE=1000 LOGSTASH_NETBOX_CACHE_TTL=30 +# Zeek log types that will be ignored (dropped) by LogStash +LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout # Logstash memory allowance and other Java options LS_JAVA_OPTS=-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file diff --git a/config/netbox-common.env.example b/config/netbox-common.env.example index 000500b0c..772a9cc17 100644 --- a/config/netbox-common.env.example +++ b/config/netbox-common.env.example @@ -5,6 +5,9 @@ NETBOX_DEFAULT_SITE=Malcolm # Whether or not to create catch-all IP Prefixes for private IP space NETBOX_PRELOAD_PREFIXES=false +# Customize manufacturer matching/creation with LOGSTASH_NETBOX_AUTO_POPULATE (see logstash.env) +NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER=true +NETBOX_DEFAULT_FUZZY_THRESHOLD=0.95 # Whether to disable Malcolm's NetBox instance ('true') or not ('false') NETBOX_DISABLED=true NETBOX_POSTGRES_DISABLED=true diff --git a/config/suricata-live.env.example b/config/suricata-live.env.example index 3fd9e045c..c059c7666 100644 --- a/config/suricata-live.env.example +++ b/config/suricata-live.env.example @@ -4,5 +4,10 @@ SURICATA_LIVE_CAPTURE=false # Specifies the Suricata runmode for live capture (see # https://suricata.readthedocs.io/en/latest/performance/runmodes.html) SURICATA_RUNMODE=workers +# Whether or not enable capture statistics and include them in eve.json +SURICATA_STATS_ENABLED=false +SURICATA_STATS_EVE_ENABLED=false +SURICATA_STATS_INTERVAL=30 +SURICATA_STATS_DECODER_EVENTS=false SURICATA_PCAP_PROCESSOR=false \ No newline at end of file diff --git a/config/zeek-live.env.example b/config/zeek-live.env.example index 4cf6fc0cb..944833e85 100644 --- a/config/zeek-live.env.example +++ b/config/zeek-live.env.example @@ -1,6 +1,8 @@ # Whether or not Zeek should monitor live traffic on a local # interface (PCAP_IFACE variable below specifies capture interfaces) ZEEK_LIVE_CAPTURE=false +# Set ZEEK_DISABLE_STATS to blank to generate stats.log and capture_loss.log +ZEEK_DISABLE_STATS=true ZEEK_PCAP_PROCESSOR=false ZEEK_CRON=true diff --git a/config/zeek.env.example b/config/zeek.env.example index ab208f6a2..966a1bfa2 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -51,6 +51,8 @@ EXTRACTED_FILE_PIPELINE_VERBOSITY= EXTRACTED_FILE_HTTP_SERVER_ENABLE=false # Whether or not Zeek-extracted files served over HTTP will be archived in a Zip file EXTRACTED_FILE_HTTP_SERVER_ZIP=false +# Whether or not to use libmagic to show MIME types for Zeek-extracted files served +EXTRACTED_FILE_HTTP_SERVER_MAGIC=false # HTTP server will look in subdirectories for requested filename (e.g., in "/quarantined" and "/preserved") EXTRACTED_FILE_HTTP_SERVER_RECURSIVE=true # Environment variables for tweaking Zeek at runtime (see local.zeek) @@ -76,7 +78,7 @@ ZEEK_DISABLE_ICS_BSAP= ZEEK_DISABLE_ICS_DNP3= ZEEK_DISABLE_ICS_ENIP= ZEEK_DISABLE_ICS_ETHERCAT= -ZEEK_DISABLE_ICS_GENISYS= +ZEEK_DISABLE_ICS_GENISYS=true ZEEK_DISABLE_ICS_OPCUA_BINARY= ZEEK_DISABLE_ICS_MODBUS= ZEEK_DISABLE_ICS_PROFINET= diff --git a/dashboards/dashboards/9ee51f94-3316-4fc5-bd89-93a52af69714.json b/dashboards/dashboards/9ee51f94-3316-4fc5-bd89-93a52af69714.json index 4e502b7c2..57ea8d429 100644 --- a/dashboards/dashboards/9ee51f94-3316-4fc5-bd89-93a52af69714.json +++ b/dashboards/dashboards/9ee51f94-3316-4fc5-bd89-93a52af69714.json @@ -1,384 +1,412 @@ -{ - "version": "1.2.0", - "objects": [ - { - "id": "9ee51f94-3316-4fc5-bd89-93a52af69714", - "type": "dashboard", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T19:46:40.335Z", - "version": "WzEzMTEsMV0=", - "attributes": { - "title": "Files", - "hits": 0, - "description": "", - "panelsJSON": "[{\"version\":\"1.2.0\",\"gridData\":{\"x\":16,\"y\":0,\"w\":32,\"h\":8,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}},\"panelRefName\":\"panel_0\"},{\"version\":\"1.2.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":27,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"1.2.0\",\"gridData\":{\"x\":0,\"y\":27,\"w\":8,\"h\":28,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_2\"},{\"version\":\"1.2.0\",\"gridData\":{\"x\":40,\"y\":8,\"w\":8,\"h\":18,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_3\"},{\"version\":\"1.2.0\",\"gridData\":{\"x\":32,\"y\":8,\"w\":8,\"h\":18,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_4\"},{\"version\":\"1.2.0\",\"gridData\":{\"x\":8,\"y\":0,\"w\":8,\"h\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"1.2.0\",\"gridData\":{\"x\":8,\"y\":8,\"w\":15,\"h\":47,\"i\":\"b932bc95-a3b3-411b-a7d2-2fe43e38cf8a\"},\"panelIndex\":\"b932bc95-a3b3-411b-a7d2-2fe43e38cf8a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"1.2.0\",\"gridData\":{\"x\":23,\"y\":8,\"w\":9,\"h\":18,\"i\":\"2d3ee44d-2d7f-4573-8b02-f6e46e550238\"},\"panelIndex\":\"2d3ee44d-2d7f-4573-8b02-f6e46e550238\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"1.2.0\",\"gridData\":{\"x\":23,\"y\":26,\"w\":25,\"h\":29,\"i\":\"fecc7359-c195-4066-a565-2effd4380b9e\"},\"panelIndex\":\"fecc7359-c195-4066-a565-2effd4380b9e\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"1.2.0\",\"gridData\":{\"x\":0,\"y\":55,\"w\":48,\"h\":35,\"i\":\"8e4863be-7d69-4354-9eb4-4e30a7c983d6\"},\"panelIndex\":\"8e4863be-7d69-4354-9eb4-4e30a7c983d6\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"}]", - "optionsJSON": "{\"useMargins\":true}", - "version": 1, - "timeRestore": false, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"*\"},\"filter\":[]}" - } - }, - "references": [ - { - "name": "panel_0", - "type": "visualization", - "id": "aaa4fbb0-d5fe-4ef9-be76-405b977bcd5b" - }, - { - "name": "panel_1", - "type": "visualization", - "id": "df9e399b-efa5-4e33-b0ac-a7668a8ac2b3" - }, - { - "name": "panel_2", - "type": "visualization", - "id": "66d5d357-edce-450d-b5be-a5a00190e153" - }, - { - "name": "panel_3", - "type": "visualization", - "id": "d3a0ac2e-73cf-462e-8b03-e6ff3b8612b7" - }, - { - "name": "panel_4", - "type": "visualization", - "id": "9ba4473b-66f4-4aea-b19e-4309ec6534b8" - }, - { - "name": "panel_5", - "type": "visualization", - "id": "AWDG9goqxQT5EBNmq4BP" - }, - { - "name": "panel_6", - "type": "visualization", - "id": "269ec200-7fa6-11ec-998f-a1f630163497" - }, - { - "name": "panel_7", - "type": "visualization", - "id": "b49ab0c0-7fa9-11ec-998f-a1f630163497" - }, - { - "name": "panel_8", - "type": "visualization", - "id": "91157aa0-7fa8-11ec-998f-a1f630163497" - }, - { - "name": "panel_9", - "type": "search", - "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" - } - ], - "migrationVersion": { - "dashboard": "7.9.3" - } - }, - { - "id": "aaa4fbb0-d5fe-4ef9-be76-405b977bcd5b", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T18:58:50.233Z", - "version": "WzUwMSwxXQ==", - "attributes": { - "visState": "{\"title\":\"Files - Log Count Over Time\",\"type\":\"line\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER per 12 hours\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"interval\":\"auto\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\" \"}}],\"listeners\":{}}", - "description": "", - "title": "Files - Log Count Over Time", - "uiStateJSON": "{}", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "type": "search", - "name": "search_0", - "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "df9e399b-efa5-4e33-b0ac-a7668a8ac2b3", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T18:59:16.564Z", - "version": "Wzc4NiwxXQ==", - "attributes": { - "title": "Network Logs", - "visState": "{\"title\":\"Network Logs\",\"type\":\"markdown\",\"params\":{\"markdown\":\"### General\\n[Overview](#/dashboard/0ad3d7c2-3441-485e-9dfe-dbb22e84e576) \\n[Security Overview](#/dashboard/95479950-41f2-11ea-88fa-7151df485405) \\n[ICS/IoT Security Overview](#/dashboard/4a4bde20-4760-11ea-949c-bbb5a9feecbf) \\n[Severity](#/dashboard/d2dd0180-06b1-11ec-8c6b-353266ade330) \\n[Connections](#/dashboard/abdd7550-2c7c-40dc-947e-f6d186a158c4) \\n[Actions and Results](#/dashboard/a33e0a50-afcd-11ea-993f-b7d8522a8bed) \\n[Files](#/dashboard/9ee51f94-3316-4fc5-bd89-93a52af69714) \\n[Executables](#/dashboard/0a490422-0ce9-44bf-9a2d-19329ddde8c3) \\n[Software](#/dashboard/87d990cc-9e0b-41e5-b8fe-b10ae1da0c85) \\n[Zeek Known Summary](#/dashboard/89d1cc50-974c-11ed-bb6b-3fb06c879b11) \\n[Zeek Intelligence](#/dashboard/36ed695f-edcc-47c1-b0ec-50d20c93ce0f) \\n[Zeek Notices](#/dashboard/f1f09567-fc7f-450b-a341-19d2f2bb468b) \\n[Zeek Weird](#/dashboard/1fff49f6-0199-4a0f-820b-721aff9ff1f1) \\n[Signatures](#/dashboard/665d1610-523d-11e9-a30e-e3576242f3ed) \\n[Suricata Alerts](#/dashboard/5694ca60-cbdf-11ec-a50a-5fedd672f5c5) \\n[Asset Interaction Analysis](#/dashboard/677ee170-809e-11ed-8d5b-07069f823b6f) \\n[↪ NetBox](/netbox/) \\n[↪ Arkime](/arkime/) \\n\\n### Common Protocols\\n[DCE/RPC](#/dashboard/432af556-c5c0-4cc3-8166-b274b4e3a406) ● [DHCP](#/dashboard/2d98bb8e-214c-4374-837b-20e1bcd63a5e) ● [DNS](#/dashboard/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9) ● [FTP](#/dashboard/078b9aa5-9bd4-4f02-ae5e-cf80fa6f887b) / [TFTP](#/dashboard/bf5efbb0-60f1-11eb-9d60-dbf0411cfc48) ● [HTTP](#/dashboard/37041ee1-79c0-4684-a436-3173b0e89876) ● [IRC](#/dashboard/76f2f912-80da-44cd-ab66-6a73c8344cc3) ● [Kerberos](#/dashboard/82da3101-2a9c-4ae2-bb61-d447a3fbe673) ● [LDAP](#/dashboard/05e3e000-f118-11e9-acda-83a8e29e1a24) ● [MQTT](#/dashboard/87a32f90-ef58-11e9-974e-9d600036d105) ● [MySQL](#/dashboard/50ced171-1b10-4c3f-8b67-2db9635661a6) ● [NTLM](#/dashboard/543118a9-02d7-43fe-b669-b8652177fc37) ● [NTP](#/dashboard/af5df620-eeb6-11e9-bdef-65a192b7f586) ● [OSPF](#/dashboard/1cc01ff0-5205-11ec-a62c-7bc80e88f3f0) ● [QUIC](#/dashboard/11ddd980-e388-11e9-b568-cf17de8e860c) ● [RADIUS](#/dashboard/ae79b7d1-4281-4095-b2f6-fa7eafda9970) ● [RDP](#/dashboard/7f41913f-cba8-43f5-82a8-241b7ead03e0) ● [RFB](#/dashboard/f77bf097-18a8-465c-b634-eb2acc7a4f26) ● [SIP](#/dashboard/0b2354ae-0fe9-4fd9-b156-1c3870e5c7aa) ● [SMB](#/dashboard/42e831b9-41a9-4f35-8b7d-e1566d368773) ● [SMTP](#/dashboard/bb827f8e-639e-468c-93c8-9f5bc132eb8f) ● [SNMP](#/dashboard/4e5f106e-c60a-4226-8f64-d534abb912ab) ● [SSH](#/dashboard/caef3ade-d289-4d05-a511-149f3e97f238) ● [SSL](#/dashboard/7f77b58a-df3e-4cc2-b782-fd7f8bad8ffb) / [X.509 Certificates](#/dashboard/024062a6-48d6-498f-a91a-3bf2da3a3cd3) ● [STUN](#/dashboard/fa477130-2b8a-11ec-a9f2-3911c8571bfd) ● [Syslog](#/dashboard/92985909-dc29-4533-9e80-d3182a0ecf1d) ● [TDS](#/dashboard/bed185a0-ef82-11e9-b38a-2db3ee640e88) / [TDS RPC](#/dashboard/32587740-ef88-11e9-b38a-2db3ee640e88) / [TDS SQL](#/dashboard/fa141950-ef89-11e9-b38a-2db3ee640e88) ● [Telnet / rlogin / rsh](#/dashboard/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2) ● [Tunnels](#/dashboard/11be6381-beef-40a7-bdce-88c5398392fc)\\n\\n### ICS/IoT Protocols\\n[BACnet](#/dashboard/2bec1490-eb94-11e9-a384-0fcf32210194) ● [BSAP](#/dashboard/ca5799a0-56b5-11eb-b749-576de068f8ad) ● [DNP3](#/dashboard/870a5862-6c26-4a08-99fd-0c06cda85ba3) ● [EtherCAT](#/dashboard/4a073440-b286-11eb-a4d4-09fa12a6ebd4) ● [EtherNet/IP](#/dashboard/29a1b290-eb98-11e9-a384-0fcf32210194) ● [GENISYS](#/dashboard/03207c00-d07e-11ec-b4a7-d1b4003706b7) ● [Modbus](#/dashboard/152f29dc-51a2-4f53-93e9-6e92765567b8) ● [OPCUA Binary](#/dashboard/dd87edd0-796a-11ec-9ce6-b395c1ff58f4) ● [PROFINET](#/dashboard/a7514350-eba6-11e9-a384-0fcf32210194) ● [S7comm](#/dashboard/e76d05c0-eb9f-11e9-a384-0fcf32210194) ● [Synchrophasor](#/dashboard/2cc56240-e460-11ed-a9d5-9f591c284cb4) ● [Best Guess](#/dashboard/12e3a130-d83b-11eb-a0b0-f328ce09b0b7)\",\"type\":\"markdown\",\"fontSize\":10,\"openLinksInNewTab\":false},\"aggs\":[]}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":{\"query_string\":{\"query\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" - } - }, - "references": [], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "66d5d357-edce-450d-b5be-a5a00190e153", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T19:38:19.745Z", - "version": "WzExOTksMV0=", - "attributes": { - "title": "Files - Files By Size (Bytes)", - "visState": "{\"title\":\"Files - Files By Size (Bytes)\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.files.seen_bytes\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Bytes Seen\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":20,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "d3a0ac2e-73cf-462e-8b03-e6ff3b8612b7", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T18:58:50.233Z", - "version": "WzUwNCwxXQ==", - "attributes": { - "visState": "{\"title\":\"FIles - Destination IP Address\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"destination.ip\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"IP Address\"}}],\"listeners\":{}}", - "description": "", - "title": "FIles - Destination IP Address", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "type": "search", - "name": "search_0", - "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "9ba4473b-66f4-4aea-b19e-4309ec6534b8", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T18:58:50.233Z", - "version": "WzUwNSwxXQ==", - "attributes": { - "visState": "{\"title\":\"FIles - Source IP Address\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"File IP Address\"}}],\"listeners\":{}}", - "description": "", - "title": "FIles - Source IP Address", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "type": "search", - "name": "search_0", - "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "AWDG9goqxQT5EBNmq4BP", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T18:58:50.233Z", - "version": "WzUwOCwxXQ==", - "attributes": { - "title": "Files - Log Count", - "visState": "{\"title\":\"Files - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}", - "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "type": "search", - "name": "search_0", - "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "269ec200-7fa6-11ec-998f-a1f630163497", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T19:23:41.331Z", - "version": "WzExMDcsMV0=", - "attributes": { - "title": "Files - Source", - "visState": "{\"title\":\"Files - Source\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"file.source\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"file.source\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", - "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "b49ab0c0-7fa9-11ec-998f-a1f630163497", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T19:45:39.788Z", - "version": "WzEyOTksMV0=", - "attributes": { - "title": "Files - MIME Type", - "visState": "{\"title\":\"Files - MIME Type\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"file.mime_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Mime Type\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "91157aa0-7fa8-11ec-998f-a1f630163497", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T19:37:30.697Z", - "version": "WzExODMsMV0=", - "attributes": { - "title": "Files - Paths", - "visState": "{\"title\":\"Files - Paths\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Log Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"network.protocol\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Protocol\"},\"schema\":\"bucket\"},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"file.path\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Path\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":20,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459", - "type": "search", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-27T18:59:05.412Z", - "version": "WzY4OCwxXQ==", - "attributes": { - "title": "Files - Logs", - "description": "", - "hits": 0, - "columns": [ - "source.ip", - "destination.ip", - "file.source", - "file.mime_type", - "file.path", - "event.id" - ], - "sort": [ - [ - "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER", - "desc" - ] - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"filter\":[],\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"event.dataset:files\"}},\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" - } - ], - "migrationVersion": { - "search": "7.9.3" - } - } - ] +{ + "version": "2.11.1", + "objects": [ + { + "id": "9ee51f94-3316-4fc5-bd89-93a52af69714", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:31:54.606Z", + "version": "Wzk1MywxXQ==", + "attributes": { + "title": "Files", + "hits": 0, + "description": "", + "panelsJSON": "[{\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}},\"gridData\":{\"h\":10,\"i\":\"2\",\"w\":32,\"x\":16,\"y\":0},\"panelIndex\":\"2\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":27,\"i\":\"3\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"3\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":0,\"direction\":\"desc\"}}},\"gridData\":{\"h\":28,\"i\":\"6\",\"w\":8,\"x\":0,\"y\":27},\"panelIndex\":\"6\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"7\",\"w\":8,\"x\":40,\"y\":10},\"panelIndex\":\"7\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"8\",\"w\":8,\"x\":32,\"y\":10},\"panelIndex\":\"8\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":6,\"i\":\"11\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"11\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":4,\"i\":\"67954b42-513c-47af-af19-e2382ad27cf9\",\"w\":8,\"x\":8,\"y\":6},\"panelIndex\":\"67954b42-513c-47af-af19-e2382ad27cf9\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":47,\"i\":\"b932bc95-a3b3-411b-a7d2-2fe43e38cf8a\",\"w\":15,\"x\":8,\"y\":10},\"panelIndex\":\"b932bc95-a3b3-411b-a7d2-2fe43e38cf8a\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"gridData\":{\"h\":18,\"i\":\"2d3ee44d-2d7f-4573-8b02-f6e46e550238\",\"w\":9,\"x\":23,\"y\":10},\"panelIndex\":\"2d3ee44d-2d7f-4573-8b02-f6e46e550238\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}},\"gridData\":{\"h\":29,\"i\":\"fecc7359-c195-4066-a565-2effd4380b9e\",\"w\":25,\"x\":23,\"y\":28},\"panelIndex\":\"fecc7359-c195-4066-a565-2effd4380b9e\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_9\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":35,\"i\":\"8e4863be-7d69-4354-9eb4-4e30a7c983d6\",\"w\":48,\"x\":0,\"y\":57},\"panelIndex\":\"8e4863be-7d69-4354-9eb4-4e30a7c983d6\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_10\"}]", + "optionsJSON": "{\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"*\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "visualization", + "id": "aaa4fbb0-d5fe-4ef9-be76-405b977bcd5b" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "df9e399b-efa5-4e33-b0ac-a7668a8ac2b3" + }, + { + "name": "panel_2", + "type": "visualization", + "id": "66d5d357-edce-450d-b5be-a5a00190e153" + }, + { + "name": "panel_3", + "type": "visualization", + "id": "d3a0ac2e-73cf-462e-8b03-e6ff3b8612b7" + }, + { + "name": "panel_4", + "type": "visualization", + "id": "9ba4473b-66f4-4aea-b19e-4309ec6534b8" + }, + { + "name": "panel_5", + "type": "visualization", + "id": "AWDG9goqxQT5EBNmq4BP" + }, + { + "name": "panel_6", + "type": "visualization", + "id": "1642f6f0-c44c-11ee-876e-5d93490b24bb" + }, + { + "name": "panel_7", + "type": "visualization", + "id": "269ec200-7fa6-11ec-998f-a1f630163497" + }, + { + "name": "panel_8", + "type": "visualization", + "id": "b49ab0c0-7fa9-11ec-998f-a1f630163497" + }, + { + "name": "panel_9", + "type": "visualization", + "id": "91157aa0-7fa8-11ec-998f-a1f630163497" + }, + { + "name": "panel_10", + "type": "search", + "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "aaa4fbb0-d5fe-4ef9-be76-405b977bcd5b", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:00.991Z", + "version": "WzU3NSwxXQ==", + "attributes": { + "visState": "{\"title\":\"Files - Log Count Over Time\",\"type\":\"line\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"firstPacket per 12 hours\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"firstPacket\",\"interval\":\"auto\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\" \"}}],\"listeners\":{}}", + "description": "", + "title": "Files - Log Count Over Time", + "uiStateJSON": "{}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "type": "search", + "name": "search_0", + "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "df9e399b-efa5-4e33-b0ac-a7668a8ac2b3", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:27.382Z", + "version": "Wzg2MSwxXQ==", + "attributes": { + "title": "Network Logs", + "visState": "{\"title\":\"Network Logs\",\"type\":\"markdown\",\"params\":{\"markdown\":\"### General\\n[Overview](#/dashboard/0ad3d7c2-3441-485e-9dfe-dbb22e84e576) \\n[Security Overview](#/dashboard/95479950-41f2-11ea-88fa-7151df485405) \\n[ICS/IoT Security Overview](#/dashboard/4a4bde20-4760-11ea-949c-bbb5a9feecbf) \\n[Severity](#/dashboard/d2dd0180-06b1-11ec-8c6b-353266ade330) \\n[Connections](#/dashboard/abdd7550-2c7c-40dc-947e-f6d186a158c4) \\n[Actions and Results](#/dashboard/a33e0a50-afcd-11ea-993f-b7d8522a8bed) \\n[Files](#/dashboard/9ee51f94-3316-4fc5-bd89-93a52af69714) \\n[Executables](#/dashboard/0a490422-0ce9-44bf-9a2d-19329ddde8c3) \\n[Software](#/dashboard/87d990cc-9e0b-41e5-b8fe-b10ae1da0c85) \\n[Zeek Known Summary](#/dashboard/89d1cc50-974c-11ed-bb6b-3fb06c879b11) \\n[Zeek Intelligence](#/dashboard/36ed695f-edcc-47c1-b0ec-50d20c93ce0f) \\n[Zeek Notices](#/dashboard/f1f09567-fc7f-450b-a341-19d2f2bb468b) \\n[Zeek Weird](#/dashboard/1fff49f6-0199-4a0f-820b-721aff9ff1f1) \\n[Signatures](#/dashboard/665d1610-523d-11e9-a30e-e3576242f3ed) \\n[Suricata Alerts](#/dashboard/5694ca60-cbdf-11ec-a50a-5fedd672f5c5) \\n[Asset Interaction Analysis](#/dashboard/677ee170-809e-11ed-8d5b-07069f823b6f) \\n[↪ NetBox](/netbox/) \\n[↪ Arkime](/arkime/) \\n\\n### Common Protocols\\n[DCE/RPC](#/dashboard/432af556-c5c0-4cc3-8166-b274b4e3a406) ● [DHCP](#/dashboard/2d98bb8e-214c-4374-837b-20e1bcd63a5e) ● [DNS](#/dashboard/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9) ● [FTP](#/dashboard/078b9aa5-9bd4-4f02-ae5e-cf80fa6f887b) / [TFTP](#/dashboard/bf5efbb0-60f1-11eb-9d60-dbf0411cfc48) ● [HTTP](#/dashboard/37041ee1-79c0-4684-a436-3173b0e89876) ● [IRC](#/dashboard/76f2f912-80da-44cd-ab66-6a73c8344cc3) ● [Kerberos](#/dashboard/82da3101-2a9c-4ae2-bb61-d447a3fbe673) ● [LDAP](#/dashboard/05e3e000-f118-11e9-acda-83a8e29e1a24) ● [MQTT](#/dashboard/87a32f90-ef58-11e9-974e-9d600036d105) ● [MySQL](#/dashboard/50ced171-1b10-4c3f-8b67-2db9635661a6) ● [NTLM](#/dashboard/543118a9-02d7-43fe-b669-b8652177fc37) ● [NTP](#/dashboard/af5df620-eeb6-11e9-bdef-65a192b7f586) ● [OSPF](#/dashboard/1cc01ff0-5205-11ec-a62c-7bc80e88f3f0) ● [QUIC](#/dashboard/11ddd980-e388-11e9-b568-cf17de8e860c) ● [RADIUS](#/dashboard/ae79b7d1-4281-4095-b2f6-fa7eafda9970) ● [RDP](#/dashboard/7f41913f-cba8-43f5-82a8-241b7ead03e0) ● [RFB](#/dashboard/f77bf097-18a8-465c-b634-eb2acc7a4f26) ● [SIP](#/dashboard/0b2354ae-0fe9-4fd9-b156-1c3870e5c7aa) ● [SMB](#/dashboard/42e831b9-41a9-4f35-8b7d-e1566d368773) ● [SMTP](#/dashboard/bb827f8e-639e-468c-93c8-9f5bc132eb8f) ● [SNMP](#/dashboard/4e5f106e-c60a-4226-8f64-d534abb912ab) ● [SSH](#/dashboard/caef3ade-d289-4d05-a511-149f3e97f238) ● [SSL](#/dashboard/7f77b58a-df3e-4cc2-b782-fd7f8bad8ffb) / [X.509 Certificates](#/dashboard/024062a6-48d6-498f-a91a-3bf2da3a3cd3) ● [STUN](#/dashboard/fa477130-2b8a-11ec-a9f2-3911c8571bfd) ● [Syslog](#/dashboard/92985909-dc29-4533-9e80-d3182a0ecf1d) ● [TDS](#/dashboard/bed185a0-ef82-11e9-b38a-2db3ee640e88) / [TDS RPC](#/dashboard/32587740-ef88-11e9-b38a-2db3ee640e88) / [TDS SQL](#/dashboard/fa141950-ef89-11e9-b38a-2db3ee640e88) ● [Telnet / rlogin / rsh](#/dashboard/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2) ● [Tunnels](#/dashboard/11be6381-beef-40a7-bdce-88c5398392fc)\\n\\n### ICS/IoT Protocols\\n[BACnet](#/dashboard/2bec1490-eb94-11e9-a384-0fcf32210194) ● [BSAP](#/dashboard/ca5799a0-56b5-11eb-b749-576de068f8ad) ● [DNP3](#/dashboard/870a5862-6c26-4a08-99fd-0c06cda85ba3) ● [EtherCAT](#/dashboard/4a073440-b286-11eb-a4d4-09fa12a6ebd4) ● [EtherNet/IP](#/dashboard/29a1b290-eb98-11e9-a384-0fcf32210194) ● [GENISYS](#/dashboard/03207c00-d07e-11ec-b4a7-d1b4003706b7) ● [Modbus](#/dashboard/152f29dc-51a2-4f53-93e9-6e92765567b8) ● [OPCUA Binary](#/dashboard/dd87edd0-796a-11ec-9ce6-b395c1ff58f4) ● [PROFINET](#/dashboard/a7514350-eba6-11e9-a384-0fcf32210194) ● [S7comm](#/dashboard/e76d05c0-eb9f-11e9-a384-0fcf32210194) ● [Synchrophasor](#/dashboard/2cc56240-e460-11ed-a9d5-9f591c284cb4) ● [Best Guess](#/dashboard/12e3a130-d83b-11eb-a0b0-f328ce09b0b7)\",\"type\":\"markdown\",\"fontSize\":10,\"openLinksInNewTab\":false},\"aggs\":[]}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":{\"query_string\":{\"query\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "66d5d357-edce-450d-b5be-a5a00190e153", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:00.991Z", + "version": "WzU3NywxXQ==", + "attributes": { + "title": "Files - Files By Size (Bytes)", + "visState": "{\"title\":\"Files - Files By Size (Bytes)\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.files.seen_bytes\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Bytes Seen\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":20,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "d3a0ac2e-73cf-462e-8b03-e6ff3b8612b7", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:00.991Z", + "version": "WzU3OCwxXQ==", + "attributes": { + "visState": "{\"title\":\"FIles - Destination IP Address\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"destination.ip\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"IP Address\"}}],\"listeners\":{}}", + "description": "", + "title": "FIles - Destination IP Address", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "type": "search", + "name": "search_0", + "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "9ba4473b-66f4-4aea-b19e-4309ec6534b8", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:00.991Z", + "version": "WzU3OSwxXQ==", + "attributes": { + "visState": "{\"title\":\"FIles - Source IP Address\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"File IP Address\"}}],\"listeners\":{}}", + "description": "", + "title": "FIles - Source IP Address", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "type": "search", + "name": "search_0", + "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "AWDG9goqxQT5EBNmq4BP", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:00.991Z", + "version": "WzU4MCwxXQ==", + "attributes": { + "title": "Files - Log Count", + "visState": "{\"title\":\"Files - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "type": "search", + "name": "search_0", + "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "1642f6f0-c44c-11ee-876e-5d93490b24bb", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:31:30.082Z", + "version": "Wzk1MiwxXQ==", + "attributes": { + "title": "Browse Extracted Files Link", + "visState": "{\"title\":\"Browse Extracted Files Link\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":12,\"openLinksInNewTab\":true,\"markdown\":\"[📁 Browse extracted files](/extracted-files/) (if [file extraction and scanning](/readme/docs/file-scanning.html#ZeekFileExtraction) is enabled)\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "269ec200-7fa6-11ec-998f-a1f630163497", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:00.991Z", + "version": "WzU4MSwxXQ==", + "attributes": { + "title": "Files - Source", + "visState": "{\"title\":\"Files - Source\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"file.source\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"file.source\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "b49ab0c0-7fa9-11ec-998f-a1f630163497", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:00.991Z", + "version": "WzU4MiwxXQ==", + "attributes": { + "title": "Files - MIME Type", + "visState": "{\"title\":\"Files - MIME Type\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"file.mime_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Mime Type\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "arkime_sessions3-*" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "91157aa0-7fa8-11ec-998f-a1f630163497", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:00.991Z", + "version": "WzU4MywxXQ==", + "attributes": { + "title": "Files - Paths", + "visState": "{\"title\":\"Files - Paths\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Log Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"network.protocol\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Protocol\"},\"schema\":\"bucket\"},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"file.path\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Path\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":20,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "arkime_sessions3-*" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "0aca5333-3b1c-4cda-afb4-f7dd86910459", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-02-05T17:21:16.253Z", + "version": "Wzc2NCwxXQ==", + "attributes": { + "title": "Files - Logs", + "description": "", + "hits": 0, + "columns": [ + "source.ip", + "destination.ip", + "file.source", + "file.mime_type", + "file.path", + "event.id" + ], + "sort": [ + [ + "firstPacket", + "desc" + ] + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"filter\":[],\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"event.dataset:files\"}},\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "arkime_sessions3-*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + } + ] } \ No newline at end of file diff --git a/dashboards/templates/composable/component/suricata_stats.json b/dashboards/templates/composable/component/suricata_stats.json new file mode 100644 index 000000000..03fb77b2c --- /dev/null +++ b/dashboards/templates/composable/component/suricata_stats.json @@ -0,0 +1,201 @@ +{ + "template": { + "mappings": { + "properties": { + "suricata": { + "properties": { + "stats": { + "properties": { + "app_layer.expectations": { "type": "long" }, + "app_layer.flow.dcerpc_tcp": { "type": "long" }, + "app_layer.flow.dcerpc_udp": { "type": "long" }, + "app_layer.flow.dhcp": { "type": "long" }, + "app_layer.flow.dnp3": { "type": "long" }, + "app_layer.flow.dns_tcp": { "type": "long" }, + "app_layer.flow.dns_udp": { "type": "long" }, + "app_layer.flow.enip_tcp": { "type": "long" }, + "app_layer.flow.enip_udp": { "type": "long" }, + "app_layer.flow.failed_tcp": { "type": "long" }, + "app_layer.flow.failed_udp": { "type": "long" }, + "app_layer.flow.ftp": { "type": "long" }, + "app_layer.flow.ftp-data": { "type": "long" }, + "app_layer.flow.http": { "type": "long" }, + "app_layer.flow.ikev2": { "type": "long" }, + "app_layer.flow.imap": { "type": "long" }, + "app_layer.flow.krb5_tcp": { "type": "long" }, + "app_layer.flow.krb5_udp": { "type": "long" }, + "app_layer.flow.modbus": { "type": "long" }, + "app_layer.flow.mqtt": { "type": "long" }, + "app_layer.flow.nfs_tcp": { "type": "long" }, + "app_layer.flow.nfs_udp": { "type": "long" }, + "app_layer.flow.ntp": { "type": "long" }, + "app_layer.flow.rdp": { "type": "long" }, + "app_layer.flow.rfb": { "type": "long" }, + "app_layer.flow.sip": { "type": "long" }, + "app_layer.flow.smb": { "type": "long" }, + "app_layer.flow.smtp": { "type": "long" }, + "app_layer.flow.snmp": { "type": "long" }, + "app_layer.flow.ssh": { "type": "long" }, + "app_layer.flow.tftp": { "type": "long" }, + "app_layer.flow.tls": { "type": "long" }, + "app_layer.tx.dcerpc_tcp": { "type": "long" }, + "app_layer.tx.dcerpc_udp": { "type": "long" }, + "app_layer.tx.dhcp": { "type": "long" }, + "app_layer.tx.dnp3": { "type": "long" }, + "app_layer.tx.dns_tcp": { "type": "long" }, + "app_layer.tx.dns_udp": { "type": "long" }, + "app_layer.tx.enip_tcp": { "type": "long" }, + "app_layer.tx.enip_udp": { "type": "long" }, + "app_layer.tx.ftp": { "type": "long" }, + "app_layer.tx.ftp-data": { "type": "long" }, + "app_layer.tx.http": { "type": "long" }, + "app_layer.tx.ikev2": { "type": "long" }, + "app_layer.tx.imap": { "type": "long" }, + "app_layer.tx.krb5_tcp": { "type": "long" }, + "app_layer.tx.krb5_udp": { "type": "long" }, + "app_layer.tx.modbus": { "type": "long" }, + "app_layer.tx.mqtt": { "type": "long" }, + "app_layer.tx.nfs_tcp": { "type": "long" }, + "app_layer.tx.nfs_udp": { "type": "long" }, + "app_layer.tx.ntp": { "type": "long" }, + "app_layer.tx.rdp": { "type": "long" }, + "app_layer.tx.rfb": { "type": "long" }, + "app_layer.tx.sip": { "type": "long" }, + "app_layer.tx.smb": { "type": "long" }, + "app_layer.tx.smtp": { "type": "long" }, + "app_layer.tx.snmp": { "type": "long" }, + "app_layer.tx.ssh": { "type": "long" }, + "app_layer.tx.tftp": { "type": "long" }, + "app_layer.tx.tls": { "type": "long" }, + "capture.errors": { "type": "long" }, + "capture.kernel_drops": { "type": "long" }, + "capture.kernel_packets": { "type": "long" }, + "decoder.avg_pkt_size": { "type": "long" }, + "decoder.bytes": { "type": "long" }, + "decoder.chdlc": { "type": "long" }, + "decoder.erspan": { "type": "long" }, + "decoder.ethernet": { "type": "long" }, + "decoder.geneve": { "type": "long" }, + "decoder.gre": { "type": "long" }, + "decoder.icmpv4": { "type": "long" }, + "decoder.icmpv6": { "type": "long" }, + "decoder.ieee8021ah": { "type": "long" }, + "decoder.invalid": { "type": "long" }, + "decoder.ipv4": { "type": "long" }, + "decoder.ipv4_in_ipv6": { "type": "long" }, + "decoder.ipv6": { "type": "long" }, + "decoder.ipv6_in_ipv6": { "type": "long" }, + "decoder.max_mac_addrs_dst": { "type": "long" }, + "decoder.max_mac_addrs_src": { "type": "long" }, + "decoder.max_pkt_size": { "type": "long" }, + "decoder.mpls": { "type": "long" }, + "decoder.null": { "type": "long" }, + "decoder.pkts": { "type": "long" }, + "decoder.ppp": { "type": "long" }, + "decoder.pppoe": { "type": "long" }, + "decoder.raw": { "type": "long" }, + "decoder.sctp": { "type": "long" }, + "decoder.sll": { "type": "long" }, + "decoder.tcp": { "type": "long" }, + "decoder.teredo": { "type": "long" }, + "decoder.udp": { "type": "long" }, + "decoder.vlan": { "type": "long" }, + "decoder.vlan_qinq": { "type": "long" }, + "decoder.vntag": { "type": "long" }, + "decoder.vxlan": { "type": "long" }, + "defrag.ipv4.fragments": { "type": "long" }, + "defrag.ipv4.reassembled": { "type": "long" }, + "defrag.ipv4.timeouts": { "type": "long" }, + "defrag.ipv6.fragments": { "type": "long" }, + "defrag.ipv6.reassembled": { "type": "long" }, + "defrag.ipv6.timeouts": { "type": "long" }, + "defrag.max_frag_hits": { "type": "long" }, + "detect.alert": { "type": "long" }, + "detect.alert_queue_overflow": { "type": "long" }, + "detect.alerts_suppressed": { "type": "long" }, + "detect.engines": { + "type": "nested", + "properties": { + "id": { "type": "long" }, + "last_reload": { "type": "date" }, + "rules_failed": { "type": "long" }, + "rules_loaded": { "type": "long" } + } + }, + "file_store.open_files": { "type": "long" }, + "flow.emerg_mode_entered": { "type": "long" }, + "flow.emerg_mode_over": { "type": "long" }, + "flow.get_used": { "type": "long" }, + "flow.get_used_eval": { "type": "long" }, + "flow.get_used_eval_busy": { "type": "long" }, + "flow.get_used_eval_reject": { "type": "long" }, + "flow.get_used_failed": { "type": "long" }, + "flow.icmpv4": { "type": "long" }, + "flow.icmpv6": { "type": "long" }, + "flow.memcap": { "type": "long" }, + "flow.memuse": { "type": "long" }, + "flow.mgr.bypassed_pruned": { "type": "long" }, + "flow.mgr.closed_pruned": { "type": "long" }, + "flow.mgr.est_pruned": { "type": "long" }, + "flow.mgr.flows_checked": { "type": "long" }, + "flow.mgr.flows_evicted": { "type": "long" }, + "flow.mgr.flows_evicted_needs_work": { "type": "long" }, + "flow.mgr.flows_notimeout": { "type": "long" }, + "flow.mgr.flows_timeout": { "type": "long" }, + "flow.mgr.flows_timeout_inuse": { "type": "long" }, + "flow.mgr.full_hash_pass": { "type": "long" }, + "flow.mgr.new_pruned": { "type": "long" }, + "flow.mgr.rows_maxlen": { "type": "long" }, + "flow.spare": { "type": "long" }, + "flow.tcp": { "type": "long" }, + "flow.tcp_reuse": { "type": "long" }, + "flow.udp": { "type": "long" }, + "flow.wrk.flows_evicted": { "type": "long" }, + "flow.wrk.flows_evicted_needs_work": { "type": "long" }, + "flow.wrk.flows_evicted_pkt_inject": { "type": "long" }, + "flow.wrk.flows_injected": { "type": "long" }, + "flow.wrk.spare_sync": { "type": "long" }, + "flow.wrk.spare_sync_avg": { "type": "long" }, + "flow.wrk.spare_sync_empty": { "type": "long" }, + "flow.wrk.spare_sync_incomplete": { "type": "long" }, + "flow_bypassed.bytes": { "type": "long" }, + "flow_bypassed.closed": { "type": "long" }, + "flow_bypassed.local_bytes": { "type": "long" }, + "flow_bypassed.local_capture_bytes": { "type": "long" }, + "flow_bypassed.local_capture_pkts": { "type": "long" }, + "flow_bypassed.local_pkts": { "type": "long" }, + "flow_bypassed.pkts": { "type": "long" }, + "ftp.memcap": { "type": "long" }, + "ftp.memuse": { "type": "long" }, + "http.memcap": { "type": "long" }, + "http.memuse": { "type": "long" }, + "tcp.insert_data_normal_fail": { "type": "long" }, + "tcp.insert_data_overlap_fail": { "type": "long" }, + "tcp.insert_list_fail": { "type": "long" }, + "tcp.invalid_checksum": { "type": "long" }, + "tcp.memuse": { "type": "long" }, + "tcp.midstream_pickups": { "type": "long" }, + "tcp.no_flow": { "type": "long" }, + "tcp.overlap": { "type": "long" }, + "tcp.overlap_diff_data": { "type": "long" }, + "tcp.pkt_on_wrong_thread": { "type": "long" }, + "tcp.pseudo": { "type": "long" }, + "tcp.pseudo_failed": { "type": "long" }, + "tcp.reassembly_gap": { "type": "long" }, + "tcp.reassembly_memuse": { "type": "long" }, + "tcp.rst": { "type": "long" }, + "tcp.segment_memcap_drop": { "type": "long" }, + "tcp.sessions": { "type": "long" }, + "tcp.ssn_memcap_drop": { "type": "long" }, + "tcp.stream_depth_reached": { "type": "long" }, + "tcp.syn": { "type": "long" }, + "tcp.synack": { "type": "long" }, + "uptime": { "type": "long" } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json index ee6a176ca..30f8d0f4e 100644 --- a/dashboards/templates/composable/component/zeek.json +++ b/dashboards/templates/composable/component/zeek.json @@ -2,6 +2,11 @@ "template": { "mappings": { "properties": { + "zeek.analyzer.cause": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "zeek.analyzer.analyzer_kind": { "type": "keyword" }, + "zeek.analyzer.analyzer_name": { "type": "keyword" }, + "zeek.analyzer.failure_reason": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "zeek.analyzer.failure_data": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "zeek.conn.conn_state": { "type": "keyword" }, "zeek.conn.conn_state_description": { "type": "keyword" }, "zeek.conn.duration": { "type": "float" }, diff --git a/dashboards/templates/composable/component/zeek_diagnostic.json b/dashboards/templates/composable/component/zeek_diagnostic.json new file mode 100644 index 000000000..eaa65e284 --- /dev/null +++ b/dashboards/templates/composable/component/zeek_diagnostic.json @@ -0,0 +1,65 @@ +{ + "template": { + "mappings": { + "properties": { + "zeek": { + "properties": { + "broker.event_type": { "type": "keyword" }, + "broker.event_action": { "type": "keyword" }, + "broker.peer_ip": { "type": "keyword" }, + "broker.peer_port": { "type": "integer" }, + "broker.peer_message": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "capture_loss.ts_delta": { "type": "float" }, + "capture_loss.peer": { "type": "keyword" }, + "capture_loss.gaps": { "type": "long" }, + "capture_loss.acks": { "type": "long" }, + "capture_loss.percent_lost": { "type": "float" }, + "cluster.node": { "type": "keyword" }, + "cluster.node_message": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "config.value_name": { "type": "keyword" }, + "config.value_old": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "config.value_new": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "config.location": { "type": "keyword" }, + "packet_filter.node": { "type": "keyword" }, + "packet_filter.filter": { "type": "keyword" }, + "packet_filter.init": { "type": "keyword" }, + "packet_filter.success": { "type": "keyword" }, + "packet_filter.failure_reason": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "print.vals": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "reporter.node": { "type": "keyword" }, + "reporter.filter": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "reporter.init": { "type": "keyword" }, + "reporter.success": { "type": "keyword" }, + "reporter.failure_reason": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "stats.peer": { "type": "keyword" }, + "stats.mem": { "type": "long" }, + "stats.pkts_proc": { "type": "long" }, + "stats.bytes_recv": { "type": "long" }, + "stats.pkts_dropped": { "type": "long" }, + "stats.pkts_link": { "type": "long" }, + "stats.pkt_lag": { "type": "float" }, + "stats.pkts_filtered": { "type": "long" }, + "stats.events_proc": { "type": "long" }, + "stats.events_queued": { "type": "long" }, + "stats.active_tcp_conns": { "type": "long" }, + "stats.active_udp_conns": { "type": "long" }, + "stats.active_icmp_conns": { "type": "long" }, + "stats.tcp_conns": { "type": "long" }, + "stats.udp_conns": { "type": "long" }, + "stats.icmp_conns": { "type": "long" }, + "stats.timers": { "type": "long" }, + "stats.active_timers": { "type": "long" }, + "stats.files": { "type": "long" }, + "stats.active_files": { "type": "long" }, + "stats.dns_requests": { "type": "long" }, + "stats.active_dns_requests": { "type": "long" }, + "stats.reassem_tcp_size": { "type": "long" }, + "stats.reassem_file_size": { "type": "long" }, + "stats.reassem_frag_size": { "type": "long" }, + "stats.reassem_unknown_size": { "type": "long" } + } + } + } + } + } +} \ No newline at end of file diff --git a/dashboards/templates/malcolm_beats_template.json b/dashboards/templates/malcolm_beats_template.json index fdbad390e..4c9da40e4 100644 --- a/dashboards/templates/malcolm_beats_template.json +++ b/dashboards/templates/malcolm_beats_template.json @@ -21,7 +21,9 @@ "ecs_url", "ecs_user", "ecs_user_agent", - "custom_miscbeat" + "custom_miscbeat", + "custom_suricata_stats", + "custom_zeek_diagnostic" ], "template" :{ "settings" : { diff --git a/dashboards/templates/malcolm_template.json b/dashboards/templates/malcolm_template.json index adb4e4e1c..09bff2b8d 100644 --- a/dashboards/templates/malcolm_template.json +++ b/dashboards/templates/malcolm_template.json @@ -74,6 +74,7 @@ "related.mac": { "type": "keyword" }, "related.oui": { "type": "keyword" }, "related.password": { "type": "keyword", "ignore_above": 256, "fields": { "text": { "type": "text" } } }, + "related.device_id": { "type": "integer" }, "related.device_name": { "type": "keyword" }, "related.device_type": { "type": "keyword" }, "related.manufacturer": { "type": "keyword" }, diff --git a/docker-compose-standalone.yml b/docker-compose-dev.yml similarity index 83% rename from docker-compose-standalone.yml rename to docker-compose-dev.yml index 0d670754a..0639b407a 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-dev.yml @@ -12,7 +12,10 @@ x-logging: services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:24.01.0 + build: + context: . + dockerfile: Dockerfiles/opensearch.Dockerfile + image: ghcr.io/idaholab/malcolm/opensearch:24.02.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -54,7 +57,10 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.01.0 + build: + context: . + dockerfile: Dockerfiles/dashboards-helper.Dockerfile + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -84,7 +90,10 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:24.01.0 + build: + context: . + dockerfile: Dockerfiles/dashboards.Dockerfile + image: ghcr.io/idaholab/malcolm/dashboards:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -112,7 +121,10 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:24.01.0 + build: + context: . + dockerfile: Dockerfiles/logstash.Dockerfile + image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -148,6 +160,10 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./.opensearch.secondary.curlrc:/var/local/curlrc/.opensearch.secondary.curlrc:ro + - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.orig.yml:ro + - ./logstash/pipelines:/usr/share/logstash/malcolm-pipelines.available:ro + - ./logstash/patterns:/usr/share/logstash/malcolm-patterns:ro + - ./logstash/ruby:/usr/share/logstash/malcolm-ruby:ro - ./logstash/maps/malcolm_severity.yaml:/etc/malcolm_severity.yaml:ro - ./logstash/certs/ca.crt:/certs/ca.crt:ro - ./logstash/certs/server.crt:/certs/server.crt:ro @@ -159,7 +175,10 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.01.0 + build: + context: . + dockerfile: Dockerfiles/filebeat.Dockerfile + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -194,7 +213,10 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:24.01.0 + build: + context: . + dockerfile: Dockerfiles/arkime.Dockerfile + image: ghcr.io/idaholab/malcolm/arkime:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -210,7 +232,6 @@ services: - ./config/upload-common.env - ./config/auth.env - ./config/arkime.env - - ./config/arkime-offline.env - ./config/arkime-secret.env environment: VIRTUAL_HOST : 'arkime.malcolm.local' @@ -219,8 +240,11 @@ services: volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./arkime/rules:/opt/arkime/rules:ro - ./pcap:/data/pcap + - ./arkime/rules:/opt/arkime/rules:ro + - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro + - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro + - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro healthcheck: test: ["CMD", "curl", "--insecure", "--silent", "--fail", "https://localhost:8005/_ns_/nstest.html"] interval: 90s @@ -228,7 +252,10 @@ services: retries: 3 start_period: 210s arkime-live: - image: ghcr.io/idaholab/malcolm/arkime:24.01.0 + build: + context: . + dockerfile: Dockerfiles/arkime.Dockerfile + image: ghcr.io/idaholab/malcolm/arkime:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -258,10 +285,16 @@ services: volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./arkime/rules:/opt/arkime/rules:ro - ./pcap:/data/pcap + - ./arkime/rules:/opt/arkime/rules:ro + - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro + - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro + - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro zeek: - image: ghcr.io/idaholab/malcolm/zeek:24.01.0 + build: + context: . + dockerfile: Dockerfiles/zeek.Dockerfile + image: ghcr.io/idaholab/malcolm/zeek:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -284,6 +317,7 @@ services: - ./pcap:/pcap - ./zeek-logs/upload:/zeek/upload - ./zeek-logs/extract_files:/zeek/extract_files + - ./zeek/config/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro - ./zeek/intel:/opt/zeek/share/zeek/site/intel - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro healthcheck: @@ -293,7 +327,10 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:24.01.0 + build: + context: . + dockerfile: Dockerfiles/zeek.Dockerfile + image: ghcr.io/idaholab/malcolm/zeek:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -318,10 +355,14 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./zeek-logs/live:/zeek/live - ./zeek-logs/extract_files:/zeek/extract_files + - ./zeek/config/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro - ./zeek/intel:/opt/zeek/share/zeek/site/intel - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro suricata: - image: ghcr.io/idaholab/malcolm/suricata:24.01.0 + build: + context: . + dockerfile: Dockerfiles/suricata.Dockerfile + image: ghcr.io/idaholab/malcolm/suricata:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -349,7 +390,10 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:24.01.0 + build: + context: . + dockerfile: Dockerfiles/suricata.Dockerfile + image: ghcr.io/idaholab/malcolm/suricata:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -364,7 +408,7 @@ services: # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - # SYS_NICE - to set process nice values and scheduling policies for capture + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling - SYS_NICE env_file: - ./config/process.env @@ -379,7 +423,10 @@ services: - ./suricata/rules:/opt/suricata/rules:ro - ./suricata/include-configs:/opt/suricata/include-configs:ro file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:24.01.0 + build: + context: . + dockerfile: Dockerfiles/file-monitor.Dockerfile + image: ghcr.io/idaholab/malcolm/file-monitor:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -391,6 +438,7 @@ services: env_file: - ./config/process.env - ./config/ssl.env + - ./config/dashboards.env - ./config/zeek.env - ./config/zeek-secret.env environment: @@ -407,7 +455,10 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:24.01.0 + build: + context: . + dockerfile: Dockerfiles/pcap-capture.Dockerfile + image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -436,7 +487,10 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.01.0 + build: + context: . + dockerfile: Dockerfiles/pcap-monitor.Dockerfile + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -464,7 +518,10 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:24.01.0 + build: + context: . + dockerfile: Dockerfiles/file-upload.Dockerfile + image: ghcr.io/idaholab/malcolm/file-upload:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -491,9 +548,12 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.01.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.02.0 profiles: ["malcolm"] logging: *default-logging + build: + context: . + dockerfile: Dockerfiles/htadmin.Dockerfile restart: "no" stdin_open: false tty: true @@ -518,9 +578,12 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.01.0 + image: ghcr.io/idaholab/malcolm/freq:24.02.0 profiles: ["malcolm"] logging: *default-logging + build: + context: . + dockerfile: Dockerfiles/freq.Dockerfile restart: "no" stdin_open: false tty: true @@ -542,9 +605,12 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.01.0 + image: ghcr.io/idaholab/malcolm/netbox:24.02.0 profiles: ["malcolm"] logging: *default-logging + build: + context: . + dockerfile: Dockerfiles/netbox.Dockerfile restart: "no" stdin_open: false tty: true @@ -576,9 +642,12 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.01.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.02.0 profiles: ["malcolm"] logging: *default-logging + build: + context: . + dockerfile: Dockerfiles/postgresql.Dockerfile restart: "no" stdin_open: false tty: true @@ -602,9 +671,12 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.01.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.0 profiles: ["malcolm"] logging: *default-logging + build: + context: . + dockerfile: Dockerfiles/redis.Dockerfile restart: "no" stdin_open: false tty: true @@ -632,9 +704,12 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.01.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.0 profiles: ["malcolm"] logging: *default-logging + build: + context: . + dockerfile: Dockerfiles/redis.Dockerfile restart: "no" stdin_open: false tty: true @@ -661,9 +736,12 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.01.0 + image: ghcr.io/idaholab/malcolm/api:24.02.0 profiles: ["malcolm"] logging: *default-logging + build: + context: . + dockerfile: Dockerfiles/api.Dockerfile command: gunicorn --bind 0:5000 manage:app restart: "no" stdin_open: false @@ -688,7 +766,10 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.01.0 + build: + context: . + dockerfile: Dockerfiles/nginx.Dockerfile + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -700,6 +781,8 @@ services: env_file: - ./config/process.env - ./config/ssl.env + - ./config/opensearch.env + - ./config/dashboards.env - ./config/auth-common.env - ./config/nginx.env depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index b72ace183..ae9444ecc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,10 +12,7 @@ x-logging: services: opensearch: - build: - context: . - dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:24.01.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.02.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -57,10 +54,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - build: - context: . - dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.01.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -90,10 +84,7 @@ services: retries: 3 start_period: 30s dashboards: - build: - context: . - dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:24.01.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -121,10 +112,7 @@ services: retries: 3 start_period: 210s logstash: - build: - context: . - dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:24.01.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -160,10 +148,6 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./.opensearch.secondary.curlrc:/var/local/curlrc/.opensearch.secondary.curlrc:ro - - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.orig.yml:ro - - ./logstash/pipelines:/usr/share/logstash/malcolm-pipelines.available:ro - - ./logstash/patterns:/usr/share/logstash/malcolm-patterns:ro - - ./logstash/ruby:/usr/share/logstash/malcolm-ruby:ro - ./logstash/maps/malcolm_severity.yaml:/etc/malcolm_severity.yaml:ro - ./logstash/certs/ca.crt:/certs/ca.crt:ro - ./logstash/certs/server.crt:/certs/server.crt:ro @@ -175,10 +159,7 @@ services: retries: 3 start_period: 600s filebeat: - build: - context: . - dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.01.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -213,10 +194,7 @@ services: retries: 3 start_period: 60s arkime: - build: - context: . - dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.01.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -232,6 +210,7 @@ services: - ./config/upload-common.env - ./config/auth.env - ./config/arkime.env + - ./config/arkime-offline.env - ./config/arkime-secret.env environment: VIRTUAL_HOST : 'arkime.malcolm.local' @@ -240,11 +219,8 @@ services: volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./pcap:/data/pcap - ./arkime/rules:/opt/arkime/rules:ro - - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro - - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro - - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro + - ./pcap:/data/pcap healthcheck: test: ["CMD", "curl", "--insecure", "--silent", "--fail", "https://localhost:8005/_ns_/nstest.html"] interval: 90s @@ -252,10 +228,7 @@ services: retries: 3 start_period: 210s arkime-live: - build: - context: . - dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.01.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -285,16 +258,10 @@ services: volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./pcap:/data/pcap - ./arkime/rules:/opt/arkime/rules:ro - - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro - - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro - - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro + - ./pcap:/data/pcap zeek: - build: - context: . - dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.01.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -317,7 +284,6 @@ services: - ./pcap:/pcap - ./zeek-logs/upload:/zeek/upload - ./zeek-logs/extract_files:/zeek/extract_files - - ./zeek/config/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro - ./zeek/intel:/opt/zeek/share/zeek/site/intel - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro healthcheck: @@ -327,10 +293,7 @@ services: retries: 3 start_period: 60s zeek-live: - build: - context: . - dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.01.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -341,7 +304,7 @@ services: # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE env_file: - ./config/process.env @@ -355,14 +318,10 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./zeek-logs/live:/zeek/live - ./zeek-logs/extract_files:/zeek/extract_files - - ./zeek/config/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro - ./zeek/intel:/opt/zeek/share/zeek/site/intel - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro suricata: - build: - context: . - dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.01.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -390,10 +349,7 @@ services: retries: 3 start_period: 120s suricata-live: - build: - context: . - dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.01.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -408,7 +364,7 @@ services: # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + # SYS_NICE - to set process nice values and scheduling policies for capture - SYS_NICE env_file: - ./config/process.env @@ -423,10 +379,7 @@ services: - ./suricata/rules:/opt/suricata/rules:ro - ./suricata/include-configs:/opt/suricata/include-configs:ro file-monitor: - build: - context: . - dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:24.01.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -438,6 +391,7 @@ services: env_file: - ./config/process.env - ./config/ssl.env + - ./config/dashboards.env - ./config/zeek.env - ./config/zeek-secret.env environment: @@ -454,10 +408,7 @@ services: retries: 3 start_period: 60s pcap-capture: - build: - context: . - dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:24.01.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -486,10 +437,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - build: - context: . - dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.01.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -517,10 +465,7 @@ services: retries: 3 start_period: 90s upload: - build: - context: . - dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:24.01.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -547,12 +492,9 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.01.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.02.0 profiles: ["malcolm"] logging: *default-logging - build: - context: . - dockerfile: Dockerfiles/htadmin.Dockerfile restart: "no" stdin_open: false tty: true @@ -577,12 +519,9 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.01.0 + image: ghcr.io/idaholab/malcolm/freq:24.02.0 profiles: ["malcolm"] logging: *default-logging - build: - context: . - dockerfile: Dockerfiles/freq.Dockerfile restart: "no" stdin_open: false tty: true @@ -604,12 +543,9 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.01.0 + image: ghcr.io/idaholab/malcolm/netbox:24.02.0 profiles: ["malcolm"] logging: *default-logging - build: - context: . - dockerfile: Dockerfiles/netbox.Dockerfile restart: "no" stdin_open: false tty: true @@ -641,12 +577,9 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.01.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.02.0 profiles: ["malcolm"] logging: *default-logging - build: - context: . - dockerfile: Dockerfiles/postgresql.Dockerfile restart: "no" stdin_open: false tty: true @@ -670,12 +603,9 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.01.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.0 profiles: ["malcolm"] logging: *default-logging - build: - context: . - dockerfile: Dockerfiles/redis.Dockerfile restart: "no" stdin_open: false tty: true @@ -703,12 +633,9 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.01.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.0 profiles: ["malcolm"] logging: *default-logging - build: - context: . - dockerfile: Dockerfiles/redis.Dockerfile restart: "no" stdin_open: false tty: true @@ -735,12 +662,9 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.01.0 + image: ghcr.io/idaholab/malcolm/api:24.02.0 profiles: ["malcolm"] logging: *default-logging - build: - context: . - dockerfile: Dockerfiles/api.Dockerfile command: gunicorn --bind 0:5000 manage:app restart: "no" stdin_open: false @@ -765,10 +689,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - build: - context: . - dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.01.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -780,6 +701,8 @@ services: env_file: - ./config/process.env - ./config/ssl.env + - ./config/opensearch.env + - ./config/dashboards.env - ./config/auth-common.env - ./config/nginx.env depends_on: diff --git a/docs/README.md b/docs/README.md index 560f87caa..6883a6d8f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -72,6 +72,10 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation * [Screenshots](dashboards.md#PrebuiltVisualizationsGallery) + [Building your own visualizations and dashboards](dashboards.md#BuildDashboard) * [Screenshots](dashboards.md#NewVisualizationsGallery) + - [Anomaly Detection](anomaly-detection.md#AnomalyDetection) + - [Reporting](reporting.md#Reporting) + - [Alerting](alerting.md#Alerting) + + [Email Sender Accounts](alerting.md#AlertingEmail) * [Search Queries in Arkime and OpenSearch Dashboards](queries-cheat-sheet.md#SearchCheatSheet) * Other Malcolm features - [Custom Rules and Scripts](custom-rules.md#CustomRulesAndScripts) @@ -80,15 +84,15 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation + [YARA](custom-rules.md#YARA) + [Other Customizations](custom-rules.md#Other) - [Automatic file extraction and scanning](file-scanning.md#ZeekFileExtraction) - - [OpenSearch index management](index-management.md#IndexManagement) + + [User interface](file-scanning.md#ZeekFileExtractionUI) + - [Index management](index-management.md# + + [OpenSearch index management](index-management.md#OpenSearchIndexManagement) + + [Using ILM/ISM with Arkime](index-management.md#ArkimeIndexPolicies) - [Event severity scoring](severity.md#Severity) + [Customizing event severity scoring](severity.md#SeverityConfig) - [Zeek Intelligence Framework](zeek-intel.md#ZeekIntel) + [STIX™ and TAXII™](zeek-intel.md#ZeekIntelSTIX) + [MISP](zeek-intel.md#ZeekIntelMISP) - - [Anomaly Detection](anomaly-detection.md#AnomalyDetection) - - [Alerting](alerting.md#Alerting) - + [Email Sender Accounts](alerting.md#AlertingEmail) - ["Best Guess" Fingerprinting for ICS Protocols](ics-best-guess.md#ICSBestGuess) - [Asset Interaction Analysis](asset-interaction-analysis.md#AssetInteractionAnalysis) + [Enriching network traffic metadata via NetBox lookups](asset-interaction-analysis.md#NetBoxEnrichment) @@ -96,6 +100,7 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation + Populating the NetBox inventory * [Manually](asset-interaction-analysis.md#NetBoxPopManual) * [Via passively-gathered network traffic metadata](asset-interaction-analysis.md#NetBoxPopPassive) + - [Matching device manufacturers to OUIs](asset-interaction-analysis.md#NetBoxPopPassiveOUIMatch) * [Via active discovery](asset-interaction-analysis.md#NetBoxPopActive) + [Compare NetBox inventory with database of known vulnerabilities](asset-interaction-analysis.md#NetBoxVuln) + [Preloading NetBox inventory](asset-interaction-analysis.md#NetBoxPreload) diff --git a/docs/api-version.md b/docs/api-version.md index 50f4af4e5..76364f95c 100644 --- a/docs/api-version.md +++ b/docs/api-version.md @@ -47,6 +47,6 @@ Returns version information about Malcolm and version/[health](https://opensearc } }, "sha": "77574975", - "version": "24.01.0" + "version": "24.02.0" } ``` diff --git a/docs/asset-interaction-analysis.md b/docs/asset-interaction-analysis.md index c3e0fc678..ab8e798d1 100644 --- a/docs/asset-interaction-analysis.md +++ b/docs/asset-interaction-analysis.md @@ -5,6 +5,7 @@ * Populating the NetBox inventory - [Manually](#NetBoxPopManual) - [Via passively-gathered network traffic metadata](#NetBoxPopPassive) + + [Matching device manufacturers to OUIs](#NetBoxPopPassiveOUIMatch) - [Via active discovery](#NetBoxPopActive) * [Compare NetBox inventory with database of known vulnerabilities](#NetBoxVuln) * [Preloading NetBox inventory](#NetBoxPreload) @@ -40,6 +41,7 @@ As Zeek logs and Suricata alerts are parsed and enriched (if the `LOGSTASH_NETBO * `source.…` same as `destination.…` * collected as `related` fields (the [same approach](https://www.elastic.co/guide/en/ecs/current/ecs-related.html) used in ECS) - `related.device_type` + - `related.device_id` - `related.device_name` - `related.manufacturer` - `related.role` @@ -104,6 +106,17 @@ Although network devices can be automatically created using this method, [servic See [idaholab/Malcolm#135](https://github.com/idaholab/Malcolm/issues/135) for more information on this feature. +### Matching device manufacturers to OUIs + +Malcolm's NetBox inventory is prepopulated with a collection of [community-sourced device type definitions](https://github.com/netbox-community/devicetype-library) which can then be augmented by users [manually](#NetBoxPopManual) or through [preloading](#NetBoxPreload). During passive autopopulation device manufacturer is inferred from organizationally unique identifiers (OUIs), which make up the first three octets of a MAC address. The IEEE Standards Association maintains the [registry of OUIs](https://standards-oui.ieee.org/), which is not necessarily very internally consistent with how organizations specify the name associated with their OUI entry. In other words, there's not a foolproof programattic way for Malcolm to map MAC address OUI organization names to NetBox manufacturer names, barring creating and maintaining a manual mapping (which would be very large and difficult to keep up-to-date). + +Malcolm's [NetBox lookup code]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/ruby/netbox_enrich.rb) used in the log enrichment pipeline attempts to match OUI organization names against the list of NetBox's manufacturers using ["fuzzy string matching"](https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance), a technique in which two strings of characters are compared and assigned a similarity score between `0` (completely dissimilar) and `1` (identical). The `NETBOX_DEFAULT_FUZZY_THRESHOLD` [environment variable in `netbox-common.env`](malcolm-config.md#MalcolmConfigEnvVars) can be used to tune the threshold for determining a match. A fairly high value is recommended (above `0.85`; `0.95` is the default) to avoid autopopulating the NetBox inventory with devices with manufacturers that don't actually exist in the network being monitored. + +Users may select between two behaviors for when the match threshold is not met (i.e., no manufacturer is found in the NetBox database which closely matches the OUI organization name). This behavior is specified by the `NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER` [environment variable in `netbox-common.env`](malcolm-config.md#MalcolmConfigEnvVars): + +* `NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER=false` - the autopopulated device will be created with the manufacturer set to `Unspecified` +* `NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER=true` - the autopopulated device will be created along with a new manufacturer entry in the NetBox database set to the OUI organization name + ## Populate NetBox inventory via active discovery See [idaholab/Malcolm#136](https://github.com/idaholab/Malcolm/issues/136). diff --git a/docs/contributing-local-modifications.md b/docs/contributing-local-modifications.md index 8f2491351..438f33a19 100644 --- a/docs/contributing-local-modifications.md +++ b/docs/contributing-local-modifications.md @@ -7,7 +7,7 @@ There are several ways to customize Malcolm's runtime behavior via local changes Some configuration changes can be put in place by modifying local copies of configuration files and then using a [Docker bind mount](https://docs.docker.com/storage/bind-mounts/) to overlay the modified file onto the running Malcolm container. This is already done for many files and directories used to persist Malcolm configuration and data. For example, the default list of bind mounted files and directories for each Malcolm service is as follows: ``` -$ grep -P "^( - ./| [\w-]+:)" docker-compose-standalone.yml +$ grep -P "^( - ./| [\w-]+:)" docker-compose.yml opensearch: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro diff --git a/docs/contributing-new-image.md b/docs/contributing-new-image.md index 5cb3597ff..3dac78092 100644 --- a/docs/contributing-new-image.md +++ b/docs/contributing-new-image.md @@ -4,7 +4,7 @@ A new service can be added to Malcolm by following the following steps: 1. Create a new subdirectory for the service (under the Malcolm working copy base directory) containing whatever source or configuration files are necessary to build and run the service 1. Create the service's Dockerfile in the [Dockerfiles]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/Dockerfiles) directory of the Malcolm working copy -1. Add a new section for the service under `services:` in the `docker-compose.yml` and `docker-compose-standalone.yml` files +1. Add a new section for the service under `services:` in the `docker-compose.yml` and `docker-compose-dev.yml` files 1. To enable automatic builds for the service on GitHub, create a new [workflow]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/), using an existing workflow as an example ## Networking and firewall diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index 62ed8e26b..167235639 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.01.0 release]({{ site.github.repository_url }}/releases/tag/v24.01.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.02.0 release]({{ site.github.repository_url }}/releases/tag/v24.02.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/development.md b/docs/development.md index 7b338341a..b59bc3b49 100644 --- a/docs/development.md +++ b/docs/development.md @@ -37,8 +37,8 @@ Checking out the [Malcolm source code]({{ site.github.repository_url }}/tree/{{ and the following files of special note: -* `docker-compose.yml` - the configuration file used by `docker compose` to build, start, and stop an instance of the Malcolm appliance -* `docker-compose-standalone.yml` - similar to `docker-compose.yml`, only used for the ["packaged"](#Packager) installation of Malcolm +* `docker-compose-dev.yml` - the configuration file used by `docker compose` to build, start, and stop an instance of the Malcolm appliance +* `docker-compose.yml` - similar to `docker-compose-dev.yml`, only used for the ["packaged"](#Packager) installation of Malcolm ## Building from source @@ -121,14 +121,7 @@ To start, stop, restart, etc. Malcolm: - wipe (stop Malcolm and clear its database) - auth_setup (change authentication-related settings) -A minute or so after starting Malcolm, the following services will be accessible: - - Arkime: https://localhost/ - - OpenSearch Dashboards: https://localhost/dashboards/ - - PCAP upload (web): https://localhost/upload/ - - PCAP upload (sftp): sftp://USERNAME@127.0.0.1:8022/files/ - - NetBox: https://localhost/netbox/ - - Account management: https://localhost/auth/ - - Documentation: https://localhost/readme/ +Malcolm services can be accessed at https:/// ``` The above example will result in the following artifacts for distribution as explained in the script's output: diff --git a/docs/download.md b/docs/download.md index ba86b6edd..2ce9b8a65 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-24.01.0.iso](/iso/malcolm-24.01.0.iso) (5.2GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/malcolm-24.01.0.iso.sha256.txt) | +| [malcolm-24.02.0.iso](/iso/malcolm-24.02.0.iso) (5.2GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/malcolm-24.02.0.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,15 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-24.01.0.iso](/iso/hedgehog-24.01.0.iso) (2.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.01.0.iso.sha256.txt) | +| [hedgehog-24.02.0.iso](/iso/hedgehog-24.02.0.iso) (2.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.02.0.iso.sha256.txt) | + +### Raspberry Pi 4 Image + +[Instructions are provided](hedgehog-raspi-build.md#HedgehogRaspiBuild) to generate the Hedgehog Linux Raspberry Pi image from source. While official downloads of the Hedgehog Linux image are not provided, an **unofficial build** of the image for the latest stable release is available for download here. This image is compatible with Raspberry Pi 4 models. + +| Image | SHA256 | +|---|---| +| [raspi_4_bookworm.img.xz](/iso/raspi_4_bookworm.img.xz) (1.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/raspi_4_bookworm.img.xz.sha256.txt) | ## Warning diff --git a/docs/file-scanning.md b/docs/file-scanning.md index 30dfbd8a6..06cc4150c 100644 --- a/docs/file-scanning.md +++ b/docs/file-scanning.md @@ -1,5 +1,8 @@ # Automatic file extraction and scanning +* [Automatic file extraction and scanning](#ZeekFileExtraction) + - [User interface](#ZeekFileExtractionUI) + Malcolm can leverage Zeek's knowledge of network protocols to automatically detect file transfers and extract those files from PCAPs as Zeek processes them. This behavior can be enabled globally by modifying the `ZEEK_EXTRACTOR_MODE` [variable in `zeek.env`](malcolm-config.md#MalcolmConfigEnvVars), or on a per-upload basis for PCAP files uploaded via the [browser-based upload form](upload.md#Upload) when **Analyze with Zeek** is selected. To specify which files should be extracted, the following values are acceptable in `ZEEK_EXTRACTOR_MODE`: @@ -44,3 +47,19 @@ The `EXTRACTED_FILE_HTTP_SERVER_…` [environment variables in `zeek.env` and `z - downloaded files are downloaded as-is, without archival or compression: + `EXTRACTED_FILE_HTTP_SERVER_ZIP=false` + `EXTRACTED_FILE_HTTP_SERVER_KEY=` + +## User interface + +The files extracted by Zeek and the data about those files can be accessed through several of Malcolm's user interfaces. + +* The [Files dashboard](dashboards.md#PrebuiltVisualizations) summarizes the file transfers observed in network traffic: + +![The files dashboard displays metrics about the files transferred over the network](./images/screenshots/dashboards_files_source.png) + +* Viewing logs from Zeek's `files.log` (e.g., `event.provider == zeek && event.dataset == files`), the Arkime [session](arkime.md#ArkimeSessions) detail's **Extracted Filename** field can be clicked for a context menu item to **Download** the extracted file, if it was preserved as described above. + +![Arkime's session details for files.log entries](./images/screenshots/arkime_sessions_files_log_dl.png) + +* Malcolm provides an extracted files directory listing to browse and download Zeek-extracted files. This interface is available at at **https://localhost/extracted-files/** if connecting locally. The Zeek `uid` and `fuid` values associated with these files and the sessions from which they were extracted are listed in the **IDs** column as filter links back into Dashboards. + +![The extracted files directory interface](./images/screenshots/extracted_files_dl_ui.png) diff --git a/docs/hedgehog-hardening.md b/docs/hedgehog-hardening.md index 9db45a9b9..75307fd17 100644 --- a/docs/hedgehog-hardening.md +++ b/docs/hedgehog-hardening.md @@ -1,4 +1,4 @@ -# Appendix D - Hardening +# Appendix E - Hardening Hedgehog Linux uses the [harbian-audit](https://github.com/hardenedlinux/harbian-audit) benchmarks which target the following guidelines for establishing a secure configuration posture: diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 4b228dc12..1c4e7faa0 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-24.01.0.iso" +Finished, created "/sensor-build/hedgehog-24.02.0.iso" … ``` diff --git a/docs/hedgehog-raspi-build.md b/docs/hedgehog-raspi-build.md new file mode 100644 index 000000000..82e4dc188 --- /dev/null +++ b/docs/hedgehog-raspi-build.md @@ -0,0 +1,121 @@ +# Appendix B - Generating a Raspberry Pi Image + +Hedgehog Linux can [also be built]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/sensor-raspi) for the Raspberry Pi platform, although this capability is still considered experimental. + +* [Building the Image](#HedgehogRaspiBuild) +* [Writing the Image to Flash Media](#HedgehogRaspiBurn) +* [Setting the `root` and `sensor` Passwords](#HedgehogRaspiPassword) +* [Configuration](#HedgehogRaspiConfig) +* [Performance Considerations](#HedgehogRaspiPerformance) + +## Building the Image + +Official downloads of the Hedgehog Linux Raspberry Pi image are not provided: however, it can be built easily on an internet-connected Linux host with Vagrant: + +* [Vagrant](https://www.vagrantup.com/) + - [`vagrant-sshfs`](https://github.com/dustymabe/vagrant-sshfs) plugin + - [`bento/debian-12`](https://app.vagrantup.com/bento/boxes/debian-12) Vagrant box + +The build should work with either the [VirtualBox](https://www.virtualbox.org/) provider or the [libvirt](https://libvirt.org/) provider: + +* [VirtualBox](https://www.virtualbox.org/) [provider](https://www.vagrantup.com/docs/providers/virtualbox) + - [`vagrant-vbguest`](https://github.com/dotless-de/vagrant-vbguest) plugin +* [libvirt](https://libvirt.org/) + - [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) provider plugin + - [`vagrant-mutate`](https://github.com/sciurus/vagrant-mutate) plugin to convert [`bento/debian-12`](https://app.vagrantup.com/bento/boxes/debian-12) Vagrant box to `libvirt` format + +To perform a clean build the Hedgehog Linux Raspberry Pi image, navigate to your local [Malcolm]({{ site.github.repository_url }}/) working copy and run: + +``` +$ ./sensor-raspi/build_via_vagrant.sh -f -z +… +Starting build machine... +Bringing machine 'vagrant-hedgehog-raspi-build' up with 'virtualbox' provider... +… +``` + +As this build process is cross-compiling for the ARM64 architecture, building the image is likely to take more than five hours depending on your system. As the build finishes, you will see the following message indicating success: + +``` +… +2024-01-21 05:11:44 INFO All went fine. +2024-01-21 05:11:44 DEBUG Ending, all OK +… +``` + +## Writing the Image to Flash Media + +The resulting `.img.xz` file can be written to a microSD card using the [Raspberry Pi Imager](https://www.raspberrypi.com/documentation/computers/getting-started.html#raspberry-pi-imager) or `dd`. + +![Using the Raspberry Pi Imager](./images/screenshots/raspi_imager_hedgehog.png) + +## Setting the `root` and `sensor` Passwords + +The provided image will allow login (requiring physical access) with the `sensor` account using a default password of `Hedgehog_Linux` or the `root` account with a default password of `Hedgehog_Linux_Root`. It is **highly** recommended for users to use the `passwd` utility to change both of these passwords prior to configuring networking on the device. + +``` + + ,cc:,.. .:' + :dddddddoc,. ;,. oddo:. .c;. + :dddddddddddo;:ddc:dddddd; ldddl, + .dddddddddddddddddxdddddddo:odddddo' cl;. + ........ :ddddddddddddddddOkdddddddddxdddddd;,dddd' + .;lddddddddddolcddddddddddddddddk0kddddddddOxdddddddddddo. + 'dddddddddddddddddxkdddddddddddddx00xdddddddxkddddddoodddd, + .odddddddddddddddddO0OxdddddddddddO0Oddddddddoccloddc':xxd; + .:dddddddddddddddddxO00kdddddddddx00kdddddo;'....',;,'dddc. .,;,. + .cdddxOkkxdddddddddxO00kddddddddO00ddddo,..cxxxl'...........;O0000: + .',,,,,,':ddddkO00OxddddddddxO00kdddddddOOddddc...l0000l............',o0c + cddddddddddddddddxO00kddddddddx000xdddddddddddo'...:dxo,..............'' + 'lddddddddddddddddxO0Odddddddddk00xdddddddddddc'...................... + 'lddddddddddddddddddddddddddddxkdddddddddddddl,.............':lc:;. + .:dxkkkxxddddddddddddddddddddocc:;;;;;;;::cll,............,:,... + ;clooooddxkOOOdddoc:;,'''',:ooc;'................................. + odddddddddddddl:,...........'................................... + cdddddddddl:'............................................. + .,coddoc,........................................... + .'........................................... + ............................................ + ................. ............. ........ + .................. .......... ....... + .......... ...... ........ ...... + ........ ..... ...... .... + ..... .... .... .. + + HH HH EEEE DDDDD GGGGG EEEE HH HH OOOO GGGGG + HH HH EE DD DD GG EE HH HH OO OO GG + HHHHHHH EEEEE DD DD GGGGGGG EEEEE HHHHHHH OO OO GGGGGGG + HH HH EE DD DD GG GG EE HH HH OO OO GG GG + HH HH EEEE DDDDD GGGGGG EEEE HH HH OOOO GGGGGG + + LL II NN NN UU UU XX XX + LL II NNN NN UU UU XXX + LL II NN NNN UU UU XXX + LLLLL II NN NN UUUU XX XX + +Hedgehog-rpi-4 login: sensor +Password: + +sensor@Hedgehog-rpi-4:~$ su - +Password: +root@Hedgehog-rpi-4:~# passwd +New password: +Retype new password: +passwd: password updated successfully +root@Hedgehog-rpi-4:~# passwd sensor +New password: +Retype new password: +passwd: password updated successfully +``` + +## Configuration + +Once Hedgehog Linux has booted, [configuration](malcolm-hedgehog-e2e-iso-install.md#HedgehogInstallAndConfig) can proceed as usual using the `configure-interfaces` and `configure-capture` tools. + +## Performance Considerations + +While these instructions will build an image for various Raspberry Pi models, Hedgehog Linux resource requirements will likely only be satisfied by the 8GB versions of the Raspberry Pi model 4 and higher. + +Using faster storage (e.g., SATA solid-state drive connected to the Pi's USB 3.0 port using a USB 3.0 SATA to USB adapter, NVMe M.2 SSD, etc.) for the Hedgehog Linux OS drive and capture artifact directories will result in much better performance than booting from a microSD card. + +Users wishing to push the performance of Hedgehog Linux on Raspberry Pi may be required to disable certain features in order to operate within the constraints imposed by the Pi's available resources. For example the **ClamAV** engine used in [file extraction and scanning](malcolm-hedgehog-e2e-iso-install.md#HedgehogZeekFileExtraction) consumes a large percentage of a Raspberry Pi's system memory and could be disabled to make available those resources for other processes. Further resources could be freed up by disabling [`arkime-capture`](malcolm-hedgehog-e2e-iso-install.md#Hedgehogarkime-capture) (unselecting it from the [autostart services](malcolm-hedgehog-e2e-iso-install.md#HedgehogConfigAutostart)) which would allow Hedgehog Linux to still provide network traffic metadata generated by Zeek and Suricata at the cost of not generating Arkime session records and not storing the underlying full PCAP. \ No newline at end of file diff --git a/docs/hedgehog-ssh.md b/docs/hedgehog-ssh.md index 4a5f515d4..29e5ac073 100644 --- a/docs/hedgehog-ssh.md +++ b/docs/hedgehog-ssh.md @@ -1,4 +1,4 @@ -# Appendix B - Configuring SSH access +# Appendix C - Configuring SSH access SSH access to the sensor's non-privileged sensor account is only available using secure key-based authentication which can be enabled by adding a public SSH key to the **/home/sensor/.ssh/authorized_keys** file as illustrated below: diff --git a/docs/hedgehog-troubleshooting.md b/docs/hedgehog-troubleshooting.md index 736f50d40..deb359b22 100644 --- a/docs/hedgehog-troubleshooting.md +++ b/docs/hedgehog-troubleshooting.md @@ -1,4 +1,4 @@ -# Appendix C - Troubleshooting +# Appendix D - Troubleshooting Should the sensor not function as expected, first try rebooting the device. If the behavior continues, here are a few things that may help you diagnose the problem (items which may require Linux command line use are marked with **†**) diff --git a/docs/hedgehog-upgrade.md b/docs/hedgehog-upgrade.md index f106bb030..37d8a1a53 100644 --- a/docs/hedgehog-upgrade.md +++ b/docs/hedgehog-upgrade.md @@ -1,4 +1,4 @@ -# Appendix E - Upgrades +# Appendix F - Upgrades At this time there is not an "official" upgrade procedure to get from one release of Hedgehog Linux to the next. Upgrading the underlying operating system packages is generally straightforward, but not all of the Hedgehog Linux components are packaged into .deb archives yet as they should be, so for now it's a manual (and kind of nasty) process to Frankenstein an upgrade into existance. The author of this project intends to remedy this at some future point when time and resources allow. @@ -208,7 +208,7 @@ commands: ``` chown root:netdev /usr/sbin/netsniff-ng && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng chown root:netdev /opt/zeek/bin/zeek && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/zeek/bin/zeek chown root:netdev /sbin/ethtool && \ diff --git a/docs/hedgehog.md b/docs/hedgehog.md index e5c696a9f..f3399d5e3 100644 --- a/docs/hedgehog.md +++ b/docs/hedgehog.md @@ -32,8 +32,9 @@ Hedgehog Linux is a Debian-based operating system built to - [Autostart services](malcolm-hedgehog-e2e-iso-install.md#HedgehogConfigAutostart) + [Zeek Intelligence Framework](hedgehog-config-zeek-intel.md#HedgehogZeekIntel) * [Appendix A - Generating the ISO](hedgehog-iso-build.md#HedgehogISOBuild) -* [Appendix B - Configuring SSH access](hedgehog-ssh.md#HedgehogConfigSSH) -* [Appendix C - Troubleshooting](hedgehog-troubleshooting.md#HedgehogTroubleshooting) -* [Appendix D - Hardening](hedgehog-hardening.md#HedgehogHardening) +* [Appendix B - Generating a Raspberry Pi Image](hedgehog-raspi-build.md#HedgehogRaspiBuild) +* [Appendix C - Configuring SSH access](hedgehog-ssh.md#HedgehogConfigSSH) +* [Appendix D - Troubleshooting](hedgehog-troubleshooting.md#HedgehogTroubleshooting) +* [Appendix E - Hardening](hedgehog-hardening.md#HedgehogHardening) - [Compliance exceptions](hedgehog-hardening.md#HedgehogComplianceExceptions) -* [Appendix E - Upgrades](hedgehog-upgrade.md#HedgehogUpgradePlan) +* [Appendix F - Upgrades](hedgehog-upgrade.md#HedgehogUpgradePlan) diff --git a/docs/host-config-macos.md b/docs/host-config-macos.md index 200b7c485..d2df3cf6e 100644 --- a/docs/host-config-macos.md +++ b/docs/host-config-macos.md @@ -49,7 +49,7 @@ Compose is now a Docker plugin. For Docker to find this plugin, symlink it: Some changes should be made for performance ([this link](http://markshust.com/2018/01/30/performance-tuning-docker-mac) gives a good succinct overview). -* **Resource allocation** - For best results, Mac users should be running a quad-core MacBook Pro with 16GB RAM and an SSD, or desktop equivalent. Malcolm can run on older MacBook Pro machines (e.g., 2013 with 8GB RAM), but users are encouraged to bring a higher level of processing power. In the system tray, select **Docker** → **Preferences** → **Advanced**. Set the resources available to Docker to at least 4 CPUs and 8GB of RAM (>= 16GB is preferable). +* **Resource allocation** - For best results, Mac users should be running recent system with at least 16GB RAM and an SSD. In the system tray, select **Docker** → **Preferences** → **Advanced**. Set the resources available to Docker to at least 4 CPUs and at least 16GB RAM (even more is preferable). * **Volume mount performance** - Users can speed up performance of volume mounts by removing unused paths from **Docker** → **Preferences** → **File Sharing**. For example, if volumes are mounted under the home directory only, users could share /Users but remove other paths. diff --git a/docs/images/hedgehog/logo/hedgehog-ascii-text.txt b/docs/images/hedgehog/logo/hedgehog-ascii-text.txt new file mode 100644 index 000000000..c3a56cfe9 --- /dev/null +++ b/docs/images/hedgehog/logo/hedgehog-ascii-text.txt @@ -0,0 +1,40 @@ + + ,cc:,.. .:' + :dddddddoc,. ;,. oddo:. .c;. + :dddddddddddo;:ddc:dddddd; ldddl, + .dddddddddddddddddxdddddddo:odddddo' cl;. + ........ :ddddddddddddddddOkdddddddddxdddddd;,dddd' + .;lddddddddddolcddddddddddddddddk0kddddddddOxdddddddddddo. + 'dddddddddddddddddxkdddddddddddddx00xdddddddxkddddddoodddd, + .odddddddddddddddddO0OxdddddddddddO0Oddddddddoccloddc':xxd; + .:dddddddddddddddddxO00kdddddddddx00kdddddo;'....',;,'dddc. .,;,. + .cdddxOkkxdddddddddxO00kddddddddO00ddddo,..cxxxl'...........;O0000: + .',,,,,,':ddddkO00OxddddddddxO00kdddddddOOddddc...l0000l............',o0c + cddddddddddddddddxO00kddddddddx000xdddddddddddo'...:dxo,..............'' + 'lddddddddddddddddxO0Odddddddddk00xdddddddddddc'...................... + 'lddddddddddddddddddddddddddddxkdddddddddddddl,.............':lc:;. + .:dxkkkxxddddddddddddddddddddocc:;;;;;;;::cll,............,:,... + ;clooooddxkOOOdddoc:;,'''',:ooc;'................................. + odddddddddddddl:,...........'................................... + cdddddddddl:'............................................. + .,coddoc,........................................... + .'........................................... + ............................................ + ................. ............. ........ + .................. .......... ....... + .......... ...... ........ ...... + ........ ..... ...... .... + ..... .... .... .. + + HH HH EEEE DDDDD GGGGG EEEE HH HH OOOO GGGGG + HH HH EE DD DD GG EE HH HH OO OO GG + HHHHHHH EEEEE DD DD GGGGGGG EEEEE HHHHHHH OO OO GGGGGGG + HH HH EE DD DD GG GG EE HH HH OO OO GG GG + HH HH EEEE DDDDD GGGGGG EEEE HH HH OOOO GGGGGG + + LL II NN NN UU UU XX XX + LL II NNN NN UU UU XXX + LL II NN NNN UU UU XXX + LLLLL II NN NN UUUU XX XX + + diff --git a/docs/images/screenshots/arkime_sessions_files_browse.png b/docs/images/screenshots/arkime_sessions_files_browse.png new file mode 100644 index 000000000..3b281d2a5 Binary files /dev/null and b/docs/images/screenshots/arkime_sessions_files_browse.png differ diff --git a/docs/images/screenshots/arkime_sessions_files_log_dl.png b/docs/images/screenshots/arkime_sessions_files_log_dl.png new file mode 100644 index 000000000..5262b2188 Binary files /dev/null and b/docs/images/screenshots/arkime_sessions_files_log_dl.png differ diff --git a/docs/images/screenshots/dashboards_files_source.png b/docs/images/screenshots/dashboards_files_source.png index 634d0fcf7..14a25105a 100644 Binary files a/docs/images/screenshots/dashboards_files_source.png and b/docs/images/screenshots/dashboards_files_source.png differ diff --git a/docs/images/screenshots/dashboards_fs_integrity.png b/docs/images/screenshots/dashboards_fs_integrity.png new file mode 100644 index 000000000..c27f57c04 Binary files /dev/null and b/docs/images/screenshots/dashboards_fs_integrity.png differ diff --git a/docs/images/screenshots/dashboards_hosts_overview.png b/docs/images/screenshots/dashboards_hosts_overview.png new file mode 100644 index 000000000..f893e5672 Binary files /dev/null and b/docs/images/screenshots/dashboards_hosts_overview.png differ diff --git a/docs/images/screenshots/dashboards_journald_logs.png b/docs/images/screenshots/dashboards_journald_logs.png new file mode 100644 index 000000000..df9233450 Binary files /dev/null and b/docs/images/screenshots/dashboards_journald_logs.png differ diff --git a/docs/images/screenshots/dashboards_nginx_access_and_errors.png b/docs/images/screenshots/dashboards_nginx_access_and_errors.png new file mode 100644 index 000000000..6dc97649f Binary files /dev/null and b/docs/images/screenshots/dashboards_nginx_access_and_errors.png differ diff --git a/docs/images/screenshots/dashboards_nginx_overview.png b/docs/images/screenshots/dashboards_nginx_overview.png new file mode 100644 index 000000000..f393232bd Binary files /dev/null and b/docs/images/screenshots/dashboards_nginx_overview.png differ diff --git a/docs/images/screenshots/dashboards_sensor_audit_logs.png b/docs/images/screenshots/dashboards_sensor_audit_logs.png new file mode 100644 index 000000000..c6685707e Binary files /dev/null and b/docs/images/screenshots/dashboards_sensor_audit_logs.png differ diff --git a/docs/images/screenshots/dashboards_sensor_temperature.png b/docs/images/screenshots/dashboards_sensor_temperature.png new file mode 100644 index 000000000..6e0aa8886 Binary files /dev/null and b/docs/images/screenshots/dashboards_sensor_temperature.png differ diff --git a/docs/images/screenshots/dashboards_system_overview.png b/docs/images/screenshots/dashboards_system_overview.png new file mode 100644 index 000000000..fe8b1607d Binary files /dev/null and b/docs/images/screenshots/dashboards_system_overview.png differ diff --git a/docs/images/screenshots/extracted_files_dl_ui.png b/docs/images/screenshots/extracted_files_dl_ui.png new file mode 100644 index 000000000..a32a06d0d Binary files /dev/null and b/docs/images/screenshots/extracted_files_dl_ui.png differ diff --git a/docs/images/screenshots/raspi_imager_hedgehog.png b/docs/images/screenshots/raspi_imager_hedgehog.png new file mode 100644 index 000000000..da74185d7 Binary files /dev/null and b/docs/images/screenshots/raspi_imager_hedgehog.png differ diff --git a/docs/index-management.md b/docs/index-management.md index 9431fa5ad..607dcd60d 100644 --- a/docs/index-management.md +++ b/docs/index-management.md @@ -1,7 +1,11 @@ -# OpenSearch index management +# OpenSearch index management Malcolm releases prior to v6.2.0 used environment variables to configure OpenSearch [Index State Management](https://opensearch.org/docs/latest/im-plugin/ism/index/) [policies](https://opensearch.org/docs/latest/im-plugin/ism/policies/). Since then, OpenSearch Dashboards has developed and released plugins with UIs for [Index State Management](https://opensearch.org/docs/latest/im-plugin/ism/index/) and [Snapshot Management](https://opensearch.org/docs/latest/opensearch/snapshots/sm-dashboards/). Because these plugins provide a more comprehensive and user-friendly interface for these features, the old environment variable-based configuration code has been removed from Malcolm; with the exception of the code that uses the `OPENSEARCH_INDEX_SIZE_PRUNE_LIMIT` and `OPENSEARCH_INDEX_SIZE_PRUNE_NAME_SORT` [variables in `dashboards-helper.env`](malcolm-config.md#MalcolmConfigEnvVars), which deals with deleting the oldest network session metadata indices when the database exceeds a certain size. -Note that OpenSearch index state management and snapshot management only deals with disk space consumed by OpenSearch indices: it does not have anything to do with PCAP file storage. The `MANAGE_PCAP_FILES` environment variable in the [`arkime.env` file](malcolm-config.md#MalcolmConfigEnvVars) can be used to allow Arkime to prune old PCAP files based on available disk space. \ No newline at end of file +Note that OpenSearch index state management and snapshot management only deals with disk space consumed by OpenSearch indices: it does not have anything to do with PCAP file storage. The `MANAGE_PCAP_FILES` environment variable in the [`arkime.env` file](malcolm-config.md#MalcolmConfigEnvVars) can be used to allow Arkime to prune old PCAP files based on available disk space. + +# Using ILM/ISM with Arkime + +Arkime allows setting [index management policies](https://arkime.com/faq#ilm) with its sessions and history indices. The Malcolm environment variables for configuring this behavior are set in [`arkime.env`](malcolm-config.md#MalcolmConfigEnvVars). These variables can be used for both [OpenSearch and Elasticsearch instances](opensearch-instances.md#OpenSearchInstance) (OpenSearch [Index State Management (ISM)](https://opensearch.org/docs/latest/im-plugin/ism/index/) and [Elasticsearch Index Lifecycle Management (ILM)](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-lifecycle-management.html), respectively). \ No newline at end of file diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 0c3cd1da6..3b41df9a5 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.01.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.01.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.01.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.01.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.01.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.01.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.01.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.01.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.01.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.01.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.01.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.01.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.01.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.01.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.01.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.01.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.01.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.01.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.01.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.01.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.01.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.01.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.02.0 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.0 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.02.0 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.02.0 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.02.0 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.02.0 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.02.0 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.02.0 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.02.0 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.02.0 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.02.0 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.02.0 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.02.0 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.02.0 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.02.0 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.0 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.02.0 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.02.0 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.02.0 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.02.0 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.02.0 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.02.0 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -367,6 +367,8 @@ Require encrypted HTTPS connections? (Y / n): y 3: None Select authentication method (Basic): 1 +Enable index management policies (ILM/ISM) in Arkime? (y / N): n + Should Malcolm delete the oldest database indices and/or PCAP files based on available storage? (y / N): y Delete the oldest indices when the database exceeds a certain size? (y / N): y @@ -551,28 +553,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.01.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.01.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.01.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.01.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.01.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.01.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.01.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.01.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.01.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.01.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.01.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.01.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.01.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.01.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.01.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.01.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.01.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.01.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.01.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.01.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.01.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.01.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.02.0 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.02.0 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.02.0 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.02.0 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.02.0 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.02.0 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.02.0 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.02.0 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.02.0 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.02.0 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.02.0 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.02.0 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.02.0 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.0 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.02.0 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.02.0 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.02.0 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.02.0 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.02.0 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.0 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.02.0 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.02.0 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index b103bd901..7caac0274 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -14,6 +14,14 @@ Although the configuration script automates many of the following configuration - `ARKIME_ROTATE_INDEX` - how often (based on network traffic timestamp) to [create a new index](https://arkime.com/settings#rotateIndex) in OpenSearch - `MANAGE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will be marked as available for deletion by Arkime if available storage space becomes too low (default `false`) - `MAXMIND_GEOIP_DB_LICENSE_KEY` - Malcolm uses MaxMind's free GeoLite2 databases for GeoIP lookups. As of December 30, 2019, these databases are [no longer available](https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/) for download via a public URL. Instead, they must be downloaded using a MaxMind license key (available without charge [from MaxMind](https://www.maxmind.com/en/geolite2/signup)). The license key can be specified here for GeoIP database downloads during build- and run-time. + - The following variables configure [Arkime's use](index-management.md#ArkimeIndexPolicies) of OpenSearch [Index State Management (ISM)](https://opensearch.org/docs/latest/im-plugin/ism/index/) or Elasticsearch [Index Lifecycle Management (ILM)](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-lifecycle-management.html): + + `INDEX_MANAGEMENT_ENABLED` - if set to `true`, Malcolm's instance of Arkime will [use these features](https://arkime.com/faq#ilm) when indexing data + + `INDEX_MANAGEMENT_OPTIMIZATION_PERIOD` - the period in hours or days that Arkime will keep records in the **hot** state (default `30d`) + + `INDEX_MANAGEMENT_RETENTION_TIME` - the period in hours or days that Arkime will keep records before deleting them (default `90d`) + + `INDEX_MANAGEMENT_OLDER_SESSION_REPLICAS` - the number of replicas for older sessions indices (default `0`) + + `INDEX_MANAGEMENT_HISTORY_RETENTION_WEEKS` - the retention time period (weeks) for Arkime history data (default `13`) + + `INDEX_MANAGEMENT_SEGMENTS` - the number of segments Arlime will use to optimize sessions (default `1`) + + `INDEX_MANAGEMENT_HOT_WARM_ENABLED` - whether or not Arkime should use a hot/warm design (storing non-session data in a warm index); setting up hot/warm index policies also requires configuration on the local nodes in accordance with the [Arkime documentation](https://arkime.com/faq#ilm) * **`auth-common.env`** - [authentication](authsetup.md)-related settings - `NGINX_BASIC_AUTH` - if set to `true`, use [TLS-encrypted HTTP basic](authsetup.md#AuthBasicAccountManagement) authentication (default); if set to `false`, use [Lightweight Directory Access Protocol (LDAP)](authsetup.md#AuthLDAP) authentication * **`auth.env`** - stores the Malcolm administrator's username and password hash for its nginx reverse proxy @@ -40,6 +48,10 @@ Although the configuration script automates many of the following configuration - `TOTAL_MEGABYTES_SEVERITY_THRESHOLD` - when [severity scoring](severity.md#Severity) is enabled, this variable indicates the size threshold (in megabytes) for assigning severity to large connections or file transfers (default `1000`) * **`netbox-common.env`**, `netbox.env`, `netbox-secret.env`, `netbox-postgres.env`, `netbox-redis-cache.env` and `netbox-redis.env` - settings related to [NetBox](https://netbox.dev/) and [Asset Interaction Analysis](asset-interaction-analysis.md#AssetInteractionAnalysis) - `NETBOX_DISABLED` - if set to `true`, Malcolm will **not** start and manage a [NetBox](asset-interaction-analysis.md#AssetInteractionAnalysis) instance (default `true`) + - `NETBOX_DEFAULT_SITE` - specifies the default NetBox [site name](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/) for use when [enriching network traffic metadata via NetBox lookups](asset-interaction-analysis.md#NetBoxEnrichment) (default `Malcolm`) + - `NETBOX_PRELOAD_PREFIXES` - if set to `true`, Malcolm's NetBox initialization will automatically create "catch-all" prefixes for private IP address space (i.e., one each for `10.0.0.0/8`, `172.16.0.0/12`, and `192.168.0.0/16`, respectively) for use when [enriching network traffic metadata via NetBox lookups](asset-interaction-analysis.md#NetBoxEnrichment) + - `NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER` - if set to `true`, new manufacturer entries will be created in the NetBox database when [matching device manufacturers to OUIs](asset-interaction-analysis.md#NetBoxPopPassiveOUIMatch) (default `true`) + - `NETBOX_DEFAULT_FUZZY_THRESHOLD` - fuzzy-matching threshold for [matching device manufacturers to OUIs](asset-interaction-analysis.md#NetBoxPopPassiveOUIMatch) (default `0.95`) * **`nginx.env`** - settings specific to Malcolm's nginx reverse proxy - `NGINX_LOG_ACCESS_AND_ERRORS` - if set to `true`, all access to Malcolm via its [web interfaces](quickstart.md#UserInterfaceURLs) will be logged to OpenSearch (default `false`) - `NGINX_SSL` - if set to `true`, require HTTPS connections to Malcolm's `nginx-proxy` container (default); if set to `false`, use unencrypted HTTP connections (using unsecured HTTP connections is **NOT** recommended unless you are running Malcolm behind another reverse proxy such as Traefik, Caddy, etc.) diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index 1d7308a8d..666bfc743 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -173,6 +173,20 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest - When using LDAP authentication, this question allows users to configure [LDAP connection security](authsetup.md#AuthLDAPSecurity) * **Store PCAP, log and index files locally under /home/user/Malcolm?** - Malcolm generates a number of large file sets during normal operation: PCAP files, Zeek or Suricata logs, OpenSearch indices, etc. By default all of these are stored in subdirectories in the Malcolm installation directory. This question allows users to specify alternative storage location(s) (for example, a separate dedicated drive or RAID volume) for these artifacts. +* **Enable index management policies (ILM/ISM) in Arkime?** + - Choose **Y** to proceed to the following related questions about [using ILM/ISM with Arkime](index-management.md#ArkimeIndexPolicies). + - **Should Arkime use a hot/warm design in which non-session data is stored in a warm index?** + - This quesion allows users to specify if Arkime should store non-session indices (`arkime-history`) indices in a warm index. This requires additional configuration as demonstrated in the [Arkime documentation](https://arkime.com/faq#ilm). + - **How long should Arkime keep an index in the hot node? (e.g. 25h, 5d, etc.)** + - This question allows users to specify how long an Arkime index should remain in the **hot** state before moving into a **warm** state. + - **How long should Arkime retain SPI data before deleting it? (e.g. 25h, 90d, etc.)** + - This question is used to set the maximum age at which Arkime session indices are deleted. + - **How many segments should Arkime use to optimize?** + - This question asks for the number of segments to use for optimization. + - **How many replicas should Arkime maintain for older session indices?** + - This defines how many additional copies of older session indices Arkime should store. + - **How many weeks of history should Arkime keep?",** + - This defines the retention period (in weeks) for `arkime-history` indices. * **Should Malcolm delete the oldest database indices and/or PCAP files based on available storage?** - Choose **Y** to proceed to the following related questions about managing the data storage used by Malcolm. - **Delete the oldest indices when the database exceeds a certain size?** diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index fff839df5..da5e35fe0 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-24.01.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-24.02.0.iso" … ``` diff --git a/docs/malcolm-upgrade.md b/docs/malcolm-upgrade.md index b27f80f3b..aa79503c5 100644 --- a/docs/malcolm-upgrade.md +++ b/docs/malcolm-upgrade.md @@ -40,15 +40,15 @@ If Malcolm was installed from [pre-packaged installation files]({{ site.github.r * `tar xf malcolm_YYYYMMDD_HHNNSS_xxxxxxx.tar.gz` 1. backup current Malcolm scripts, configuration files and certificates * `mkdir -p ./upgrade_backup_$(date +%Y-%m-%d)` - * `cp -r filebeat/ htadmin/ logstash/ nginx/ config/ docker-compose.yml ./scripts ./README.md ./upgrade_backup_$(date +%Y-%m-%d)/` + * `cp -r filebeat/ htadmin/ logstash/ nginx/ config/ docker-compose*.yml ./scripts ./README.md ./upgrade_backup_$(date +%Y-%m-%d)/` 1. replace scripts and local documentation in the existing installation with the new ones * `rm -rf ./scripts ./README.md` * `cp -r ./malcolm_YYYYMMDD_HHNNSS_xxxxxxx/scripts ./malcolm_YYYYMMDD_HHNNSS_xxxxxxx/README.md ./` -1. replace (overwrite) `docker-compose.yml` file with new version - * `cp ./malcolm_YYYYMMDD_HHNNSS_xxxxxxx/docker-compose.yml ./docker-compose.yml` +1. replace (overwrite) `docker-compose*.yml` file with new versions + * `cp ./malcolm_YYYYMMDD_HHNNSS_xxxxxxx/docker-compose*.yml ./` 1. re-run `./scripts/configure` as described in [Malcolm Configuration](malcolm-config.md#ConfigAndTuning) * to do an in-depth comparison of the previous version's settings with the new setings: - + using a file comparison tool (e.g., `diff`, `meld`, `Beyond Compare`, etc.), compare `docker-compose.yml` and the `docker-compare.yml` file backed up in Step 3, and manually migrate over any customizations in file + + using a file comparison tool (e.g., `diff`, `meld`, `Beyond Compare`, etc.), compare `docker-compose.yml` and the `docker-compose.yml` files backed up in Step 3, and manually migrate over any customizations in file + compare the contents of each `.env` file Malcolm's `./config/` directory with its corresponding `.env.example` file. the author uses this command which uses [difftastic](https://github.com/Wilfred/difftastic), [bat](https://github.com/sharkdp/bat), [unbuffer](https://manpages.debian.org/stretch/expect/unbuffer.1.en.html), and [cmp](https://en.wikipedia.org/wiki/Cmp_(Unix)). ```bash for FILE in *.env; do \ diff --git a/docs/quickstart.md b/docs/quickstart.md index e90cbe9cf..de66729d6 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.01.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.01.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.01.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.01.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.01.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.01.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.01.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.01.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.01.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.01.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.01.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.01.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.01.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.01.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.01.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.01.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.01.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.01.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.01.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.02.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.02.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.02.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.02.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.02.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.02.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.02.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.02.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.02.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.02.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.02.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.02.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.02.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.02.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.02.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.02.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.02.0 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/reporting.md b/docs/reporting.md new file mode 100644 index 000000000..e66f6401b --- /dev/null +++ b/docs/reporting.md @@ -0,0 +1,7 @@ +# Reporting + +Malcolm uses the Reporting plugin for [OpenSearch Dashboards](https://github.com/opensearch-project/reporting). Reports can be generated on-demand or defined using [visualizations and dashboards](dashboards.md#DashboardsVisualizations), [the discover view](dashboards.md#Discover), or [Notebooks](https://opensearch.org/docs/latest/observing-your-data/notebooks/) pages. See [Reporting](https://opensearch.org/docs/latest/reporting/report-dashboard-index/) in the OpenSearch documentation for usage instructions. + +## Known Issues + +The Malcolm development team is [aware of an issue](https://github.com/idaholab/Malcolm/issues/249) exporting some dashboards as PDF and PNG reports using the Mozilla Firefox web browser. While the root cause and fix are being investigated, users that encounter this bug may attempt the action using another web browser. diff --git a/docs/system-requirements.md b/docs/system-requirements.md index b5534716d..df0f4bfe3 100644 --- a/docs/system-requirements.md +++ b/docs/system-requirements.md @@ -2,6 +2,6 @@ Malcolm runs on top of [Docker](https://www.docker.com/), which runs on recent releases of Linux, Apple [macOS](host-config-macos.md#HostSystemConfigMac), and [Microsoft Windows](host-config-windows.md#HostSystemConfigWindows) 10 and up. Malcolm can also be deployed in the cloud [with Kubernetes](kubernetes.md#Kubernetes). -To quote the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html), "If there is one resource that you will run out of first, it will likely be memory." The same is true for Malcolm: and users will want at least 16GB of RAM to run Malcolm comfortably. For processing large volumes of traffic, Malcolm developers recommend a minimum of 16 cores and 16 gigabytes of RAM on a dedicated server. Malcolm can run on less, but more is better. Of course, users will want as much hard drive space as possible, as the amount of PCAP data a machine can analyze and store will be limited by its hard drive. +To quote the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html), "If there is one resource that you will run out of first, it will likely be memory." Malcolm developers recommend a minimum of 8 cores and 16 gigabytes of RAM on a dedicated server. Malcolm can run on less, but more is better. Of course, users will want as much hard drive space as possible, as the amount of PCAP data a machine can analyze and store will be limited by its hard drive. Arkime's wiki has documents ([here](https://github.com/arkime/arkime#hardware-requirements) and [here](https://github.com/arkime/arkime/wiki/FAQ#what-kind-of-capture-machines-should-we-buy) and [here](https://github.com/arkime/arkime/wiki/FAQ#how-many-elasticsearch-nodes-or-machines-do-i-need) and a [calculator here](https://arkime.com/estimators)) that may be helpful, although not everything in those documents will apply to a Docker-based setup such as Malcolm. \ No newline at end of file diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 5478e91ec..179901b8f 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -132,6 +132,8 @@ Select authentication method (Basic): 1 Store PCAP, log and index files locally under /home/user/Malcolm? (Y / n): y +Enable index management policies (ILM/ISM) in Arkime? (y / N): n + Should Malcolm delete the oldest database indices and/or PCAP files based on available storage? n Automatically analyze all PCAP files with Suricata? (Y / n): y @@ -255,25 +257,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.01.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.01.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.01.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.01.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.01.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.01.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.01.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.01.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.01.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.01.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.01.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.01.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.01.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.01.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.01.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.01.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.01.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.01.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.01.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.02.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.02.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.02.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.02.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.02.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.02.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.02.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.02.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.02.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.02.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.02.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.02.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.02.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.02.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.02.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.02.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.02.0 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/file-monitor/scripts/.gitignore b/file-monitor/scripts/.gitignore new file mode 100644 index 000000000..a2d7c8915 --- /dev/null +++ b/file-monitor/scripts/.gitignore @@ -0,0 +1 @@ +malcolm_utils.py diff --git a/file-monitor/scripts/extracted_files_http_server.py b/file-monitor/scripts/extracted_files_http_server.py new file mode 100755 index 000000000..a8a30bc0a --- /dev/null +++ b/file-monitor/scripts/extracted_files_http_server.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Multithreaded simple HTTP directory server. +# +# The files can optionally be archived in a ZIP file, with or without a password, or +# be aes-256-cbc encrypted in a way that's compatible with: +# openssl enc -aes-256-cbc -d -in encrypted.data -out decrypted.data + +import argparse +import dominate +import hashlib +import magic +import os +import re +import sys +from Crypto.Cipher import AES +from datetime import datetime, timedelta, UTC +from dominate.tags import * +from http.server import HTTPServer, SimpleHTTPRequestHandler +from socketserver import ThreadingMixIn +from stat import S_IFREG +from stream_zip import ZIP_32, stream_zip +from threading import Thread + +from malcolm_utils import ( + eprint, + EVP_BytesToKey, + EVP_KEY_SIZE, + OPENSSL_ENC_MAGIC, + PKCS5_SALT_LEN, + remove_prefix, + sizeof_fmt, + str2bool, + temporary_filename, +) + +################################################################################################### +args = None +debug = False +script_name = os.path.basename(__file__) +script_path = os.path.dirname(os.path.realpath(__file__)) +orig_path = os.getcwd() +filename_truncate_len = 20 + + +################################################################################################### +# a function for performing "natural" (case insensitive) sort +def natural_sort_key(s, _nsre=re.compile('([0-9]+)')): + return [int(text) if text.isdigit() else text.lower() for text in _nsre.split(s)] + + +################################################################################################### +# return the names and flags for Zipping a list of files +def LocalFilesForZip(names): + now = datetime.now(UTC) + + def contents(name): + with open(name, 'rb') as f: + while chunk := f.read(65536): + yield chunk + + return ((os.path.join('.', os.path.basename(name)), now, S_IFREG | 0o600, ZIP_32, contents(name)) for name in names) + + +################################################################################################### +# a simple HTTP request handler for listing directories of files and serving those files for download +class HTTPHandler(SimpleHTTPRequestHandler): + # return full path based on server base path and requested path + def translate_path(self, path): + path = SimpleHTTPRequestHandler.translate_path(self, path) + relpath = os.path.relpath(path, os.getcwd()) + fullpath = os.path.join(self.server.base_path, relpath) + return fullpath, relpath + + # override do_GET for fancy directory listing and so that files are encrypted/zipped, if requested + def do_GET(self): + global debug + global args + + fullpath, relpath = self.translate_path(self.path) + fileBaseName = os.path.basename(fullpath) + + tomorrowStr = (datetime.now(UTC) + timedelta(days=1)).isoformat().split('.')[0] + + # HTTP-FUID-UID-TIMESTAMP.ext + carvedFileRegex = re.compile( + r'^(?P[^-]+)-(?PF[a-zA-Z0-9]+|unknown)-(?PC[a-zA-Z0-9]+|unknown)-(?P\d+)(?P\..+)?$' + ) + # UID-FUID-whatever + carvedFileRegexAlt = re.compile(r'^(?PC[a-zA-Z0-9]+)_(?PF[a-zA-Z0-9]+)') + # XOR decrypted from FEieEe1f1SI6YJk4H5 + xorRegex = re.compile(r'^(?PXOR) decrypted from (?PF[a-zA-Z0-9]+)') + + if os.path.isdir(fullpath) and (args.links or (not os.path.islink(fullpath))): + # directory listing + self.send_response(200) + self.send_header('Content-type', "text/html") + self.end_headers() + + pageTitle = f"Directory listing for {fileBaseName if fileBaseName != '.' else '/'}" + doc = dominate.document(title=pageTitle) + + # + with doc.head: + meta(charset="utf-8") + meta(name="viewport", content="width=device-width, initial-scale=1, shrink-to-fit=no") + link(rel="icon", href=f"{args.assetsDirRespReplacer}favicon.ico", type="image/x-icon") + link(rel="stylesheet", href=f"{args.assetsDirRespReplacer}css/bootstrap-icons.css", type="text/css") + link(rel="stylesheet", href=f"{args.assetsDirRespReplacer}css/google-fonts.css", type="text/css") + link(rel="stylesheet", href=f"{args.assetsDirRespReplacer}css/styles.css", type="text/css") + + # + with doc: + # header decoration + with nav(cls='navbar navbar-light bg-light static-top'): + div(cls='container') + header(cls='masthead') + with section(cls="features-icons bg-light"): + with div(cls='container'): + h1(pageTitle, cls='mb-5', style='text-align: center') + with div(cls='container').add(div(cls="row")).add(div(cls="col-lg-12")): + with table(cls='table-bordered', width='100%').add(tbody()): + # header row + t = tr(style="text-align: center") + t.add( + th( + f"Download{' (AE-2 zipped)' if (args.zip and args.key) else ' (zipped)' if args.zip else ' (encrypted)' if args.key else ''}" + ), + th("Type" if args.magic else "Extension"), + th("Size"), + ) + if args.malcolm: + t.add( + th("Source"), + th("IDs"), + th("Timestamp"), + ) + if fileBaseName != '.': + t = tr() + t.add( + td(a(i(cls="bi bi-arrow-90deg-up"), href=f'..')), + td("Directory"), + td(''), + ) + if args.malcolm: + t.add(th(), th(), th()) + + # content rows (files and directories) + for dirpath, dirnames, filenames in os.walk(fullpath): + # list directories first + for dirname in sorted(dirnames, key=natural_sort_key): + try: + child = os.path.join(dirpath, dirname) + if args.links or (not os.path.islink(child)): + t = tr() + t.add( + td(a(dirname, href=f'{dirname}/')), + td("Directory"), + td(''), + ) + if args.malcolm: + t.add(th(), th(), th()) + except Exception as e: + eprint(f'Error with directory "{dirname}"": {e}') + + # list files after directories + for filename in sorted(filenames, key=natural_sort_key): + try: + child = os.path.join(dirpath, filename) + if args.links or (not os.path.islink(child)): + t = tr() + + # calculate some of the stuff for representing Malcolm files + timestamp = None + timestampStr = '' + timestampStartFilterStr = '' + fmatch = None + fsource = '' + fids = list() + if args.malcolm: + # determine if filename is in a pattern we recognize + fmatch = carvedFileRegex.search(filename) + if fmatch is None: + fmatch = carvedFileRegexAlt.search(filename) + if fmatch is not None: + # format timestamp as ISO date/time + timestampStr = fmatch.groupdict().get('timestamp', '') + try: + timestamp = datetime.strptime(timestampStr, '%Y%m%d%H%M%S') + timestampStr = timestamp.isoformat() + timestampStartFilterStr = ( + (timestamp - timedelta(days=1)) + .isoformat() + .split('.')[0] + ) + except Exception as te: + if timestampStr: + eprint(f'Error with time "{str(timestampStr)}": {te}') + # put UIDs and FUIDs into a single event.id-filterable column + fids = list( + [ + x + for x in [ + fmatch.groupdict().get('uid', ''), + fmatch.groupdict().get('fuid', ''), + ] + if x and x != 'unknown' + ] + ) + # massage source a little bit (remove '' and handle + # 'XOR decrypted from...') + fsource = fmatch.groupdict().get('source', '') + if fsource == '': + fsource = '' + elif xorMatch := xorRegex.search(fsource): + fsource = xorMatch.groupdict().get('source', '') + fids.append(xorMatch.groupdict().get('fuid', '')) + + # only request mime type for files if specified in arguments + fileinfo = ( + magic.from_file(os.path.realpath(child), mime=True) + if args.magic + else os.path.splitext(filename)[1] + ) + + # show filename, file type (with link to IANA if MIME type is shown), and file size + t.add( + td( + a( + ( + (filename[:filename_truncate_len] + '...') + if len(filename) > filename_truncate_len + else filename + ), + href=f'{filename}', + ), + title=filename, + ), + ( + td( + a( + fileinfo, + href=f'https://www.iana.org/assignments/media-types/{fileinfo}', + ), + ) + if args.magic + else td(fileinfo) + ), + td(sizeof_fmt(os.path.getsize(child)), style="text-align: right"), + ) + + # show special malcolm columns if requested + if args.malcolm and fmatch is not None: + # list carve source, IDs, and timestamp + t.add( + td( + fsource, + style="text-align: center", + ), + td( + [ + a( + fid, + href=f'/arkime/idark2dash/filter?start={timestampStartFilterStr}&stop={tomorrowStr}&field=event.id&value={fid}', + ) + for fid in fids + ], + style="text-align: center", + ), + td( + ( + timestamp.strftime("%Y-%m-%d %H:%M:%S") + if timestamp + else timestampStr + ), + title=timestampStr, + style="text-align: center", + ), + ) + else: + # file name format was not recognized, so extra columns are empty + t.add(th(), th(), th()) + + except Exception as e: + eprint(f'Error with file "{filename}": {e}') + + # our "walk" is not recursive right now, we only need to go one level deep + break + + # footer decoration + with footer(cls='footer bg-light').add(div(cls='container')).add(div(cls='row')): + with div(cls="col-lg-6 h-100 text-center text-lg-start my-auto"): + p( + "Malcolm © 2024 Battelle Energy Alliance, LLC; developed at INL and released through the cooperation of the Cybersecurity and Infrastructure Security Agency of the U.S. Department of Homeland Security.", + cls="text-muted small mb-4 mb-lg-0", + ) + + with div(cls="col-lg-6 h-100 text-center text-lg-end my-auto").add(ul(cls="list-inline mb-0")): + li(cls="list-inline-item").add(a(href=f'/')).add(i(cls="bi bi-house fs-3", title="Malcolm")) + li(cls="list-inline-item").add(a(href=f'/readme/')).add( + i(cls="bi bi-question-circle fs-3", title="Documentation") + ) + li(cls="list-inline-item").add( + a(href=f'/dashboards/app/dashboards#/view/9ee51f94-3316-4fc5-bd89-93a52af69714') + ).add(i(cls="bi bi-bar-chart-line fs-3", title="Dashboards")) + li(cls="list-inline-item").add(a(href=f'/arkime/sessions/')).add( + i(cls="bi bi-table fs-3", title="Arkime") + ) + li(cls="list-inline-item").add(a(href=f'https://github.com/cisagov/Malcolm/')).add( + i(cls="bi-github fs-3", title="GitHub") + ) + + script(type="text/javascript", src=f"{args.assetsDirRespReplacer}js/bootstrap.bundle.min.js") + script(type="text/javascript", src=f"{args.assetsDirRespReplacer}js/scripts.js") + + # send directory listing HTML to web client + self.wfile.write(str.encode(str(doc))) + + else: + # serve a file for download + + # handle special case of requesting assets (css, js, etc.) + satisfied = False + tmpPath = os.path.join('/', relpath) + if ( + (not os.path.isfile(fullpath)) + and (not os.path.islink(fullpath)) + and tmpPath.startswith(args.assetsDirReqReplacer) + and os.path.isdir(str(args.assetsDir)) + ): + # an asset was requested, so translate it into the real asset's path + if ( + (fullpath := os.path.join(args.assetsDir, remove_prefix(tmpPath, args.assetsDirReqReplacer))) + and os.path.isfile(fullpath) + and (args.links or (not os.path.islink(fullpath))) + ): + # serve the asset file + satisfied = True + ctype = self.guess_type(fullpath) + with open(fullpath, 'rb') as fhandle: + fs = os.fstat(fhandle.fileno()) + self.send_response(200) + self.send_header('Content-type', self.guess_type(fullpath)) + self.send_header("Content-Length", str(fs[6])) + self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + self.end_headers() + while chunk := fhandle.read(1024): + self.wfile.write(chunk) + + # handle regular file downloads + if not satisfied: + # if the file doesn't exist as specified but recursive is enabled, go deeper to find the file + if args.recursive and (not os.path.isfile(fullpath)) and (not os.path.islink(fullpath)): + for root, dirs, files in os.walk(os.path.dirname(fullpath)): + if fileBaseName in files: + fullpath = os.path.join(root, fileBaseName) + break + + if os.path.isfile(fullpath) and (args.links or (not os.path.islink(fullpath))): + if args.zip: + # ZIP file (streamed, AES-encrypted with password or unencrypted) + self.send_response(200) + self.send_header('Content-type', "application/zip") + self.send_header('Content-Disposition', f'attachment; filename={fileBaseName}.zip') + self.end_headers() + for chunk in stream_zip(LocalFilesForZip([fullpath]), password=args.key if args.key else None): + self.wfile.write(chunk) + + elif args.key: + # openssl-compatible encrypted file + self.send_response(200) + self.send_header('Content-type', 'application/octet-stream') + self.send_header('Content-Disposition', f'attachment; filename={fileBaseName}.encrypted') + self.end_headers() + salt = os.urandom(PKCS5_SALT_LEN) + key, iv = EVP_BytesToKey( + EVP_KEY_SIZE, AES.block_size, hashlib.sha256, salt, args.key.encode('utf-8') + ) + cipher = AES.new(key, AES.MODE_CBC, iv) + encrypted = b"" + encrypted += OPENSSL_ENC_MAGIC + encrypted += salt + self.wfile.write(encrypted) + with open(fullpath, 'rb') as f: + padding = b'' + while True: + chunk = f.read(cipher.block_size) + if len(chunk) < cipher.block_size: + remaining = cipher.block_size - len(chunk) + padding = bytes([remaining] * remaining) + self.wfile.write(cipher.encrypt(chunk + padding)) + if padding: + break + + else: + # original file, unencrypted + SimpleHTTPRequestHandler.do_GET(self) + + else: + self.send_error(404, "Not Found") + + +################################################################################################### +# +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + def __init__(self, base_path, server_address, RequestHandlerClass=HTTPHandler): + self.base_path = base_path + HTTPServer.__init__(self, server_address, RequestHandlerClass) + + +################################################################################################### +# +def serve_on_port(path: str, port: int): + server = ThreadingHTTPServer(path, ("", port)) + print(f"serving {path} at port {port}") + server.serve_forever() + + +################################################################################################### +# main +def main(): + global args + global debug + global orig_path + + defaultDebug = os.getenv('EXTRACTED_FILE_HTTP_SERVER_DEBUG', 'false') + defaultZip = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ZIP', 'false') + defaultRecursive = os.getenv('EXTRACTED_FILE_HTTP_SERVER_RECURSIVE', 'false') + defaultMagic = os.getenv('EXTRACTED_FILE_HTTP_SERVER_MAGIC', 'false') + defaultLinks = os.getenv('EXTRACTED_FILE_HTTP_SERVER_LINKS', 'false') + defaultMalcolm = os.getenv('EXTRACTED_FILE_HTTP_SERVER_MALCOLM', 'false') + defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440)) + defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'infected') + defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path) + defaultAssetsDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR', '/opt/assets') + defaultAssetsDirReqReplacer = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR_REQ_REPLACER', '/assets') + defaultAssetsDirRespReplacer = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR_RESP_REPLACER', '/assets') + + parser = argparse.ArgumentParser( + description=script_name, add_help=False, usage='{} '.format(script_name) + ) + parser.add_argument( + '-v', + '--verbose', + dest='debug', + type=str2bool, + nargs='?', + const=True, + default=defaultDebug, + metavar='true|false', + help=f"Verbose/debug output ({defaultDebug})", + ) + parser.add_argument( + '-p', + '--port', + dest='port', + help=f"Server port ({defaultPort})", + metavar='', + type=int, + default=defaultPort, + ) + parser.add_argument( + '-d', + '--directory', + dest='serveDir', + help=f'Directory to serve ({defaultDir})', + metavar='', + type=str, + default=defaultDir, + ) + parser.add_argument( + '-a', + '--assets-directory', + dest='assetsDir', + help=f'Directory hosting assets ({defaultAssetsDir})', + metavar='', + type=str, + default=defaultAssetsDir, + ) + parser.add_argument( + '--assets-directory-req-replacer', + dest='assetsDirReqReplacer', + help=f'Virtual directory name for requests to redirect to assets directory ({defaultAssetsDirReqReplacer})', + metavar='', + type=str, + default=defaultAssetsDirReqReplacer, + ) + parser.add_argument( + '--assets-directory-resp-replacer', + dest='assetsDirRespReplacer', + help=f'Virtual directory name for responses to indicate files in the assets directory ({defaultAssetsDirRespReplacer})', + metavar='', + type=str, + default=defaultAssetsDirRespReplacer, + ) + parser.add_argument( + '-m', + '--magic', + dest='magic', + type=str2bool, + nargs='?', + const=True, + default=defaultMagic, + metavar='true|false', + help=f"Get file MIME type ({defaultMagic})", + ) + parser.add_argument( + '-k', + '--key', + dest='key', + help="File encryption key (for ZIP file if -z/--zip, otherwise openssl-compatible encryption", + metavar='', + type=str, + default=defaultKey, + ) + parser.add_argument( + '-z', + '--zip', + dest='zip', + type=str2bool, + nargs='?', + const=True, + default=defaultZip, + metavar='true|false', + help=f"Zip file ({defaultZip})", + ) + parser.add_argument( + '-r', + '--recursive', + dest='recursive', + type=str2bool, + nargs='?', + const=True, + default=defaultRecursive, + metavar='true|false', + help=f"Recursively look for requested file if not found ({defaultRecursive})", + ) + parser.add_argument( + '-l', + '--links', + dest='links', + type=str2bool, + nargs='?', + const=True, + default=defaultLinks, + metavar='true|false', + help=f"Serve symlinks in addition to regular files ({defaultLinks})", + ) + parser.add_argument( + '--malcolm', + dest='malcolm', + type=str2bool, + nargs='?', + const=True, + default=defaultMalcolm, + metavar='true|false', + help=f"Include columns for Zeek-extracted files in Malcolm ({defaultMalcolm})", + ) + try: + parser.error = parser.exit + args = parser.parse_args() + except SystemExit: + parser.print_help() + exit(2) + + debug = args.debug + if debug: + eprint(os.path.join(script_path, script_name)) + eprint("Arguments: {}".format(sys.argv[1:])) + eprint("Arguments: {}".format(args)) + else: + sys.tracebacklimit = 0 + + if args.assetsDirReqReplacer: + args.assetsDirReqReplacer = os.path.join(args.assetsDirReqReplacer, '') + if args.assetsDirRespReplacer: + args.assetsDirRespReplacer = os.path.join(args.assetsDirRespReplacer, '') + + Thread(target=serve_on_port, args=[args.serveDir, args.port]).start() + + +################################################################################################### +if __name__ == '__main__': + main() diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf index 2a9a1fe39..4ca505d7a 100644 --- a/file-monitor/supervisord.conf +++ b/file-monitor/supervisord.conf @@ -155,6 +155,10 @@ command=/usr/local/bin/extracted_files_http_server.py --zip %(ENV_EXTRACTED_FILE_HTTP_SERVER_ZIP)s --recursive %(ENV_EXTRACTED_FILE_HTTP_SERVER_RECURSIVE)s --directory /zeek/extract_files + --assets-directory "%(ENV_EXTRACTED_FILE_HTTP_SERVER_ASSETS_DIR)s" + --assets-directory-req-replacer /assets + --assets-directory-resp-replacer /extracted-files/assets + --malcolm autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s autorestart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s startsecs=0 diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index ee401cec9..8f82de56e 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:24.01.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index 54301d123..dc9fe4aca 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:24.01.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index 003c00476..54ac4699e 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:24.01.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index ce67e2cc4..f3fe9213a 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.01.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 59bac4763..9358a7980 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:24.01.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -79,7 +79,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index 2c60ade37..cd8462b80 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:24.01.0 + image: ghcr.io/idaholab/malcolm/api:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index 90b818d0d..4f8598940 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.01.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index b3f39c623..34352453c 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:24.01.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index 5568fe7df..c19818116 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:24.01.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index f7ddf38ff..ed4ee049c 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:24.01.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -49,6 +49,8 @@ spec: name: process-env - configMapRef: name: ssl-env + - configMapRef: + name: dashboards-env - configMapRef: name: zeek-env - secretRef: @@ -81,7 +83,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index 70a1ce8fc..553f04d46 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.01.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 8151d9f47..47b4943a4 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:24.01.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -115,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 738c27dc9..27e54661d 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:24.01.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 3d382482c..011f7baf7 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:24.01.0 + image: ghcr.io/idaholab/malcolm/redis:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index f244be897..2b345240f 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:24.01.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 4cb746385..37577f687 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:24.01.0 + image: ghcr.io/idaholab/malcolm/netbox:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index f7babad77..f39cb01d7 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:24.01.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index c0f804807..f0ddd2ff8 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:24.01.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 431c2ee5a..1af8ac204 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:24.01.0 + image: ghcr.io/idaholab/malcolm/zeek:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index c323398ef..bd1ca6e31 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:24.01.0 + image: ghcr.io/idaholab/malcolm/suricata:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index c91ab8440..a84945410 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/idaholab/malcolm/arkime:24.01.0 + image: ghcr.io/idaholab/malcolm/arkime:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index 8e45e1a91..6fcd077cd 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:24.01.0 + image: ghcr.io/idaholab/malcolm/freq:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index bebc05efa..35dbe4986 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.01.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.0 imagePullPolicy: Always stdin: false tty: true @@ -58,6 +58,10 @@ spec: name: process-env - configMapRef: name: ssl-env + - configMapRef: + name: opensearch-env + - configMapRef: + name: dashboards-env - configMapRef: name: auth-common-env - configMapRef: @@ -95,7 +99,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.01.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 imagePullPolicy: Always stdin: false tty: true diff --git a/logstash/maps/malcolm_severity.yaml b/logstash/maps/malcolm_severity.yaml index 3f21a1271..8c5c78e21 100644 --- a/logstash/maps/malcolm_severity.yaml +++ b/logstash/maps/malcolm_severity.yaml @@ -18,6 +18,7 @@ "File transfer": 0 "File transfer (high concern)": 75 "File transfer (medium concern)": 50 +"MITRE ATT&CK for ICS framework technique": 80 "MITRE ATT&CK framework technique": 80 "Notice (other)": 60 "Notice (protocol)": 60 diff --git a/logstash/maps/notice_authors.yaml b/logstash/maps/notice_authors.yaml index 35f0d0f54..2985e4247 100644 --- a/logstash/maps/notice_authors.yaml +++ b/logstash/maps/notice_authors.yaml @@ -1,4 +1,5 @@ "ATTACK": "MITRE" +"ATTACKICS": "MITRE" "Corelight": "Corelight" "CVE_2020_0601": "Johanna Amann" "CVE_2020_13777": "Johanna Amann" diff --git a/logstash/pipelines/beats/01_input_beats.conf b/logstash/pipelines/beats/01_input_beats.conf index d3a26348d..68ce49e11 100644 --- a/logstash/pipelines/beats/01_input_beats.conf +++ b/logstash/pipelines/beats/01_input_beats.conf @@ -6,14 +6,15 @@ input { filter { # this pipeline only needs to see logs from other misc. beats used by Malcolm - if ("_malcolm_beats" in [tags]) { - mutate { id => "mutate_filebeat_malcolm_beats_forward_tag_remove" - remove_tag => [ "_malcolm_beats" ] } + if ("_malcolm_beats" in [tags]) or + ("_zeekdiagnostic" in [tags]) or + ("_suricatastats" in [tags]) { + + # rename message to event.original + mutate { id => "mutate_rename_beats_message" + rename => { "[message]" => "[event][original]" } } + } else { drop { id => "drop_not_malcolm_beats" } } - - # rename message to event.original - mutate { id => "mutate_rename_beats_message" - rename => { "[message]" => "[event][original]" } } } diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index b1ed20bc4..7f883c163 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -1,6 +1,7 @@ -######################## # parsing of logs/metrics specific to the operation of -# Malcolm and Hedgehog Linux itself (i.e., not captured -# network traffic metadata, but operational metadata) +######################## +# parsing of logs/metrics specific to the operation of +# Malcolm and Hedgehog Linux itself (i.e., not captured +# network traffic metadata, but operational metadata) # # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ####################### @@ -60,7 +61,6 @@ filter { id => "grok_beat_nginx_access" patterns_dir => "/usr/share/logstash/malcolm-patterns" match => { "[event][original]" => "%{NGINX_ACCESS}" } - # remove_tag => ["_grokparsefailure"] } } if ([event][dataset] == "nginx.error") { @@ -68,7 +68,6 @@ filter { id => "grok_beat_nginx_error" patterns_dir => "/usr/share/logstash/malcolm-patterns" match => { "[event][original]" => "%{NGINX_ERROR}" } - # remove_tag => ["_grokparsefailure"] } } @@ -781,6 +780,64 @@ filter { } # [miscbeat][winstat] + } else if ("_zeekdiagnostic" in [tags]) { + #------------------------------------------------- + # Zeek diagnostic logs + # https://docs.zeek.org/en/master/script-reference/log-files.html#zeek-diagnostics + + if ([zeek][stats]) { + # remove zero values from zeek stats + ruby { + id => "ruby_zeek_remove_zero_stats" + path => "/usr/share/logstash/malcolm-ruby/compact_event_hash.rb" + script_params => { + "field" => "[zeek][stats]" + "discard_zeroes" => "true" + } + } + } + + mutate { id => "mutate_add_field_event_module_zeek_diagnostic" + add_field => { "[event][module]" => "zeek" } } + + mutate { id => "mutate_remove_fields_zeek_diagnostic" + remove_field => [ "[event][original]", + "[firstPacket]", + "[lastPacket]", + "[timestamp]", + "[zeek][ts]", + "[length]", + "[event][duration]" ] } + + } else if ("_suricatastats" in [tags]) { + #------------------------------------------------- + # Suricata statistics + # https://docs.suricata.io/en/suricata-6.0.2/configuration/suricata-yaml.html#stats + + if ([suricata][stats]) { + # remove zero values from suricata stats + ruby { + id => "ruby_suricata_remove_zero_stats" + path => "/usr/share/logstash/malcolm-ruby/compact_event_hash.rb" + script_params => { + "field" => "[suricata][stats]" + "discard_zeroes" => "true" + } + } + } + + mutate { id => "mutate_add_field_event_module_suricata_stats" + add_field => { "[event][module]" => "suricata" } } + + mutate { id => "mutate_remove_fields_suricata_stats" + remove_field => [ "[event][original]", + "[firstPacket]", + "[lastPacket]", + "[timestamp]", + "[suricata][timestamp]", + "[length]", + "[event][duration]" ] } + } # event type (filebeat.nginx, miscbeat, etc) } \ No newline at end of file diff --git a/logstash/pipelines/beats/98_finalize.conf b/logstash/pipelines/beats/98_finalize.conf index b3b3dff30..4db017bea 100644 --- a/logstash/pipelines/beats/98_finalize.conf +++ b/logstash/pipelines/beats/98_finalize.conf @@ -72,19 +72,6 @@ filter { ] } - # remove tags we'd rather not see - mutate { id => "mutate_beats_tags_remove" - remove_tag => [ "beats_input_codec_plain_applied", - "beats_input_raw_event", - "_malcolm_miscbeat", - "_dateparsefailure", - "_grokparsefailure", - "_jsonparsefailure", - "_dissectfailure", - "_ouilookupfailure", - "_geoip_lookup_failure" ] } - - # event.provider if (![event][provider]) { mutate { id => "mutate_add_field_event_provider_beats" add_field => { "[event][provider]" => "malcolm" } } } diff --git a/logstash/pipelines/enrichment/21_netbox.conf b/logstash/pipelines/enrichment/21_netbox.conf index a4370ab8b..88937f796 100644 --- a/logstash/pipelines/enrichment/21_netbox.conf +++ b/logstash/pipelines/enrichment/21_netbox.conf @@ -5,20 +5,30 @@ filter { # Do enrichment based on NetBox lookups: # - source.ip -> source.device and source.segment # - destination.ip -> destination.device and destination.segment - # - TODO: source.mac -> source.device - # - TODO: destination.mac -> destination.device - # The LOGSTASH_NETBOX_ENRICHMENT environment variable is checked inside netbox_enrich.rb + # - source.mac -> source.device + # - destination.mac -> destination.device + # Which log types get enriched is based on the LOGSTASH_NETBOX_ENRICHMENT_DATASETS env. variable + # The LOGSTASH_NETBOX_ENRICHMENT env. variable is checked inside netbox_enrich.rb # and will short-circuit unles this feature is enabled. - # - # Enrich zeek conn.log, notice.log, weird.log, signatures.log, software.log, known*.log and all non-zeek data sources - if (([event][provider] != "zeek") or - ([event][dataset] == "conn") or - ([event][dataset] == "notice") or - ([event][dataset] == "weird") or - ([event][dataset] == "signatures") or - ([event][dataset] == "software") or - ([event][dataset] =~ /^known/)) { + ruby { + id => "ruby_determine_netbox_suitability" + # @logtypes = {"suricata"=>["alert"], "zeek"=>["conn", "known_hosts", "known_services", "notice", "signatures", "software", "weird"]} + init => "logtypesStr = ENV['LOGSTASH_NETBOX_ENRICHMENT_DATASETS'] || 'suricata.alert,zeek.conn,zeek.known_hosts,zeek.known_services,zeek.notice,zeek.signatures,zeek.software,zeek.weird' ; logtypesArr = logtypesStr.gsub(/\s+/, '').split(','); @logtypes = logtypesArr.group_by { |logtype| logtype.split('.').first }.transform_values { |values| values.map { |v| v.split('.')[1] } }" + code => " + provider = event.get('[event][provider]').to_s + dataset = event.get('[event][dataset]').to_s + if (@logtypes.is_a?(Hash) && + !@logtypes.empty? && + (@logtypes.has_key?('all') || + (!provider.empty? && !dataset.empty? && @logtypes.has_key?(provider) && @logtypes[provider].is_a?(Array) && @logtypes[provider].include?(dataset)))) + then + event.set('[@metadata][do_netbox_enrichment]', true) + end + " + } + + if ([@metadata][do_netbox_enrichment]) { if ([source][ip]) and (([network][direction] == "internal") or ([network][direction] == "outbound")) { ruby { @@ -53,6 +63,7 @@ filter { "default_dtype_env" => "NETBOX_DEFAULT_DEVICE_TYPE" "default_role_env" => "NETBOX_DEFAULT_ROLE" "autopopulate_fuzzy_threshold_env" => "NETBOX_DEFAULT_FUZZY_THRESHOLD" + "autopopulate_create_manuf_env" => "NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER" "source_oui" => "[source][oui]" "source_mac" => "[source][mac]" } @@ -93,6 +104,7 @@ filter { "default_dtype_env" => "NETBOX_DEFAULT_DEVICE_TYPE" "default_role_env" => "NETBOX_DEFAULT_ROLE" "autopopulate_fuzzy_threshold_env" => "NETBOX_DEFAULT_FUZZY_THRESHOLD" + "autopopulate_create_manuf_env" => "NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER" "source_oui" => "[destination][oui]" "source_mac" => "[destination][mac]" } @@ -129,6 +141,10 @@ filter { merge => { "[related][device_name]" => "[source][device][name]" } } } if ([destination][device][name]) { mutate { id => "mutate_merge_destination_device_name_related" merge => { "[related][device_name]" => "[destination][device][name]" } } } + if ([source][device][id]) { mutate { id => "mutate_merge_source_device_id_related" + merge => { "[related][device_id]" => "[source][device][id]" } } } + if ([destination][device][id]) { mutate { id => "mutate_merge_destination_device_id_related" + merge => { "[related][device_id]" => "[destination][device][id]" } } } # network.name (based on info from [destination][segment][name] and [source][segment][name]) if ([destination][segment][name]) { mutate { id => "mutate_add_field_ecs_network_name_resp" diff --git a/logstash/pipelines/enrichment/23_severity.conf b/logstash/pipelines/enrichment/23_severity.conf index 6f8506c32..27d65747a 100644 --- a/logstash/pipelines/enrichment/23_severity.conf +++ b/logstash/pipelines/enrichment/23_severity.conf @@ -38,7 +38,7 @@ filter { if ([source][geo][country_iso_code]) or ([destination][geo][country_iso_code]) or ([dns][GEO]) { ruby { id => "ruby_add_field_severity_geo" - init => "countriesStr = ENV['SENSITIVE_COUNTRY_CODES'] || 'AM,AZ,BY,CN,CU,DZ,GE,HK,IL,IN,IQ,IR,KG,KP,KZ,LY,MD,MO,PK,RU,SD,SS,SY,TJ,TM,TW,UA,UZ' ; $countries = countriesStr.gsub(/\s+/, '').upcase.split(',')" + init => "countriesStr = ENV['SENSITIVE_COUNTRY_CODES'] || 'AM,AZ,BY,CN,CU,DZ,GE,HK,IL,IN,IQ,IR,KG,KP,KZ,LY,MD,MO,PK,RU,SD,SS,SY,TJ,TM,TW,UA,UZ' ; @countries = countriesStr.gsub(/\s+/, '').upcase.split(',')" code => " srcGEOs = event.get('[source][geo][country_iso_code]') dstGEOs = event.get('[destination][geo][country_iso_code]') @@ -46,7 +46,7 @@ filter { allGEOs = [srcGEOs.nil? ? [] : (srcGEOs.kind_of?(Array) ? srcGEOs : [srcGEOs]), dstGEOs.nil? ? [] : (dstGEOs.kind_of?(Array) ? dstGEOs : [dstGEOs]), dnsGEOs.nil? ? [] : (dnsGEOs.kind_of?(Array) ? dnsGEOs : [dnsGEOs])].flatten - if (!((allGEOs & $countries).empty?)) then + if (!((allGEOs & @countries).empty?)) then sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]')) if !sevtags.kind_of?(Array) then newtags = Array.new @@ -193,13 +193,13 @@ filter { if ([event][freq_score_v1]) or ([event][freq_score_v2]) { ruby { id => "ruby_add_field_severity_domain_entropy" - init => "$freqSeverityThreshold = ENV['FREQ_SEVERITY_THRESHOLD'] || '3.0'" + init => "@freqSeverityThreshold = ENV['FREQ_SEVERITY_THRESHOLD'] || '3.0'" code => " freqs1 = event.get('[event][freq_score_v1]') freqs2 = event.get('[event][freq_score_v2]') lowestFreqScore = [freqs1.nil? ? 100 : (freqs1.kind_of?(Array) ? freqs1.min : freqs1), freqs2.nil? ? 100 : (freqs2.kind_of?(Array) ? freqs2.min : freqs2)].min - if (lowestFreqScore < Float($freqSeverityThreshold)) then + if (lowestFreqScore < Float(@freqSeverityThreshold)) then sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]')) if !sevtags.kind_of?(Array) then newtags = Array.new @@ -216,13 +216,13 @@ filter { if ([totDataBytes]) or ([network][bytes]) { ruby { id => "ruby_add_field_severity_total_bytes" - init => "mbSeverityThreshold = ENV['TOTAL_MEGABYTES_SEVERITY_THRESHOLD'] || '1000' ; $bytesSeverityThreshold = Integer(mbSeverityThreshold) * 1000000" + init => "mbSeverityThreshold = ENV['TOTAL_MEGABYTES_SEVERITY_THRESHOLD'] || '1000' ; @bytesSeverityThreshold = Integer(mbSeverityThreshold) * 1000000" code => " totDataBytes = event.get('[totDataBytes]') totBytes = event.get('[network][bytes]') highBytes = [totDataBytes.nil? ? 0 : Integer(totDataBytes), totBytes.nil? ? 0 : Integer(totBytes)].max - if ($bytesSeverityThreshold > 0) and (highBytes >= $bytesSeverityThreshold) then + if (@bytesSeverityThreshold > 0) and (highBytes >= @bytesSeverityThreshold) then sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]')) if !sevtags.kind_of?(Array) then newtags = Array.new @@ -239,9 +239,9 @@ filter { if ([length]) { ruby { id => "ruby_add_field_severity_duration" - init => "secSeverityThreshold = ENV['CONNECTION_SECONDS_SEVERITY_THRESHOLD'] || '3600' ; $msSeverityThreshold = Integer(secSeverityThreshold) * 1000" + init => "secSeverityThreshold = ENV['CONNECTION_SECONDS_SEVERITY_THRESHOLD'] || '3600' ; @msSeverityThreshold = Integer(secSeverityThreshold) * 1000" code => " - if ($msSeverityThreshold > 0) and (event.get('[length]') >= $msSeverityThreshold) then + if (@msSeverityThreshold > 0) and (event.get('[length]') >= @msSeverityThreshold) then sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]')) if !sevtags.kind_of?(Array) then newtags = Array.new diff --git a/logstash/pipelines/enrichment/96_make_unique.conf b/logstash/pipelines/enrichment/96_make_unique.conf index 913f54514..1e5367017 100644 --- a/logstash/pipelines/enrichment/96_make_unique.conf +++ b/logstash/pipelines/enrichment/96_make_unique.conf @@ -98,6 +98,15 @@ filter { } } } + if ([related][device_id]) { + ruby { + id => "ruby_related_device_id_uniq" + path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb" + script_params => { + "field" => "[related][device_id]" + } + } + } if ([related][device_name]) { ruby { id => "ruby_related_device_name_uniq" diff --git a/logstash/pipelines/enrichment/98_finalize.conf b/logstash/pipelines/enrichment/98_finalize.conf index 0d979c734..2a8be2578 100644 --- a/logstash/pipelines/enrichment/98_finalize.conf +++ b/logstash/pipelines/enrichment/98_finalize.conf @@ -59,28 +59,4 @@ filter { "[message]" ] } - - # remove tags we'd rather not see - - mutate { id => "mutate_enrichment_tags_remove" - remove_tag => [ "beats_input_codec_plain_applied", - "beats_input_raw_event", - "_dateparsefailure", - "_grokparsefailure", - "_jsonparsefailure", - "_dissectfailure", - "_ouilookupfailure", - "_geoip_lookup_failure", - "_filebeat_suricata", - "_filebeat_suricata_hedgehog_live", - "_filebeat_suricata_live", - "_filebeat_suricata_malcolm_live", - "_filebeat_suricata_malcolm_upload", - "_filebeat_suricata_upload", - "_filebeat_zeek", - "_filebeat_zeek_hedgehog_live", - "_filebeat_zeek_live", - "_filebeat_zeek_malcolm_live", - "_filebeat_zeek_malcolm_upload", - "_filebeat_zeek_upload" ] } } \ No newline at end of file diff --git a/logstash/pipelines/output/98_finalize.conf b/logstash/pipelines/output/98_finalize.conf new file mode 100644 index 000000000..6cf43e49e --- /dev/null +++ b/logstash/pipelines/output/98_finalize.conf @@ -0,0 +1,31 @@ +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +filter { + # remove tags we'd rather not see globally + mutate { id => "mutate_final_tags_remove" + remove_tag => [ "_dateparsefailure", + "_dissectfailure", + "_filebeat_suricata", + "_filebeat_suricata_hedgehog_live", + "_filebeat_suricata_live", + "_filebeat_suricata_malcolm_live", + "_filebeat_suricata_malcolm_upload", + "_filebeat_suricata_upload", + "_filebeat_zeek", + "_filebeat_zeek_hedgehog_live", + "_filebeat_zeek_live", + "_filebeat_zeek_malcolm_live", + "_filebeat_zeek_malcolm_upload", + "_filebeat_zeek_upload", + "_geoip_lookup_failure", + "_grokparsefailure", + "_jsonparsefailure", + "_malcolm_miscbeat", + "_malcolm_beats", + "_ouilookupfailure", + "_suricatastats", + "_zeekdiagnostic", + "beats_input_codec_plain_applied", + "beats_input_raw_event" ] } + +} \ No newline at end of file diff --git a/logstash/pipelines/suricata/11_suricata_logs.conf b/logstash/pipelines/suricata/11_suricata_logs.conf index 6fbd8b175..e38a996c8 100644 --- a/logstash/pipelines/suricata/11_suricata_logs.conf +++ b/logstash/pipelines/suricata/11_suricata_logs.conf @@ -9,10 +9,6 @@ filter { source => "message" target => "suricata" } - # drop fields we don't need - mutate { id => "mutate_remove_field_suricata_message_and_stats" - remove_field => [ "[suricata][stats]" ] } - # tags may have been specified, like: eve-123456789-1-(tagA,tagB,tagC).json, extract the tags ruby { id => "ruby_suricata_tags_extract" @@ -54,6 +50,10 @@ filter { if ([suricata][event_type] == "alert") { mutate { id => "mutate_add_field_suricata_ecs_event_kind_alert" add_field => { "[event][kind]" => "alert" } } + } else if ([suricata][event_type] == "stats") { + mutate { id => "mutate_add_field_suricata_ecs_event_kind_stats" + add_field => { "[event][kind]" => "metric" } + add_tag => [ "_suricatastats" ] } } else { mutate { id => "mutate_add_field_suricata_ecs_event_kind_event" add_field => { "[event][kind]" => "event" } } @@ -82,7 +82,7 @@ filter { ruby { id => "ruby_suricata_timestamp_calc" - init => "require 'time'; require 'date';" + init => "require 'time'; require 'date'" code => " timpStamp = DateTime.parse(event.get('[suricata][timestamp]')).to_time timeStampMs = (1000*timpStamp.to_f).round(0) @@ -435,7 +435,6 @@ filter { remove_field => [ "[suricata][files][stored]", "[suricata][fileinfo][stored]", - "[suricata][drop]", "[suricata][packet]", "[suricata][packet_info]", "[suricata][payload_printable]", diff --git a/logstash/pipelines/suricata/99_suricata_forward.conf b/logstash/pipelines/suricata/99_suricata_forward.conf index bdbab18bb..1f5794e66 100644 --- a/logstash/pipelines/suricata/99_suricata_forward.conf +++ b/logstash/pipelines/suricata/99_suricata_forward.conf @@ -1,5 +1,13 @@ output { - pipeline { - send_to => ["log-enrichment"] + # the Suricata stats logs are not network traffic metadata to be enriched, + # and belong more with the supporting runtime-oriented logs + if "_suricatastats" in [tags] { + pipeline { + send_to => ["beats-parse"] + } + } else { + pipeline { + send_to => ["log-enrichment"] + } } } diff --git a/logstash/pipelines/zeek/10_zeek_prep.conf b/logstash/pipelines/zeek/10_zeek_prep.conf index 6e0785a35..5201f75a7 100644 --- a/logstash/pipelines/zeek/10_zeek_prep.conf +++ b/logstash/pipelines/zeek/10_zeek_prep.conf @@ -23,24 +23,14 @@ filter { end" } - # report types we're going to ignore - if (([log_source] == "analyzer") or - ([log_source] == "bsap_ip_unknown") or - ([log_source] == "bsap_serial_unknown") or - ([log_source] == "ecat_arp_info") or - ([log_source] == "reporter") or - ([log_source] == "broker") or - ([log_source] == "cluster") or - ([log_source] == "capture_loss") or - ([log_source] == "communication") or - ([log_source] == "packet_filter") or - ([log_source] == "png") or - ([log_source] == "stats") or - ([log_source] == "stderr") or - ([log_source] == "stdout") or - ([log_source] == "loaded_scripts")) { - drop { id => "drop_zeek_ignored_source" } + # Zeek logs we're going to ignore + ruby { + id => "ruby_zeek_log_type_determine_drop" + init => "logtypesStr = ENV['LOGSTASH_ZEEK_IGNORED_LOGS'] || 'analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout' ; @logtypes = logtypesStr.gsub(/\s+/, '').split(',')" + code => "event.set('[@metadata][drop_zeek_log]', true) if @logtypes.include?(event.get('[log_source]').to_s)" } + if [@metadata][drop_zeek_log] { drop { id => "drop_zeek_ignored_source" } } + # remove some tags pulled from the filename we might not want if ([@metadata][zeek_log_tags]) { diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 249450c3d..6cec9e898 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -154,8 +154,8 @@ filter { } ruby { id => "ruby_zip_zeek_conn" - init => "$zeek_conn_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'local_orig', 'local_resp', 'missed_bytes', 'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'tunnel_parents', 'vlan', 'inner_vlan', 'orig_l2_addr', 'resp_l2_addr', 'community_id' ]" - code => "event.set('[zeek_cols]', $zeek_conn_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_conn_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'local_orig', 'local_resp', 'missed_bytes', 'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'tunnel_parents', 'vlan', 'inner_vlan', 'orig_l2_addr', 'resp_l2_addr', 'community_id' ]" + code => "event.set('[zeek_cols]', @zeek_conn_field_names.zip(event.get('[message]')).to_h)" } } @@ -225,8 +225,8 @@ filter { } ruby { id => "ruby_zip_zeek_bacnet" - init => "$zeek_bacnet_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'bvlc_function', 'pdu_type', 'pdu_service', 'invoke_id', 'result_code' ]" - code => "event.set('[zeek_cols]', $zeek_bacnet_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bacnet_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'bvlc_function', 'pdu_type', 'pdu_service', 'invoke_id', 'result_code' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_field_names.zip(event.get('[message]')).to_h)" } } @@ -258,8 +258,8 @@ filter { } ruby { id => "ruby_zip_zeek_bestguess" - init => "$zeek_bestguess_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'name', 'category' ]" - code => "event.set('[zeek_cols]', $zeek_bestguess_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bestguess_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'name', 'category' ]" + code => "event.set('[zeek_cols]', @zeek_bestguess_field_names.zip(event.get('[message]')).to_h)" } } @@ -286,8 +286,8 @@ filter { } ruby { id => "ruby_zip_zeek_bsap_ip_header" - init => "$zeek_bsap_ip_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'num_msg', 'type_name' ]" - code => "event.set('[zeek_cols]', $zeek_bsap_ip_header_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bsap_ip_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'num_msg', 'type_name' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_ip_header_field_names.zip(event.get('[message]')).to_h)" } } @@ -320,8 +320,8 @@ filter { } ruby { id => "ruby_zip_zeek_bsap_ip_rdb" - init => "$zeek_bsap_ip_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_size', 'mes_seq', 'res_seq', 'data_len', 'sequence', 'app_func_code', 'node_status', 'func_code', 'variable_count', 'variables', 'variable_value' ]" - code => "event.set('[zeek_cols]', $zeek_bsap_ip_rdb_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bsap_ip_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_size', 'mes_seq', 'res_seq', 'data_len', 'sequence', 'app_func_code', 'node_status', 'func_code', 'variable_count', 'variables', 'variable_value' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_ip_rdb_field_names.zip(event.get('[message]')).to_h)" } } @@ -354,8 +354,8 @@ filter { } ruby { id => "ruby_zip_zeek_bsap_serial_header" - init => "$zeek_bsap_serial_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ser', 'dadd', 'sadd', 'ctl', 'dfun', 'seq', 'sfun', 'nsb', 'type_name' ]" - code => "event.set('[zeek_cols]', $zeek_bsap_serial_header_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bsap_serial_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ser', 'dadd', 'sadd', 'ctl', 'dfun', 'seq', 'sfun', 'nsb', 'type_name' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_serial_header_field_names.zip(event.get('[message]')).to_h)" } } @@ -388,8 +388,8 @@ filter { } ruby { id => "ruby_zip_zeek_bsap_serial_rdb" - init => "$zeek_bsap_serial_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'func_code', 'variable_count', 'variables', 'variable_value' ]" - code => "event.set('[zeek_cols]', $zeek_bsap_serial_rdb_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bsap_serial_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'func_code', 'variable_count', 'variables', 'variable_value' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_field_names.zip(event.get('[message]')).to_h)" } } @@ -422,8 +422,8 @@ filter { } ruby { id => "ruby_zip_zeek_bsap_serial_rdb_ext" - init => "$zeek_bsap_serial_rdb_ext_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'dfun', 'seq', 'sfun', 'nsb', 'extfun', 'data' ]" - code => "event.set('[zeek_cols]', $zeek_bsap_serial_rdb_ext_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bsap_serial_rdb_ext_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'dfun', 'seq', 'sfun', 'nsb', 'extfun', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_ext_field_names.zip(event.get('[message]')).to_h)" } } @@ -462,8 +462,8 @@ filter { } ruby { id => "ruby_zip_zeek_bacnet_device_control" - init => "$zeek_bacnet_device_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'time_duration', 'device_state', 'password', 'result', 'result_code' ]" - code => "event.set('[zeek_cols]', $zeek_bacnet_device_control_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bacnet_device_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'time_duration', 'device_state', 'password', 'result', 'result_code' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_device_control_field_names.zip(event.get('[message]')).to_h)" } } @@ -496,8 +496,8 @@ filter { } ruby { id => "ruby_zip_zeek_bacnet_discovery" - init => "$zeek_bacnet_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_service', 'object_type', 'instance_number', 'vendor', 'range', 'object_name' ]" - code => "event.set('[zeek_cols]', $zeek_bacnet_discovery_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bacnet_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_service', 'object_type', 'instance_number', 'vendor', 'range', 'object_name' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_discovery_field_names.zip(event.get('[message]')).to_h)" } } @@ -530,8 +530,8 @@ filter { } ruby { id => "ruby_zip_zeek_bacnet_property" - init => "$zeek_bacnet_property_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'object_type', 'instance_number', 'property', 'array_index', 'value' ]" - code => "event.set('[zeek_cols]', $zeek_bacnet_property_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_bacnet_property_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'object_type', 'instance_number', 'property', 'array_index', 'value' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_property_field_names.zip(event.get('[message]')).to_h)" } } @@ -566,8 +566,8 @@ filter { } ruby { id => "ruby_zip_zeek_cip" - init => "$zeek_cip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cip_sequence_count', 'direction', 'cip_service_code', 'cip_service', 'cip_status_code', 'cip_status', 'cip_extended_status_code', 'cip_extended_status', 'class_id', 'class_name', 'instance_id', 'attribute_id' ]" - code => "event.set('[zeek_cols]', $zeek_cip_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_cip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cip_sequence_count', 'direction', 'cip_service_code', 'cip_service', 'cip_status_code', 'cip_status', 'cip_extended_status_code', 'cip_extended_status', 'class_id', 'class_name', 'instance_id', 'attribute_id' ]" + code => "event.set('[zeek_cols]', @zeek_cip_field_names.zip(event.get('[message]')).to_h)" } } @@ -599,8 +599,8 @@ filter { } ruby { id => "ruby_zip_zeek_cip_identity" - init => "$zeek_cip_identity_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'encapsulation_version', 'socket_address', 'socket_port', 'vendor_id', 'vendor_name', 'device_type_id', 'device_type_name', 'product_code', 'device_status', 'serial_number', 'product_name', 'device_state' ]" - code => "event.set('[zeek_cols]', $zeek_cip_identity_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_cip_identity_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'encapsulation_version', 'socket_address', 'socket_port', 'vendor_id', 'vendor_name', 'device_type_id', 'device_type_name', 'product_code', 'device_status', 'serial_number', 'product_name', 'device_state' ]" + code => "event.set('[zeek_cols]', @zeek_cip_identity_field_names.zip(event.get('[message]')).to_h)" } } @@ -631,8 +631,8 @@ filter { } ruby { id => "ruby_zip_zeek_cip_io" - init => "$zeek_cip_io_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'connection_id', 'sequence_number', 'data_length', 'io_data' ]" - code => "event.set('[zeek_cols]', $zeek_cip_io_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_cip_io_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'connection_id', 'sequence_number', 'data_length', 'io_data' ]" + code => "event.set('[zeek_cols]', @zeek_cip_io_field_names.zip(event.get('[message]')).to_h)" } } @@ -662,8 +662,8 @@ filter { } ruby { id => "ruby_zip_zeek_dce_rpc" - init => "$zeek_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rtt', 'named_pipe', 'endpoint', 'operation' ]" - code => "event.set('[zeek_cols]', $zeek_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rtt', 'named_pipe', 'endpoint', 'operation' ]" + code => "event.set('[zeek_cols]', @zeek_dce_rpc_field_names.zip(event.get('[message]')).to_h)" } } @@ -733,8 +733,8 @@ filter { } ruby { id => "ruby_zip_zeek_dhcp" - init => "$zeek_dhcp_field_names = [ 'ts', 'uid', 'orig_h', 'resp_h', 'orig_l2_addr', 'host_name', 'client_fqdn', 'domain', 'requested_ip', 'assigned_ip', 'lease_time', 'client_message', 'server_message', 'msg_types', 'duration', 'client_software', 'server_software' ]" - code => "event.set('[zeek_cols]', $zeek_dhcp_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_dhcp_field_names = [ 'ts', 'uid', 'orig_h', 'resp_h', 'orig_l2_addr', 'host_name', 'client_fqdn', 'domain', 'requested_ip', 'assigned_ip', 'lease_time', 'client_message', 'server_message', 'msg_types', 'duration', 'client_software', 'server_software' ]" + code => "event.set('[zeek_cols]', @zeek_dhcp_field_names.zip(event.get('[message]')).to_h)" } } @@ -785,8 +785,8 @@ filter { } ruby { id => "ruby_zip_zeek_dnp3" - init => "$zeek_dnp3_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fc_request', 'fc_reply', 'iin' ]" - code => "event.set('[zeek_cols]', $zeek_dnp3_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_dnp3_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fc_request', 'fc_reply', 'iin' ]" + code => "event.set('[zeek_cols]', @zeek_dnp3_field_names.zip(event.get('[message]')).to_h)" } } @@ -816,8 +816,8 @@ filter { } ruby { id => "ruby_zip_zeek_dnp3_control" - init => "$zeek_dnp3_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'block_type', 'function_code', 'index_number', 'trip_control_code', 'operation_type', 'execute_count', 'on_time', 'off_time', 'status_code' ]" - code => "event.set('[zeek_cols]', $zeek_dnp3_control_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_dnp3_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'block_type', 'function_code', 'index_number', 'trip_control_code', 'operation_type', 'execute_count', 'on_time', 'off_time', 'status_code' ]" + code => "event.set('[zeek_cols]', @zeek_dnp3_control_field_names.zip(event.get('[message]')).to_h)" } } @@ -847,8 +847,8 @@ filter { } ruby { id => "ruby_zip_zeek_dnp3_objects" - init => "$zeek_dnp3_objects_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'function_code', 'object_type', 'object_count', 'range_low', 'range_high' ]" - code => "event.set('[zeek_cols]', $zeek_dnp3_objects_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_dnp3_objects_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'function_code', 'object_type', 'object_count', 'range_low', 'range_high' ]" + code => "event.set('[zeek_cols]', @zeek_dnp3_objects_field_names.zip(event.get('[message]')).to_h)" } } @@ -878,8 +878,8 @@ filter { } ruby { id => "ruby_zip_zeek_dns" - init => "$zeek_dns_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'trans_id', 'rtt', 'query', 'qclass', 'qclass_name', 'qtype', 'qtype_name', 'rcode', 'rcode_name', 'AA', 'TC', 'RD', 'RA', 'Z', 'answers', 'TTLs', 'rejected' ]" - code => "event.set('[zeek_cols]', $zeek_dns_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_dns_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'trans_id', 'rtt', 'query', 'qclass', 'qclass_name', 'qtype', 'qtype_name', 'rcode', 'rcode_name', 'AA', 'TC', 'RD', 'RA', 'Z', 'answers', 'TTLs', 'rejected' ]" + code => "event.set('[zeek_cols]', @zeek_dns_field_names.zip(event.get('[message]')).to_h)" } } @@ -912,8 +912,8 @@ filter { } ruby { id => "ruby_zip_zeek_dpd" - init => "$zeek_dpd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'failure_reason' ]" - code => "event.set('[zeek_cols]', $zeek_dpd_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_dpd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'failure_reason' ]" + code => "event.set('[zeek_cols]', @zeek_dpd_field_names.zip(event.get('[message]')).to_h)" } } @@ -953,8 +953,8 @@ filter { } ruby { id => "ruby_zip_zeek_enip" - init => "$zeek_enip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'enip_command', 'length', 'session_handle', 'enip_status', 'sender_context', 'options' ]" - code => "event.set('[zeek_cols]', $zeek_enip_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_enip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'enip_command', 'length', 'session_handle', 'enip_status', 'sender_context', 'options' ]" + code => "event.set('[zeek_cols]', @zeek_enip_field_names.zip(event.get('[message]')).to_h)" } } @@ -984,8 +984,8 @@ filter { } ruby { id => "ruby_zip_zeek_ecat_registers" - init => "$zeek_ecat_registers_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'command', 'server_addr', 'register_type', 'register_addr', 'data' ]" - code => "event.set('[zeek_cols]', $zeek_ecat_registers_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ecat_registers_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'command', 'server_addr', 'register_type', 'register_addr', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_registers_field_names.zip(event.get('[message]')).to_h)" } } @@ -1017,8 +1017,8 @@ filter { } ruby { id => "ruby_zip_zeek_ecat_log_address" - init => "$zeek_ecat_log_address_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'log_addr', 'length', 'command', 'data' ]" - code => "event.set('[zeek_cols]', $zeek_ecat_log_address_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ecat_log_address_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'log_addr', 'length', 'command', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_log_address_field_names.zip(event.get('[message]')).to_h)" } } @@ -1050,8 +1050,8 @@ filter { } ruby { id => "ruby_zip_zeek_ecat_dev_info" - init => "$zeek_ecat_dev_info_field_names = [ 'ts', 'server_id', 'revision', 'dev_type', 'build', 'fmmucnt', 'smcount', 'ports', 'dpram', 'features' ]" - code => "event.set('[zeek_cols]', $zeek_ecat_dev_info_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ecat_dev_info_field_names = [ 'ts', 'server_id', 'revision', 'dev_type', 'build', 'fmmucnt', 'smcount', 'ports', 'dpram', 'features' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_dev_info_field_names.zip(event.get('[message]')).to_h)" } } @@ -1083,8 +1083,8 @@ filter { } ruby { id => "ruby_zip_zeek_ecat_aoe_info" - init => "$zeek_ecat_aoe_info_field_names = [ 'ts', 'resp_l2_addr', 'resp_port', 'orig_l2_addr', 'orig_port', 'command', 'state', 'data' ]" - code => "event.set('[zeek_cols]', $zeek_ecat_aoe_info_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ecat_aoe_info_field_names = [ 'ts', 'resp_l2_addr', 'resp_port', 'orig_l2_addr', 'orig_port', 'command', 'state', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_aoe_info_field_names.zip(event.get('[message]')).to_h)" } } @@ -1116,8 +1116,8 @@ filter { } ruby { id => "ruby_zip_zeek_ecat_coe_info" - init => "$zeek_ecat_coe_info_field_names = [ 'ts', 'number', 'type', 'req_resp', 'index', 'subindex', 'dataoffset' ]" - code => "event.set('[zeek_cols]', $zeek_ecat_coe_info_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ecat_coe_info_field_names = [ 'ts', 'number', 'type', 'req_resp', 'index', 'subindex', 'dataoffset' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_coe_info_field_names.zip(event.get('[message]')).to_h)" } } @@ -1149,8 +1149,8 @@ filter { } ruby { id => "ruby_zip_zeek_ecat_foe_info" - init => "$zeek_ecat_foe_info_field_names = [ 'ts', 'opcode', 'reserved', 'packet_num', 'error_code', 'filename', 'data' ]" - code => "event.set('[zeek_cols]', $zeek_ecat_foe_info_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ecat_foe_info_field_names = [ 'ts', 'opcode', 'reserved', 'packet_num', 'error_code', 'filename', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_foe_info_field_names.zip(event.get('[message]')).to_h)" } } @@ -1182,8 +1182,8 @@ filter { } ruby { id => "ruby_zip_zeek_ecat_soe_info" - init => "$zeek_ecat_soe_info_field_names = [ 'ts', 'opcode', 'incomplete', 'error', 'drive_num', 'element', 'index' ]" - code => "event.set('[zeek_cols]', $zeek_ecat_soe_info_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ecat_soe_info_field_names = [ 'ts', 'opcode', 'incomplete', 'error', 'drive_num', 'element', 'index' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_soe_info_field_names.zip(event.get('[message]')).to_h)" } } @@ -1219,8 +1219,8 @@ filter { } ruby { id => "ruby_zip_zeek_ecat_arp_info" - init => "$zeek_ecat_arp_info_field_names = [ 'ts', 'arp_type', 'orig_l2_addr', 'resp_l2_addr', 'orig_proto_addr', 'orig_hw_addr', 'resp_proto_addr', 'resp_hw_addr' ]" - code => "event.set('[zeek_cols]', $zeek_ecat_arp_info_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ecat_arp_info_field_names = [ 'ts', 'arp_type', 'orig_l2_addr', 'resp_l2_addr', 'orig_proto_addr', 'orig_hw_addr', 'resp_proto_addr', 'resp_hw_addr' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_arp_info_field_names.zip(event.get('[message]')).to_h)" } } @@ -1255,7 +1255,7 @@ filter { id => "dissect_zeek_v51_files_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][uid]} %{[zeek_cols][id.orig_h]} %{[zeek_cols][id.orig_p]} %{[zeek_cols][id.resp_h]} %{[zeek_cols][id.resp_p]} %{[zeek_cols][source]} %{[zeek_cols][depth]} %{[zeek_cols][analyzers]} %{[zeek_cols][mime_type]} %{[zeek_cols][filename]} %{[zeek_cols][duration]} %{[zeek_cols][local_orig]} %{[zeek_cols][is_orig]} %{[zeek_cols][seen_bytes]} %{[zeek_cols][total_bytes]} %{[zeek_cols][missing_bytes]} %{[zeek_cols][overflow_bytes]} %{[zeek_cols][timedout]} %{[zeek_cols][parent_fuid]} %{[zeek_cols][md5]} %{[zeek_cols][sha1]} %{[zeek_cols][sha256]} %{[zeek_cols][extracted]} %{[zeek_cols][extracted_cutoff]} %{[zeek_cols][extracted_size]} %{[zeek_cols][ftime]}" + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][source]} %{[zeek_cols][depth]} %{[zeek_cols][analyzers]} %{[zeek_cols][mime_type]} %{[zeek_cols][filename]} %{[zeek_cols][duration]} %{[zeek_cols][local_orig]} %{[zeek_cols][is_orig]} %{[zeek_cols][seen_bytes]} %{[zeek_cols][total_bytes]} %{[zeek_cols][missing_bytes]} %{[zeek_cols][overflow_bytes]} %{[zeek_cols][timedout]} %{[zeek_cols][parent_fuid]} %{[zeek_cols][md5]} %{[zeek_cols][sha1]} %{[zeek_cols][sha256]} %{[zeek_cols][extracted]} %{[zeek_cols][extracted_cutoff]} %{[zeek_cols][extracted_size]} %{[zeek_cols][ftime]}" } } @@ -1303,8 +1303,8 @@ filter { } ruby { id => "ruby_zip_zeek_files" - init => "$zeek_files_field_names = [ 'ts', 'fuid', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'source', 'depth', 'analyzers', 'mime_type', 'filename', 'duration', 'local_orig', 'is_orig', 'seen_bytes', 'total_bytes', 'missing_bytes', 'overflow_bytes', 'timedout', 'parent_fuid', 'md5', 'sha1', 'sha256', 'extracted', 'extracted_cutoff', 'extracted_size', 'ftime' ]" - code => "event.set('[zeek_cols]', $zeek_files_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_files_field_names = [ 'ts', 'fuid', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'source', 'depth', 'analyzers', 'mime_type', 'filename', 'duration', 'local_orig', 'is_orig', 'seen_bytes', 'total_bytes', 'missing_bytes', 'overflow_bytes', 'timedout', 'parent_fuid', 'md5', 'sha1', 'sha256', 'extracted', 'extracted_cutoff', 'extracted_size', 'ftime' ]" + code => "event.set('[zeek_cols]', @zeek_files_field_names.zip(event.get('[message]')).to_h)" } } @@ -1365,8 +1365,8 @@ filter { } ruby { id => "ruby_zip_zeek_ftp" - init => "$zeek_ftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'password', 'command', 'arg', 'mime_type', 'file_size', 'reply_code', 'reply_msg', 'data_channel_passive', 'data_channel_orig_h', 'data_channel_resp_h', 'data_channel_resp_p', 'fuid' ]" - code => "event.set('[zeek_cols]', $zeek_ftp_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'password', 'command', 'arg', 'mime_type', 'file_size', 'reply_code', 'reply_msg', 'data_channel_passive', 'data_channel_orig_h', 'data_channel_resp_h', 'data_channel_resp_p', 'fuid' ]" + code => "event.set('[zeek_cols]', @zeek_ftp_field_names.zip(event.get('[message]')).to_h)" } } @@ -1398,8 +1398,8 @@ filter { } ruby { id => "ruby_zip_zeek_genisys" - init => "$zeek_genisys_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'header', 'server', 'direction', 'crc_transmitted', 'crc_calculated', 'payload_raw' ]" - code => "event.set('[zeek_cols]', $zeek_genisys_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_genisys_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'header', 'server', 'direction', 'crc_transmitted', 'crc_calculated', 'payload_raw' ]" + code => "event.set('[zeek_cols]', @zeek_genisys_field_names.zip(event.get('[message]')).to_h)" } } @@ -1431,8 +1431,8 @@ filter { } ruby { id => "ruby_zip_zeek_gquic" - init => "$zeek_gquic_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'server_name', 'user_agent', 'tag_count', 'cyu', 'cyutags' ]" - code => "event.set('[zeek_cols]', $zeek_gquic_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_gquic_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'server_name', 'user_agent', 'tag_count', 'cyu', 'cyutags' ]" + code => "event.set('[zeek_cols]', @zeek_gquic_field_names.zip(event.get('[message]')).to_h)" } } @@ -1511,8 +1511,8 @@ filter { } ruby { id => "ruby_zip_zeek_http" - init => "$zeek_http_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'host', 'uri', 'referrer', 'version', 'user_agent', 'origin', 'request_body_len', 'response_body_len', 'status_code', 'status_msg', 'info_code', 'info_msg', 'tags', 'user', 'password', 'proxied', 'orig_fuids', 'orig_filenames', 'orig_mime_types', 'resp_fuids', 'resp_filenames', 'resp_mime_types', 'post_username', 'post_password_plain', 'post_password_md5', 'post_password_sha1', 'post_password_sha256' ]" - code => "event.set('[zeek_cols]', $zeek_http_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_http_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'host', 'uri', 'referrer', 'version', 'user_agent', 'origin', 'request_body_len', 'response_body_len', 'status_code', 'status_msg', 'info_code', 'info_msg', 'tags', 'user', 'password', 'proxied', 'orig_fuids', 'orig_filenames', 'orig_mime_types', 'resp_fuids', 'resp_filenames', 'resp_mime_types', 'post_username', 'post_password_plain', 'post_password_md5', 'post_password_sha1', 'post_password_sha256' ]" + code => "event.set('[zeek_cols]', @zeek_http_field_names.zip(event.get('[message]')).to_h)" } } @@ -1544,8 +1544,8 @@ filter { } ruby { id => "ruby_zip_zeek_intel" - init => "$zeek_intel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'seen_indicator', 'seen_indicator_type', 'seen_where', 'seen_node', 'matched', 'sources', 'fuid', 'file_mime_type', 'file_desc', 'cif_tags', 'cif_confidence', 'cif_source', 'cif_description', 'cif_firstseen', 'cif_lastseen' ]" - code => "event.set('[zeek_cols]', $zeek_intel_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_intel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'seen_indicator', 'seen_indicator_type', 'seen_where', 'seen_node', 'matched', 'sources', 'fuid', 'file_mime_type', 'file_desc', 'cif_tags', 'cif_confidence', 'cif_source', 'cif_description', 'cif_firstseen', 'cif_lastseen' ]" + code => "event.set('[zeek_cols]', @zeek_intel_field_names.zip(event.get('[message]')).to_h)" } } @@ -1569,8 +1569,8 @@ filter { } ruby { id => "ruby_zip_zeek_ipsec" - init => "$zeek_ipsec_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'is_orig', 'initiator_spi', 'responder_spi', 'maj_ver', 'min_ver', 'exchange_type', 'flag_e', 'flag_c', 'flag_a', 'flag_i', 'flag_v', 'flag_r', 'message_id', 'vendor_ids', 'notify_messages', 'transforms', 'ke_dh_groups', 'proposals', 'protocol_id', 'certificates', 'transform_attributes', 'length', 'hash', 'doi', 'situation' ]" - code => "event.set('[zeek_cols]', $zeek_ipsec_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ipsec_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'is_orig', 'initiator_spi', 'responder_spi', 'maj_ver', 'min_ver', 'exchange_type', 'flag_e', 'flag_c', 'flag_a', 'flag_i', 'flag_v', 'flag_r', 'message_id', 'vendor_ids', 'notify_messages', 'transforms', 'ke_dh_groups', 'proposals', 'protocol_id', 'certificates', 'transform_attributes', 'length', 'hash', 'doi', 'situation' ]" + code => "event.set('[zeek_cols]', @zeek_ipsec_field_names.zip(event.get('[message]')).to_h)" } } @@ -1601,8 +1601,8 @@ filter { } ruby { id => "ruby_zip_zeek_irc" - init => "$zeek_irc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'nick', 'user', 'command', 'value', 'addl', 'dcc_file_name', 'dcc_file_size', 'dcc_mime_type', 'fuid' ]" - code => "event.set('[zeek_cols]', $zeek_irc_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_irc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'nick', 'user', 'command', 'value', 'addl', 'dcc_file_name', 'dcc_file_size', 'dcc_mime_type', 'fuid' ]" + code => "event.set('[zeek_cols]', @zeek_irc_field_names.zip(event.get('[message]')).to_h)" } } @@ -1634,8 +1634,8 @@ filter { } ruby { id => "ruby_zip_zeek_cotp" - init => "$zeek_cotp_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_code', 'pdu_name' ]" - code => "event.set('[zeek_cols]', $zeek_cotp_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_cotp_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_code', 'pdu_name' ]" + code => "event.set('[zeek_cols]', @zeek_cotp_field_names.zip(event.get('[message]')).to_h)" } } @@ -1668,8 +1668,8 @@ filter { } ruby { id => "ruby_zip_zeek_kerberos" - init => "$zeek_kerberos_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'request_type', 'cname', 'sname', 'success', 'error_msg', 'from', 'till', 'cipher', 'forwardable', 'renewable', 'client_cert_subject', 'client_cert_fuid', 'server_cert_subject', 'server_cert_fuid' ]" - code => "event.set('[zeek_cols]', $zeek_kerberos_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_kerberos_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'request_type', 'cname', 'sname', 'success', 'error_msg', 'from', 'till', 'cipher', 'forwardable', 'renewable', 'client_cert_subject', 'client_cert_fuid', 'server_cert_subject', 'server_cert_fuid' ]" + code => "event.set('[zeek_cols]', @zeek_kerberos_field_names.zip(event.get('[message]')).to_h)" } } @@ -1698,8 +1698,8 @@ filter { } ruby { id => "ruby_zip_zeek_known_certs" - init => "$zeek_known_certs_field_names = [ 'ts', 'orig_h', 'orig_p', 'subject', 'resp_h', 'issuer_subject', 'serial' ]" - code => "event.set('[zeek_cols]', $zeek_known_certs_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_known_certs_field_names = [ 'ts', 'orig_h', 'orig_p', 'subject', 'resp_h', 'issuer_subject', 'serial' ]" + code => "event.set('[zeek_cols]', @zeek_known_certs_field_names.zip(event.get('[message]')).to_h)" } } @@ -1731,8 +1731,8 @@ filter { } ruby { id => "ruby_zip_zeek_known_hosts" - init => "$zeek_known_hosts_field_names = [ 'ts', 'orig_h' ]" - code => "event.set('[zeek_cols]', $zeek_known_hosts_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_known_hosts_field_names = [ 'ts', 'orig_h' ]" + code => "event.set('[zeek_cols]', @zeek_known_hosts_field_names.zip(event.get('[message]')).to_h)" } } @@ -1756,8 +1756,8 @@ filter { } ruby { id => "ruby_zip_zeek_known_modbus" - init => "$zeek_known_modbus_field_names = [ 'ts', 'orig_h', 'device_type' ]" - code => "event.set('[zeek_cols]', $zeek_known_modbus_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_known_modbus_field_names = [ 'ts', 'orig_h', 'device_type' ]" + code => "event.set('[zeek_cols]', @zeek_known_modbus_field_names.zip(event.get('[message]')).to_h)" } } @@ -1793,8 +1793,8 @@ filter { } ruby { id => "ruby_zip_zeek_known_services" - init => "$zeek_known_services_field_names = [ 'ts', 'resp_h', 'resp_p', 'proto', 'service' ]" - code => "event.set('[zeek_cols]', $zeek_known_services_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_known_services_field_names = [ 'ts', 'resp_h', 'resp_p', 'proto', 'service' ]" + code => "event.set('[zeek_cols]', @zeek_known_services_field_names.zip(event.get('[message]')).to_h)" } } @@ -1840,8 +1840,8 @@ filter { } ruby { id => "ruby_zip_zeek_ldap" - init => "$zeek_ldap_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'version', 'operation', 'result_code', 'result_message', 'object', 'argument' ]" - code => "event.set('[zeek_cols]', $zeek_ldap_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ldap_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'version', 'operation', 'result_code', 'result_message', 'object', 'argument' ]" + code => "event.set('[zeek_cols]', @zeek_ldap_field_names.zip(event.get('[message]')).to_h)" } } @@ -1874,8 +1874,8 @@ filter { } ruby { id => "ruby_zip_zeek_ldap_search" - init => "$zeek_ldap_search_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'scope', 'deref', 'base_object', 'result_count', 'result_code', 'result_message', 'filter', 'attributes' ]" - code => "event.set('[zeek_cols]', $zeek_ldap_search_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ldap_search_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'scope', 'deref', 'base_object', 'result_count', 'result_code', 'result_message', 'filter', 'attributes' ]" + code => "event.set('[zeek_cols]', @zeek_ldap_search_field_names.zip(event.get('[message]')).to_h)" } } @@ -1907,8 +1907,8 @@ filter { } ruby { id => "ruby_zip_zeek_login" - init => "$zeek_login_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'service', 'success', 'confused', 'user', 'client_user', 'password' ]" - code => "event.set('[zeek_cols]', $zeek_login_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_login_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'service', 'success', 'confused', 'user', 'client_user', 'password' ]" + code => "event.set('[zeek_cols]', @zeek_login_field_names.zip(event.get('[message]')).to_h)" } } @@ -1932,8 +1932,8 @@ filter { } ruby { id => "ruby_zip_zeek_modbus" - init => "$zeek_modbus_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'exception' ]" - code => "event.set('[zeek_cols]', $zeek_modbus_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_modbus_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'exception' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_field_names.zip(event.get('[message]')).to_h)" } } @@ -1964,8 +1964,8 @@ filter { } ruby { id => "ruby_zip_zeek_modbus_detailed" - init => "$zeek_modbus_detailed_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'quantity', 'values' ]" - code => "event.set('[zeek_cols]', $zeek_modbus_detailed_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_modbus_detailed_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'quantity', 'values' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_detailed_field_names.zip(event.get('[message]')).to_h)" } } @@ -1998,8 +1998,8 @@ filter { } ruby { id => "ruby_zip_zeek_modbus_mask_write_register" - init => "$zeek_modbus_mask_write_register_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'and_mask', 'or_mask' ]" - code => "event.set('[zeek_cols]', $zeek_modbus_mask_write_register_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_modbus_mask_write_register_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'and_mask', 'or_mask' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_mask_write_register_field_names.zip(event.get('[message]')).to_h)" } } @@ -2032,8 +2032,8 @@ filter { } ruby { id => "ruby_zip_zeek_modbus_read_device_identification" - init => "$zeek_modbus_read_device_identification_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'mei_type', 'conformity_level_code', 'conformity_level', 'device_id_code', 'object_id_code', 'object_id', 'object_value' ]" - code => "event.set('[zeek_cols]', $zeek_modbus_read_device_identification_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_modbus_read_device_identification_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'mei_type', 'conformity_level_code', 'conformity_level', 'device_id_code', 'object_id_code', 'object_id', 'object_value' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_read_device_identification_field_names.zip(event.get('[message]')).to_h)" } } @@ -2066,8 +2066,8 @@ filter { } ruby { id => "ruby_zip_zeek_modbus_read_write_multiple_registers" - init => "$zeek_modbus_read_write_multiple_registers_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'write_start_address', 'write_registers', 'read_start_address', 'read_quantity', 'read_registers' ]" - code => "event.set('[zeek_cols]', $zeek_modbus_read_write_multiple_registers_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_modbus_read_write_multiple_registers_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'write_start_address', 'write_registers', 'read_start_address', 'read_quantity', 'read_registers' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_read_write_multiple_registers_field_names.zip(event.get('[message]')).to_h)" } } @@ -2099,8 +2099,8 @@ filter { } ruby { id => "ruby_zip_zeek_mqtt_connect" - init => "$zeek_mqtt_connect_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto_name', 'proto_version', 'client_id', 'connect_status', 'will_topic', 'will_payload' ]" - code => "event.set('[zeek_cols]', $zeek_mqtt_connect_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_mqtt_connect_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto_name', 'proto_version', 'client_id', 'connect_status', 'will_topic', 'will_payload' ]" + code => "event.set('[zeek_cols]', @zeek_mqtt_connect_field_names.zip(event.get('[message]')).to_h)" } } @@ -2132,8 +2132,8 @@ filter { } ruby { id => "ruby_zip_zeek_mqtt_publish" - init => "$zeek_mqtt_publish_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'from_client', 'retain', 'qos', 'status', 'topic', 'payload', 'payload_len' ]" - code => "event.set('[zeek_cols]', $zeek_mqtt_publish_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_mqtt_publish_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'from_client', 'retain', 'qos', 'status', 'topic', 'payload', 'payload_len' ]" + code => "event.set('[zeek_cols]', @zeek_mqtt_publish_field_names.zip(event.get('[message]')).to_h)" } } @@ -2165,8 +2165,8 @@ filter { } ruby { id => "ruby_zip_zeek_mqtt_subscribe" - init => "$zeek_mqtt_subscribe_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'action', 'topics', 'qos_levels', 'granted_qos_level', 'ack' ]" - code => "event.set('[zeek_cols]', $zeek_mqtt_subscribe_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_mqtt_subscribe_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'action', 'topics', 'qos_levels', 'granted_qos_level', 'ack' ]" + code => "event.set('[zeek_cols]', @zeek_mqtt_subscribe_field_names.zip(event.get('[message]')).to_h)" } } @@ -2201,8 +2201,8 @@ filter { } ruby { id => "ruby_zip_zeek_mysql" - init => "$zeek_mysql_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cmd', 'arg', 'success', 'rows', 'response' ]" - code => "event.set('[zeek_cols]', $zeek_mysql_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_mysql_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cmd', 'arg', 'success', 'rows', 'response' ]" + code => "event.set('[zeek_cols]', @zeek_mysql_field_names.zip(event.get('[message]')).to_h)" } } @@ -2232,8 +2232,8 @@ filter { } ruby { id => "ruby_zip_zeek_notice" - init => "$zeek_notice_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'file_mime_type', 'file_desc', 'proto', 'note', 'msg', 'sub', 'src', 'dst', 'p', 'n', 'peer_descr', 'actions', 'email_dest', 'suppress_for', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude' ]" - code => "event.set('[zeek_cols]', $zeek_notice_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_notice_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'file_mime_type', 'file_desc', 'proto', 'note', 'msg', 'sub', 'src', 'dst', 'p', 'n', 'peer_descr', 'actions', 'email_dest', 'suppress_for', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude' ]" + code => "event.set('[zeek_cols]', @zeek_notice_field_names.zip(event.get('[message]')).to_h)" } } @@ -2274,8 +2274,8 @@ filter { } ruby { id => "ruby_zip_zeek_ntlm" - init => "$zeek_ntlm_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'host', 'domain', 'server_nb_computer', 'server_dns_computer', 'server_tree', 'success' ]" - code => "event.set('[zeek_cols]', $zeek_ntlm_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ntlm_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'host', 'domain', 'server_nb_computer', 'server_dns_computer', 'server_tree', 'success' ]" + code => "event.set('[zeek_cols]', @zeek_ntlm_field_names.zip(event.get('[message]')).to_h)" } } @@ -2304,8 +2304,8 @@ filter { } ruby { id => "ruby_zip_zeek_ntp" - init => "$zeek_ntp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'mode', 'stratum', 'poll', 'precision', 'root_delay', 'root_disp', 'ref_id', 'ref_time', 'org_time', 'rec_time', 'xmt_time', 'num_exts' ]" - code => "event.set('[zeek_cols]', $zeek_ntp_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ntp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'mode', 'stratum', 'poll', 'precision', 'root_delay', 'root_disp', 'ref_id', 'ref_time', 'org_time', 'rec_time', 'xmt_time', 'num_exts' ]" + code => "event.set('[zeek_cols]', @zeek_ntp_field_names.zip(event.get('[message]')).to_h)" } } @@ -2338,8 +2338,8 @@ filter { ruby { id => "ruby_zip_zeek_ocsp" - init => "$zeek_ocsp_field_names = [ 'ts', 'fuid', 'hashAlgorithm', 'issuerNameHash', 'issuerKeyHash', 'serialNumber', 'certStatus', 'revoketime', 'revokereason', 'thisUpdate', 'nextUpdate' ]" - code => "event.set('[zeek_cols]', $zeek_ocsp_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ocsp_field_names = [ 'ts', 'fuid', 'hashAlgorithm', 'issuerNameHash', 'issuerKeyHash', 'serialNumber', 'certStatus', 'revoketime', 'revokereason', 'thisUpdate', 'nextUpdate' ]" + code => "event.set('[zeek_cols]', @zeek_ocsp_field_names.zip(event.get('[message]')).to_h)" } } @@ -2367,8 +2367,8 @@ filter { ruby { id => "ruby_zip_zeek_ospf" - init => "$zeek_ospf_field_names = [ 'ts', 'orig_h', 'resp_h', 'ospf_type', 'version', 'router_id', 'area_id', 'interface_id', 'netmask', 'desig_router', 'backup_router', 'neighbors', 'lsa_type', 'link_state_id', 'advert_router', 'routers', 'link_id', 'link_data', 'link_type', 'neighbor_router_id', 'metrics', 'fwd_addrs', 'route_tags', 'neighbor_interface_id', 'prefix', 'metric', 'dest_router_id', 'link_prefixes', 'intra_prefixes' ]" - code => "event.set('[zeek_cols]', $zeek_ospf_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ospf_field_names = [ 'ts', 'orig_h', 'resp_h', 'ospf_type', 'version', 'router_id', 'area_id', 'interface_id', 'netmask', 'desig_router', 'backup_router', 'neighbors', 'lsa_type', 'link_state_id', 'advert_router', 'routers', 'link_id', 'link_data', 'link_type', 'neighbor_router_id', 'metrics', 'fwd_addrs', 'route_tags', 'neighbor_interface_id', 'prefix', 'metric', 'dest_router_id', 'link_prefixes', 'intra_prefixes' ]" + code => "event.set('[zeek_cols]', @zeek_ospf_field_names.zip(event.get('[message]')).to_h)" } } @@ -2400,8 +2400,8 @@ filter { } ruby { id => "ruby_zip_zeek_pe" - init => "$zeek_pe_field_names = [ 'ts', 'fuid', 'machine', 'compile_ts', 'os', 'subsystem', 'is_exe', 'is_64bit', 'uses_aslr', 'uses_dep', 'uses_code_integrity', 'uses_seh', 'has_import_table', 'has_export_table', 'has_cert_table', 'has_debug_data', 'section_names' ]" - code => "event.set('[zeek_cols]', $zeek_pe_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_pe_field_names = [ 'ts', 'fuid', 'machine', 'compile_ts', 'os', 'subsystem', 'is_exe', 'is_64bit', 'uses_aslr', 'uses_dep', 'uses_code_integrity', 'uses_seh', 'has_import_table', 'has_export_table', 'has_cert_table', 'has_debug_data', 'section_names' ]" + code => "event.set('[zeek_cols]', @zeek_pe_field_names.zip(event.get('[message]')).to_h)" } } @@ -2425,8 +2425,8 @@ filter { } ruby { id => "ruby_zip_zeek_profinet" - init => "$zeek_profinet_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'operation_type', 'block_version', 'slot_number', 'subslot_number', 'index' ]" - code => "event.set('[zeek_cols]', $zeek_profinet_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_profinet_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'operation_type', 'block_version', 'slot_number', 'subslot_number', 'index' ]" + code => "event.set('[zeek_cols]', @zeek_profinet_field_names.zip(event.get('[message]')).to_h)" } } @@ -2456,8 +2456,8 @@ filter { } ruby { id => "ruby_zip_zeek_profinet_dce_rpc" - init => "$zeek_profinet_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'packet_type', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'operation' ]" - code => "event.set('[zeek_cols]', $zeek_profinet_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_profinet_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'packet_type', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'operation' ]" + code => "event.set('[zeek_cols]', @zeek_profinet_dce_rpc_field_names.zip(event.get('[message]')).to_h)" } } @@ -2487,8 +2487,8 @@ filter { } ruby { id => "ruby_zip_zeek_radius" - init => "$zeek_radius_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'mac', 'framed_addr', 'tunnel_client', 'connect_info', 'reply_msg', 'result', 'ttl' ]" - code => "event.set('[zeek_cols]', $zeek_radius_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_radius_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'mac', 'framed_addr', 'tunnel_client', 'connect_info', 'reply_msg', 'result', 'ttl' ]" + code => "event.set('[zeek_cols]', @zeek_radius_field_names.zip(event.get('[message]')).to_h)" } } @@ -2554,8 +2554,8 @@ filter { } ruby { id => "ruby_zip_zeek_rdp" - init => "$zeek_rdp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cookie', 'result', 'security_protocol', 'client_channels', 'keyboard_layout', 'client_build', 'client_name', 'client_dig_product_id', 'desktop_width', 'desktop_height', 'requested_color_depth', 'cert_type', 'cert_count', 'cert_permanent', 'encryption_level', 'encryption_method' ]" - code => "event.set('[zeek_cols]', $zeek_rdp_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_rdp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cookie', 'result', 'security_protocol', 'client_channels', 'keyboard_layout', 'client_build', 'client_name', 'client_dig_product_id', 'desktop_width', 'desktop_height', 'requested_color_depth', 'cert_type', 'cert_count', 'cert_permanent', 'encryption_level', 'encryption_method' ]" + code => "event.set('[zeek_cols]', @zeek_rdp_field_names.zip(event.get('[message]')).to_h)" } } @@ -2588,8 +2588,8 @@ filter { } ruby { id => "ruby_zip_zeek_rfb" - init => "$zeek_rfb_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'client_major_version', 'client_minor_version', 'server_major_version', 'server_minor_version', 'authentication_method', 'auth', 'share_flag', 'desktop_name', 'width', 'height' ]" - code => "event.set('[zeek_cols]', $zeek_rfb_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_rfb_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'client_major_version', 'client_minor_version', 'server_major_version', 'server_minor_version', 'authentication_method', 'auth', 'share_flag', 'desktop_name', 'width', 'height' ]" + code => "event.set('[zeek_cols]', @zeek_rfb_field_names.zip(event.get('[message]')).to_h)" } } @@ -2618,8 +2618,8 @@ filter { } ruby { id => "ruby_zip_zeek_s7comm" - init => "$zeek_s7comm_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_code', 'rosctr_name', 'pdu_reference', 'function_code', 'function_name', 'subfunction_code', 'subfunction_name', 'error_class', 'error_code' ]" - code => "event.set('[zeek_cols]', $zeek_s7comm_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_s7comm_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_code', 'rosctr_name', 'pdu_reference', 'function_code', 'function_name', 'subfunction_code', 'subfunction_name', 'error_class', 'error_code' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_field_names.zip(event.get('[message]')).to_h)" } } @@ -2652,8 +2652,8 @@ filter { } ruby { id => "ruby_zip_zeek_s7comm_plus" - init => "$zeek_s7comm_plus_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'opcode', 'opcode_name', 'function_code', 'function_name' ]" - code => "event.set('[zeek_cols]', $zeek_s7comm_plus_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_s7comm_plus_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'opcode', 'opcode_name', 'function_code', 'function_name' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_plus_field_names.zip(event.get('[message]')).to_h)" } } @@ -2686,8 +2686,8 @@ filter { } ruby { id => "ruby_zip_zeek_s7comm_read_szl" - init => "$zeek_s7comm_read_szl_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_reference', 'method', 'szl_id', 'szl_id_name', 'szl_index', 'return_code', 'return_code_name' ]" - code => "event.set('[zeek_cols]', $zeek_s7comm_read_szl_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_s7comm_read_szl_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_reference', 'method', 'szl_id', 'szl_id_name', 'szl_index', 'return_code', 'return_code_name' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_read_szl_field_names.zip(event.get('[message]')).to_h)" } } @@ -2720,8 +2720,8 @@ filter { } ruby { id => "ruby_zip_zeek_s7comm_upload_download" - init => "$zeek_s7comm_upload_download_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_name', 'pdu_reference', 'function_name', 'function_status', 'session_id', 'blocklength', 'filename', 'block_type', 'block_number', 'destination_filesystem' ]" - code => "event.set('[zeek_cols]', $zeek_s7comm_upload_download_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_s7comm_upload_download_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_name', 'pdu_reference', 'function_name', 'function_status', 'session_id', 'blocklength', 'filename', 'block_type', 'block_number', 'destination_filesystem' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_upload_download_field_names.zip(event.get('[message]')).to_h)" } } @@ -2753,8 +2753,8 @@ filter { } ruby { id => "ruby_zip_zeek_signatures" - init => "$zeek_signatures_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'note', 'signature_id', 'event_message', 'sub_message', 'signature_count', 'host_count' ]" - code => "event.set('[zeek_cols]', $zeek_signatures_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_signatures_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'note', 'signature_id', 'event_message', 'sub_message', 'signature_count', 'host_count' ]" + code => "event.set('[zeek_cols]', @zeek_signatures_field_names.zip(event.get('[message]')).to_h)" } } @@ -2778,8 +2778,8 @@ filter { } ruby { id => "ruby_zip_zeek_sip" - init => "$zeek_sip_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'uri', 'date', 'request_from', 'request_to', 'response_from', 'response_to', 'reply_to', 'call_id', 'seq', 'subject', 'request_path', 'response_path', 'user_agent', 'status_code', 'status_msg', 'warning', 'request_body_len', 'response_body_len', 'content_type' ]" - code => "event.set('[zeek_cols]', $zeek_sip_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_sip_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'uri', 'date', 'request_from', 'request_to', 'response_from', 'response_to', 'reply_to', 'call_id', 'seq', 'subject', 'request_path', 'response_path', 'user_agent', 'status_code', 'status_msg', 'warning', 'request_body_len', 'response_body_len', 'content_type' ]" + code => "event.set('[zeek_cols]', @zeek_sip_field_names.zip(event.get('[message]')).to_h)" } } @@ -2811,8 +2811,8 @@ filter { } ruby { id => "ruby_zip_zeek_smb_cmd" - init => "$zeek_smb_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command', 'sub_command', 'argument', 'status', 'rtt', 'version', 'user', 'tree', 'tree_service', 'referenced_file.ts', 'referenced_file.uid', 'referenced_file.orig_h', 'referenced_file.orig_p', 'referenced_file.resp_h', 'referenced_file.resp_p', 'referenced_file.fuid', 'referenced_file.action', 'referenced_file.path', 'referenced_file.name', 'referenced_file.size', 'referenced_file.prev_name', 'referenced_file.times_modified', 'referenced_file.times_accessed', 'referenced_file.times_created', 'referenced_file.times_changed', 'referenced_file.data_offset_req', 'referenced_file.data_len_req', 'referenced_file.data_len_rsp' ]" - code => "event.set('[zeek_cols]', $zeek_smb_cmd_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_smb_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command', 'sub_command', 'argument', 'status', 'rtt', 'version', 'user', 'tree', 'tree_service', 'referenced_file.ts', 'referenced_file.uid', 'referenced_file.orig_h', 'referenced_file.orig_p', 'referenced_file.resp_h', 'referenced_file.resp_p', 'referenced_file.fuid', 'referenced_file.action', 'referenced_file.path', 'referenced_file.name', 'referenced_file.size', 'referenced_file.prev_name', 'referenced_file.times_modified', 'referenced_file.times_accessed', 'referenced_file.times_created', 'referenced_file.times_changed', 'referenced_file.data_offset_req', 'referenced_file.data_len_req', 'referenced_file.data_len_rsp' ]" + code => "event.set('[zeek_cols]', @zeek_smb_cmd_field_names.zip(event.get('[message]')).to_h)" } } @@ -2893,8 +2893,8 @@ filter { } ruby { id => "ruby_zip_zeek_smb_files" - init => "$zeek_smb_files_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'action', 'path', 'name', 'size', 'prev_name', 'times_modified', 'times_accessed', 'times_created', 'times_changed', 'data_offset_req', 'data_len_req', 'data_len_rsp' ]" - code => "event.set('[zeek_cols]', $zeek_smb_files_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_smb_files_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'action', 'path', 'name', 'size', 'prev_name', 'times_modified', 'times_accessed', 'times_created', 'times_changed', 'data_offset_req', 'data_len_req', 'data_len_rsp' ]" + code => "event.set('[zeek_cols]', @zeek_smb_files_field_names.zip(event.get('[message]')).to_h)" } } @@ -2929,8 +2929,8 @@ filter { } ruby { id => "ruby_zip_zeek_smb_mapping" - init => "$zeek_smb_mapping_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'path', 'resource_type', 'native_file_system', 'share_type' ]" - code => "event.set('[zeek_cols]', $zeek_smb_mapping_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_smb_mapping_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'path', 'resource_type', 'native_file_system', 'share_type' ]" + code => "event.set('[zeek_cols]', @zeek_smb_mapping_field_names.zip(event.get('[message]')).to_h)" } } @@ -2962,8 +2962,8 @@ filter { } ruby { id => "ruby_zip_zeek_smtp" - init => "$zeek_smtp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'helo', 'mailfrom', 'rcptto', 'date', 'from', 'to', 'cc', 'reply_to', 'msg_id', 'in_reply_to', 'subject', 'x_originating_ip', 'first_received', 'second_received', 'last_reply', 'path', 'user_agent', 'tls', 'fuid', 'is_webmail' ]" - code => "event.set('[zeek_cols]', $zeek_smtp_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_smtp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'helo', 'mailfrom', 'rcptto', 'date', 'from', 'to', 'cc', 'reply_to', 'msg_id', 'in_reply_to', 'subject', 'x_originating_ip', 'first_received', 'second_received', 'last_reply', 'path', 'user_agent', 'tls', 'fuid', 'is_webmail' ]" + code => "event.set('[zeek_cols]', @zeek_smtp_field_names.zip(event.get('[message]')).to_h)" } } @@ -2995,8 +2995,8 @@ filter { } ruby { id => "ruby_zip_zeek_snmp" - init => "$zeek_snmp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'duration', 'version', 'community', 'get_requests', 'get_bulk_requests', 'get_responses', 'set_requests', 'display_string', 'up_since' ]" - code => "event.set('[zeek_cols]', $zeek_snmp_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_snmp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'duration', 'version', 'community', 'get_requests', 'get_bulk_requests', 'get_responses', 'set_requests', 'display_string', 'up_since' ]" + code => "event.set('[zeek_cols]', @zeek_snmp_field_names.zip(event.get('[message]')).to_h)" } } @@ -3028,8 +3028,8 @@ filter { } ruby { id => "ruby_zip_zeek_socks" - init => "$zeek_socks_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'user', 'password', 'server_status', 'request_host', 'request_name', 'request_port', 'bound_host', 'bound_name', 'bound_port' ]" - code => "event.set('[zeek_cols]', $zeek_socks_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_socks_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'user', 'password', 'server_status', 'request_host', 'request_name', 'request_port', 'bound_host', 'bound_name', 'bound_port' ]" + code => "event.set('[zeek_cols]', @zeek_socks_field_names.zip(event.get('[message]')).to_h)" } } @@ -3058,8 +3058,8 @@ filter { } ruby { id => "ruby_zip_zeek_software" - init => "$zeek_software_field_names = [ 'ts', 'orig_h', 'orig_p', 'software_type', 'name', 'version_major', 'version_minor', 'version_minor2', 'version_minor3', 'version_addl', 'unparsed_version', 'url' ]" - code => "event.set('[zeek_cols]', $zeek_software_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_software_field_names = [ 'ts', 'orig_h', 'orig_p', 'software_type', 'name', 'version_major', 'version_minor', 'version_minor2', 'version_minor3', 'version_addl', 'unparsed_version', 'url' ]" + code => "event.set('[zeek_cols]', @zeek_software_field_names.zip(event.get('[message]')).to_h)" } } @@ -3083,8 +3083,8 @@ filter { } ruby { id => "ruby_zip_zeek_wireguard" - init => "$zeek_wireguard_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'established', 'initiations', 'responses' ]" - code => "event.set('[zeek_cols]', $zeek_wireguard_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_wireguard_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'established', 'initiations', 'responses' ]" + code => "event.set('[zeek_cols]', @zeek_wireguard_field_names.zip(event.get('[message]')).to_h)" } } @@ -3159,8 +3159,8 @@ filter { } ruby { id => "ruby_zip_zeek_ssh" - init => "$zeek_ssh_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'auth_success', 'auth_attempts', 'direction', 'client', 'server', 'cipher_alg', 'mac_alg', 'compression_alg', 'kex_alg', 'host_key_alg', 'host_key', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude', 'hasshVersion', 'hassh', 'hasshServer', 'cshka', 'hasshAlgorithms', 'sshka', 'hasshServerAlgorithms' ]" - code => "event.set('[zeek_cols]', $zeek_ssh_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ssh_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'auth_success', 'auth_attempts', 'direction', 'client', 'server', 'cipher_alg', 'mac_alg', 'compression_alg', 'kex_alg', 'host_key_alg', 'host_key', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude', 'hasshVersion', 'hassh', 'hasshServer', 'cshka', 'hasshAlgorithms', 'sshka', 'hasshServerAlgorithms' ]" + code => "event.set('[zeek_cols]', @zeek_ssh_field_names.zip(event.get('[message]')).to_h)" } } @@ -3232,8 +3232,8 @@ filter { } ruby { id => "ruby_zip_zeek_ssl" - init => "$zeek_ssl_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ssl_version', 'cipher', 'curve', 'server_name', 'resumed', 'last_alert', 'next_protocol', 'established', 'ssl_history', 'cert_chain_fps', 'client_cert_chain_fps', 'sni_matches_cert', 'validation_status', 'ja3', 'ja3s' ]" - code => "event.set('[zeek_cols]', $zeek_ssl_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_ssl_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ssl_version', 'cipher', 'curve', 'server_name', 'resumed', 'last_alert', 'next_protocol', 'established', 'ssl_history', 'cert_chain_fps', 'client_cert_chain_fps', 'sni_matches_cert', 'validation_status', 'ja3', 'ja3s' ]" + code => "event.set('[zeek_cols]', @zeek_ssl_field_names.zip(event.get('[message]')).to_h)" } } @@ -3265,8 +3265,8 @@ filter { } ruby { id => "ruby_zip_zeek_stun" - init => "$zeek_stun_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'trans_id', 'method', 'class', 'attr_type', 'attr_val' ]" - code => "event.set('[zeek_cols]', $zeek_stun_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_stun_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'trans_id', 'method', 'class', 'attr_type', 'attr_val' ]" + code => "event.set('[zeek_cols]', @zeek_stun_field_names.zip(event.get('[message]')).to_h)" } } @@ -3297,8 +3297,8 @@ filter { } ruby { id => "ruby_zip_zeek_stun_nat" - init => "$zeek_stun_nat_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'wan_addr', 'wan_port', 'lan_addr' ]" - code => "event.set('[zeek_cols]', $zeek_stun_nat_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_stun_nat_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'wan_addr', 'wan_port', 'lan_addr' ]" + code => "event.set('[zeek_cols]', @zeek_stun_nat_field_names.zip(event.get('[message]')).to_h)" } } @@ -3330,8 +3330,8 @@ filter { } ruby { id => "ruby_zip_zeek_synchrophasor" - init => "$zeek_synchrophasor_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'version', 'data_stream_id', 'history', 'frame_size_min', 'frame_size_max', 'frame_size_tot', 'data_frame_count', 'data_rate' ]" - code => "event.set('[zeek_cols]', $zeek_synchrophasor_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_synchrophasor_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'version', 'data_stream_id', 'history', 'frame_size_min', 'frame_size_max', 'frame_size_tot', 'data_frame_count', 'data_rate' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_field_names.zip(event.get('[message]')).to_h)" } } @@ -3365,8 +3365,8 @@ filter { } ruby { id => "ruby_zip_zeek_synchrophasor_cmd" - init => "$zeek_synchrophasor_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'command', 'extframe' ]" - code => "event.set('[zeek_cols]', $zeek_synchrophasor_cmd_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_synchrophasor_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'command', 'extframe' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_cmd_field_names.zip(event.get('[message]')).to_h)" } } @@ -3399,8 +3399,8 @@ filter { } ruby { id => "ruby_zip_zeek_synchrophasor_cfg" - init => "$zeek_synchrophasor_cfg_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'cont_idx', 'pmu_count_expected', 'pmu_count_actual', 'data_rate', 'cfg_frame_id' ]" - code => "event.set('[zeek_cols]', $zeek_synchrophasor_cfg_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_synchrophasor_cfg_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'cont_idx', 'pmu_count_expected', 'pmu_count_actual', 'data_rate', 'cfg_frame_id' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_field_names.zip(event.get('[message]')).to_h)" } } @@ -3433,8 +3433,8 @@ filter { } ruby { id => "ruby_zip_zeek_synchrophasor_cfg_detail" - init => "$zeek_synchrophasor_cfg_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'cfg_frame_id', 'pmu_idx', 'svc_class', 'station_name', 'data_source_id', 'global_pmuid', 'phasor_shape', 'phasor_format', 'analog_format', 'freq_format', 'phnmr', 'annmr', 'dgnmr', 'phnam', 'annam', 'dgnam', 'phasor_conv_phunit', 'phasor_conv_phvalue', 'phasor_conv_upsampled_interpolation', 'phasor_conv_upsampled_extrapolation', 'phasor_conv_downsampled_reselection', 'phasor_conv_downsampled_fir_filter', 'phasor_conv_downsampled_no_fir_filter', 'phasor_conv_filtered_without_changing_sampling', 'phasor_conv_calibration_mag_adj', 'phasor_conv_calibration_phas_adj', 'phasor_conv_rotation_phase_adj', 'phasor_conv_pseudo_phasor_val', 'phasor_conv_mod_appl', 'phasor_conv_phasor_component', 'phasor_conv_phasor_type', 'phasor_conv_user_def', 'phasor_conv_scale_factor', 'phasor_conv_angle_adj', 'analog_conv_analog_flags', 'analog_conv_user_defined_scaling', 'analog_conv_mag_scale', 'analog_conv_offset', 'digital_conv_normal_status_mask', 'digital_conv_valid_inputs_mask', 'pmu_lat', 'pmu_lon', 'pmu_elev', 'window', 'group_delay', 'fnom', 'cfgcnt' ]" - code => "event.set('[zeek_cols]', $zeek_synchrophasor_cfg_detail_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_synchrophasor_cfg_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'cfg_frame_id', 'pmu_idx', 'svc_class', 'station_name', 'data_source_id', 'global_pmuid', 'phasor_shape', 'phasor_format', 'analog_format', 'freq_format', 'phnmr', 'annmr', 'dgnmr', 'phnam', 'annam', 'dgnam', 'phasor_conv_phunit', 'phasor_conv_phvalue', 'phasor_conv_upsampled_interpolation', 'phasor_conv_upsampled_extrapolation', 'phasor_conv_downsampled_reselection', 'phasor_conv_downsampled_fir_filter', 'phasor_conv_downsampled_no_fir_filter', 'phasor_conv_filtered_without_changing_sampling', 'phasor_conv_calibration_mag_adj', 'phasor_conv_calibration_phas_adj', 'phasor_conv_rotation_phase_adj', 'phasor_conv_pseudo_phasor_val', 'phasor_conv_mod_appl', 'phasor_conv_phasor_component', 'phasor_conv_phasor_type', 'phasor_conv_user_def', 'phasor_conv_scale_factor', 'phasor_conv_angle_adj', 'analog_conv_analog_flags', 'analog_conv_user_defined_scaling', 'analog_conv_mag_scale', 'analog_conv_offset', 'digital_conv_normal_status_mask', 'digital_conv_valid_inputs_mask', 'pmu_lat', 'pmu_lon', 'pmu_elev', 'window', 'group_delay', 'fnom', 'cfgcnt' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_detail_field_names.zip(event.get('[message]')).to_h)" } } @@ -3467,8 +3467,8 @@ filter { } ruby { id => "ruby_zip_zeek_synchrophasor_data" - init => "$zeek_synchrophasor_data_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'pmu_count_expected', 'pmu_count_actual', 'data_frame_id' ]" - code => "event.set('[zeek_cols]', $zeek_synchrophasor_data_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_synchrophasor_data_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'pmu_count_expected', 'pmu_count_actual', 'data_frame_id' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_field_names.zip(event.get('[message]')).to_h)" } } @@ -3501,8 +3501,8 @@ filter { } ruby { id => "ruby_zip_zeek_synchrophasor_data_detail" - init => "$zeek_synchrophasor_data_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'data_frame_id', 'pmu_idx', 'trigger_reason', 'unlocked_time', 'pmu_time_quality', 'data_modified', 'config_change', 'pmu_trigger_pickup', 'data_sorting_type', 'pmu_sync_error', 'data_error_indicator', 'est_rectangular_real', 'est_rectangular_imaginary', 'est_polar_magnitude', 'est_polar_angle', 'freq_dev_mhz', 'rocof', 'analog_data', 'digital' ]" - code => "event.set('[zeek_cols]', $zeek_synchrophasor_data_detail_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_synchrophasor_data_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'data_frame_id', 'pmu_idx', 'trigger_reason', 'unlocked_time', 'pmu_time_quality', 'data_modified', 'config_change', 'pmu_trigger_pickup', 'data_sorting_type', 'pmu_sync_error', 'data_error_indicator', 'est_rectangular_real', 'est_rectangular_imaginary', 'est_polar_magnitude', 'est_polar_angle', 'freq_dev_mhz', 'rocof', 'analog_data', 'digital' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_detail_field_names.zip(event.get('[message]')).to_h)" } } @@ -3535,8 +3535,8 @@ filter { } ruby { id => "ruby_zip_zeek_synchrophasor_hdr" - init => "$zeek_synchrophasor_hdr_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'data' ]" - code => "event.set('[zeek_cols]', $zeek_synchrophasor_hdr_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_synchrophasor_hdr_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_hdr_field_names.zip(event.get('[message]')).to_h)" } } @@ -3568,8 +3568,8 @@ filter { } ruby { id => "ruby_zip_zeek_syslog" - init => "$zeek_syslog_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'facility', 'severity', 'message' ]" - code => "event.set('[zeek_cols]', $zeek_syslog_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_syslog_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'facility', 'severity', 'message' ]" + code => "event.set('[zeek_cols]', @zeek_syslog_field_names.zip(event.get('[message]')).to_h)" } } @@ -3598,8 +3598,8 @@ filter { } ruby { id => "ruby_zip_zeek_tds" - init => "$zeek_tds_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command' ]" - code => "event.set('[zeek_cols]', $zeek_tds_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_tds_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command' ]" + code => "event.set('[zeek_cols]', @zeek_tds_field_names.zip(event.get('[message]')).to_h)" } } @@ -3631,8 +3631,8 @@ filter { } ruby { id => "ruby_zip_zeek_tds_rpc" - init => "$zeek_tds_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'procedure_name', 'parameter' ]" - code => "event.set('[zeek_cols]', $zeek_tds_rpc_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_tds_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'procedure_name', 'parameter' ]" + code => "event.set('[zeek_cols]', @zeek_tds_rpc_field_names.zip(event.get('[message]')).to_h)" } } @@ -3664,8 +3664,8 @@ filter { } ruby { id => "ruby_zip_zeek_tds_sql_batch" - init => "$zeek_tds_sql_batch_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_type', 'query' ]" - code => "event.set('[zeek_cols]', $zeek_tds_sql_batch_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_tds_sql_batch_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_type', 'query' ]" + code => "event.set('[zeek_cols]', @zeek_tds_sql_batch_field_names.zip(event.get('[message]')).to_h)" } } @@ -3697,8 +3697,8 @@ filter { } ruby { id => "ruby_zip_zeek_tftp" - init => "$zeek_tftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'wrq', 'fname', 'mode', 'uid_data', 'size', 'block_sent', 'block_acked', 'error_code', 'error_msg' ]" - code => "event.set('[zeek_cols]', $zeek_tftp_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_tftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'wrq', 'fname', 'mode', 'uid_data', 'size', 'block_sent', 'block_acked', 'error_code', 'error_msg' ]" + code => "event.set('[zeek_cols]', @zeek_tftp_field_names.zip(event.get('[message]')).to_h)" } } @@ -3730,8 +3730,8 @@ filter { } ruby { id => "ruby_zip_zeek_tunnel" - init => "$zeek_tunnel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'tunnel_type', 'action' ]" - code => "event.set('[zeek_cols]', $zeek_tunnel_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_tunnel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'tunnel_type', 'action' ]" + code => "event.set('[zeek_cols]', @zeek_tunnel_field_names.zip(event.get('[message]')).to_h)" } } @@ -3769,8 +3769,8 @@ filter { } ruby { id => "ruby_zip_zeek_weird" - init => "$zeek_weird_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'name', 'addl', 'notice', 'peer' ]" - code => "event.set('[zeek_cols]', $zeek_weird_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_weird_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'name', 'addl', 'notice', 'peer' ]" + code => "event.set('[zeek_cols]', @zeek_weird_field_names.zip(event.get('[message]')).to_h)" } } @@ -3795,8 +3795,8 @@ filter { } ruby { id => "ruby_zip_zeek_x509" - init => "$zeek_x509_field_names = [ 'ts', 'fuid', 'certificate_version', 'certificate_serial', 'certificate_subject', 'certificate_issuer', 'certificate_not_valid_before', 'certificate_not_valid_after', 'certificate_key_alg', 'certificate_sig_alg', 'certificate_key_type', 'certificate_key_length', 'certificate_exponent', 'certificate_curve', 'san_dns', 'san_uri', 'san_email', 'san_ip', 'basic_constraints_ca', 'basic_constraints_path_len', 'host_cert', 'client_cert' ]" - code => "event.set('[zeek_cols]', $zeek_x509_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_x509_field_names = [ 'ts', 'fuid', 'certificate_version', 'certificate_serial', 'certificate_subject', 'certificate_issuer', 'certificate_not_valid_before', 'certificate_not_valid_after', 'certificate_key_alg', 'certificate_sig_alg', 'certificate_key_type', 'certificate_key_length', 'certificate_exponent', 'certificate_curve', 'san_dns', 'san_uri', 'san_email', 'san_ip', 'basic_constraints_ca', 'basic_constraints_path_len', 'host_cert', 'client_cert' ]" + code => "event.set('[zeek_cols]', @zeek_x509_field_names.zip(event.get('[message]')).to_h)" } } @@ -3826,8 +3826,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary" - init => "$zeek_opcua_binary_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'msg_type', 'is_final', 'msg_size', 'error', 'reason', 'version', 'rcv_buf_size', 'snd_buf_size', 'max_msg_size', 'max_chunk_cnt', 'endpoint_url', 'sec_channel_id', 'sec_policy_uri_len', 'sec_policy_uri', 'snd_cert_len', 'snd_cert', 'rcv_cert_len', 'rcv_cert', 'seq_number', 'request_id', 'encoding_mask', 'namespace_idx', 'identifier', 'identifier_str', 'req_hdr_node_id_type', 'req_hdr_node_id_namespace_idx', 'req_hdr_node_id_numeric', 'req_hdr_node_id_string', 'req_hdr_node_id_guid', 'req_hdr_node_id_opaque', 'req_hdr_timestamp', 'req_hdr_request_handle', 'req_hdr_return_diag', 'req_hdr_audit_entry_id', 'req_hdr_timeout_hint', 'req_hdr_add_hdr_type_id', 'req_hdr_add_hdr_enc_mask', 'res_hdr_timestamp', 'res_hdr_request_handle', 'status_code_link_id', 'res_hdr_service_diag_encoding', 'res_hdr_add_hdr_type_id', 'res_hdr_add_hdr_enc_mask' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'msg_type', 'is_final', 'msg_size', 'error', 'reason', 'version', 'rcv_buf_size', 'snd_buf_size', 'max_msg_size', 'max_chunk_cnt', 'endpoint_url', 'sec_channel_id', 'sec_policy_uri_len', 'sec_policy_uri', 'snd_cert_len', 'snd_cert', 'rcv_cert_len', 'rcv_cert', 'seq_number', 'request_id', 'encoding_mask', 'namespace_idx', 'identifier', 'identifier_str', 'req_hdr_node_id_type', 'req_hdr_node_id_namespace_idx', 'req_hdr_node_id_numeric', 'req_hdr_node_id_string', 'req_hdr_node_id_guid', 'req_hdr_node_id_opaque', 'req_hdr_timestamp', 'req_hdr_request_handle', 'req_hdr_return_diag', 'req_hdr_audit_entry_id', 'req_hdr_timeout_hint', 'req_hdr_add_hdr_type_id', 'req_hdr_add_hdr_enc_mask', 'res_hdr_timestamp', 'res_hdr_request_handle', 'status_code_link_id', 'res_hdr_service_diag_encoding', 'res_hdr_add_hdr_type_id', 'res_hdr_add_hdr_enc_mask' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_field_names.zip(event.get('[message]')).to_h)" } } @@ -3861,8 +3861,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_activate_session" - init => "$zeek_opcua_binary_activate_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_algorithm', 'client_signature', 'client_software_cert_link_id', 'opcua_locale_link_id', 'ext_obj_type_id_encoding_mask', 'ext_obj_type_id_namespace_idx', 'ext_obj_type_id_numeric', 'ext_obj_type_id_string', 'ext_obj_type_id_guid', 'ext_obj_type_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding', 'ext_obj_policy_id', 'ext_obj_user_name', 'ext_obj_password', 'ext_obj_encryption_algorithom', 'ext_obj_certificate_data', 'ext_obj_token_data', 'user_token_algorithm', 'user_token_signature', 'server_nonce', 'status_code_link_id', 'activate_session_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_activate_session_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_activate_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_algorithm', 'client_signature', 'client_software_cert_link_id', 'opcua_locale_link_id', 'ext_obj_type_id_encoding_mask', 'ext_obj_type_id_namespace_idx', 'ext_obj_type_id_numeric', 'ext_obj_type_id_string', 'ext_obj_type_id_guid', 'ext_obj_type_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding', 'ext_obj_policy_id', 'ext_obj_user_name', 'ext_obj_password', 'ext_obj_encryption_algorithom', 'ext_obj_certificate_data', 'ext_obj_token_data', 'user_token_algorithm', 'user_token_signature', 'server_nonce', 'status_code_link_id', 'activate_session_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_field_names.zip(event.get('[message]')).to_h)" } } @@ -3896,8 +3896,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_activate_session_client_software_cert" - init => "$zeek_opcua_binary_activate_session_client_software_cert_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'client_software_cert_link_id', 'cert_data', 'cert_signature' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_activate_session_client_software_cert_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_activate_session_client_software_cert_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'client_software_cert_link_id', 'cert_data', 'cert_signature' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_client_software_cert_field_names.zip(event.get('[message]')).to_h)" } } @@ -3931,8 +3931,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_activate_session_locale_id" - init => "$zeek_opcua_binary_activate_session_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_locale_link_id', 'local_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_activate_session_locale_id_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_activate_session_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_locale_link_id', 'local_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_locale_id_field_names.zip(event.get('[message]')).to_h)" } } @@ -3966,8 +3966,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_aggregate_filter" - init => "$zeek_opcua_binary_aggregate_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'start_time', 'start_time_str', 'aggregate_type_encoding_mask', 'aggregate_type_namespace_idx', 'aggregate_type_numeric', 'aggregate_type_string', 'aggregate_type_guid', 'aggregate_type_opaque', 'processing_interval', 'use_server_capabilities_default', 'treat_uncertain_as_bad', 'percent_data_good', 'percent_data_bad', 'use_slopped_extrapolation', 'revised_start_time', 'revised_start_time_str', 'revised_processing_interval', 'revised_use_server_capabilities_default', 'revised_treat_uncertain_as_bad', 'revised_percent_data_good', 'revised_percent_data_bad', 'revised_use_slopped_extrapolation' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_aggregate_filter_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_aggregate_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'start_time', 'start_time_str', 'aggregate_type_encoding_mask', 'aggregate_type_namespace_idx', 'aggregate_type_numeric', 'aggregate_type_string', 'aggregate_type_guid', 'aggregate_type_opaque', 'processing_interval', 'use_server_capabilities_default', 'treat_uncertain_as_bad', 'percent_data_good', 'percent_data_bad', 'use_slopped_extrapolation', 'revised_start_time', 'revised_start_time_str', 'revised_processing_interval', 'revised_use_server_capabilities_default', 'revised_treat_uncertain_as_bad', 'revised_percent_data_good', 'revised_percent_data_bad', 'revised_use_slopped_extrapolation' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_aggregate_filter_field_names.zip(event.get('[message]')).to_h)" } } @@ -4001,8 +4001,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand" - init => "$zeek_opcua_binary_event_filter_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'alias', 'browse_path_element_link_id', 'attribute', 'index_range' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'alias', 'browse_path_element_link_id', 'attribute', 'index_range' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_field_names.zip(event.get('[message]')).to_h)" } } @@ -4036,8 +4036,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" - init => "$zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_path_element_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'is_inverse', 'include_subtypes', 'target_name_namespace_idx', 'target_name' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_path_element_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'is_inverse', 'include_subtypes', 'target_name_namespace_idx', 'target_name' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" } } @@ -4071,8 +4071,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_browse" - init => "$zeek_opcua_binary_browse_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'browse_service_type', 'browse_view_id_encoding_mask', 'browse_view_id_namespace_idx', 'browse_view_id_numeric', 'browse_view_id_string', 'browse_view_id_guid', 'browse_view_id_opaque', 'browse_view_description_timestamp', 'browse_view_description_view_version', 'req_max_ref_nodes', 'browse_description_link_id', 'browse_next_release_continuation_point', 'browse_next_link_id', 'browse_response_link_id', 'browse_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_browse_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_browse_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'browse_service_type', 'browse_view_id_encoding_mask', 'browse_view_id_namespace_idx', 'browse_view_id_numeric', 'browse_view_id_string', 'browse_view_id_guid', 'browse_view_id_opaque', 'browse_view_description_timestamp', 'browse_view_description_view_version', 'req_max_ref_nodes', 'browse_description_link_id', 'browse_next_release_continuation_point', 'browse_next_link_id', 'browse_response_link_id', 'browse_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_field_names.zip(event.get('[message]')).to_h)" } } @@ -4106,8 +4106,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_browse_description" - init => "$zeek_opcua_binary_browse_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_description_link_id', 'browse_description_encoding_mask', 'browse_description_namespace_idx', 'browse_description_numeric', 'browse_description_string', 'browse_description_guid', 'browse_description_opaque', 'browse_direction', 'browse_description_ref_encoding_mask', 'browse_description_ref_namespace_idx', 'browse_description_ref_numeric', 'browse_description_ref_string', 'browse_description_ref_guid', 'browse_description_ref_opaque', 'browse_description_include_subtypes', 'browse_node_class_mask', 'browse_result_mask' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_browse_description_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_browse_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_description_link_id', 'browse_description_encoding_mask', 'browse_description_namespace_idx', 'browse_description_numeric', 'browse_description_string', 'browse_description_guid', 'browse_description_opaque', 'browse_direction', 'browse_description_ref_encoding_mask', 'browse_description_ref_namespace_idx', 'browse_description_ref_numeric', 'browse_description_ref_string', 'browse_description_ref_guid', 'browse_description_ref_opaque', 'browse_description_include_subtypes', 'browse_node_class_mask', 'browse_result_mask' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_description_field_names.zip(event.get('[message]')).to_h)" } } @@ -4141,8 +4141,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_browse_response_references" - init => "$zeek_opcua_binary_browse_response_references_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_reference_link_id', 'browse_response_ref_encoding_mask', 'browse_response_ref_namespace_idx', 'browse_response_ref_numeric', 'browse_response_ref_string', 'browse_response_ref_guid', 'browse_response_ref_opaque', 'browse_response_is_forward', 'browse_response_ref_type_encoding_mask', 'browse_response_ref_type_namespace_idx', 'browse_response_ref_type_numeric', 'browse_response_ref_type_string', 'browse_response_ref_type_guid', 'browse_response_ref_type_opaque', 'browse_response_ref_type_namespace_uri', 'browse_response_ref_type_server_idx', 'browse_response_ref_name_idx', 'browse_response_ref_name', 'browse_response_display_name_mask', 'browse_response_display_name_locale', 'browse_response_display_name_text', 'browse_response_node_class', 'browse_response_type_def_encoding_mask', 'browse_response_type_def_namespace_idx', 'browse_response_type_def_numeric', 'browse_response_type_def_string', 'browse_response_type_def_guid', 'browse_response_type_def_opaque', 'browse_response_type_def_namespace_uri', 'browse_response_type_def_server_idx' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_browse_response_references_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_browse_response_references_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_reference_link_id', 'browse_response_ref_encoding_mask', 'browse_response_ref_namespace_idx', 'browse_response_ref_numeric', 'browse_response_ref_string', 'browse_response_ref_guid', 'browse_response_ref_opaque', 'browse_response_is_forward', 'browse_response_ref_type_encoding_mask', 'browse_response_ref_type_namespace_idx', 'browse_response_ref_type_numeric', 'browse_response_ref_type_string', 'browse_response_ref_type_guid', 'browse_response_ref_type_opaque', 'browse_response_ref_type_namespace_uri', 'browse_response_ref_type_server_idx', 'browse_response_ref_name_idx', 'browse_response_ref_name', 'browse_response_display_name_mask', 'browse_response_display_name_locale', 'browse_response_display_name_text', 'browse_response_node_class', 'browse_response_type_def_encoding_mask', 'browse_response_type_def_namespace_idx', 'browse_response_type_def_numeric', 'browse_response_type_def_string', 'browse_response_type_def_guid', 'browse_response_type_def_opaque', 'browse_response_type_def_namespace_uri', 'browse_response_type_def_server_idx' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_response_references_field_names.zip(event.get('[message]')).to_h)" } } @@ -4176,8 +4176,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_browse_request_continuation_point" - init => "$zeek_opcua_binary_browse_request_continuation_point_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_next_link_id', 'continuation_point' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_browse_request_continuation_point_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_browse_request_continuation_point_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_next_link_id', 'continuation_point' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_request_continuation_point_field_names.zip(event.get('[message]')).to_h)" } } @@ -4211,8 +4211,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_browse_result" - init => "$zeek_opcua_binary_browse_result_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_response_link_id', 'status_code_link_id', 'browse_result_continuation_point', 'browse_reference_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_browse_result_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_browse_result_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_response_link_id', 'status_code_link_id', 'browse_result_continuation_point', 'browse_reference_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_result_field_names.zip(event.get('[message]')).to_h)" } } @@ -4246,8 +4246,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_close_session" - init => "$zeek_opcua_binary_close_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'del_subscriptions' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_close_session_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_close_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'del_subscriptions' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_close_session_field_names.zip(event.get('[message]')).to_h)" } } @@ -4281,8 +4281,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause" - init => "$zeek_opcua_binary_event_filter_where_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'where_clause_link_id', 'content_filter_element_link_id', 'content_filter_status_code_link_id', 'content_filter_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_where_clause_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_where_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'where_clause_link_id', 'content_filter_element_link_id', 'content_filter_status_code_link_id', 'content_filter_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_field_names.zip(event.get('[message]')).to_h)" } } @@ -4316,8 +4316,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause_elements" - init => "$zeek_opcua_binary_event_filter_where_clause_elements_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_element_link_id', 'filter_operator', 'content_filter_filter_operand_type_id_node_id_encoding_mask', 'content_filter_filter_operand_type_id_node_id_namespace_idx', 'content_filter_filter_operand_type_id_node_id_numeric', 'content_filter_filter_operand_type_id_node_id_string', 'content_filter_filter_operand_type_id_node_id_guid', 'content_filter_filter_operand_type_id_node_id_opaque', 'content_filter_filter_operand_type_id_string', 'content_filter_filter_operand_type_id_encoding', 'content_filter_filter_operand_link_id', 'content_filter_operand_status_code_link_id', 'content_filter_operand_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_where_clause_elements_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_where_clause_elements_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_element_link_id', 'filter_operator', 'content_filter_filter_operand_type_id_node_id_encoding_mask', 'content_filter_filter_operand_type_id_node_id_namespace_idx', 'content_filter_filter_operand_type_id_node_id_numeric', 'content_filter_filter_operand_type_id_node_id_string', 'content_filter_filter_operand_type_id_node_id_guid', 'content_filter_filter_operand_type_id_node_id_opaque', 'content_filter_filter_operand_type_id_string', 'content_filter_filter_operand_type_id_encoding', 'content_filter_filter_operand_link_id', 'content_filter_operand_status_code_link_id', 'content_filter_operand_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_elements_field_names.zip(event.get('[message]')).to_h)" } } @@ -4351,8 +4351,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_create_monitored_items" - init => "$zeek_opcua_binary_create_monitored_items_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'subscription_id', 'timestamps_to_return', 'timestamps_to_return_str', 'create_item_link_id', 'create_monitored_items_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_create_monitored_items_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_create_monitored_items_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'subscription_id', 'timestamps_to_return', 'timestamps_to_return_str', 'create_item_link_id', 'create_monitored_items_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_field_names.zip(event.get('[message]')).to_h)" } } @@ -4386,8 +4386,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_create_monitored_items_create_item" - init => "$zeek_opcua_binary_create_monitored_items_create_item_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'create_item_link_id', 'item_to_monitor_node_id_encoding_mask', 'item_to_monitor_node_id_namespace_idx', 'item_to_monitor_node_id_numeric', 'item_to_monitor_node_id_string', 'item_to_monitor_node_id_guid', 'item_to_monitor_node_id_opaque', 'item_to_monitor_attribute_id', 'item_to_monitor_index_range', 'item_to_monitor_namespace_idx', 'item_to_monitor_name', 'monitoring_mode', 'monitoring_parameters_client_handle', 'monitoring_parameters_sampling_interval', 'monitoring_parameters_queue_size', 'monitoring_parameters_discard_oldest', 'monitoring_parameters_filter_info_type_id_node_id_encoding_mask', 'monitoring_parameters_filter_info_type_id_node_id_namespace_idx', 'monitoring_parameters_filter_info_type_id_node_id_numeric', 'monitoring_parameters_filter_info_type_id_node_id_string', 'monitoring_parameters_filter_info_type_id_node_id_guid', 'monitoring_parameters_filter_info_type_id_node_id_opaque', 'monitoring_parameters_filter_info_type_id_string', 'monitoring_parameters_filter_info_type_id_encoding', 'filter_info_details_link_id', 'monitoring_parameters_status_code_link_id', 'monitored_item_index_id', 'monitoring_parameters_revised_sampling_interval', 'monitoring_parameters_revised_queue_size' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_create_monitored_items_create_item_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_create_monitored_items_create_item_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'create_item_link_id', 'item_to_monitor_node_id_encoding_mask', 'item_to_monitor_node_id_namespace_idx', 'item_to_monitor_node_id_numeric', 'item_to_monitor_node_id_string', 'item_to_monitor_node_id_guid', 'item_to_monitor_node_id_opaque', 'item_to_monitor_attribute_id', 'item_to_monitor_index_range', 'item_to_monitor_namespace_idx', 'item_to_monitor_name', 'monitoring_mode', 'monitoring_parameters_client_handle', 'monitoring_parameters_sampling_interval', 'monitoring_parameters_queue_size', 'monitoring_parameters_discard_oldest', 'monitoring_parameters_filter_info_type_id_node_id_encoding_mask', 'monitoring_parameters_filter_info_type_id_node_id_namespace_idx', 'monitoring_parameters_filter_info_type_id_node_id_numeric', 'monitoring_parameters_filter_info_type_id_node_id_string', 'monitoring_parameters_filter_info_type_id_node_id_guid', 'monitoring_parameters_filter_info_type_id_node_id_opaque', 'monitoring_parameters_filter_info_type_id_string', 'monitoring_parameters_filter_info_type_id_encoding', 'filter_info_details_link_id', 'monitoring_parameters_status_code_link_id', 'monitored_item_index_id', 'monitoring_parameters_revised_sampling_interval', 'monitoring_parameters_revised_queue_size' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_create_item_field_names.zip(event.get('[message]')).to_h)" } } @@ -4421,8 +4421,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_create_session" - init => "$zeek_opcua_binary_create_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'server_uri', 'endpoint_url', 'session_name', 'client_nonce', 'client_cert_size', 'client_cert', 'req_session_timeout', 'max_res_msg_size', 'session_id_encoding_mask', 'session_id_namespace_idx', 'session_id_numeric', 'session_id_string', 'session_id_guid', 'session_id_opaque', 'auth_token_encoding_mask', 'auth_token_namespace_idx', 'auth_token_numeric', 'auth_token_string', 'auth_token_guid', 'auth_token_opaque', 'revised_session_timeout', 'server_nonce', 'server_cert_size', 'server_cert', 'endpoint_link_id', 'algorithm', 'signature', 'max_req_msg_size' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_create_session_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_create_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'server_uri', 'endpoint_url', 'session_name', 'client_nonce', 'client_cert_size', 'client_cert', 'req_session_timeout', 'max_res_msg_size', 'session_id_encoding_mask', 'session_id_namespace_idx', 'session_id_numeric', 'session_id_string', 'session_id_guid', 'session_id_opaque', 'auth_token_encoding_mask', 'auth_token_namespace_idx', 'auth_token_numeric', 'auth_token_string', 'auth_token_guid', 'auth_token_opaque', 'revised_session_timeout', 'server_nonce', 'server_cert_size', 'server_cert', 'endpoint_link_id', 'algorithm', 'signature', 'max_req_msg_size' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_field_names.zip(event.get('[message]')).to_h)" } } @@ -4456,8 +4456,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_create_session_discovery" - init => "$zeek_opcua_binary_create_session_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_uri', 'discovery_profile_url' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_create_session_discovery_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_create_session_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_uri', 'discovery_profile_url' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_discovery_field_names.zip(event.get('[message]')).to_h)" } } @@ -4491,8 +4491,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_create_session_endpoints" - init => "$zeek_opcua_binary_create_session_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_link_id', 'endpoint_url', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_create_session_endpoints_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_create_session_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_link_id', 'endpoint_url', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_endpoints_field_names.zip(event.get('[message]')).to_h)" } } @@ -4526,8 +4526,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_create_session_user_token" - init => "$zeek_opcua_binary_create_session_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_create_session_user_token_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_create_session_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_user_token_field_names.zip(event.get('[message]')).to_h)" } } @@ -4561,8 +4561,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_create_subscription" - init => "$zeek_opcua_binary_create_subscription_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'requested_publishing_interval', 'requested_lifetime_count', 'requested_max_keep_alive_count', 'max_notifications_per_publish', 'publishing_enabled', 'priority', 'subscription_id', 'revised_publishing_interval', 'revised_lifetime_count', 'revised_max_keep_alive_count' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_create_subscription_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_create_subscription_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'requested_publishing_interval', 'requested_lifetime_count', 'requested_max_keep_alive_count', 'max_notifications_per_publish', 'publishing_enabled', 'priority', 'subscription_id', 'revised_publishing_interval', 'revised_lifetime_count', 'revised_max_keep_alive_count' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_subscription_field_names.zip(event.get('[message]')).to_h)" } } @@ -4596,8 +4596,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_data_change_filter" - init => "$zeek_opcua_binary_data_change_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'trigger', 'deadband_type', 'deadband_value' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_data_change_filter_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_data_change_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'trigger', 'deadband_type', 'deadband_value' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_data_change_filter_field_names.zip(event.get('[message]')).to_h)" } } @@ -4631,8 +4631,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_diag_info_detail" - init => "$zeek_opcua_binary_diag_info_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'diag_info_link_id', 'root_object_id', 'source', 'source_str', 'inner_diag_level', 'has_symbolic_id', 'symbolic_id', 'symbolic_id_str', 'has_namespace_uri', 'namespace_uri', 'namespace_uri_str', 'has_locale', 'locale', 'locale_str', 'has_locale_txt', 'locale_txt', 'locale_txt_str', 'has_addl_info', 'addl_info', 'has_inner_stat_code', 'inner_stat_code', 'has_inner_diag_info' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_diag_info_detail_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_diag_info_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'diag_info_link_id', 'root_object_id', 'source', 'source_str', 'inner_diag_level', 'has_symbolic_id', 'symbolic_id', 'symbolic_id_str', 'has_namespace_uri', 'namespace_uri', 'namespace_uri_str', 'has_locale', 'locale', 'locale_str', 'has_locale_txt', 'locale_txt', 'locale_txt_str', 'has_addl_info', 'addl_info', 'has_inner_stat_code', 'inner_stat_code', 'has_inner_diag_info' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_diag_info_detail_field_names.zip(event.get('[message]')).to_h)" } } @@ -4666,8 +4666,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter_element_operand" - init => "$zeek_opcua_binary_event_filter_element_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'element_index' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_element_operand_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_element_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'element_index' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_element_operand_field_names.zip(event.get('[message]')).to_h)" } } @@ -4701,8 +4701,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter" - init => "$zeek_opcua_binary_event_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'select_clause_link_id', 'where_clause_content_filter_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'select_clause_link_id', 'where_clause_content_filter_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_field_names.zip(event.get('[message]')).to_h)" } } @@ -4736,8 +4736,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_get_endpoints" - init => "$zeek_opcua_binary_get_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'endpoint_url', 'locale_link_id', 'profile_uri_link_id', 'endpoint_description_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_get_endpoints_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_get_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'endpoint_url', 'locale_link_id', 'profile_uri_link_id', 'endpoint_description_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_field_names.zip(event.get('[message]')).to_h)" } } @@ -4771,8 +4771,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_get_endpoints_description" - init => "$zeek_opcua_binary_get_endpoints_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_description_link_id', 'endpoint_uri', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_get_endpoints_description_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_get_endpoints_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_description_link_id', 'endpoint_uri', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_description_field_names.zip(event.get('[message]')).to_h)" } } @@ -4806,8 +4806,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_get_endpoints_discovery" - init => "$zeek_opcua_binary_get_endpoints_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_url' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_get_endpoints_discovery_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_get_endpoints_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_url' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_discovery_field_names.zip(event.get('[message]')).to_h)" } } @@ -4841,8 +4841,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_get_endpoints_locale_id" - init => "$zeek_opcua_binary_get_endpoints_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'locale_link_id', 'locale_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_get_endpoints_locale_id_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_get_endpoints_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'locale_link_id', 'locale_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_locale_id_field_names.zip(event.get('[message]')).to_h)" } } @@ -4876,8 +4876,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_get_endpoints_profile_uri" - init => "$zeek_opcua_binary_get_endpoints_profile_uri_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'profile_uri_link_id', 'profile_uri' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_get_endpoints_profile_uri_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_get_endpoints_profile_uri_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'profile_uri_link_id', 'profile_uri' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_profile_uri_field_names.zip(event.get('[message]')).to_h)" } } @@ -4911,8 +4911,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_get_endpoints_user_token" - init => "$zeek_opcua_binary_get_endpoints_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_get_endpoints_user_token_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_get_endpoints_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_user_token_field_names.zip(event.get('[message]')).to_h)" } } @@ -4946,8 +4946,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter_literal_operand" - init => "$zeek_opcua_binary_event_filter_literal_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'literal_operand_variant_link' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_literal_operand_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_literal_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'literal_operand_variant_link' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_literal_operand_field_names.zip(event.get('[message]')).to_h)" } } @@ -4981,8 +4981,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_opensecure_channel" - init => "$zeek_opcua_binary_opensecure_channel_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_proto_ver', 'sec_token_request_type', 'message_security_mode', 'client_nonce', 'req_lifetime', 'server_proto_ver', 'sec_token_sec_channel_id', 'sec_token_id', 'sec_token_created_at', 'sec_token_revised_time', 'server_nonce' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_opensecure_channel_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_opensecure_channel_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_proto_ver', 'sec_token_request_type', 'message_security_mode', 'client_nonce', 'req_lifetime', 'server_proto_ver', 'sec_token_sec_channel_id', 'sec_token_id', 'sec_token_created_at', 'sec_token_revised_time', 'server_nonce' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_opensecure_channel_field_names.zip(event.get('[message]')).to_h)" } } @@ -5016,8 +5016,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_read" - init => "$zeek_opcua_binary_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'max_age', 'timestamps_to_return', 'timestamps_to_return_str', 'nodes_to_read_link_id', 'read_results_link_id', 'diag_info_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_read_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'max_age', 'timestamps_to_return', 'timestamps_to_return_str', 'nodes_to_read_link_id', 'read_results_link_id', 'diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_field_names.zip(event.get('[message]')).to_h)" } } @@ -5051,8 +5051,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_read_nodes_to_read" - init => "$zeek_opcua_binary_read_nodes_to_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'nodes_to_read_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'attribute_id', 'attribute_id_str', 'index_range', 'data_encoding_name_idx', 'data_encoding_name' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_read_nodes_to_read_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_read_nodes_to_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'nodes_to_read_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'attribute_id', 'attribute_id_str', 'index_range', 'data_encoding_name_idx', 'data_encoding_name' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_nodes_to_read_field_names.zip(event.get('[message]')).to_h)" } } @@ -5086,8 +5086,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_read_results" - init => "$zeek_opcua_binary_read_results_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'results_link_id', 'level', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'read_results_variant_metadata_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_read_results_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_read_results_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'results_link_id', 'level', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'read_results_variant_metadata_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_results_field_names.zip(event.get('[message]')).to_h)" } } @@ -5121,8 +5121,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter_select_clause" - init => "$zeek_opcua_binary_event_filter_select_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'select_clause_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range', 'select_clause_status_code_link_id', 'select_clause_diagnostic_info_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_select_clause_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_select_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'select_clause_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range', 'select_clause_status_code_link_id', 'select_clause_diagnostic_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_select_clause_field_names.zip(event.get('[message]')).to_h)" } } @@ -5156,8 +5156,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand" - init => "$zeek_opcua_binary_event_filter_simple_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_simple_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_field_names.zip(event.get('[message]')).to_h)" } } @@ -5191,8 +5191,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" - init => "$zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'simple_attribute_operand_browse_path_link_id', 'browse_path_src', 'namespace_index', 'name' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'simple_attribute_operand_browse_path_link_id', 'browse_path_src', 'namespace_index', 'name' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" } } @@ -5226,8 +5226,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_status_code_detail" - init => "$zeek_opcua_binary_status_code_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'status_code_link_id', 'source', 'source_str', 'source_level', 'status_code', 'severity', 'severity_str', 'sub_code', 'sub_code_str', 'structure_changed', 'semantics_changed', 'info_type', 'info_type_str', 'limit_bits', 'limit_bits_str', 'overflow', 'historian_bits', 'historian_bits_str', 'historianpartial', 'historianextradata', 'historianmultivalue' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_status_code_detail_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_status_code_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'status_code_link_id', 'source', 'source_str', 'source_level', 'status_code', 'severity', 'severity_str', 'sub_code', 'sub_code_str', 'structure_changed', 'semantics_changed', 'info_type', 'info_type_str', 'limit_bits', 'limit_bits_str', 'overflow', 'historian_bits', 'historian_bits_str', 'historianpartial', 'historianextradata', 'historianmultivalue' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_status_code_detail_field_names.zip(event.get('[message]')).to_h)" } } @@ -5261,8 +5261,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_variant_array_dims" - init => "$zeek_opcua_binary_variant_array_dims_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'array_dim_link_id', 'dimension' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_variant_array_dims_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_variant_array_dims_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'array_dim_link_id', 'dimension' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_array_dims_field_names.zip(event.get('[message]')).to_h)" } } @@ -5296,8 +5296,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_variant_data" - init => "$zeek_opcua_binary_variant_data_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_link_id', 'variant_data_value_signed_numeric', 'variant_data_value_unsigned_numeric', 'variant_data_value_string', 'variant_data_node_id_encoding_mask', 'variant_data_node_id_namespace_idx', 'variant_data_node_id_numeric', 'variant_data_node_id_string', 'variant_data_node_id_guid', 'variant_data_node_id_opaque', 'variant_data_node_id_namespace_uri', 'variant_data_node_id_server_idx', 'variant_data_value_time', 'variant_data_encoding_name_idx', 'variant_data_encoding_name', 'variant_data_mask', 'variant_data_locale', 'variant_data_text', 'variant_data_value_decimal', 'variant_data_status_code_link_id', 'variant_data_diag_info_link_id', 'variant_data_ext_obj_link_id', 'variant_metadata_data_link_id', 'variant_data_value_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_variant_data_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_variant_data_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_link_id', 'variant_data_value_signed_numeric', 'variant_data_value_unsigned_numeric', 'variant_data_value_string', 'variant_data_node_id_encoding_mask', 'variant_data_node_id_namespace_idx', 'variant_data_node_id_numeric', 'variant_data_node_id_string', 'variant_data_node_id_guid', 'variant_data_node_id_opaque', 'variant_data_node_id_namespace_uri', 'variant_data_node_id_server_idx', 'variant_data_value_time', 'variant_data_encoding_name_idx', 'variant_data_encoding_name', 'variant_data_mask', 'variant_data_locale', 'variant_data_text', 'variant_data_value_decimal', 'variant_data_status_code_link_id', 'variant_data_diag_info_link_id', 'variant_data_ext_obj_link_id', 'variant_metadata_data_link_id', 'variant_data_value_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_field_names.zip(event.get('[message]')).to_h)" } } @@ -5331,8 +5331,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_variant_data_value" - init => "$zeek_opcua_binary_variant_data_value_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_value_source_link', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'variant_metadata_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_variant_data_value_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_variant_data_value_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_value_source_link', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'variant_metadata_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_value_field_names.zip(event.get('[message]')).to_h)" } } @@ -5366,8 +5366,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_variant_extension_object" - init => "$zeek_opcua_binary_variant_extension_object_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'ext_obj_link_id', 'ext_obj_node_id_encoding_mask', 'ext_obj_node_id_namespace_idx', 'ext_obj_node_id_numeric', 'ext_obj_node_id_string', 'ext_obj_node_id_guid', 'ext_obj_node_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_variant_extension_object_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_variant_extension_object_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'ext_obj_link_id', 'ext_obj_node_id_encoding_mask', 'ext_obj_node_id_namespace_idx', 'ext_obj_node_id_numeric', 'ext_obj_node_id_string', 'ext_obj_node_id_guid', 'ext_obj_node_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_extension_object_field_names.zip(event.get('[message]')).to_h)" } } @@ -5401,8 +5401,8 @@ filter { } ruby { id => "ruby_zip_zeek_opcua_binary_variant_metadata" - init => "$zeek_opcua_binary_variant_metadata_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_source_data_link_id', 'variant_data_source', 'variant_data_source_str', 'dara_variant_encoding_mask', 'data_variant_data_type', 'data_variant_data_type_str', 'built_in_data_type', 'built_in_data_type_str', 'variant_data_link_id', 'variant_data_array_dim', 'variant_data_array_multi_dim_link_id' ]" - code => "event.set('[zeek_cols]', $zeek_opcua_binary_variant_metadata_field_names.zip(event.get('[message]')).to_h)" + init => "@zeek_opcua_binary_variant_metadata_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_source_data_link_id', 'variant_data_source', 'variant_data_source_str', 'dara_variant_encoding_mask', 'data_variant_data_type', 'data_variant_data_type_str', 'built_in_data_type', 'built_in_data_type_str', 'variant_data_link_id', 'variant_data_array_dim', 'variant_data_array_multi_dim_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_metadata_field_names.zip(event.get('[message]')).to_h)" } } @@ -5439,6 +5439,266 @@ filter { } # if / else if for opcua log types + } else if ([log_source] == "analyzer") { + ############################################################################################################################# + # analyzer.log + # Zeek Logging analyzer confirmations and violations into analyzer.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/analyzer/logging.zeek.html + + dissect { + id => "dissect_zeek_diagnostic_analyzer" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][cause]} %{[zeek_cols][analyzer_kind]} %{[zeek_cols][analyzer_name]} %{[zeek_cols][uid]} %{[zeek_cols][fuid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][failure_reason]} %{[zeek_cols][failure_data]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_analyzer" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_analyzer" + init => "@zeek_diagnostic_analyzer_field_names = [ 'ts', 'cause', 'analyzer_kind', 'analyzer_name', 'uid', 'fuid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'failure_reason', 'failure_data' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_analyzer_field_names.zip(event.get('[message]')).to_h)" + } + } + + # we are *not* adding the _zeekdiagnostic even though it could arguably be classified as such, the reason being that + # the UID/FUID and IP/ports make it suitable to be searched with the network data + + } else if ([log_source] == "broker") { + ############################################################################################################################# + # broker.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/broker/log.zeek.html + + dissect { + id => "dissect_zeek_diagnostic_broker" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][event_type]} %{[zeek_cols][event_action]} %{[zeek_cols][peer_ip]} %{[zeek_cols][peer_port]} %{[zeek_cols][peer_message]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_broker" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_broker" + init => "@zeek_diagnostic_broker_field_names = [ 'ts', 'event_type', 'event_action', 'peer_ip', 'peer_port', 'peer_message' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_broker_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_broker" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "capture_loss") { + ############################################################################################################################# + # capture_loss.log + # Reports analysis of missing traffic. Zeek bases its conclusions on analysis of TCP sequence numbers. + # https://docs.zeek.org/en/master/logs/capture-loss-and-reporter.html + + dissect { + id => "dissect_zeek_diagnostic_capture_loss" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][ts_delta]} %{[zeek_cols][peer]} %{[zeek_cols][gaps]} %{[zeek_cols][acks]} %{[zeek_cols][percent_lost]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_capture_loss" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_capture_loss" + init => "@zeek_diagnostic_capture_loss_field_names = [ 'ts', 'ts_delta', 'peer', 'gaps', 'acks', 'percent_lost' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_capture_loss_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_capture_loss" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "cluster") { + ############################################################################################################################# + # cluster.log + # Logging for establishing and controlling a cluster of Zeek instances + # https://docs.zeek.org/en/master/scripts/base/frameworks/cluster/main.zeek.html#type-Cluster::Info + + dissect { + id => "dissect_zeek_diagnostic_cluster" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][node_message]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_cluster" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_cluster" + init => "@zeek_diagnostic_cluster_field_names = [ 'ts', 'node', 'node_message' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_cluster_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_cluster" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "config") { + ############################################################################################################################# + # config.log + # Logging for Zeek configuration changes + # https://docs.zeek.org/en/master/scripts/base/frameworks/config/main.zeek.html#type-Config::Info + + dissect { + id => "dissect_zeek_diagnostic_config" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][value_name]} %{[zeek_cols][value_old]} %{[zeek_cols][value_new]} %{[zeek_cols][location]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_config" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_config" + init => "@zeek_diagnostic_config_field_names = [ 'ts', 'value_name', 'value_old', 'value_new', 'location' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_config_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_config" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "packet_filter") { + ############################################################################################################################# + # packet_filter.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/packet-filter/main.zeek.html#type-PacketFilter::Info + + dissect { + id => "dissect_zeek_diagnostic_packet_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][filter]} %{[zeek_cols][init]} %{[zeek_cols][success]} %{[zeek_cols][failure_reason]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_packet_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_packet_filter" + init => "@zeek_diagnostic_packet_filter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_packet_filter_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_packet_filter" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "print") { + ############################################################################################################################# + # print.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/logging/main.zeek.html#type-Log::PrintLogInfo + + dissect { + id => "dissect_zeek_diagnostic_print" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][vals]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_print" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_print" + init => "@zeek_diagnostic_print_field_names = [ 'ts', 'vals' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_print_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "split_zeek_diagnostic_print_vals" + split => { "[zeek_cols][vals]" => "," } } + + mutate { id => "mutate_add_tag_zeek_diagnostic_print" + add_tag => [ "_zeekdiagnostic" ] } + + + } else if ([log_source] == "reporter") { + ############################################################################################################################# + # reporter.log + # https://docs.zeek.org/en/master/scripts/base/frameworks/reporter/main.zeek.html#type-Reporter::Info + + dissect { + id => "dissect_zeek_diagnostic_reporter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][level]} %{[zeek_cols][msg]} %{[zeek_cols][location]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_reporter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_reporter" + init => "@zeek_diagnostic_reporter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_reporter_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_reporter" + add_tag => [ "_zeekdiagnostic" ] } + + } else if ([log_source] == "stats") { + ############################################################################################################################# + # stats.log + # https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info + + dissect { + id => "dissect_zeek_diagnostic_stats" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][peer]} %{[zeek_cols][mem]} %{[zeek_cols][pkts_proc]} %{[zeek_cols][bytes_recv]} %{[zeek_cols][pkts_dropped]} %{[zeek_cols][pkts_link]} %{[zeek_cols][pkt_lag]} %{[zeek_cols][pkts_filtered]} %{[zeek_cols][events_proc]} %{[zeek_cols][events_queued]} %{[zeek_cols][active_tcp_conns]} %{[zeek_cols][active_udp_conns]} %{[zeek_cols][active_icmp_conns]} %{[zeek_cols][tcp_conns]} %{[zeek_cols][udp_conns]} %{[zeek_cols][icmp_conns]} %{[zeek_cols][timers]} %{[zeek_cols][active_timers]} %{[zeek_cols][files]} %{[zeek_cols][active_files]} %{[zeek_cols][dns_requests]} %{[zeek_cols][active_dns_requests]} %{[zeek_cols][reassem_tcp_size]} %{[zeek_cols][reassem_file_size]} %{[zeek_cols][reassem_frag_size]} %{[zeek_cols][reassem_unknown_size]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_stats" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_stats" + init => "@zeek_diagnostic_stats_field_names = [ 'ts', 'peer', 'mem', 'pkts_proc', 'bytes_recv', 'pkts_dropped', 'pkts_link', 'pkt_lag', 'pkts_filtered', 'events_proc', 'events_queued', 'active_tcp_conns', 'active_udp_conns', 'active_icmp_conns', 'tcp_conns', 'udp_conns', 'icmp_conns', 'timers', 'active_timers', 'files', 'active_files', 'dns_requests', 'active_dns_requests', 'reassem_tcp_size', 'reassem_file_size', 'reassem_frag_size', 'reassem_unknown_size' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_stats_field_names.zip(event.get('[message]')).to_h)" + } + } + + mutate { id => "mutate_add_tag_zeek_diagnostic_stats" + add_tag => [ "_zeekdiagnostic" ] } + } else { # some other unknown zeek log file. should start with ts at least! csv { diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index 9eebdebc8..a56866eee 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -259,10 +259,10 @@ filter { ruby { id => "ruby_bsap_ip_rdb_variables" - init => "$bsap_ip_rdb_fields = [ :var, :val ]" + init => "@bsap_ip_rdb_fields = [ :var, :val ]" code => " vars = event.get('[zeek][bsap_ip_rdb][variables]').to_s.split(',').zip( - event.get('[zeek][bsap_ip_rdb][variable_value]').to_s.split(',')).map{ |x| $bsap_ip_rdb_fields.zip(x).to_h } + event.get('[zeek][bsap_ip_rdb][variable_value]').to_s.split(',')).map{ |x| @bsap_ip_rdb_fields.zip(x).to_h } event.set('[zeek][bsap_ip_rdb][variables]', vars) event.set('[zeek][bsap_ip_rdb][variable_count]', vars.length) " @@ -278,10 +278,10 @@ filter { ruby { id => "ruby_bsap_serial_rdb_variables" - init => "$bsap_serial_rdb_fields = [ :var, :val ]" + init => "@bsap_serial_rdb_fields = [ :var, :val ]" code => " vars = event.get('[zeek][bsap_serial_rdb][variables]').to_s.split(',').zip( - event.get('[zeek][bsap_serial_rdb][variable_value]').to_s.split(',')).map{ |x| $bsap_serial_rdb_fields.zip(x).to_h } + event.get('[zeek][bsap_serial_rdb][variable_value]').to_s.split(',')).map{ |x| @bsap_serial_rdb_fields.zip(x).to_h } event.set('[zeek][bsap_serial_rdb][variables]', vars) event.set('[zeek][bsap_serial_rdb][variable_count]', vars.length) " @@ -385,12 +385,12 @@ filter { # reference: https://raw.githubusercontent.com/wireshark/wireshark/master/epan/dissectors/packet-dnp.c (search IIN) ruby { id => "ruby_parse_dnp3_iin" - init => "$zeek_dnp3_iin_flags = [ 'Function Code not Implemented', 'Requested Objects Unknown', 'Parameters Invalid or Out of Range', 'Event Buffer Overflow', 'Operation Already Executing', 'Configuration Corrupt', 'Reserved', 'Reserved', 'Broadcast Msg Rx', 'Class 1 Data Available', 'Class 2 Data Available', 'Class 3 Data Available', 'Time Sync Required', 'Digital Outputs in Local', 'Device Trouble', 'Device Restart' ]" + init => "@zeek_dnp3_iin_flags = [ 'Function Code not Implemented', 'Requested Objects Unknown', 'Parameters Invalid or Out of Range', 'Event Buffer Overflow', 'Operation Already Executing', 'Configuration Corrupt', 'Reserved', 'Reserved', 'Broadcast Msg Rx', 'Class 1 Data Available', 'Class 2 Data Available', 'Class 3 Data Available', 'Time Sync Required', 'Digital Outputs in Local', 'Device Trouble', 'Device Restart' ]" code => " iinNum = event.get('[zeek][dnp3][iin]').to_i if (iinNum > 0) then iinFlags = Array.new - $zeek_dnp3_iin_flags.each_with_index do |val, idx| + @zeek_dnp3_iin_flags.each_with_index do |val, idx| iinFlags.push(val) if (iinNum[idx] == 1) end event.set('[zeek][dnp3][iin_flags]', iinFlags) @@ -587,9 +587,9 @@ filter { ruby { id => "ruby_genisys_payload" - init => "$genisys_fields = [ :address, :data ]" + init => "@genisys_fields = [ :address, :data ]" code => " - vars = event.get('[zeek][genisys][payload_raw]').to_s.split(',').map{ |x| x.split('=') }.map{ |x| $genisys_fields.zip(x).to_h } + vars = event.get('[zeek][genisys][payload_raw]').to_s.split(',').map{ |x| x.split('=') }.map{ |x| @genisys_fields.zip(x).to_h } event.set('[zeek][genisys][payload]', vars) unless vars.nil? or (vars.length == 0) " } @@ -2484,6 +2484,9 @@ filter { if ([zeek][notice]) or ([zeek][signatures]) or ([zeek][weird]) { mutate { id => "mutate_add_field_ecs_event_kind_alert" add_field => { "[event][kind]" => "alert" } } + } else if ("_zeekdiagnostic" in [tags]) and ([zeek][stats]) { + mutate { id => "mutate_add_field_ecs_event_kind_metric" + add_field => { "[event][kind]" => "metric" } } } else { mutate { id => "mutate_add_field_ecs_event_kind_event" add_field => { "[event][kind]" => "event" } } diff --git a/logstash/pipelines/zeek/19_severity.conf b/logstash/pipelines/zeek/19_severity.conf index aedf410ae..3cef10472 100644 --- a/logstash/pipelines/zeek/19_severity.conf +++ b/logstash/pipelines/zeek/19_severity.conf @@ -36,23 +36,26 @@ filter { } # assign severity to notice based on category - if ([zeek][notice]) { - if ([zeek][notice][category] == "ATTACK") { + if ([event][dataset] == "notice") { + if ("ATTACK" in [rule][category]) { mutate { id => "mutate_add_field_severity_notice_mitre_attack" add_field => { "[event][severity_tags]" => "MITRE ATT&CK framework technique" } } - } else if ([zeek][notice][category] == "Scan") { + } else if ("ATTACKICS" in [rule][category]) { + mutate { id => "mutate_add_field_severity_notice_mitre_attack_ics" + add_field => { "[event][severity_tags]" => "MITRE ATT&CK for ICS framework technique" } } + } else if ("Scan" in [rule][category]) { mutate { id => "mutate_add_field_severity_notice_scan" add_field => { "[event][severity_tags]" => "Notice (scan)" } } - } else if (([zeek][notice][category] == "FTP") or - ([zeek][notice][category] == "HTTP") or - ([zeek][notice][category] == "HTTPATTACKS") or - ([zeek][notice][category] == "SSL")) { + } else if (("FTP" in [rule][category]) or + ("HTTP" in [rule][category]) or + ("HTTPATTACKS" in [rule][category]) or + ("SSL" in [rule][category])) { mutate { id => "mutate_add_field_severity_notice_protocol" add_field => { "[event][severity_tags]" => "Notice (protocol)" } } - } else if (([zeek][notice][category] =~ /^CVE/) or - ([zeek][notice][category] == "EternalSafety") or - ([zeek][notice][category] == "Ripple20") or - ([zeek][notice][category] == "Zerologon")) { + } else if (([vulnerability][enumeration] == "CVE") or + ("EternalSafety" in [rule][category]) or + ("Ripple20" in [rule][category]) or + ("Zerologon" in [rule][category])) { mutate { id => "mutate_add_field_severity_notice_vuln" add_field => { "[event][severity_tags]" => "Notice (vulnerability)" } } } else { diff --git a/logstash/pipelines/zeek/99_zeek_forward.conf b/logstash/pipelines/zeek/99_zeek_forward.conf index bdbab18bb..ccdecf119 100644 --- a/logstash/pipelines/zeek/99_zeek_forward.conf +++ b/logstash/pipelines/zeek/99_zeek_forward.conf @@ -1,5 +1,13 @@ output { - pipeline { - send_to => ["log-enrichment"] + # the Zeek diagnostic logs are not network traffic metadata to be enriched, + # and belong more with the supporting runtime-oriented logs + if "_zeekdiagnostic" in [tags] { + pipeline { + send_to => ["beats-parse"] + } + } else { + pipeline { + send_to => ["log-enrichment"] + } } } diff --git a/logstash/ruby/compact_event.rb b/logstash/ruby/compact_event.rb index 08d360a57..b01157893 100644 --- a/logstash/ruby/compact_event.rb +++ b/logstash/ruby/compact_event.rb @@ -2,6 +2,11 @@ def concurrency :shared end +def register(params) + _discard_zeroes_str = params["discard_zeroes"] + @discard_zeroes = [1, true, '1', 'true', 't', 'on', 'enabled'].include?(_discard_zeroes_str.to_s.downcase) +end + def compact(h) h.inject({}) do |result, (k, v)| case v @@ -10,6 +15,8 @@ def compact(h) result[k] = c unless c.empty? when String result[k] = v unless (v.empty? || (v == "-") || (v == "?") || (v == "(empty)") || (v == "(none)") || (v == "(null)") || (v == "unset") || (v == "Nul")) + when Numeric + result[k] = v unless (@discard_zeroes && v.zero?) when Array c = v.delete_if{|e| e.nil? || (e.is_a?(String) && (e.empty? || (e == "-") || (e == "?") || (e == "(empty)") || (e == "(none)") || (e == "(null)") || (e == "unset") || (e == "Nul")))} result[k] = c unless c.empty? diff --git a/logstash/ruby/compact_event_hash.rb b/logstash/ruby/compact_event_hash.rb index 0706a3746..69d2d06eb 100644 --- a/logstash/ruby/compact_event_hash.rb +++ b/logstash/ruby/compact_event_hash.rb @@ -4,6 +4,8 @@ def concurrency def register(params) @field = params["field"] + _discard_zeroes_str = params["discard_zeroes"] + @discard_zeroes = [1, true, '1', 'true', 't', 'on', 'enabled'].include?(_discard_zeroes_str.to_s.downcase) end def compact(h) @@ -14,6 +16,8 @@ def compact(h) result[k] = c unless c.empty? when String result[k] = v unless (v.empty? || (v == "-") || (v == "(empty)")) + when Numeric + result[k] = v unless (@discard_zeroes && v.zero?) when Array c = v.delete_if{|e| e.nil? || (e.is_a?(String) && (e.empty? || (e == "-") || (e == "(empty)")))} result[k] = c unless c.empty? diff --git a/logstash/ruby/netbox_enrich.rb b/logstash/ruby/netbox_enrich.rb index 4eabb0bb4..77cd68480 100644 --- a/logstash/ruby/netbox_enrich.rb +++ b/logstash/ruby/netbox_enrich.rb @@ -183,7 +183,7 @@ def register(params) _autopopulate_fuzzy_threshold_str = ENV[_autopopulate_fuzzy_threshold_str_env] end if _autopopulate_fuzzy_threshold_str.nil? || _autopopulate_fuzzy_threshold_str.empty? - @autopopulate_fuzzy_threshold = 0.80 + @autopopulate_fuzzy_threshold = 0.95 else @autopopulate_fuzzy_threshold = _autopopulate_fuzzy_threshold_str.to_f end @@ -207,6 +207,26 @@ def register(params) # end of autopopulation arguments + # used for massaging OUI/manufacturer names for matching + @name_cleaning_patterns = [ /\ba[sbg]\b/, + /\b(beijing|shenzhen)\b/, + /\bbv\b/, + /\bco(rp(oration|orate)?)?\b/, + /\b(computer|network|electronic|solution|system)s?\b/, + /\bglobal\b/, + /\bgmbh\b/, + /\binc(orporated)?\b/, + /\bint(ernationa)?l?\b/, + /\bkft\b/, + /\blimi?ted\b/, + /\bllc\b/, + /\b(co)?ltda?\b/, + /\bpt[ey]\b/, + /\bpvt\b/, + /\boo\b/, + /\bsa\b/, + /\bsr[ol]s?\b/, + /\btech(nolog(y|ie|iya)s?)?\b/ ] end def filter(event) @@ -232,11 +252,11 @@ def filter(event) _autopopulate_default_role = (@default_role.nil? || @default_role.empty?) ? "Unspecified" : @default_role _autopopulate_default_dtype = (@default_dtype.nil? || @default_dtype.empty?) ? "Unspecified" : @default_dtype _autopopulate_default_site = (@lookup_site.nil? || @lookup_site.empty?) ? "default" : @lookup_site - _autopopulate_fuzzy_threshold = @autopopulate_fuzzy_threshold - _autopopulate_create_manuf = @autopopulate_create_manuf && !_autopopulate_oui.nil? && !_autopopulate_oui.empty? _autopopulate_hostname = event.get("#{@source_hostname}") _autopopulate_mac = event.get("#{@source_mac}") _autopopulate_oui = event.get("#{@source_oui}") + _autopopulate_fuzzy_threshold = @autopopulate_fuzzy_threshold + _autopopulate_create_manuf = @autopopulate_create_manuf && !_autopopulate_oui.nil? && !_autopopulate_oui.empty? _result = @cache_hash.getset(_lookup_type){ LruRedux::TTL::ThreadSafeCache.new(_cache_size, _cache_ttl) @@ -385,6 +405,7 @@ def filter(event) # looks like this is not a virtual machine (or we can't tell) so assume its' a regular device _autopopulate_manuf = @manuf_hash.getset(_autopopulate_oui) { _fuzzy_matcher = FuzzyStringMatch::JaroWinkler.create( :pure ) + _autopopulate_oui_cleaned = clean_manuf_string(_autopopulate_oui.to_s) _manufs = Array.new # fetch the manufacturers to do the comparison. this is a lot of work # and not terribly fast but once the hash it populated it shouldn't happen too often @@ -398,12 +419,15 @@ def filter(event) _tmp_manufs = _manufs_response.fetch(:results, []) _tmp_manufs.each do |_manuf| _tmp_name = _manuf.fetch(:name, _manuf.fetch(:display, nil)) - _manufs << { :name => _tmp_name, - :id => _manuf.fetch(:id, nil), - :url => _manuf.fetch(:url, nil), - :match => _fuzzy_matcher.getDistance(_tmp_name.to_s.downcase, _autopopulate_oui.to_s.downcase), - :vm => false - } + _tmp_distance = _fuzzy_matcher.getDistance(clean_manuf_string(_tmp_name.to_s), _autopopulate_oui_cleaned) + if (_tmp_distance >= _autopopulate_fuzzy_threshold) then + _manufs << { :name => _tmp_name, + :id => _manuf.fetch(:id, nil), + :url => _manuf.fetch(:url, nil), + :match => _tmp_distance, + :vm => false + } + end end _query[:offset] += _tmp_manufs.length() break unless (_tmp_manufs.length() >= _page_size) @@ -416,11 +440,13 @@ def filter(event) _exception_error = true end # return the manuf with the highest match + # puts('0. %{key}: %{matches}' % { key: _autopopulate_oui_cleaned, matches: JSON.generate(_manufs) })-] !_manufs&.empty? ? _manufs.max_by{|k| k[:match] } : nil } end # virtual machine vs. regular device end # _autopopulate_oui specified + # puts('1. %{key}: %{found}' % { key: _autopopulate_oui, found: JSON.generate(_autopopulate_manuf) }) if !_autopopulate_manuf.is_a?(Hash) # no match was found at ANY match level (empty database or no OUI specified), set default ("unspecified") manufacturer _autopopulate_manuf = { :name => _autopopulate_create_manuf ? _autopopulate_oui : _autopopulate_default_manuf, @@ -428,6 +454,7 @@ def filter(event) :vm => false, :id => nil} end + # puts('2. %{key}: %{found}' % { key: _autopopulate_oui, found: JSON.generate(_autopopulate_manuf) }) # make sure the site and role exists @@ -539,6 +566,7 @@ def filter(event) _autopopulate_manuf[:match] = 1.0 end end + # puts('3. %{key}: %{found}' % { key: _autopopulate_oui, found: JSON.generate(_autopopulate_manuf) }) if !_autopopulate_manuf.fetch(:id, nil)&.nonzero? # the manufacturer is still not found, create it @@ -550,6 +578,7 @@ def filter(event) _autopopulate_manuf[:id] = _manuf_create_response.fetch(:id, nil) _autopopulate_manuf[:match] = 1.0 end + # puts('4. %{key}: %{created}' % { key: _autopopulate_manuf, created: JSON.generate(_manuf_create_response) }) end # at this point we *must* have the manufacturer ID @@ -598,12 +627,22 @@ def filter(event) _autopopulate_device = _device_create_response end + else + # didn't figure out the device type ID, make sure we're not setting something half-populated + _autopopulate_dtype = nil end # _autopopulate_dtype[:id] is valid + else + # didn't figure out the manufacturer ID, make sure we're not setting something half-populated + _autopopulate_manuf = nil end # _autopopulate_manuf[:id] is valid end # virtual machine vs. regular device + else + # didn't figure out the IDs, make sure we're not setting something half-populated + _autopopulate_site = nil + _autopopulate_role = nil end # site and role are valid rescue Faraday::Error @@ -612,6 +651,7 @@ def filter(event) end if !_autopopulate_device.nil? + # puts('5. %{key}: %{found}' % { key: _autopopulate_oui, found: JSON.generate(_autopopulate_manuf) }) # we created a device, so send it back out as the result for the event as well _devices << { :name => _autopopulate_device&.fetch(:name, _autopopulate_device&.fetch(:display, nil)), :id => _autopopulate_device&.fetch(:id, nil), @@ -772,6 +812,22 @@ def crush(thing) end end +def clean_manuf_string(val) + # 0. downcase + # 1. replace commas with spaces + # 2. remove all punctuation (except parens) + # 3. squash whitespace down to one space + # 4. remove each of @name_cleaning_patterns (LLC, LTD, Inc., etc.) + # 5. remove all punctuation (even parens) + # 6. strip leading and trailing spaces + new_val = val.downcase.gsub(',', ' ').gsub(/[^\(\)A-Za-z0-9\s]/, '').gsub(/\s+/, ' ') + @name_cleaning_patterns.each do |pat| + new_val = new_val.gsub(pat, '') + end + new_val = new_val.gsub(/[^A-Za-z0-9\s]/, '').gsub(/\s+/, ' ').lstrip.rstrip + new_val +end + ############################################################################### # tests diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index 9d3f94f09..9e6ed08ce 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -71,7 +71,7 @@ if [ -d "$WORKDIR" ]; then chown -R root:root * # configure installation options - YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-compose-standalone.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" + YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" [[ -n $YML_IMAGE_VERSION ]] && IMAGE_VERSION="$YML_IMAGE_VERSION" sed -i "s@^\(title-text[[:space:]]*:\).*@\1 \"Malcolm $IMAGE_VERSION $(date +'%Y-%m-%d %H:%M:%S')\"@g" ./config/bootloaders/grub-pc/live-theme/theme.txt cp ./config/includes.binary/install/preseed_multipar.cfg ./config/includes.binary/install/preseed_multipar_crypto.cfg @@ -125,7 +125,7 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/zeek/custom/" mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/MISP/" mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/STIX/" - cp ./docker-compose-standalone.yml "$MALCOLM_DEST_DIR/docker-compose.yml" + cp ./docker-compose.yml "$MALCOLM_DEST_DIR/docker-compose.yml" cp ./net-map.json "$MALCOLM_DEST_DIR/" cp ./scripts/install.py "$MALCOLM_DEST_DIR/scripts/" cp ./scripts/control.py "$MALCOLM_DEST_DIR/scripts/" @@ -168,8 +168,8 @@ if [ -d "$WORKDIR" ]; then echo "VARIANT=\"Hedgehog Linux (Malcolm) v${IMAGE_VERSION}\"" >> "$MALCOLM_DEST_DIR"/.os-info echo "VARIANT_ID=\"hedgehog-malcolm\"" >> "$MALCOLM_DEST_DIR"/.os-info echo "ID_LIKE=\"debian\"" >> "$MALCOLM_DEST_DIR"/.os-info - echo "HOME_URL=\"https://malcolm.fyi\"" >> "$MALCOLM_DEST_DIR"/.os-info - echo "DOCUMENTATION_URL=\"https://malcolm.fyi/documentation/\"" >> "$MALCOLM_DEST_DIR"/.os-info + echo "HOME_URL=\"https://${IMAGE_PUBLISHER}.github.io/Malcolm\"" >> "$MALCOLM_DEST_DIR"/.os-info + echo "DOCUMENTATION_URL=\"https://${IMAGE_PUBLISHER}.github.io/Malcolm\"" >> "$MALCOLM_DEST_DIR"/.os-info echo "SUPPORT_URL=\"https://github.com/${IMAGE_PUBLISHER}\"" >> "$MALCOLM_DEST_DIR"/.os-info echo "BUG_REPORT_URL=\"https://github.com/${IMAGE_PUBLISHER}/malcolm/issues\"" >> "$MALCOLM_DEST_DIR"/.os-info diff --git a/malcolm-iso/config/archives/mozilla.key.binary b/malcolm-iso/config/archives/mozilla.key.binary new file mode 100644 index 000000000..a8236db78 --- /dev/null +++ b/malcolm-iso/config/archives/mozilla.key.binary @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +xsBNBGCRt7MBCADkYJHHQQoL6tKrW/LbmfR9ljz7ib2aWno4JO3VKQvLwjyUMPpq +/SXXMOnx8jXwgWizpPxQYDRJ0SQXS9ULJ1hXRL/OgMnZAYvYDeV2jBnKsAIEdiG/ +e1qm8P4W9qpWJc+hNq7FOT13RzGWRx57SdLWSXo0KeY38r9lvjjOmT/cuOcmjwlD +T9XYf/RSO+yJ/AsyMdAr+ZbDeQUd9HYJiPdI04lGaGM02MjDMnx+monc+y54t+Z+ +ry1WtQdzoQt9dHlIPlV1tR+xV5DHHsejCZxu9TWzzSlL5wfBBeEz7R/OIzivGJpW +QdJzd+2QDXSRg9q2XYWP5ZVtSgjVVJjNlb6ZABEBAAHNVEFydGlmYWN0IFJlZ2lz +dHJ5IFJlcG9zaXRvcnkgU2lnbmVyIDxhcnRpZmFjdC1yZWdpc3RyeS1yZXBvc2l0 +b3J5LXNpZ25lckBnb29nbGUuY29tPsLAjgQTAQoAOBYhBDW6oLM+nrOW9ZyoOMC6 +XObcYxWjBQJgkbezAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEMC6XObc +YxWj+igIAMFh6DrAYMeq9sbZ1ZG6oAMrinUheGQbEqe76nIDQNsZnhDwZ2wWqgVC +7DgOMqlhQmOmzm7M6Nzmq2dvPwq3xC2OeI9fQyzjT72deBTzLP7PJok9PJFOMdLf +ILSsUnmMsheQt4DUO0jYAX2KUuWOIXXJaZ319QyoRNBPYa5qz7qXS7wHLOY89IDq +fHt6Aud8ER5zhyOyhytcYMeaGC1g1IKWmgewnhEq02FantMJGlmmFi2eA0EPD02G +C3742QGqRxLwjWsm5/TpyuU24EYKRGCRm7QdVIo3ugFSetKrn0byOxWGBvtu4fH8 +XWvZkRT+u+yzH1s5yFYBqc2JTrrJvRU= +=QnvN +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/malcolm-iso/config/archives/mozilla.key.chroot b/malcolm-iso/config/archives/mozilla.key.chroot new file mode 100644 index 000000000..a8236db78 --- /dev/null +++ b/malcolm-iso/config/archives/mozilla.key.chroot @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +xsBNBGCRt7MBCADkYJHHQQoL6tKrW/LbmfR9ljz7ib2aWno4JO3VKQvLwjyUMPpq +/SXXMOnx8jXwgWizpPxQYDRJ0SQXS9ULJ1hXRL/OgMnZAYvYDeV2jBnKsAIEdiG/ +e1qm8P4W9qpWJc+hNq7FOT13RzGWRx57SdLWSXo0KeY38r9lvjjOmT/cuOcmjwlD +T9XYf/RSO+yJ/AsyMdAr+ZbDeQUd9HYJiPdI04lGaGM02MjDMnx+monc+y54t+Z+ +ry1WtQdzoQt9dHlIPlV1tR+xV5DHHsejCZxu9TWzzSlL5wfBBeEz7R/OIzivGJpW +QdJzd+2QDXSRg9q2XYWP5ZVtSgjVVJjNlb6ZABEBAAHNVEFydGlmYWN0IFJlZ2lz +dHJ5IFJlcG9zaXRvcnkgU2lnbmVyIDxhcnRpZmFjdC1yZWdpc3RyeS1yZXBvc2l0 +b3J5LXNpZ25lckBnb29nbGUuY29tPsLAjgQTAQoAOBYhBDW6oLM+nrOW9ZyoOMC6 +XObcYxWjBQJgkbezAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEMC6XObc +YxWj+igIAMFh6DrAYMeq9sbZ1ZG6oAMrinUheGQbEqe76nIDQNsZnhDwZ2wWqgVC +7DgOMqlhQmOmzm7M6Nzmq2dvPwq3xC2OeI9fQyzjT72deBTzLP7PJok9PJFOMdLf +ILSsUnmMsheQt4DUO0jYAX2KUuWOIXXJaZ319QyoRNBPYa5qz7qXS7wHLOY89IDq +fHt6Aud8ER5zhyOyhytcYMeaGC1g1IKWmgewnhEq02FantMJGlmmFi2eA0EPD02G +C3742QGqRxLwjWsm5/TpyuU24EYKRGCRm7QdVIo3ugFSetKrn0byOxWGBvtu4fH8 +XWvZkRT+u+yzH1s5yFYBqc2JTrrJvRU= +=QnvN +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/malcolm-iso/config/archives/mozilla.list.binary b/malcolm-iso/config/archives/mozilla.list.binary new file mode 100644 index 000000000..ab034cedd --- /dev/null +++ b/malcolm-iso/config/archives/mozilla.list.binary @@ -0,0 +1 @@ +deb https://packages.mozilla.org/apt mozilla main diff --git a/malcolm-iso/config/archives/mozilla.list.chroot b/malcolm-iso/config/archives/mozilla.list.chroot new file mode 100644 index 000000000..ab034cedd --- /dev/null +++ b/malcolm-iso/config/archives/mozilla.list.chroot @@ -0,0 +1 @@ +deb https://packages.mozilla.org/apt mozilla main diff --git a/malcolm-iso/config/archives/mozilla.pref.binary b/malcolm-iso/config/archives/mozilla.pref.binary new file mode 100644 index 000000000..bf474df29 --- /dev/null +++ b/malcolm-iso/config/archives/mozilla.pref.binary @@ -0,0 +1,5 @@ + +Package: * +Pin: origin packages.mozilla.org +Pin-Priority: 1000 + diff --git a/malcolm-iso/config/archives/mozilla.pref.chroot b/malcolm-iso/config/archives/mozilla.pref.chroot new file mode 100644 index 000000000..bf474df29 --- /dev/null +++ b/malcolm-iso/config/archives/mozilla.pref.chroot @@ -0,0 +1,5 @@ + +Package: * +Pin: origin packages.mozilla.org +Pin-Priority: 1000 + diff --git a/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot b/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot deleted file mode 100755 index 98b7a4782..000000000 --- a/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. - -export LC_ALL=C.UTF-8 -export LANG=C.UTF-8 - -curl -o /tmp/firefox.tar.bz2 -L "https://download.mozilla.org/?product=firefox-latest-ssl&os=linux64&lang=en-US" -if [ $(file -b --mime-type /tmp/firefox.tar.bz2) = 'application/x-bzip2' ]; then - mkdir -p /opt - rm -rvf /opt/firefox - tar -xvf /tmp/firefox.tar.bz2 -C /opt/ - rm -vf /tmp/firefox.tar.bz2 - if [[ -f /opt/firefox/firefox ]]; then - rm -vf /usr/local/bin/firefox - ln -vrs /opt/firefox/firefox /usr/local/bin/firefox - dpkg -s firefox-esr >/dev/null 2>&1 && apt-get -y --purge remove firefox-esr || true - cat << 'EOF' > /usr/share/applications/firefox.desktop -[Desktop Entry] -Name=Firefox -Comment=Web Browser -GenericName=Web Browser -X-GNOME-FullName=Firefox Web Browser -Exec=/opt/firefox/firefox %u -Terminal=false -X-MultipleArgs=false -Type=Application -Icon=/opt/firefox/browser/chrome/icons/default/default128.png -Categories=Network;WebBrowser; -MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/vnd.mozilla.xul+xml;application/rss+xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https; -StartupWMClass=Firefox -StartupNotify=true -EOF - fi -fi # /tmp/firefox.tar.bz2 check - -rm -f /tmp/firefox.tar.bz2 diff --git a/malcolm-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot b/malcolm-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot index ecce06369..71916acf7 100755 --- a/malcolm-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot @@ -19,4 +19,4 @@ EOF sed -i "1i #!/bin/sh" /etc/rc.local -chmod +x /etc/rc.local /usr/local/bin/*.sh /usr/local/bin/configure-*.py +chmod o+rx /etc/rc.local /usr/local/bin/*.sh /usr/local/bin/configure-*.py diff --git a/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot index 99d3535c4..cf1e1aff1 100755 --- a/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot @@ -27,6 +27,10 @@ apt-get -y --purge remove bluez-firmware \ apt-get -y autoremove apt-get clean +# hold packages we don't want to update with an apt-get upgrade +# we built htpdate from source for HTTPS support, so leave it +dpkg -s htpdate >/dev/null 2>&1 && apt-mark hold htpdate + # remove any residual configs dpkg -l | awk '/^rc/ { print $2 }' | xargs -r -l dpkg --purge diff --git a/malcolm-iso/config/hooks/normal/0992-localepurge.hook.chroot b/malcolm-iso/config/hooks/normal/0992-localepurge.hook.chroot index 08f4e6a07..63f833d0a 100755 --- a/malcolm-iso/config/hooks/normal/0992-localepurge.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0992-localepurge.hook.chroot @@ -25,6 +25,7 @@ localepurge cat > /etc/environment << EOF LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 +TERM=linux PYTHONDONTWRITEBYTECODE=1 EOF diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16343116651.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16343116651.desktop index e7fcbfff7..24fae04ba 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16343116651.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16343116651.desktop @@ -3,11 +3,11 @@ Name=Firefox Comment=Web Browser GenericName=Web Browser X-GNOME-FullName=Firefox Web Browser -Exec=/opt/firefox/firefox https://localhost/ +Exec=/usr/bin/firefox https://localhost/ Terminal=false X-MultipleArgs=false Type=Application -Icon=/opt/firefox/browser/chrome/icons/default/default128.png +Icon=/usr/lib/firefox/browser/chrome/icons/default/default128.png Categories=Network;WebBrowser; MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/vnd.mozilla.xul+xml;application/rss+xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https; StartupWMClass=Firefox diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16637019741.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16637019741.desktop index f27606a80..bb5641c96 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16637019741.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16637019741.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - NetBox -Exec=/opt/firefox/firefox https://localhost/netbox/ +Exec=/usr/bin/firefox https://localhost/netbox/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16343117498.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16343117498.desktop index 216c3b8a3..da0ea8849 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16343117498.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16343117498.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - User Management -Exec=/opt/firefox/firefox https://localhost/auth/ +Exec=/usr/bin/firefox https://localhost/auth/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16343116973.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16343116973.desktop index 3229599b2..e91fe4141 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16343116973.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16343116973.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 -Name=Malcolm - README -Exec=/opt/firefox/firefox https://localhost/readme/ +Name=Malcolm - Landing Page +Exec=/usr/bin/firefox https://localhost/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16343117306.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16343117306.desktop index 2b60e44a9..c021aea2f 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16343117306.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16343117306.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - Upload -Exec=/opt/firefox/firefox https://localhost/upload/ +Exec=/usr/bin/firefox https://localhost/upload/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16343117175.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16343117175.desktop index ec580ab30..2452570b6 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16343117175.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16343117175.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - OpenSearch Dashboards -Exec=/opt/firefox/firefox https://localhost/dashboards/ +Exec=/usr/bin/firefox https://localhost/dashboards/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-23/16343117599.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-23/16343117599.desktop index 5fc645455..b8716360b 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-23/16343117599.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-23/16343117599.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - CyberChef -Exec=/opt/firefox/firefox https://localhost/cyberchef.html +Exec=/usr/bin/firefox https://localhost/cyberchef.html Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-24/16343117074.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-24/16343117074.desktop index 81b3b9615..653bd2024 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-24/16343117074.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-24/16343117074.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - Arkime -Exec=/opt/firefox/firefox https://localhost/ +Exec=/usr/bin/firefox https://localhost/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-arkime.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-arkime.desktop index 09e00c6e0..d4b601988 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-arkime.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-arkime.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - Arkime -Exec=/opt/firefox/firefox https://localhost/ +Exec=/usr/bin/firefox https://localhost/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-cyberchef.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-cyberchef.desktop index 205909da6..784bab363 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-cyberchef.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-cyberchef.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - CyberChef -Exec=/opt/firefox/firefox https://localhost/cyberchef.html +Exec=/usr/bin/firefox https://localhost/cyberchef.html Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-dashboards.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-dashboards.desktop index db03c6360..3d9fbc9c9 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-dashboards.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-dashboards.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - OpenSearch Dashboards -Exec=/opt/firefox/firefox https://localhost/dashboards/ +Exec=/usr/bin/firefox https://localhost/dashboards/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-netbox.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-netbox.desktop index 6871a79bd..3ea40dfda 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-netbox.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-netbox.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - NetBox -Exec=/opt/firefox/firefox https://localhost/netbox/ +Exec=/usr/bin/firefox https://localhost/netbox/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-readme.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-readme.desktop index e25936bd7..8c395cf59 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-readme.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-readme.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 -Name=Malcolm - README -Exec=/opt/firefox/firefox https://localhost/readme/ +Name=Malcolm - Landing Page +Exec=/usr/bin/firefox https://localhost/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-upload.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-upload.desktop index 1860f2dd2..3b4530d34 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-upload.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-upload.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Malcolm - Upload -Exec=/opt/firefox/firefox https://localhost/upload/ +Exec=/usr/bin/firefox https://localhost/upload/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-users.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-users.desktop index 411365af7..be5389b99 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-users.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-users.desktop @@ -1,6 +1,6 @@ Version=1.0 Name=Malcolm - User Management -Exec=/opt/firefox/firefox https://localhost/auth/ +Exec=/usr/bin/firefox https://localhost/auth/ Terminal=false X-MultipleArgs=false Type=Application diff --git a/malcolm-iso/config/package-lists/desktopmanager.list.chroot b/malcolm-iso/config/package-lists/desktopmanager.list.chroot index 6446b24c0..7bcd60ad7 100644 --- a/malcolm-iso/config/package-lists/desktopmanager.list.chroot +++ b/malcolm-iso/config/package-lists/desktopmanager.list.chroot @@ -1,6 +1,7 @@ arandr dconf-cli file-roller +firefox fonts-symbola galculator gnome-themes-extra diff --git a/malcolm-iso/config/package-lists/net.list.chroot b/malcolm-iso/config/package-lists/net.list.chroot index e28188a2e..8cfb4e837 100644 --- a/malcolm-iso/config/package-lists/net.list.chroot +++ b/malcolm-iso/config/package-lists/net.list.chroot @@ -1,6 +1,7 @@ apache2-utils ca-certificates curl +dnsutils ethtool iproute2 iputils-arping diff --git a/netbox/preload/custom_links.yml b/netbox/preload/custom_links.yml new file mode 100644 index 000000000..15eb0cb81 --- /dev/null +++ b/netbox/preload/custom_links.yml @@ -0,0 +1,30 @@ +- name: device_link_to_dashboards + link_text: 'Asset Interaction Analysis' + link_url: '/dashboards/app/dashboards#/view/677ee170-809e-11ed-8d5b-07069f823b6f?_g=(filters:!((meta:(key:related.device_id,negate:!f,params:(query:1),type:phrase),query:(match_phrase:(related.device_id:{{ object.id }})))))' + new_window: True + content_type: device +- name: prefix_link_to_dashboards + link_text: 'Asset Interaction Analysis' + link_url: "/dashboards/app/dashboards#/view/677ee170-809e-11ed-8d5b-07069f823b6f?_g=(filters:!((meta:(key:network.name,negate:!f,params:(query:1),type:phrase),query:(match_phrase:(network.name:'{{ object.description }}')))))" + new_window: True + content_type: prefix +- name: role_link_to_dashboards + link_text: 'Asset Interaction Analysis' + link_url: "/dashboards/app/dashboards#/view/677ee170-809e-11ed-8d5b-07069f823b6f?_g=(filters:!((meta:(key:related.role,negate:!f,params:(query:1),type:phrase),query:(match_phrase:(related.role:'{{ object.name }}')))))" + new_window: True + content_type: devicerole +- name: device_link_to_arkime + link_text: 'Arkime' + link_url: '/arkime/sessions?expression=(related.device_id == {{ object.id }})' + new_window: True + content_type: device +- name: prefix_link_to_arkime + link_text: 'Arkime' + link_url: '/arkime/sessions?expression=(network.name == "{{ object.description }}")' + new_window: True + content_type: prefix +- name: role_link_to_arkime + link_text: 'Arkime' + link_url: '/arkime/sessions?expression=(related.role == "{{ object.name }}")' + new_window: True + content_type: devicerole diff --git a/netbox/preload/device_roles.yml b/netbox/preload/device_roles.yml index 3068b7785..d5fefeb0b 100644 --- a/netbox/preload/device_roles.yml +++ b/netbox/preload/device_roles.yml @@ -2,6 +2,14 @@ slug: access-point color: Pink vm_role: true +- name: Active Directory + slug: active-directory + description: Central authentication in a network + vm_role: true +- name: Actuator + slug: actuator + description: Impact the physical process with standardized signals + vm_role: true - name: Application server slug: application-server color: Green @@ -24,11 +32,6 @@ slug: bridge color: Pink vm_role: true -- name: CNC - slug: cnc - description: Computer numerical control - color: Indigo - vm_role: true - name: Camera slug: camera color: Amber @@ -37,10 +40,19 @@ slug: cloud-server color: Green vm_role: true +- name: CNC + slug: cnc + description: Computer numerical control + color: Indigo + vm_role: true - name: Collaboration server slug: collaboration-server color: Green vm_role: true +- name: Database server + slug: database-server + color: Green + vm_role: true - name: DCS slug: dcs description: Distributed control system @@ -56,10 +68,6 @@ description: Domain name system server color: Dark Green vm_role: true -- name: Database server - slug: database-server - color: Green - vm_role: true - name: Domain controller slug: domain-controller color: Dark Green @@ -69,6 +77,10 @@ description: Electronic access control color: Amber vm_role: true +- name: EWS + slug: ews + description: Engineering Workstaion + vm_role: true - name: Fax slug: fax color: Cyan @@ -80,29 +92,35 @@ - name: Firewall slug: firewall color: Dark Red + description: Monitors and controls incoming and outgoing network traffic vm_role: true - name: Gateway slug: gateway color: Pink vm_role: true -- name: HMI - slug: hmi - description: Human machine interface - color: Purple - vm_role: true -- name: HVAC - slug: hvac - description: Heating, ventilation and air conditioning - color: Amber +- name: Gateway Coupler + slug: gateway-coupler + description: Handles data transmission between different protocols vm_role: true - name: Historian slug: historian color: Purple + description: Stores process values + vm_role: true +- name: HMI + slug: hmi + description: Human machine interface for field operation + color: Purple vm_role: true - name: Hub slug: hub color: Grey vm_role: true +- name: HVAC + slug: hvac + description: Heating, ventilation and air conditioning + color: Amber + vm_role: true - name: Hypervisor slug: hypervisor color: Light Green @@ -112,19 +130,27 @@ description: Intrusion detection system color: Fuchsia vm_role: true +- name: IED + slug: ied + description: Intelligent Electronic Device + vm_role: true - name: IIoT slug: iiot description: Industrial internet of things device color: Purple vm_role: true +- name: IoT + slug: iot + description: Internet of things device + color: Light Blue + vm_role: true - name: IPS slug: ips description: Intrusion prevention system color: Fuchsia vm_role: true -- name: IoT - slug: iot - description: Internet of things device +- name: Kiosk + slug: kiosk color: Light Blue vm_role: true - name: KVM @@ -132,10 +158,6 @@ description: Keyboard, video and mouse switch color: Light Blue vm_role: true -- name: Kiosk - slug: kiosk - color: Light Blue - vm_role: true - name: Lighting slug: lighting description: Lighting controls @@ -145,11 +167,6 @@ slug: load-balancer color: Pink vm_role: true -- name: MES - slug: mes - description: Manufacturing execution system - color: Indigo - vm_role: true - name: Mail server slug: mail-server color: Green @@ -158,6 +175,11 @@ slug: media-server color: Green vm_role: true +- name: MES + slug: mes + description: Manufacturing execution system + color: Indigo + vm_role: true - name: Modem slug: modem color: Pink @@ -167,14 +189,23 @@ description: Network attached storage color: Green vm_role: true +- name: Network sensor + slug: network-sensor + color: Fuchsia + vm_role: true - name: NTP server slug: ntp-server description: Network time protocol server color: Dark Green vm_role: true -- name: Network sensor - slug: network-sensor - color: Fuchsia +- name: Photocopier + slug: photocopier + color: Light Blue + vm_role: true +- name: Physical sensor + slug: physical-sensor + color: Indigo + description: Used to capture physical quantities vm_role: true - name: PLC slug: plc @@ -186,14 +217,6 @@ description: Product lifecycle management system color: Indigo vm_role: true -- name: Photocopier - slug: photocopier - color: Light Blue - vm_role: true -- name: Physical sensor - slug: physical-sensor - color: Indigo - vm_role: true - name: Print server slug: print-server color: Green @@ -206,11 +229,6 @@ slug: proxy-server color: Dark Green vm_role: true -- name: RTU - slug: rtu - description: Remote terminal unit - color: Purple - vm_role: true - name: Real-time communication server slug: real-time-communication-server color: Dark Green @@ -222,21 +240,25 @@ - name: Router slug: router color: Pink + description: Monitors and controls incoming and outgoing network traffic vm_role: true -- name: SCADA - slug: scada - description: Supervisory control and data acquisition +- name: RTU + slug: rtu + description: Remote terminal unit color: Purple vm_role: true -- name: SIEM - slug: siem - description: Security information and event management - color: Fuchsia - vm_role: true - name: Safety automation system slug: safety-automation-system color: Amber vm_role: true +- name: SCADA + slug: scada + description: Supervisory control and data acquisition + color: Purple + vm_role: true +- name: SCADA client + slug: scada-client + vm_role: true - name: Scanner slug: scanner color: Light Blue @@ -245,8 +267,22 @@ slug: server color: Green vm_role: true +- name: SIEM + slug: siem + description: Security information and event management + color: Fuchsia + vm_role: true +- name: Switch L2 + slug: switch-l2 + description: Layer 2 switch + vm_role: true +- name: Switch L3 + slug: switch-l3 + description: Layer 3 switch + vm_role: true - name: Switch slug: switch + description: Switch (layer unspecified) color: Grey vm_role: true - name: Telephony @@ -263,6 +299,10 @@ description: Variable frequency drive color: Indigo vm_role: true +- name: Virtual Machine Server + slug: vm-server + color: Light Green + vm_role: true - name: VPN server slug: vpn-server description: Virtual private network server @@ -275,8 +315,4 @@ - name: Workstation slug: workstation color: Light Green - vm_role: true -- name: Virtual Machine Server - slug: vm-server - color: Light Green - vm_role: true + vm_role: true \ No newline at end of file diff --git a/netbox/preload/prefixes_defaults.yml b/netbox/preload/prefixes_defaults.yml index 1788bb584..f95ce589d 100644 --- a/netbox/preload/prefixes_defaults.yml +++ b/netbox/preload/prefixes_defaults.yml @@ -1,6 +1,9 @@ - prefix: 10.0.0.0/8 + description: 10.0.0.0/8 site: NETBOX_DEFAULT_SITE - prefix: 172.16.0.0/12 + description: 172.16.0.0/12 site: NETBOX_DEFAULT_SITE - prefix: 192.168.0.0/16 + description: 192.168.0.0/16 site: NETBOX_DEFAULT_SITE diff --git a/nginx/landingpage/assets/img/arkime.svg b/nginx/landingpage/assets/img/arkime.svg new file mode 100644 index 000000000..76545f2b4 --- /dev/null +++ b/nginx/landingpage/assets/img/arkime.svg @@ -0,0 +1,144 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/nginx/landingpage/assets/img/elastic.svg b/nginx/landingpage/assets/img/elastic.svg new file mode 100644 index 000000000..37a349291 --- /dev/null +++ b/nginx/landingpage/assets/img/elastic.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/nginx/landingpage/css/styles.css b/nginx/landingpage/css/styles.css index 27c764bdc..5a8f8dd08 100644 --- a/nginx/landingpage/css/styles.css +++ b/nginx/landingpage/css/styles.css @@ -325,6 +325,11 @@ th { text-align: -webkit-match-parent; } +td, +th { + padding: 0.5rem; +} + thead, tbody, tfoot, @@ -10861,8 +10866,8 @@ header.masthead h1, header.masthead .h1 { } @media (min-width: 768px) { header.masthead { - padding-top: 12rem; - padding-bottom: 12rem; + padding-top: 9rem; + padding-bottom: 9rem; } header.masthead h1, header.masthead .h1 { font-size: 3rem; diff --git a/nginx/landingpage/index.html b/nginx/landingpage/index.html index bae1cbbdb..e36a58fb3 100644 --- a/nginx/landingpage/index.html +++ b/nginx/landingpage/index.html @@ -23,16 +23,7 @@

-
-
-
-
- -

-

-
-
-
+
@@ -40,14 +31,14 @@

- -

Dashboards

 + +

MALCOLM_DASHBOARDS_NAME_REPLACER

Visualize traffic or track down security concerns with dozens of pre-built dashboards, or create your own

- +

Arkime

Delve into session details including full packet payloads

diff --git a/nginx/nginx.conf b/nginx/nginx.conf index e9b621b16..87884cd6e 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -8,10 +8,14 @@ events { worker_connections 1024; } http { + include /etc/nginx/conf.d/*.conf; include /etc/nginx/mime.types; sendfile on; client_max_body_size 20m; + client_body_buffer_size 128k; + client_header_buffer_size 256k; + large_client_header_buffers 8 256k; fastcgi_buffers 16 64k; fastcgi_buffer_size 256k; @@ -139,8 +143,7 @@ http { } # Arkime -> Dashboards shortcut - location ~* ^/idark2dash(.*) { - include /etc/nginx/nginx_auth_rt.conf; + location ~* /idark2dash(.*) { set $filter_start_time now-1d; if ($arg_start != '') { set $filter_start_time \'$arg_start\'; @@ -161,11 +164,7 @@ http { set $filter_value $arg_value; } - # TODO: index and time field could be specified by environment variables - rewrite ^/idark2dash/(.*) /dashboards/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'arkime_sessions3-*',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'arkime_sessions3-*',interval:auto,query:(language:lucene,query:''),sort:!(firstPacket,desc)) redirect; - proxy_pass http://dashboards; - proxy_redirect off; - proxy_set_header Host dashboards.malcolm.local; + include /etc/nginx/nginx_idark2dash_rewrite_rt.conf; } # Dashboards -> Arkime shortcut @@ -187,12 +186,9 @@ http { proxy_set_header Host file-monitor.malcolm.local; } - # already prepended /dashboards to match the server.basePath in OpenSearch Dashboards's YML config file + # OpenSearch dashboards (or Kibana) location /dashboards { - include /etc/nginx/nginx_auth_rt.conf; - proxy_pass http://dashboards; - proxy_redirect off; - proxy_set_header Host dashboards.malcolm.local; + include /etc/nginx/nginx_dashboards_rewrite_rt.conf; } # offline region maps for dashboards diff --git a/nginx/nginx_dashboards_rewrite_dashboards.conf b/nginx/nginx_dashboards_rewrite_dashboards.conf new file mode 100644 index 000000000..6d74b03d9 --- /dev/null +++ b/nginx/nginx_dashboards_rewrite_dashboards.conf @@ -0,0 +1,4 @@ +include /etc/nginx/nginx_auth_rt.conf; +proxy_pass http://dashboards; +proxy_redirect off; +proxy_set_header Host dashboards.malcolm.local; \ No newline at end of file diff --git a/nginx/nginx_dashboards_rewrite_kibana.conf b/nginx/nginx_dashboards_rewrite_kibana.conf new file mode 100644 index 000000000..d0e7a89d7 --- /dev/null +++ b/nginx/nginx_dashboards_rewrite_kibana.conf @@ -0,0 +1 @@ +rewrite ^/dashboards/(.*) $dashboards_proxy_url/$1 redirect; \ No newline at end of file diff --git a/nginx/nginx_idark2dash_rewrite_dashboards.conf b/nginx/nginx_idark2dash_rewrite_dashboards.conf new file mode 100644 index 000000000..e8b774ae8 --- /dev/null +++ b/nginx/nginx_idark2dash_rewrite_dashboards.conf @@ -0,0 +1,5 @@ +include /etc/nginx/nginx_auth_rt.conf; +rewrite ^.*/idark2dash/(.*) $dashboards_prefix/app/dashboards#/view/0ad3d7c2-3441-485e-9dfe-dbb22e84e576?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,to:$filter_stop_time))&_a=(description:'',filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value'),type:phrase),query:(match_phrase:($filter_field:'$filter_value')))),fullScreenMode:!f,options:(useMargins:!t),query:(language:lucene,query:'*'),timeRestore:!f,viewMode:view) redirect; +proxy_pass $dashboards_proxy_pass; +proxy_redirect off; +proxy_set_header Host dashboards.malcolm.local; \ No newline at end of file diff --git a/nginx/nginx_idark2dash_rewrite_kibana.conf b/nginx/nginx_idark2dash_rewrite_kibana.conf new file mode 100644 index 000000000..47ee989a4 --- /dev/null +++ b/nginx/nginx_idark2dash_rewrite_kibana.conf @@ -0,0 +1 @@ +rewrite ^.*/idark2dash/(.*) $dashboards_proxy_url/app/dashboards#/view/0ad3d7c2-3441-485e-9dfe-dbb22e84e576?_g=(refreshInterval:(pause:!t,value:60000),time:(from:'2024-02-01T15:45:45.793Z',to:'2024-02-06T16:00:50.775Z'))&_a=(filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value'),type:phrase),query:(match_phrase:($filter_field:'$filter_value')))))? redirect; \ No newline at end of file diff --git a/nginx/nginx_readonly.conf b/nginx/nginx_readonly.conf index f19b341c5..11c6b001e 100644 --- a/nginx/nginx_readonly.conf +++ b/nginx/nginx_readonly.conf @@ -8,6 +8,7 @@ events { worker_connections 1024; } http { + include /etc/nginx/conf.d/*.conf; include /etc/nginx/mime.types; sendfile on; @@ -73,7 +74,7 @@ http { include /etc/nginx/nginx_auth_rt.conf; # Arkime -> Dashboards shortcut - location ~* ^/idark2dash(.*) { + location ~* /idark2dash(.*) { set $filter_start_time now-1d; if ($arg_start != '') { @@ -95,11 +96,7 @@ http { set $filter_value $arg_value; } - # TODO: index and time field could be specified by environment variables - rewrite ^/idark2dash/(.*) /dashboards/app/discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,mode:absolute,to:$filter_stop_time))&_a=(columns:!(_source),filters:!((meta:(alias:!n,disabled:!f,index:'arkime_sessions3-*',key:$filter_field,negate:!f,params:(query:'$filter_value',type:phrase),type:phrase,value:'$filter_value'),query:(match:($filter_field:(query:'$filter_value',type:phrase))))),index:'arkime_sessions3-*',interval:auto,query:(language:lucene,query:''),sort:!(firstPacket,desc)) redirect; - proxy_pass http://dashboards; - proxy_redirect off; - proxy_set_header Host dashboards.malcolm.local; + include /etc/nginx/nginx_idark2dash_rewrite_rt.conf; } # Dashboards -> Arkime shortcut @@ -121,11 +118,9 @@ http { proxy_set_header Host file-monitor.malcolm.local; } - # already prepended /dashboards to match the server.basePath in OpenSearch Dashboards's YML config file + # OpenSearch dashboards (or Kibana) location /dashboards { - proxy_pass http://dashboards; - proxy_redirect off; - proxy_set_header Host dashboards.malcolm.local; + include /etc/nginx/nginx_dashboards_rewrite_rt.conf; } # offline region maps for dashboards diff --git a/nginx/scripts/docker_entrypoint.sh b/nginx/scripts/docker_entrypoint.sh index 9a81dfafa..3a43d04bf 100755 --- a/nginx/scripts/docker_entrypoint.sh +++ b/nginx/scripts/docker_entrypoint.sh @@ -1,33 +1,11 @@ #!/bin/bash set -e -# Warn if the DOCKER_HOST socket does not exist -if [[ $DOCKER_HOST = unix://* ]]; then - socket_file=${DOCKER_HOST#unix://} - if ! [ -S $socket_file ]; then - cat >&2 <<-EOT - ERROR: you need to share your Docker host socket with a volume at $socket_file - Typically you should run your container with: \`-v /var/run/docker.sock:$socket_file:ro\` - See the jwilder/nginx-proxy documentation at http://git.io/vZaGJ -EOT - socketMissing=1 - fi -fi - -# Compute the DNS resolvers for use in the templates - if the IP contains ":", it's IPv6 and must be enclosed in [] -export RESOLVERS=$(awk '$1 == "nameserver" {print ($2 ~ ":")? "["$2"]": $2}' ORS=' ' /etc/resolv.conf | sed 's/ *$//g') -if [ "x$RESOLVERS" = "x" ]; then - echo "Warning: unable to determine DNS resolvers for nginx" >&2 - unset RESOLVERS -fi - -# If the user has run the default command and the socket doesn't exist, fail -if [ "$socketMissing" = 1 -a "$1" = 'supervisord' -a "$2" = '-c' -a "$3" = '/etc/supervisord.conf' ]; then - exit 1 -fi - NGINX_LANDING_INDEX_HTML=/usr/share/nginx/html/index.html +NGINX_TEMPLATES_DIR=/etc/nginx/templates +NGINX_CONFD_DIR=/etc/nginx/conf.d + # set up for HTTPS/HTTP and NGINX HTTP basic vs. LDAP/LDAPS/LDAP+StartTLS auth # "include" file that sets 'ssl on' and indicates the locations of the PEM files @@ -58,6 +36,15 @@ NGINX_RUNTIME_AUTH_CONF=/etc/nginx/nginx_auth_rt.conf # runtime "include" file for ldap config (link to either NGINX_BLANK_CONF or (possibly modified) NGINX_LDAP_USER_CONF) NGINX_RUNTIME_LDAP_CONF=/etc/nginx/nginx_ldap_rt.conf +# "include" files for idark2dash rewrite using opensearch dashboards, kibana, and runtime copy, respectively +NGINX_DASHBOARDS_IDARK2DASH_REWRITE_CONF=/etc/nginx/nginx_idark2dash_rewrite_dashboards.conf +NGINX_KIBANA_IDARK2DASH_REWRITE_CONF=/etc/nginx/nginx_idark2dash_rewrite_kibana.conf +NGINX_RUNTIME_IDARK2DASH_REWRITE_CONF=/etc/nginx/nginx_idark2dash_rewrite_rt.conf +# do the same thing for /dashboards URLs, send to kibana if they're using elasticsearch +NGINX_DASHBOARDS_DASHBOARDS_REWRITE_CONF=/etc/nginx/nginx_dashboards_rewrite_dashboards.conf +NGINX_KIBANA_DASHBOARDS_REWRITE_CONF=/etc/nginx/nginx_dashboards_rewrite_kibana.conf +NGINX_RUNTIME_DASHBOARDS_REWRITE_CONF=/etc/nginx/nginx_dashboards_rewrite_rt.conf + # config file for stunnel if using stunnel to issue LDAP StartTLS function STUNNEL_CONF=/etc/stunnel/stunnel.conf @@ -239,6 +226,7 @@ EOF fi # basic vs. ldap +# if the runtime htpasswd file doesn't exist but the "preseed" does, copy the preseed over for runtime if [[ ! -f /etc/nginx/auth/htpasswd ]] && [[ -f /tmp/auth/default/htpasswd ]]; then cp /tmp/auth/default/htpasswd /etc/nginx/auth/htpasswd [[ -n ${PUID} ]] && chown -f ${PUID} /etc/nginx/auth/htpasswd @@ -246,8 +234,79 @@ if [[ ! -f /etc/nginx/auth/htpasswd ]] && [[ -f /tmp/auth/default/htpasswd ]]; t rm -rf /tmp/auth/* || true fi -[[ -f "${NGINX_LANDING_INDEX_HTML}" ]] && sed -i "s/MALCOLM_VERSION_REPLACER/v${MALCOLM_VERSION:-unknown} (${VCS_REVISION:-} @ ${BUILD_DATE:-})/g" "${NGINX_LANDING_INDEX_HTML}" +# do environment variable substitutions from $NGINX_TEMPLATES_DIR to $NGINX_CONFD_DIR +# NGINX_DASHBOARDS_... are a special case as they have to be crafted a bit based on a few variables +set +e + +if [[ "${OPENSEARCH_PRIMARY:-opensearch-local}" == "elasticsearch-remote" ]]; then + ln -sf "$NGINX_KIBANA_IDARK2DASH_REWRITE_CONF" "$NGINX_RUNTIME_IDARK2DASH_REWRITE_CONF" + ln -sf "$NGINX_KIBANA_DASHBOARDS_REWRITE_CONF" "$NGINX_RUNTIME_DASHBOARDS_REWRITE_CONF" +else + ln -sf "$NGINX_DASHBOARDS_IDARK2DASH_REWRITE_CONF" "$NGINX_RUNTIME_IDARK2DASH_REWRITE_CONF" + ln -sf "$NGINX_DASHBOARDS_DASHBOARDS_REWRITE_CONF" "$NGINX_RUNTIME_DASHBOARDS_REWRITE_CONF" +fi + +# first parse DASHBOARDS_URL and assign the resultant urlsplit named tuple to an associative array +# going to use Python to do so as urllib will do a better job at parsing DASHBOARDS_URL than bash +DASHBOARDS_URL_PARSED="$( ( /usr/bin/env python3 -c "import sys; import json; from urllib.parse import urlsplit; [ sys.stdout.write(json.dumps(urlsplit(line)._asdict()) + '\n') for line in sys.stdin ]" 2>/dev/null <<< "${DASHBOARDS_URL:-http://dashboards:5601/dashboards}" ) | head -n 1 )" +declare -A DASHBOARDS_URL_DICT +for KEY in $(jq -r 'keys[]' 2>/dev/null <<< $DASHBOARDS_URL_PARSED); do + DASHBOARDS_URL_DICT["$KEY"]=$(jq -r ".$KEY" 2>/dev/null <<< $DASHBOARDS_URL_PARSED) +done + +# the "path" from the parsed URL is the dashboards prefix +[[ -z "${NGINX_DASHBOARDS_PREFIX:-}" ]] && \ + [[ -v DASHBOARDS_URL_DICT[path] ]] && \ + NGINX_DASHBOARDS_PREFIX="${DASHBOARDS_URL_DICT[path]}" +# if we failed to get it, use the default +[[ -z "${NGINX_DASHBOARDS_PREFIX:-}" ]] && \ + [[ "${OPENSEARCH_PRIMARY:-opensearch-local}" != "elasticsearch-remote" ]] && \ + NGINX_DASHBOARDS_PREFIX=/dashboards + +# the "path" from the parsed URL is the dashboards prefix +if [[ -z "${NGINX_DASHBOARDS_PROXY_PASS:-}" ]]; then + # if Malcolm is running in anything other than "elasticsearch-remote" mode, then + # the dashboards service is already defined in the upstream + if [[ "${OPENSEARCH_PRIMARY:-opensearch-local}" == "elasticsearch-remote" ]] && [[ -v DASHBOARDS_URL_DICT[scheme] ]] && [[ -v DASHBOARDS_URL_DICT[netloc] ]]; then + NGINX_DASHBOARDS_PROXY_PASS="${DASHBOARDS_URL_DICT[scheme]}://${DASHBOARDS_URL_DICT[netloc]}" + else + NGINX_DASHBOARDS_PROXY_PASS=http://dashboards + fi +fi +# if we failed to get it, use the default +[[ -z "${NGINX_DASHBOARDS_PROXY_PASS:-}" ]] && \ + [[ "${OPENSEARCH_PRIMARY:-opensearch-local}" != "elasticsearch-remote" ]] && \ + NGINX_DASHBOARDS_PROXY_PASS=http://dashboards + +export NGINX_DASHBOARDS_PREFIX +export NGINX_DASHBOARDS_PROXY_PASS +export NGINX_DASHBOARDS_PROXY_URL="$(echo "$(echo "$NGINX_DASHBOARDS_PROXY_PASS" | sed 's@/$@@')/$(echo "$NGINX_DASHBOARDS_PREFIX" | sed 's@^/@@')" | sed 's@/$@@')" + +# now process the environment variable substitutions +for TEMPLATE in "$NGINX_TEMPLATES_DIR"/*.conf.template; do + DOLLAR=$ envsubst < "$TEMPLATE" > "$NGINX_CONFD_DIR/$(basename "$TEMPLATE"| sed 's/\.template$//')" +done + +set -e + +# insert some build and runtime information into the landing page +if [[ -f "${NGINX_LANDING_INDEX_HTML}" ]]; then + if [[ "${OPENSEARCH_PRIMARY:-opensearch-local}" == "elasticsearch-remote" ]]; then + MALCOLM_DASHBOARDS_NAME=Kibana + MALCOLM_DASHBOARDS_URL="$NGINX_DASHBOARDS_PROXY_URL" + MALCOLM_DASHBOARDS_ICON=elastic.svg + else + MALCOLM_DASHBOARDS_NAME=Dashboards + MALCOLM_DASHBOARDS_URL="$(echo "$NGINX_DASHBOARDS_PREFIX" | sed 's@/$@@')/" + MALCOLM_DASHBOARDS_ICON=opensearch_mark_default.svg + fi + sed -i "s@MALCOLM_DASHBOARDS_NAME_REPLACER@${MALCOLM_DASHBOARDS_NAME}@g" "${NGINX_LANDING_INDEX_HTML}" + sed -i "s@MALCOLM_DASHBOARDS_URL_REPLACER@${MALCOLM_DASHBOARDS_URL}@g" "${NGINX_LANDING_INDEX_HTML}" + sed -i "s@MALCOLM_DASHBOARDS_ICON_REPLACER@${MALCOLM_DASHBOARDS_ICON}@g" "${NGINX_LANDING_INDEX_HTML}" + sed -i "s/MALCOLM_VERSION_REPLACER/v${MALCOLM_VERSION:-unknown} (${VCS_REVISION:-} @ ${BUILD_DATE:-})/g" "${NGINX_LANDING_INDEX_HTML}" +fi +# some cleanup, if necessary rm -rf /var/log/nginx/* || true # start supervisor (which will spawn nginx, stunnel, etc.) or whatever the default command is diff --git a/nginx/templates/01_template_variables.conf.template b/nginx/templates/01_template_variables.conf.template new file mode 100644 index 000000000..99cd2ad81 --- /dev/null +++ b/nginx/templates/01_template_variables.conf.template @@ -0,0 +1,19 @@ +map ${DOLLAR}host ${DOLLAR}sessions_index { + default "$MALCOLM_NETWORK_INDEX_PATTERN"; +} + +map ${DOLLAR}host ${DOLLAR}time_field { + default "$MALCOLM_NETWORK_INDEX_TIME_FIELD"; +} + +map ${DOLLAR}host ${DOLLAR}dashboards_prefix { + default "$NGINX_DASHBOARDS_PREFIX"; +} + +map ${DOLLAR}host ${DOLLAR}dashboards_proxy_pass { + default "$NGINX_DASHBOARDS_PROXY_PASS"; +} + +map ${DOLLAR}host ${DOLLAR}dashboards_proxy_url { + default "$NGINX_DASHBOARDS_PROXY_URL"; +} diff --git a/scripts/build.sh b/scripts/build.sh index 526ab83c3..7b82f9289 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -34,22 +34,21 @@ fi if [[ -f "$1" ]]; then CONFIG_FILE="$1" - DOCKER_COMPOSE_COMMAND="${DOCKER_COMPOSE_BIN[@]} --profile malcolm -f "$CONFIG_FILE"" shift # use remainder of arguments for services else - CONFIG_FILE="docker-compose.yml" - DOCKER_COMPOSE_COMMAND="${DOCKER_COMPOSE_BIN[@]} --profile malcolm" + CONFIG_FILE="docker-compose-dev.yml" fi +DOCKER_COMPOSE_COMMAND="${DOCKER_COMPOSE_BIN[@]} --profile malcolm -f "$CONFIG_FILE"" function filesize_in_image() { FILESPEC="$2" - IMAGE="$($GREP -P "^\s+image:.*$1" docker-compose-standalone.yml | awk '{print $2}' | sort -u)" + IMAGE="$($GREP -P "^\s+image:.*$1" "$CONFIG_FILE" | awk '{print $2}' | sort -u)" $DOCKER_BIN run --rm --pull never --entrypoint /bin/sh "$IMAGE" -c "stat --printf='%s' \"$FILESPEC\" 2>/dev/null || stat -c '%s' \"$FILESPEC\" 2>/dev/null" } function dirsize_in_image() { FILESPEC="$2" - IMAGE="$($GREP -P "^\s+image:.*$1" docker-compose-standalone.yml | awk '{print $2}' | sort -u)" + IMAGE="$($GREP -P "^\s+image:.*$1" "$CONFIG_FILE" | awk '{print $2}' | sort -u)" KBYTES="$($DOCKER_BIN run --rm --pull never --entrypoint /bin/sh "$IMAGE" -c "du -sk \"$FILESPEC\" 2>/dev/null | cut -f1")" echo $(($KBYTES * 1024)) } diff --git a/scripts/control.py b/scripts/control.py index caf0d0157..8c8624850 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -8,6 +8,7 @@ sys.dont_write_bytecode = True import argparse +import datetime import errno import fileinput import getpass @@ -126,6 +127,11 @@ def __exit__(self, *args): yamlImported = None dotenvImported = None MaxAskForValueCount = 100 +UsernameRegex = re.compile(r'^[a-zA-Z][a-zA-Z0-9_\-.]+$') +UsernameMinLen = 4 +UsernameMaxLen = 32 +PasswordMinLen = 8 +PasswordMaxLen = 128 ################################################################################################### try: @@ -145,27 +151,39 @@ def shutdown_handler(signum, frame): ################################################################################################### -def checkEnvFilesExist(): +def checkEnvFilesAndValues(): global args - - # first, if the configDir is completely empty, then populate from defaults - defaultConfigDir = os.path.join(MalcolmPath, 'config') - if ( - (args.configDir is not None) - and os.path.isdir(args.configDir) - and os.path.isdir(defaultConfigDir) - and (not same_file_or_dir(defaultConfigDir, args.configDir)) - and (not os.listdir(args.configDir)) - ): - for defaultEnvExampleFile in glob.glob(os.path.join(defaultConfigDir, '*.env.example')): - shutil.copy2(defaultEnvExampleFile, args.configDir) + global dotenvImported # if a specific config/*.env file doesn't exist, use the *.example.env files as defaults - envExampleFiles = glob.glob(os.path.join(args.configDir, '*.env.example')) - for envExampleFile in envExampleFiles: - envFile = envExampleFile[: -len('.example')] - if not os.path.isfile(envFile): - shutil.copyfile(envExampleFile, envFile) + if os.path.isdir(examplesConfigDir := os.path.join(MalcolmPath, 'config')): + for envExampleFile in glob.glob(os.path.join(examplesConfigDir, '*.env.example')): + envFile = os.path.join(args.configDir, os.path.basename(envExampleFile[: -len('.example')])) + if not os.path.isfile(envFile): + if args.debug: + eprint(f"Creating {envFile} from {os.path.basename(envExampleFile)}") + shutil.copyfile(envExampleFile, envFile) + + # now, example the .env and .env.example file for individual values, and create any that are + # in the .example file but missing in the .env file + for envFile in glob.glob(os.path.join(args.configDir, '*.env')): + envExampleFile = os.path.join(examplesConfigDir, os.path.basename(envFile) + '.example') + if os.path.isfile(envExampleFile): + envValues = dotenvImported.dotenv_values(envFile) + exampleValues = dotenvImported.dotenv_values(envExampleFile) + missingVars = list(set(exampleValues.keys()).difference(set(envValues.keys()))) + if missingVars: + if args.debug: + eprint(f"Missing {missingVars} in {envFile} from {os.path.basename(envExampleFile)}") + with open(envFile, "a") as envFileHandle: + print('', file=envFileHandle) + print('', file=envFileHandle) + print( + f'# missing variables created from {os.path.basename(envExampleFile)} at {str(datetime.datetime.now())}', + file=envFileHandle, + ) + for missingVar in missingVars: + print(f"{missingVar}={exampleValues[missingVar]}", file=envFileHandle) ################################################################################################### @@ -368,9 +386,9 @@ def keystore_op(service, dropPriv=False, *keystore_args, **run_process_kwargs): service, args.namespace, [x for x in cmd if x], - stdin=run_process_kwargs['stdin'] - if ('stdin' in run_process_kwargs and run_process_kwargs['stdin']) - else None, + stdin=( + run_process_kwargs['stdin'] if ('stdin' in run_process_kwargs and run_process_kwargs['stdin']) else None + ), ) err = 0 if all([deep_get(v, ['err'], 1) == 0 for k, v in podsResults.items()]) else 1 @@ -512,6 +530,7 @@ def netboxBackup(backupFileName=None): elif orchMode is OrchestrationFramework.KUBERNETES: if podsResults := PodExec( service='netbox-postgres', + container='netbox-postgres-container', namespace=args.namespace, command=[ 'pg_dump', @@ -632,18 +651,21 @@ def netboxRestore(backupFileName=None): elif orchMode is OrchestrationFramework.KUBERNETES: # copy database backup and media backup to remote temporary directory try: + service_name = "netbox" + container_name = "netbox-container" tmpRestoreDir = '/tmp' tmpRestoreFile = os.path.join( tmpRestoreDir, os.path.splitext(os.path.basename(backupFileName))[0] + '.txt' ) with gzip.open(backupFileName, 'rt') as f: if podsResults := PodExec( - service='netbox', + service=service_name, namespace=args.namespace, command=['tee', tmpRestoreFile], stdout=False, stderr=True, stdin=f.read(), + container=container_name, ): err = 0 if all([deep_get(v, ['err'], 1) == 0 for k, v in podsResults.items()]) else 1 results = list(chain(*[deep_get(v, ['output'], '') for k, v in podsResults.items()])) @@ -657,7 +679,7 @@ def netboxRestore(backupFileName=None): # perform the restore inside the container if podsResults := PodExec( - service='netbox', + service=service_name, namespace=args.namespace, command=[ '/opt/netbox/venv/bin/python', @@ -665,6 +687,7 @@ def netboxRestore(backupFileName=None): '--preload-backup', tmpRestoreFile, ], + container=container_name, ): err = 0 if all([deep_get(v, ['err'], 1) == 0 for k, v in podsResults.items()]) else 1 results = list(chain(*[deep_get(v, ['output'], '') for k, v in podsResults.items()])) @@ -679,13 +702,14 @@ def netboxRestore(backupFileName=None): finally: # cleanup on other side PodExec( - service='netbox', + service=service_name, namespace=args.namespace, command=[ 'bash', '-c', f"rm -f {tmpRestoreDir}/{os.path.splitext(backupFileName)[0]}*", ], + container=container_name, ) else: @@ -744,9 +768,11 @@ def logs(): "--color", 'auto' if coloramaImported else 'never', "--template", - '{{.Namespace}}/{{color .PodColor .PodName}}/{{color .ContainerColor .ContainerName}} | {{.Message}}{{"\\n"}}' - if args.debug - else '{{color .ContainerColor .ContainerName}} | {{.Message}}{{"\\n"}}', + ( + '{{.Namespace}}/{{color .PodColor .PodName}}/{{color .ContainerColor .ContainerName}} | {{.Message}}{{"\\n"}}' + if args.debug + else '{{color .ContainerColor .ContainerName}} | {{.Message}}{{"\\n"}}' + ), '--tail', str(args.logLineCount) if args.logLineCount else '-1', ] @@ -1224,28 +1250,28 @@ def authSetup(): loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid administrator username') while loopBreaker.increment(): username = AskForString( - "Administrator username", + f"Administrator username (between {UsernameMinLen} and {UsernameMaxLen} characters; alphanumeric, _, -, and . allowed)", default=args.authUserName, defaultBehavior=defaultBehavior, ) - if len(username) > 0: + if UsernameRegex.match(username) and (UsernameMinLen <= len(username) <= UsernameMaxLen): break loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid password') while (not args.cmdAuthSetupNonInteractive) and loopBreaker.increment(): password = AskForPassword( - f"{username} password: ", + f"{username} password (between {PasswordMinLen} and {PasswordMaxLen} characters): ", default='', defaultBehavior=defaultBehavior, ) - passwordConfirm = AskForPassword( - f"{username} password (again): ", - default='', - defaultBehavior=defaultBehavior, - ) - if password and (password == passwordConfirm): - break - eprint("Passwords do not match") + if (PasswordMinLen <= len(password) <= PasswordMaxLen): + passwordConfirm = AskForPassword( + f"{username} password (again): ", + default='', + defaultBehavior=defaultBehavior, + ) + if password and (password == passwordConfirm): + break # get previous admin username to remove from htpasswd file if it's changed authEnvFile = os.path.join(args.configDir, 'auth.env') @@ -1375,12 +1401,12 @@ def authSetup(): f.write(f'admin_user = {username}\n\n') f.write('; username field quality checks\n') f.write(';\n') - f.write('min_username_len = 4\n') - f.write('max_username_len = 32\n\n') + f.write(f'min_username_len = {UsernameMinLen}\n') + f.write(f'max_username_len = {UsernameMaxLen}\n\n') f.write('; Password field quality checks\n') f.write(';\n') - f.write('min_password_len = 8\n') - f.write('max_password_len = 128\n\n') + f.write(f'min_password_len = {PasswordMinLen}\n') + f.write(f'max_password_len = {PasswordMaxLen}\n\n') # touch the metadata file open(os.path.join(MalcolmPath, os.path.join('htadmin', 'metadata')), 'a').close() @@ -2350,7 +2376,7 @@ def main(): # the compose file references various .env files in just about every operation this script does, # so make sure they exist right off the bat - checkEnvFilesExist() + checkEnvFilesAndValues() # stop Malcolm (and wipe data if requestsed) if args.cmdRestart or args.cmdStop or args.cmdWipe: diff --git a/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh b/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh index 16a18cd6a..33845d9e4 100755 --- a/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh +++ b/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh @@ -543,7 +543,7 @@ function InstallMalcolm { CONFIRMATION=$(_GetConfirmation "Clone and setup Malcolm [Y/n]?" Y) if [[ $CONFIRMATION =~ ^[Yy] ]]; then - if _GitClone https://github.com/idaholab/Malcolm "$MALCOLM_PATH"; then + if _GitClone https://github.com/cisagov/Malcolm "$MALCOLM_PATH"; then pushd "$MALCOLM_PATH" >/dev/null 2>&1 python3 ./scripts/install.py -c -d CONFIG_PAIRS=( @@ -570,13 +570,13 @@ function InstallMalcolm { for i in ${CONFIG_PAIRS[@]}; do KEY="$(echo "$i" | cut -d':' -f1)" VALUE="$(echo "$i" | cut -d':' -f2)" - for CONFIG in docker-compose.yml docker-compose-standalone.yml; do + for CONFIG in docker-compose-dev.yml docker-compose.yml; do sed -i "s/\(^[[:space:]]*$KEY[[:space:]]*:[[:space:]]*\).*/\1$VALUE/g" "$CONFIG" done done mkdir -p ./config touch ./config/auth.env - grep image: docker-compose-standalone.yml | awk '{print $2}' | sort -u | xargs -l -r $SUDO_CMD docker pull + grep image: docker-compose.yml | awk '{print $2}' | sort -u | xargs -l -r $SUDO_CMD docker pull echo "Please run $MALCOLM_PATH/scripts/auth_setup to complete configuration" >&2 popd >/dev/null 2>&1 fi @@ -591,8 +591,8 @@ function InstallMalcolm { if [[ $CONFIRMATION =~ ^[Yy] ]]; then ((echo 'SHELL=/bin/bash') ; \ (( crontab -l | grep . | grep -v ^SHELL= ; \ - echo "@reboot sleep 60 && /bin/bash --login $LOCAL_BIN_PATH/reset_and_auto_populate.sh -r -o -n -m $MALCOLM_PATH/docker-compose-standalone.yml" ; \ - echo "15 8 * * * /bin/bash --login $LOCAL_BIN_PATH/reset_and_auto_populate.sh -w -o -n -m $MALCOLM_PATH/docker-compose-standalone.yml -d yesterday $ARTIFACTS_PATH/*.pcap" ) \ + echo "@reboot sleep 60 && /bin/bash --login $LOCAL_BIN_PATH/reset_and_auto_populate.sh -r -o -n -m $MALCOLM_PATH/docker-compose.yml" ; \ + echo "15 8 * * * /bin/bash --login $LOCAL_BIN_PATH/reset_and_auto_populate.sh -w -o -n -m $MALCOLM_PATH/docker-compose.yml -d yesterday $ARTIFACTS_PATH/*.pcap" ) \ | sort | uniq )) | crontab - fi fi @@ -602,8 +602,6 @@ function InstallMalcolm { mkdir -p "$ARTIFACTS_PATH" pushd "$ARTIFACTS_PATH" >/dev/null 2>&1 curl -sSL -J -O https://malcolm.fyi/examples/Cyberville.pcap - curl -sSL -J -O https://malcolm.fyi/examples/net-map.json - cp -f ./net-map.json "$MALCOLM_PATH"/ popd >/dev/null 2>&1 fi } diff --git a/scripts/install.py b/scripts/install.py index 95f307c8c..5e01049b9 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -645,7 +645,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): f'Enter Kibana connection URL (e.g., https://192.168.1.123:5601)', default=args.dashboardsUrl, ) - if malcolmProfile != PROFILE_MALCOLM: loopBreaker = CountUntilException(MaxAskForValueCount, f'Invalid Logstash host and port') logstashHost = '' @@ -1011,6 +1010,69 @@ def tweak_malcolm_runtime(self, malcolm_install_path): indexPruneNameSort = False arkimeManagePCAP = False arkimeFreeSpaceG = '10%' + indexManagementPolicy = False + indexManagementHotWarm = False + indexManagementOptimizationTimePeriod = '30d' + indexManagementSpiDataRetention = '90d' + indexManagementReplicas = 1 + indexManagementHistoryInWeeks = 13 + indexManagementOptimizeSessionSegments = 1 + + loopBreaker = CountUntilException( + MaxAskForValueCount, + f'Invalid ILM/ISM setting(s)', + ) + indexManagementPolicy = InstallerYesOrNo( + f'Enable index management policies (ILM/ISM) in Arkime?', default=args.indexManagementPolicy + ) + if indexManagementPolicy: + while loopBreaker.increment(): + # Set 'hot' for 'node.attr.molochtype' on new indices, warm on non sessions indices + indexManagementHotWarm = InstallerYesOrNo( + f'Should Arkime use a hot/warm design in which non-session data is stored in a warm index?', + default=args.indexManagementHotWarm, + ) + if indexManagementHotWarm: + if opensearchPrimaryMode == DatabaseMode.ElasticsearchRemote: + InstallerDisplayMessage( + f'You must configure "hot" and "warm" nodes types in the remote Elasticsearch instance (https://arkime.com/faq#ilm)' + ) + else: + InstallerDisplayMessage( + f'You must configure "hot" and "warm" nodes types in the OpenSearch instance' + ) + # Time in hours/days before (moving Arkime indexes to warm) and force merge (number followed by h or d), default 30d + indexManagementOptimizationTimePeriod = InstallerAskForString( + "How long should Arkime keep an index in the hot node? (e.g. 25h, 5d, etc.)", + default=args.indexManagementOptimizationTimePeriod, + ) + # Time in hours/days before deleting Arkime indexes (number followed by h or d), default 90d + indexManagementSpiDataRetention = InstallerAskForString( + "How long should Arkime retain SPI data before deleting it? (e.g. 25h, 90d, etc.)", + default=str(args.indexManagementSpiDataRetention), + ) + # Number of segments to optimize sessions to in the ILM policy, default 1 + indexManagementOptimizeSessionSegments = InstallerAskForString( + "How many segments should Arkime use to optimize?", + default=str(args.indexManagementOptimizeSessionSegments), + ) + # Number of replicas for older sessions indices in the ILM policy, default 0 + indexManagementReplicas = InstallerAskForString( + "How many replicas should Arkime maintain for older session indices?", + default=str(args.indexManagementReplicas), + ) + # Number of weeks of history to keep, default 13 + indexManagementHistoryInWeeks = InstallerAskForString( + "How many weeks of history should Arkime keep?", default=str(args.indexManagementHistoryInWeeks) + ) + if ( + (re.match(r"\d+(h|d)", indexManagementOptimizationTimePeriod)) + and (re.match(r"\d+(h|d)", indexManagementSpiDataRetention)) + and str(indexManagementOptimizeSessionSegments).isdigit() + and str(indexManagementReplicas).isdigit() + and str(indexManagementHistoryInWeeks).isdigit() + ): + break if InstallerYesOrNo( 'Should Malcolm delete the oldest database indices and/or PCAP files based on available storage?' @@ -1465,6 +1527,48 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'ARKIME_AUTO_ANALYZE_PCAP_FILES', TrueOrFalseNoQuote(autoArkime), ), + # Should Arkime use an ILM policy? + EnvValue( + os.path.join(args.configDir, 'arkime.env'), + 'INDEX_MANAGEMENT_ENABLED', + TrueOrFalseNoQuote(indexManagementPolicy), + ), + # Should Arkime use a hot/warm design in which non-session data is stored in a warm index? (see https://https://arkime.com/faq#ilm) + EnvValue( + os.path.join(args.configDir, 'arkime.env'), + 'INDEX_MANAGEMENT_HOT_WARM_ENABLED', + TrueOrFalseNoQuote(indexManagementHotWarm), + ), + # Time in hours/days before moving (Arkime indexes to warm) and force merge (number followed by h or d), default 30 + EnvValue( + os.path.join(args.configDir, 'arkime.env'), + 'INDEX_MANAGEMENT_OPTIMIZATION_PERIOD', + indexManagementOptimizationTimePeriod, + ), + # Time in hours/days before deleting Arkime indexes (number followed by h or d), default 90 + EnvValue( + os.path.join(args.configDir, 'arkime.env'), + 'INDEX_MANAGEMENT_RETENTION_TIME', + indexManagementSpiDataRetention, + ), + # Number of replicas for older sessions indices in the ILM policy, default 0 + EnvValue( + os.path.join(args.configDir, 'arkime.env'), + 'INDEX_MANAGEMENT_OLDER_SESSION_REPLICAS', + indexManagementReplicas, + ), + # Number of weeks of history to keep, default 13 + EnvValue( + os.path.join(args.configDir, 'arkime.env'), + 'INDEX_MANAGEMENT_HISTORY_RETENTION_WEEKS', + indexManagementHistoryInWeeks, + ), + # Number of segments to optimize sessions to in the ILM policy, default 1 + EnvValue( + os.path.join(args.configDir, 'arkime.env'), + 'INDEX_MANAGEMENT_SEGMENTS', + indexManagementOptimizeSessionSegments, + ), # authentication method: basic (true), ldap (false) or no_authentication EnvValue( os.path.join(args.configDir, 'auth-common.env'), @@ -3567,6 +3671,71 @@ def main(): default='', help=f'Delete the oldest indices when the database exceeds this threshold (e.g., 250GB, 1TB, 60٪, etc.)', ) + storageArgGroup.add_argument( + '--index-management-enable', + dest='indexManagementPolicy', + type=str2bool, + metavar="true|false", + nargs='?', + const=True, + default=False, + help="Enable index management policies (ILM/ISM) in Arkime? (see https://https://arkime.com/faq#ilm)", + ) + storageArgGroup.add_argument( + '--index-management-hot-warm-enable', + dest='indexManagementHotWarm', + type=str2bool, + metavar="true|false", + nargs='?', + const=True, + default=False, + help="Should Arkime use a hot/warm design in which non-session data is stored in a warm index?", + ) + storageArgGroup.add_argument( + '--index-management-optimization-time-period', + dest='indexManagementOptimizationTimePeriod', + required=False, + metavar='', + type=str, + default='30d', + help=f'Time in hours/days before (moving Arkime indexes to warm) and force merge (number followed by h or d), default 30d', + ) + storageArgGroup.add_argument( + '--index-management-spi-data-retention', + dest='indexManagementSpiDataRetention', + required=False, + metavar='', + type=str, + default='90d', + help=f'Time in hours/days before deleting Arkime indexes (number followed by h or d), default 90d', + ) + storageArgGroup.add_argument( + '--index-management-replicas', + dest='indexManagementReplicas', + required=False, + metavar='', + type=int, + default=0, + help='Number of replicas for older sessions indices in the ILM/ISM policy, default 0', + ) + storageArgGroup.add_argument( + '--index-management-weeks-of-history', + dest='indexManagementHistoryInWeeks', + required=False, + metavar='', + type=int, + default=13, + help='Number of weeks of history to keep, default 13', + ) + storageArgGroup.add_argument( + '--index-management-segments', + dest='indexManagementOptimizeSessionSegments', + required=False, + metavar='', + type=int, + default=1, + help='Number of segments to optimize sessions to in the ILM/ISM policy, default 1', + ) analysisArgGroup = parser.add_argument_group('Analysis options') analysisArgGroup.add_argument( diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index d4fb0f795..77be27b6b 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -95,7 +95,7 @@ if mkdir "$DESTDIR"; then mkdir $VERBOSE -p "$DESTDIR/zeek/intel/STIX/" cp $VERBOSE ./config/*.example "$DESTDIR/config/" - cp $VERBOSE ./docker-compose-standalone.yml "$DESTDIR/docker-compose.yml" + cp $VERBOSE ./docker-compose.yml "$DESTDIR/docker-compose.yml" cp $VERBOSE ./net-map.json "$DESTDIR/" cp $VERBOSE ./scripts/install.py "$DESTDIR/scripts/" cp $VERBOSE ./scripts/control.py "$DESTDIR/scripts/" @@ -171,14 +171,7 @@ if mkdir "$DESTDIR"; then echo " - wipe (stop Malcolm and clear its database)" | tee -a "$README" echo " - auth_setup (change authentication-related settings)" | tee -a "$README" echo "" | tee -a "$README" - echo "A minute or so after starting Malcolm, the following services will be accessible:" | tee -a "$README" - echo " - Arkime: https://localhost/" | tee -a "$README" - echo " - OpenSearch Dashboards: https://localhost/dashboards/" | tee -a "$README" - echo " - PCAP upload (web): https://localhost/upload/" | tee -a "$README" - echo " - PCAP upload (sftp): sftp://USERNAME@127.0.0.1:8022/files/" | tee -a "$README" - echo " - NetBox: https://localhost/netbox/" | tee -a "$README" - echo " - Account management: https://localhost/auth/" | tee -a "$README" - echo " - Documentation: https://localhost/readme/" | tee -a "$README" + echo "Malcolm services can be accessed at https:///" | tee -a "$README" popd >/dev/null 2>&1 popd >/dev/null 2>&1 popd >/dev/null 2>&1 diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index 7643f58d5..c79c3a143 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -452,9 +452,9 @@ def PodExec( stdin=None, timeout=180, maxPodsToExec=1, + container=None, ): results = {} - if namespace and (kubeImported := KubernetesDynamic()) and (client := kubeImported.client.CoreV1Api()): podsNames = GetPodNamesForService(service, namespace) @@ -469,17 +469,31 @@ def PodExec( ) if resp.status.phase != 'Pending': break - resp = kubeImported.stream.stream( - client.connect_get_namespaced_pod_exec, - podName, - namespace, - command=get_iterable(command), - stdout=stdout, - stderr=stderr, - stdin=stdin is not None, - tty=False, - _preload_content=False, - ) + if container: + resp = kubeImported.stream.stream( + client.connect_get_namespaced_pod_exec, + podName, + namespace, + container=container, + command=get_iterable(command), + stdout=stdout, + stderr=stderr, + stdin=stdin is not None, + tty=False, + _preload_content=False, + ) + else: + resp = kubeImported.stream.stream( + client.connect_get_namespaced_pod_exec, + podName, + namespace, + command=get_iterable(command), + stdout=stdout, + stderr=stderr, + stdin=stdin is not None, + tty=False, + _preload_content=False, + ) rawOutput = StringIO('') rawErrput = StringIO('') stdinRemaining = ( @@ -847,12 +861,12 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): # apply the manifests in this YAML file, otherwise skip it if containerBelongsInProfile: try: - results_dict['create_from_yaml']['result'][ - os.path.basename(yamlName) - ] = kubeImported.utils.create_from_yaml( - apiClient, - yamlName, - namespace=namespace, + results_dict['create_from_yaml']['result'][os.path.basename(yamlName)] = ( + kubeImported.utils.create_from_yaml( + apiClient, + yamlName, + namespace=namespace, + ) ) except kubeImported.client.rest.ApiException as x: if x.status != 409: diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index b95ebc41b..2515deac3 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -2,8 +2,8 @@ "aws_access_key": "XXXXXXXXXXXXXXXXXXXX", "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", - "malcolm_tag": "v24.01.0", - "malcolm_repo": "idaholab/Malcolm", + "malcolm_tag": "v24.02.0", + "malcolm_repo": "cisagov/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", "vpc_region": "us-east-1", diff --git a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh index dcc032fe8..24af9a640 100755 --- a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh +++ b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh @@ -31,7 +31,7 @@ fi # -t tag (Malcolm tag, e.g., v23.05.1) # -u UID (user UID, e.g., 1000) VERBOSE_FLAG= -MALCOLM_REPO=${MALCOLM_REPO:-idaholab/Malcolm} +MALCOLM_REPO=${MALCOLM_REPO:-cisagov/Malcolm} MALCOLM_TAG=${MALCOLM_TAG:-v23.10.0} [[ -z "$MALCOLM_UID" ]] && ( [[ $EUID -eq 0 ]] && MALCOLM_UID=1000 || MALCOLM_UID="$(id -u)" ) while getopts 'vr:t:u:' OPTION; do @@ -212,9 +212,8 @@ function InstallMalcolm { pushd "$MALCOLM_USER_HOME" >/dev/null 2>&1 mkdir -p ./Malcolm curl -fsSL "$MALCOLM_URL" | tar xzf - -C ./Malcolm --strip-components 1 - if [[ -s ./Malcolm/docker-compose-standalone.yml ]]; then + if [[ -s ./Malcolm/docker-compose.yml ]]; then pushd ./Malcolm >/dev/null 2>&1 - mv docker-compose-standalone.yml docker-compose.yml for ENVEXAMPLE in ./config/*.example; do ENVFILE="${ENVEXAMPLE%.*}"; cp "$ENVEXAMPLE" "$ENVFILE"; done echo "Pulling Docker images..." >&2 docker-compose --profile malcolm pull >/dev/null 2>&1 @@ -261,13 +260,7 @@ To start, stop, restart, etc. Malcolm: - wipe (stop Malcolm and clear its database) - auth_setup (change authentication-related settings) -A minute or so after starting Malcolm, the following services will be accessible: - - Arkime: https:/// - - OpenSearch Dashboards: https:///dashboards/ - - PCAP upload (web): https:///upload/ - - NetBox: https:///netbox/ - - Account management: https:///auth/ - - Documentation: https:///readme/ +Malcolm services can be accessed at https:/// EOT fi diff --git a/scripts/third-party-environments/virter/malcolm-setup-02-clone-install.toml b/scripts/third-party-environments/virter/malcolm-setup-02-clone-install.toml index 148577384..45df6b399 100644 --- a/scripts/third-party-environments/virter/malcolm-setup-02-clone-install.toml +++ b/scripts/third-party-environments/virter/malcolm-setup-02-clone-install.toml @@ -1,7 +1,7 @@ version = 1 [env] -MALCOLM_REPO_OWNER = "idaholab" +MALCOLM_REPO_OWNER = "cisagov" MALCOLM_REPO_NAME = "Malcolm" MALCOLM_REPO_BRANCH = "main" @@ -94,13 +94,7 @@ To start, stop, restart, etc. Malcolm: - wipe (stop Malcolm and clear its database) - auth_setup (change authentication-related settings) -A minute or so after starting Malcolm, the following services will be accessible: - - Arkime: https:/// - - OpenSearch Dashboards: https:///dashboards/ - - PCAP upload (web): https:///upload/ - - NetBox: https:///netbox/ - - Account management: https:///auth/ - - Documentation: https:///readme/ +Malcolm services can be accessed at https:/// EOT fi diff --git a/scripts/third-party-environments/virter/malcolm-virter.sh b/scripts/third-party-environments/virter/malcolm-virter.sh index 5a3cc5999..bf73608b4 100755 --- a/scripts/third-party-environments/virter/malcolm-virter.sh +++ b/scripts/third-party-environments/virter/malcolm-virter.sh @@ -7,7 +7,7 @@ ENCODING="utf-8" SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -MALCOLM_REPO_OWNER=idaholab +MALCOLM_REPO_OWNER=cisagov MALCOLM_REPO_NAME=Malcolm MALCOLM_REPO_BRANCH=main GITHUB_TOKEN=${GITHUB_TOKEN:-} diff --git a/scripts/zeek_script_to_malcolm_boilerplate.py b/scripts/zeek_script_to_malcolm_boilerplate.py index f254099e0..7375ccc8f 100755 --- a/scripts/zeek_script_to_malcolm_boilerplate.py +++ b/scripts/zeek_script_to_malcolm_boilerplate.py @@ -10,9 +10,9 @@ # The scripts are parsed into their constitutent records and &log fields. # # Each record is then printed out in the formats used by Malcolm for parsing and defining Zeek logs: -# - Logstash (https://idaholab.github.io/Malcolm/docs/contributing-logstash.html#LogstashZeek), for ./logstash/pipelines/zeek/11_zeek_parse.conf -# - Arkime (https://idaholab.github.io/Malcolm/docs/contributing-new-log-fields.html#NewFields), for ./arkime/etc/config.ini -# - OpenSearch tndex templates (https://idaholab.github.io/Malcolm/docs/contributing-new-log-fields.html#NewFields), for ./dashboards/templates/composable/component/zeek*.json +# - Logstash (https://cisagov.github.io/Malcolm/docs/contributing-logstash.html#LogstashZeek), for ./logstash/pipelines/zeek/11_zeek_parse.conf +# - Arkime (https://cisagov.github.io/Malcolm/docs/contributing-new-log-fields.html#NewFields), for ./arkime/etc/config.ini +# - OpenSearch tndex templates (https://cisagov.github.io/Malcolm/docs/contributing-new-log-fields.html#NewFields), for ./dashboards/templates/composable/component/zeek*.json # # For Logstash boilerplate, pay close attention to the comment in the logstash filter: # # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP @@ -170,7 +170,7 @@ def main(): description='\n'.join( [ 'Parse Zeek .script files and generate boilerplate for a Malcolm boilerplate for parsing and defining those Zeek logs.', - 'see https://idaholab.github.io/Malcolm/docs/contributing-guide.html', + 'see https://cisagov.github.io/Malcolm/docs/contributing-guide.html', ] ), formatter_class=argparse.RawTextHelpFormatter, diff --git a/sensor-iso/arkime/Dockerfile b/sensor-iso/arkime/Dockerfile index 80c6074ea..53a312ff0 100644 --- a/sensor-iso/arkime/Dockerfile +++ b/sensor-iso/arkime/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer="malcolm@inl.gov" ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "4.6.0" +ENV ARKIME_VERSION "5.0.0" ENV ARKIME_DIR "/opt/arkime" RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sources && \ @@ -23,6 +23,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour python3-pip \ python3-setuptools \ python3-wheel \ + re2c \ ruby \ ruby-dev \ rubygems \ diff --git a/sensor-iso/arkime/build-arkime-deb.sh b/sensor-iso/arkime/build-arkime-deb.sh index b556741ab..e1b43f15e 100755 --- a/sensor-iso/arkime/build-arkime-deb.sh +++ b/sensor-iso/arkime/build-arkime-deb.sh @@ -23,7 +23,7 @@ apt-get -q update cd /tmp git clone --recurse-submodules --branch="v$ARKIME_VERSION" "$ARKIME_URL" "./arkime-"$ARKIME_VERSION cd "./arkime-"$ARKIME_VERSION -for i in /opt/patches/*; do +for i in /opt/patches/*.patch; do patch -p 1 -r - --no-backup-if-mismatch < $i || true done @@ -31,14 +31,11 @@ export PATH="$ARKIME_DIR/bin:/tmp/arkime-$ARKIME_VERSION/node_modules/.bin:${PAT ./easybutton-build.sh --dir "$ARKIME_DIR" -npm -g config set user root - make install cp -r ./capture/plugins/lua/samples "$ARKIME_DIR"/lua -npm install license-checker -release/notice.txt.pl $ARKIME_DIR NOTICE release/CAPTURENOTICE > $ARKIME_DIR/NOTICE.txt +cat NOTICE release/CAPTURENOTICE > $ARKIME_DIR/NOTICE.txt ETC_FILES=$(shopt -s nullglob dotglob; echo /arkime-etc/*) if (( ${#ETC_FILES} )) ; then diff --git a/sensor-iso/build.sh b/sensor-iso/build.sh index a76a47c2e..201d09f78 100755 --- a/sensor-iso/build.sh +++ b/sensor-iso/build.sh @@ -5,7 +5,7 @@ IMAGE_PUBLISHER=cisagov IMAGE_VERSION=1.0.0 IMAGE_DISTRIBUTION=bookworm -BEATS_VER="8.11.4" +BEATS_VER="8.12.1" BEATS_OSS="-oss" BUILD_ERROR_CODE=1 @@ -115,14 +115,14 @@ if [ -d "$WORKDIR" ]; then rsync -a "$SCRIPT_PATH/suricata/" ./config/includes.chroot/opt/sensor/sensor_ctl/suricata/ # write out some version stuff specific to this installation version - echo "BUILD_ID=\"$(date +'%Y-%m-%d')-${IMAGE_VERSION}\"" > ./config/includes.chroot/opt/sensor/.os-info - echo "VARIANT=\"Hedgehog Linux (Sensor) v${IMAGE_VERSION}\"" >> ./config/includes.chroot/opt/sensor/.os-info - echo "VARIANT_ID=\"hedgehog-sensor\"" >> ./config/includes.chroot/opt/sensor/.os-info - echo "ID_LIKE=\"debian\"" >> ./config/includes.chroot/opt/sensor/.os-info - echo "HOME_URL=\"https://malcolm.fyi\"" >> ./config/includes.chroot/opt/sensor/.os-info - echo "DOCUMENTATION_URL=\"https://malcolm.fyi/hedgehog/\"" >> ./config/includes.chroot/opt/sensor/.os-info - echo "SUPPORT_URL=\"https://github.com/${IMAGE_PUBLISHER}\"" >> ./config/includes.chroot/opt/sensor/.os-info - echo "BUG_REPORT_URL=\"https://github.com/${IMAGE_PUBLISHER}/malcolm/issues\"" >> ./config/includes.chroot/opt/sensor/.os-info + echo "BUILD_ID=\"$(date +'%Y-%m-%d')-${IMAGE_VERSION}\"" > ./config/includes.chroot/opt/sensor/.os-info + echo "VARIANT=\"Hedgehog Linux (Sensor) v${IMAGE_VERSION}\"" >> ./config/includes.chroot/opt/sensor/.os-info + echo "VARIANT_ID=\"hedgehog-sensor\"" >> ./config/includes.chroot/opt/sensor/.os-info + echo "ID_LIKE=\"debian\"" >> ./config/includes.chroot/opt/sensor/.os-info + echo "HOME_URL=\"https://${IMAGE_PUBLISHER}.github.io/Malcolm\"" >> ./config/includes.chroot/opt/sensor/.os-info + echo "DOCUMENTATION_URL=\"https://${IMAGE_PUBLISHER}.github.io/Malcolm/docs/hedgehog.html\"" >> ./config/includes.chroot/opt/sensor/.os-info + echo "SUPPORT_URL=\"https://github.com/${IMAGE_PUBLISHER}\"" >> ./config/includes.chroot/opt/sensor/.os-info + echo "BUG_REPORT_URL=\"https://github.com/${IMAGE_PUBLISHER}/malcolm/issues\"" >> ./config/includes.chroot/opt/sensor/.os-info # environment variables to pass into chroot [[ -f "$SCRIPT_PATH/shared/environment.chroot" ]] && \ @@ -184,11 +184,8 @@ if [ -d "$WORKDIR" ]; then mv "$SCRIPT_PATH/arkime"/*.deb ./config/packages.chroot/ docker rmi -f arkime-build:latest - # clone and build Zeek .deb package in its own clean environment (rather than in hooks/) - bash "$SCRIPT_PATH/zeek/build-docker-image.sh" - docker run --rm -v "$SCRIPT_PATH"/zeek:/build zeek-build:latest -o /build -j "${BUILD_JOBS:-0}" - mv "$SCRIPT_PATH/zeek"/*.deb ./config/packages.chroot/ - docker rmi -f zeek-build:latest + # download Zeek .deb packages + bash "$SCRIPT_PATH/shared/bin/zeek-deb-download.sh" -o ./config/packages.chroot/ # reclaim some space docker system prune --volumes --force diff --git a/sensor-iso/build_via_vagrant.sh b/sensor-iso/build_via_vagrant.sh index c3db10092..73c557598 100755 --- a/sensor-iso/build_via_vagrant.sh +++ b/sensor-iso/build_via_vagrant.sh @@ -93,7 +93,7 @@ cp "$SCRIPT_PATH"/../scripts/malcolm_utils.py "$SCRIPT_PATH"/shared/bin/ mkdir "$SCRIPT_PATH"/suricata cp -r "$SCRIPT_PATH"/../suricata/rules-default "$SCRIPT_PATH"/suricata/ -YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-compose-standalone.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" +YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt [[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_LICENSE_KEY" > "$SCRIPT_PATH"/shared/maxmind_license.txt [[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot diff --git a/sensor-iso/config/archives/mozilla.key.binary b/sensor-iso/config/archives/mozilla.key.binary new file mode 100644 index 000000000..a8236db78 --- /dev/null +++ b/sensor-iso/config/archives/mozilla.key.binary @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +xsBNBGCRt7MBCADkYJHHQQoL6tKrW/LbmfR9ljz7ib2aWno4JO3VKQvLwjyUMPpq +/SXXMOnx8jXwgWizpPxQYDRJ0SQXS9ULJ1hXRL/OgMnZAYvYDeV2jBnKsAIEdiG/ +e1qm8P4W9qpWJc+hNq7FOT13RzGWRx57SdLWSXo0KeY38r9lvjjOmT/cuOcmjwlD +T9XYf/RSO+yJ/AsyMdAr+ZbDeQUd9HYJiPdI04lGaGM02MjDMnx+monc+y54t+Z+ +ry1WtQdzoQt9dHlIPlV1tR+xV5DHHsejCZxu9TWzzSlL5wfBBeEz7R/OIzivGJpW +QdJzd+2QDXSRg9q2XYWP5ZVtSgjVVJjNlb6ZABEBAAHNVEFydGlmYWN0IFJlZ2lz +dHJ5IFJlcG9zaXRvcnkgU2lnbmVyIDxhcnRpZmFjdC1yZWdpc3RyeS1yZXBvc2l0 +b3J5LXNpZ25lckBnb29nbGUuY29tPsLAjgQTAQoAOBYhBDW6oLM+nrOW9ZyoOMC6 +XObcYxWjBQJgkbezAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEMC6XObc +YxWj+igIAMFh6DrAYMeq9sbZ1ZG6oAMrinUheGQbEqe76nIDQNsZnhDwZ2wWqgVC +7DgOMqlhQmOmzm7M6Nzmq2dvPwq3xC2OeI9fQyzjT72deBTzLP7PJok9PJFOMdLf +ILSsUnmMsheQt4DUO0jYAX2KUuWOIXXJaZ319QyoRNBPYa5qz7qXS7wHLOY89IDq +fHt6Aud8ER5zhyOyhytcYMeaGC1g1IKWmgewnhEq02FantMJGlmmFi2eA0EPD02G +C3742QGqRxLwjWsm5/TpyuU24EYKRGCRm7QdVIo3ugFSetKrn0byOxWGBvtu4fH8 +XWvZkRT+u+yzH1s5yFYBqc2JTrrJvRU= +=QnvN +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/sensor-iso/config/archives/mozilla.key.chroot b/sensor-iso/config/archives/mozilla.key.chroot new file mode 100644 index 000000000..a8236db78 --- /dev/null +++ b/sensor-iso/config/archives/mozilla.key.chroot @@ -0,0 +1,19 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +xsBNBGCRt7MBCADkYJHHQQoL6tKrW/LbmfR9ljz7ib2aWno4JO3VKQvLwjyUMPpq +/SXXMOnx8jXwgWizpPxQYDRJ0SQXS9ULJ1hXRL/OgMnZAYvYDeV2jBnKsAIEdiG/ +e1qm8P4W9qpWJc+hNq7FOT13RzGWRx57SdLWSXo0KeY38r9lvjjOmT/cuOcmjwlD +T9XYf/RSO+yJ/AsyMdAr+ZbDeQUd9HYJiPdI04lGaGM02MjDMnx+monc+y54t+Z+ +ry1WtQdzoQt9dHlIPlV1tR+xV5DHHsejCZxu9TWzzSlL5wfBBeEz7R/OIzivGJpW +QdJzd+2QDXSRg9q2XYWP5ZVtSgjVVJjNlb6ZABEBAAHNVEFydGlmYWN0IFJlZ2lz +dHJ5IFJlcG9zaXRvcnkgU2lnbmVyIDxhcnRpZmFjdC1yZWdpc3RyeS1yZXBvc2l0 +b3J5LXNpZ25lckBnb29nbGUuY29tPsLAjgQTAQoAOBYhBDW6oLM+nrOW9ZyoOMC6 +XObcYxWjBQJgkbezAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEMC6XObc +YxWj+igIAMFh6DrAYMeq9sbZ1ZG6oAMrinUheGQbEqe76nIDQNsZnhDwZ2wWqgVC +7DgOMqlhQmOmzm7M6Nzmq2dvPwq3xC2OeI9fQyzjT72deBTzLP7PJok9PJFOMdLf +ILSsUnmMsheQt4DUO0jYAX2KUuWOIXXJaZ319QyoRNBPYa5qz7qXS7wHLOY89IDq +fHt6Aud8ER5zhyOyhytcYMeaGC1g1IKWmgewnhEq02FantMJGlmmFi2eA0EPD02G +C3742QGqRxLwjWsm5/TpyuU24EYKRGCRm7QdVIo3ugFSetKrn0byOxWGBvtu4fH8 +XWvZkRT+u+yzH1s5yFYBqc2JTrrJvRU= +=QnvN +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/sensor-iso/config/archives/mozilla.list.binary b/sensor-iso/config/archives/mozilla.list.binary new file mode 100644 index 000000000..ab034cedd --- /dev/null +++ b/sensor-iso/config/archives/mozilla.list.binary @@ -0,0 +1 @@ +deb https://packages.mozilla.org/apt mozilla main diff --git a/sensor-iso/config/archives/mozilla.list.chroot b/sensor-iso/config/archives/mozilla.list.chroot new file mode 100644 index 000000000..ab034cedd --- /dev/null +++ b/sensor-iso/config/archives/mozilla.list.chroot @@ -0,0 +1 @@ +deb https://packages.mozilla.org/apt mozilla main diff --git a/sensor-iso/config/archives/mozilla.pref.binary b/sensor-iso/config/archives/mozilla.pref.binary new file mode 100644 index 000000000..bf474df29 --- /dev/null +++ b/sensor-iso/config/archives/mozilla.pref.binary @@ -0,0 +1,5 @@ + +Package: * +Pin: origin packages.mozilla.org +Pin-Priority: 1000 + diff --git a/sensor-iso/config/archives/mozilla.pref.chroot b/sensor-iso/config/archives/mozilla.pref.chroot new file mode 100644 index 000000000..bf474df29 --- /dev/null +++ b/sensor-iso/config/archives/mozilla.pref.chroot @@ -0,0 +1,5 @@ + +Package: * +Pin: origin packages.mozilla.org +Pin-Priority: 1000 + diff --git a/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot b/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot deleted file mode 100755 index 98b7a4782..000000000 --- a/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. - -export LC_ALL=C.UTF-8 -export LANG=C.UTF-8 - -curl -o /tmp/firefox.tar.bz2 -L "https://download.mozilla.org/?product=firefox-latest-ssl&os=linux64&lang=en-US" -if [ $(file -b --mime-type /tmp/firefox.tar.bz2) = 'application/x-bzip2' ]; then - mkdir -p /opt - rm -rvf /opt/firefox - tar -xvf /tmp/firefox.tar.bz2 -C /opt/ - rm -vf /tmp/firefox.tar.bz2 - if [[ -f /opt/firefox/firefox ]]; then - rm -vf /usr/local/bin/firefox - ln -vrs /opt/firefox/firefox /usr/local/bin/firefox - dpkg -s firefox-esr >/dev/null 2>&1 && apt-get -y --purge remove firefox-esr || true - cat << 'EOF' > /usr/share/applications/firefox.desktop -[Desktop Entry] -Name=Firefox -Comment=Web Browser -GenericName=Web Browser -X-GNOME-FullName=Firefox Web Browser -Exec=/opt/firefox/firefox %u -Terminal=false -X-MultipleArgs=false -Type=Application -Icon=/opt/firefox/browser/chrome/icons/default/default128.png -Categories=Network;WebBrowser; -MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/vnd.mozilla.xul+xml;application/rss+xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https; -StartupWMClass=Firefox -StartupNotify=true -EOF - fi -fi # /tmp/firefox.tar.bz2 check - -rm -f /tmp/firefox.tar.bz2 diff --git a/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot b/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot index a97c39031..e40c67081 100755 --- a/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot +++ b/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot @@ -22,6 +22,7 @@ if [ -f "$CAPTURE_STORAGE_FORMAT_FILE" ]; then fi # other sensor-specific initialization prior to starting capture/forwarding jobs +echo "Running Sensor initialization" > /dev/tty0 /usr/local/bin/sensor-init.sh # enable firewall @@ -35,6 +36,7 @@ fi systemctl mask ctrl-alt-del.target if [ ! -s /var/lib/aide/aide.db ]; then + echo "Running Aide init" > /dev/tty0 > /var/lib/aide/aide.db /usr/sbin/aideinit --yes --force fi @@ -44,4 +46,4 @@ EOF sed -i "1i #!/bin/sh" /etc/rc.local -chmod +x /etc/rc.local /usr/local/bin/*.sh /usr/local/bin/configure-*.py +chmod o+rx /etc/rc.local /usr/local/bin/*.sh /usr/local/bin/configure-*.py diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 144e70778..0cf80e7da 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -5,20 +5,11 @@ # some environment variables needed for build export CCACHE_DIR="/var/spool/ccache" export CCACHE_COMPRESS=1 -export CMAKE_C_COMPILER="clang-14" -export CC="$CMAKE_C_COMPILER" -export CMAKE_CXX_COMPILER="clang++-14" -export CXX="$CMAKE_CXX_COMPILER" -export CXXFLAGS="-stdlib=libc++ -lc++abi" export PYTHONDONTWRITEBYTECODE=1 export PYTHONUNBUFFERED=1 +ARCH=$(dpkg --print-architecture) cat > /etc/environment << EOF -CMAKE_C_COMPILER="clang-14" -CC="clang-14" -CMAKE_CXX_COMPILER="clang++-14" -CXX="clang++-14" -CXXFLAGS="-stdlib=libc++ -lc++abi" PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 EOF @@ -79,14 +70,15 @@ mv ./zeek-$ZEEK_VER-hedgehog.tar.gz /opt/hedgehog_install_artifacts/ rm -Rf zeek-$ZEEK_VER* ### -# tweak some stuff for aide -chmod a-x /etc/cron.daily/aide -chattr +i /etc/cron.daily/aide -mkdir -p /etc/aide/aide.conf.d /var/lib/aide -touch /var/lib/aide/aide.db -chmod 600 /var/lib/aide/aide.db -sed -r -i "s/(Checksums\s*=\s*).*/\1 sha512/" /etc/aide/aide.conf -cat << 'EOF' >> /etc/aide/aide.conf.d/00_local_excludes +if [[ ! "${ARCH,,}" =~ ^arm ]]; then + # tweak some stuff for aide + chmod a-x /etc/cron.daily/aide + chattr +i /etc/cron.daily/aide + mkdir -p /etc/aide/aide.conf.d /var/lib/aide + touch /var/lib/aide/aide.db + chmod 600 /var/lib/aide/aide.db + sed -r -i "s/(Checksums\s*=\s*).*/\1 sha512/" /etc/aide/aide.conf + cat << 'EOF' >> /etc/aide/aide.conf.d/00_local_excludes /etc/at\.allow$ f Full-n-c-m /etc/clamav/.+\.conf$ f VarFile-n-c-m /etc/clamav$ d VarDir-n-c-m @@ -163,6 +155,7 @@ cat << 'EOF' >> /etc/aide/aide.conf.d/00_local_excludes !/var/tmp(/|$) EOF +fi ### # update suricata rules @@ -189,9 +182,42 @@ mv ./yara-rules-src-hedgehog.tar.gz /opt/hedgehog_install_artifacts/ # capa cd /tmp -curl -o ./capa.zip "${GITHUB_API_CURL_ARGS[@]}" "$(curl "${GITHUB_API_CURL_ARGS[@]}" "$(curl "${GITHUB_API_CURL_ARGS[@]}" "$CAPA_RELEASE_URL" | jq '.assets_url' | tr -d '"')" | jq '.[] | select(.browser_download_url|test("-linux\\.zip$")) | .browser_download_url' | tr -d '"')" -unzip ./capa.zip -mv ./capa /usr/local/bin/capa + +capa_assets_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$CAPA_RELEASE_URL" | jq '.assets_url' | tr -d '"')" + +if [[ "${ARCH,,}" =~ ^arm ]]; then + #Build from source for ARM... + #Not sure if there is an easier way to get the latest release tag + capa_latest_ver=$(curl "${GITHUB_API_CURL_ARGS[@]}" "$capa_assets_url" | jq ".[] | select(.name | contains(\"-linux.zip\")) | .name") + # Retrieves strings like "capa-v6.1.0-linux.zip"; below trims out the x.x.x + capa_latest_ver=$(echo ${capa_latest_ver#*v}) + capa_latest_ver=$(echo ${capa_latest_ver%%-*}) + capa_latest_src_url="https://github.com/mandiant/capa/archive/refs/tags/v${capa_latest_ver}.zip" + + python3 -m venv capa + source capa/bin/activate + cd capa + + curl "${GITHUB_API_CURL_ARGS[@]}" "${capa_latest_src_url}" -o capa.zip + unzip -q capa.zip + + cd capa-${capa_latest_ver} + python3 -m pip install -e .[build] + python scripts/cache-ruleset.py rules/ cache/ + pyinstaller .github/pyinstaller/pyinstaller.spec + mv dist/capa /usr/local/bin/capa + + deactivate + +else + # Assume 64-bit Linux otherwise + capa_zip_url=$(curl "${GITHUB_API_CURL_ARGS[@]}" "$capa_assets_url" | jq ".[] | select(.browser_download_url | contains(\"-linux.zip\")) | .browser_download_url" | tr -d '"') + curl -o capa.zip "${GITHUB_API_CURL_ARGS[@]}" "${capa_zip_url}" + unzip ./capa.zip + mv ./capa /usr/local/bin/capa + +fi + chmod 755 /usr/local/bin/capa rm -rf /tmp/capa* @@ -200,19 +226,29 @@ cp /usr/local/bin/capa /opt/hedgehog_install_artifacts/ # yq cd /tmp -curl "${GITHUB_API_CURL_ARGS[@]}" "$(curl "${GITHUB_API_CURL_ARGS[@]}" "$(curl "${GITHUB_API_CURL_ARGS[@]}" "$YQ_RELEASE_URL" | jq '.assets_url' | tr -d '"')" | jq '.[] | select(.browser_download_url|test("linux_amd64\\.tar\\.gz$")) | .browser_download_url' | tr -d '"')" | tar zxvf - ./yq_linux_amd64 -mv ./yq_linux_amd64 /usr/bin/yq +yq_assets_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$YQ_RELEASE_URL" | jq '.assets_url' | tr -d '"')" +yq_tar_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$yq_assets_url" | jq ".[] | select(.browser_download_url | contains(\"yq_linux_${ARCH}.tar.gz\")) | .browser_download_url" | tr -d '"')" +curl "${GITHUB_API_CURL_ARGS[@]}" "${yq_tar_url}" | tar -xzf - ./yq_linux_${ARCH} + +mv ./yq_linux_${ARCH} /usr/bin/yq chmod 755 /usr/bin/yq ### # supercronic -curl -o /usr/local/bin/supercronic "${GITHUB_API_CURL_ARGS[@]}" "$(curl "${GITHUB_API_CURL_ARGS[@]}" "$(curl "${GITHUB_API_CURL_ARGS[@]}" "$SUPERCRONIC_RELEASE_URL" | jq '.assets_url' | tr -d '"')" | jq '.[] | select(.browser_download_url|test("-linux-amd64$")) | .browser_download_url' | tr -d '"')" +supercronic_assets_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$SUPERCRONIC_RELEASE_URL" | jq '.assets_url' | tr -d '"')" +supercronic_bin_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$supercronic_assets_url" | jq ".[] | select(.browser_download_url | contains(\"supercronic-linux-${ARCH}\")) | .browser_download_url" | tr -d '"')" +curl -o /usr/local/bin/supercronic "${GITHUB_API_CURL_ARGS[@]}" "$supercronic_bin_url" + chmod 755 /usr/local/bin/supercronic ### # croc cd /tmp -curl "${GITHUB_API_CURL_ARGS[@]}" "$(curl "${GITHUB_API_CURL_ARGS[@]}" "$(curl "${GITHUB_API_CURL_ARGS[@]}" "$CROC_RELEASE_URL" | jq '.assets_url' | tr -d '"')" | jq '.[] | select(.browser_download_url|test("Linux-64bit\\.tar\\.gz$")) | .browser_download_url' | tr -d '"')" | tar zxvf - croc +[[ $ARCH =~ ^arm ]] && CROC_ASSET_ARCH="${ARCH^^}" || CROC_ASSET_ARCH=64bit +croc_assets_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$CROC_RELEASE_URL" | jq '.assets_url' | tr -d '"')" +croc_tar_url="$(curl "${GITHUB_API_CURL_ARGS[@]}" "$croc_assets_url" | jq ".[] | select(.browser_download_url | contains(\"_Linux-${CROC_ASSET_ARCH}.tar.gz\")) | .browser_download_url" | tr -d '"')" +curl "${GITHUB_API_CURL_ARGS[@]}" "${croc_tar_url}" | tar -xzf - croc + mv ./croc /usr/local/bin/croc chmod 755 /usr/local/bin/croc ### @@ -224,7 +260,7 @@ freshclam --stdout --quiet --no-warnings # set up capabilities for network-related tools chown root:netdev /usr/sbin/netsniff-ng && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng chown root:netdev "${ZEEK_DIR}"/bin/zeek && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek chown root:netdev /sbin/ethtool && \ diff --git a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot index ee8baa4ae..c925185fd 100755 --- a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot +++ b/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot @@ -9,7 +9,7 @@ apt-get -y --purge remove \ libc6-dbg \ ninja-build \ sparse \ - $(dpkg --get-selections | grep -Pv "(^(dpkg|libbroker|libc\+\+(abi)?|libc6|libclang|libcrypt|libdbus|libffi|libfl|libgoogle-perftools|libgcc|libkrb5|libmaxminddb|libncurses|libnsl|libobjc|libomp|libpcap|libssl|libstdc|libtinfo|libtirpc|libunwind|libxml|libyaml|libz|linux-libc|python3|zeek|zlib1g)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true + $(dpkg --get-selections | grep -Pv "(^(dpkg|libbroker|libc6|libcrypt|libdbus|libffi|libfl|libgoogle-perftools|libgcc|libkrb5|libmaxminddb|libncurses|libnsl|libobjc|libomp|libpcap|libssl|libstdc|libtinfo|libtirpc|libunwind|libxml|libyaml|libz|linux-libc|python3|zeek|zlib1g)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true rm -rf /var/spool/ccache # remove unwanted packages @@ -39,15 +39,22 @@ apt-get -y --purge remove \ apt-get -y autoremove apt-get clean +# hold packages we don't want to update with an apt-get upgrade +# we built htpdate from source for HTTPS support, so leave it +dpkg -s htpdate >/dev/null 2>&1 && apt-mark hold htpdate + # remove any residual configs dpkg -l | awk '/^rc/ { print $2 }' | xargs -r -l dpkg --purge # disable automatic running of some services (but don't abort if we fail) -systemctl disable supervisor.service || true -systemctl disable ctrl-alt-del.target || true +systemctl disable apt-daily-upgrade.service || true +systemctl disable apt-daily-upgrade.timer || true systemctl disable apt-daily.service || true systemctl disable apt-daily.timer || true -systemctl disable apt-daily-upgrade.timer || true -systemctl disable apt-daily-upgrade.service || true systemctl disable clamav-daemon.service || true systemctl disable clamav-freshclam.service || true +systemctl disable ctrl-alt-del.target || true +systemctl disable filebeat.service || true +systemctl disable supervisor.service || true +systemctl disable suricata.service || true +systemctl disable fluent-bit.service || true \ No newline at end of file diff --git a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot index 93afb4f60..3116aa3c8 100755 --- a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot +++ b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot @@ -2,21 +2,51 @@ # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. +ARCH="$(dpkg --print-architecture)" + # configure firewall sed -i "s/LOGLEVEL=.*/LOGLEVEL=off/" /etc/ufw/ufw.conf -/usr/sbin/ufw --force enable -/usr/sbin/ufw default deny incoming -/usr/sbin/ufw default allow outgoing -UFW_ALLOW_RULES=( - ntp - ssh - 9009:9013/tcp -) -for i in ${UFW_ALLOW_RULES[@]}; do - ufw allow "$i" -done -# will re-enable on boot -/usr/sbin/ufw --force disable + +if [ "${ARCH,,}" == 'arm' ] || [ "${ARCH,,}" == 'arm64' ]; then + # Known modules issue when building RPI images requires a 'restart' + # In arm builds, we're in a chroot reboot will be ignored + + run_once='/etc/init.d/run_once.sh' + + cat <<- 'EOF' > $run_once + #!/bin/bash + ufw=$(which ufw) + UFW_ALLOW_RULES=( ntp ssh 9009:9013/tcp ) + $ufw default deny incoming + $ufw default allow outgoing + for i in ${UFW_ALLOW_RULES[@]}; do $ufw allow $i; done + $ufw reload + + # Update initramfs to remove rpi-resize script from current initramfs on first boot + echo "Updating initramfs to remove rpi-resize script. This may take a few minutes..." > /dev/tty0 + /usr/sbin/update-initramfs -u + EOF + + echo "sed -i '\|$run_once|d' /etc/rc.local" >> $run_once + echo -e "rm -f $run_once\nexit 0" >> $run_once + chmod 755 $run_once + + sed -i "\|/bin/sh|a $run_once" /etc/rc.local + +else + + /usr/sbin/ufw --force enable + /usr/sbin/ufw default deny incoming + /usr/sbin/ufw default allow outgoing + UFW_ALLOW_RULES=( ntp ssh 9009:9013/tcp ) + + for i in ${UFW_ALLOW_RULES[@]}; do + ufw allow "$i" + done + + # will re-enable on boot + /usr/sbin/ufw --force disable +fi # performance parameters for networking, disk, etc. cat << 'EOF' >> /etc/sysctl.conf diff --git a/sensor-iso/config/hooks/normal/0992-localepurge.hook.chroot b/sensor-iso/config/hooks/normal/0992-localepurge.hook.chroot index c5b518054..a1a395070 100755 --- a/sensor-iso/config/hooks/normal/0992-localepurge.hook.chroot +++ b/sensor-iso/config/hooks/normal/0992-localepurge.hook.chroot @@ -25,6 +25,7 @@ localepurge cat >> /etc/environment << EOF LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 +TERM=linux EOF cat > /etc/locale.conf << EOF diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop index efffe0aba..d3f19631b 100644 --- a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop +++ b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop @@ -3,11 +3,11 @@ Name=Firefox Comment=Web Browser GenericName=Web Browser X-GNOME-FullName=Firefox Web Browser -Exec=/opt/firefox/firefox %u +Exec=/usr/bin/firefox %u Terminal=false X-MultipleArgs=false Type=Application -Icon=/opt/firefox/browser/chrome/icons/default/default128.png +Icon=/usr/lib/firefox/browser/chrome/icons/default/default128.png Categories=Network;WebBrowser; MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/vnd.mozilla.xul+xml;application/rss+xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https; StartupWMClass=Firefox diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop index d64906054..a796a095e 100644 --- a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop +++ b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Sensor Kiosk -Exec=/opt/firefox/firefox --setDefaultBrowser --no-remote --private --kiosk http://127.0.0.1:5000 +Exec=/usr/bin/firefox --setDefaultBrowser --no-remote --private --kiosk http://127.0.0.1:5000 Terminal=false X-MultipleArgs=false Type=Application diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop index b63021157..2a94dae9f 100644 --- a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop +++ b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Sensor README -Exec=/opt/firefox/firefox --setDefaultBrowser http://localhost:8420/docs/hedgehog.html +Exec=/usr/bin/firefox --setDefaultBrowser http://localhost:8420/docs/hedgehog.html Terminal=false X-MultipleArgs=false Type=Application diff --git a/sensor-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh b/sensor-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh index 363753027..a2d71255d 100755 --- a/sensor-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh +++ b/sensor-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh @@ -7,4 +7,4 @@ grep -q boot=live /proc/cmdline && exit 0 grep -q CAPTURE_INTERFACE=lo /opt/sensor/sensor_ctl/control_vars.conf && exit 0 # start firefox in kiosk mode and load the performance metrics dashboard -/opt/firefox/firefox --setDefaultBrowser --no-remote --private --kiosk http://127.0.0.1:5000 +/usr/bin/firefox --setDefaultBrowser --no-remote --private --kiosk http://127.0.0.1:5000 diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 6b2c84695..7d3b0ccee 100644 --- a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -4,6 +4,7 @@ ##! https://docs.zeek.org/en/stable/script-reference/scripts.html ##! https://github.com/zeek/zeek/blob/master/scripts/site/local.zeek +global disable_stats = (getenv("ZEEK_DISABLE_STATS") == "") ? F : T; global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == "") ? F : T; global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == "") ? F : T; global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == "") ? F : T; @@ -78,6 +79,10 @@ redef ignore_checksums = T; @if (!disable_hash_all_files) @load frameworks/files/hash-all-files @endif +@if (!disable_stats) + @load policy/misc/stats + @load policy/misc/capture-loss +@endif @load policy/protocols/conn/vlan-logging @load policy/protocols/conn/mac-logging @load policy/protocols/modbus/known-masters-slaves @@ -259,11 +264,29 @@ event zeek_init() &priority=-5 { redef SNIFFPASS::log_password_plaintext = T; redef LDAP::default_capture_password = T; @endif + redef LDAP::default_log_search_attributes = F; redef SNIFFPASS::notice_log_enable = F; redef CVE_2021_44228::log = F; -@if ((!disable_ics_all) && (!disable_ics_synchrophasor) && (synchrophasor_detailed)) - redef SYNCHROPHASOR::log_data_frame = T; - redef SYNCHROPHASOR::log_data_detail = T; - redef SYNCHROPHASOR::log_cfg_detail = T; + +@if ((!disable_ics_all) && (!disable_ics_synchrophasor) && (!synchrophasor_detailed)) + hook SYNCHROPHASOR::log_policy_sychrophasor_data_detail( + rec : SYNCHROPHASOR::Synchrophasor_Data_Detail, + id : Log::ID, + filter : Log::Filter) { + break; + } + hook SYNCHROPHASOR::log_policy_sychrophasor_config_detail( + rec : SYNCHROPHASOR::Synchrophasor_Config_Detail, + id : Log::ID, + filter : Log::Filter) { + break; + } + + hook SYNCHROPHASOR::log_policy_sychrophasor_data( + rec : SYNCHROPHASOR::Synchrophasor_Data, + id : Log::ID, + filter : Log::Filter) { + break; + } @endif diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop b/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop index b98abfcdd..3f726ebdd 100644 --- a/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop +++ b/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Sensor Kiosk -Exec=/opt/firefox/firefox --setDefaultBrowser --no-remote --private --kiosk http://127.0.0.1:5000 +Exec=/usr/bin/firefox --setDefaultBrowser --no-remote --private --kiosk http://127.0.0.1:5000 Terminal=false X-MultipleArgs=false Type=Application diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop b/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop index 93f3bca7f..a5aa4bd98 100644 --- a/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop +++ b/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop @@ -1,7 +1,7 @@ [Desktop Entry] Version=1.0 Name=Sensor README -Exec=/opt/firefox/firefox --setDefaultBrowser http://localhost:8420/docs/hedgehog.html +Exec=/usr/bin/firefox --setDefaultBrowser http://localhost:8420/docs/hedgehog.html Terminal=false X-MultipleArgs=false Type=Application diff --git a/sensor-iso/config/package-lists/build.list.chroot b/sensor-iso/config/package-lists/build.list.chroot index 041096945..5eadd0bfa 100644 --- a/sensor-iso/config/package-lists/build.list.chroot +++ b/sensor-iso/config/package-lists/build.list.chroot @@ -1,11 +1,10 @@ bison ccache checkinstall -clang cmake +gcc +g++ git -libc++-dev -libc++abi-dev libfl-dev libgoogle-perftools-dev libjansson-dev diff --git a/sensor-iso/config/package-lists/desktopmanager.list.chroot b/sensor-iso/config/package-lists/desktopmanager.list.chroot index e16b4b132..f167a1ee1 100644 --- a/sensor-iso/config/package-lists/desktopmanager.list.chroot +++ b/sensor-iso/config/package-lists/desktopmanager.list.chroot @@ -4,6 +4,7 @@ clamav-daemon clamav-freshclam dconf-cli file-roller +firefox fonts-symbola galculator gnome-themes-extra diff --git a/sensor-iso/config/package-lists/net.list.chroot b/sensor-iso/config/package-lists/net.list.chroot index 03dab33ce..ab02e33c1 100644 --- a/sensor-iso/config/package-lists/net.list.chroot +++ b/sensor-iso/config/package-lists/net.list.chroot @@ -1,6 +1,7 @@ apache2-utils ca-certificates curl +dnsutils ethtool iproute2 iputils-arping diff --git a/sensor-iso/interface/requirements.txt b/sensor-iso/interface/requirements.txt index da47d54c9..70bc6aa40 100644 --- a/sensor-iso/interface/requirements.txt +++ b/sensor-iso/interface/requirements.txt @@ -6,7 +6,7 @@ Flask-Cors==3.0.10 gunicorn==20.1.0 idna==3.4 itsdangerous==2.1.2 -Jinja2==3.1.2 +Jinja2==3.1.3 MarkupSafe==2.1.2 psutil==5.9.4 python-dotenv==1.0.0 diff --git a/sensor-iso/interface/sensor_ctl/arkime/config.ini b/sensor-iso/interface/sensor_ctl/arkime/config.ini index fd30ae188..22defcd53 100644 --- a/sensor-iso/interface/sensor_ctl/arkime/config.ini +++ b/sensor-iso/interface/sensor_ctl/arkime/config.ini @@ -55,7 +55,7 @@ tpacketv3NumThreads=2 tpacketv3BlockSize=8388608 pcapWriteMethod=simple pcapWriteSize=2560000 -simpleCompression=none +simpleCompression=zstd simpleZstdLevel=3 simpleGzipLevel=3 packetThreads=5 diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/sensor-iso/interface/sensor_ctl/control_vars.conf index 7d72f4d18..956d6daa5 100644 --- a/sensor-iso/interface/sensor_ctl/control_vars.conf +++ b/sensor-iso/interface/sensor_ctl/control_vars.conf @@ -15,8 +15,8 @@ export ARKIME_PACKET_THREADS=5 export ARKIME_PACKET_ACL= export ARKIME_ECS_PROVIDER=arkime export ARKIME_ECS_DATASET=session -export ARKIME_COMPRESSION_TYPE=none -export ARKIME_COMPRESSION_LEVEL=0 +export ARKIME_COMPRESSION_TYPE=zstd +export ARKIME_COMPRESSION_LEVEL=3 # ARKIME_VIEWER_(CERT|KEY) are under "$SUPERVISOR_PATH"/arkime/ export ARKIME_VIEWER_CERT=viewer.crt export ARKIME_VIEWER_KEY=viewer.key @@ -55,6 +55,7 @@ export ZEEK_EXTRACTOR_OVERRIDE_FILE= export EXTRACTED_FILE_MIN_BYTES=64 export EXTRACTED_FILE_MAX_BYTES=134217728 export EXTRACTED_FILE_PRESERVATION=quarantined +export ZEEK_DISABLE_STATS=true export ZEEK_DISABLE_HASH_ALL_FILES= export ZEEK_DISABLE_LOG_PASSWORDS= export ZEEK_DISABLE_SSL_VALIDATE_CERTS= @@ -76,7 +77,7 @@ export ZEEK_DISABLE_ICS_BSAP= export ZEEK_DISABLE_ICS_DNP3= export ZEEK_DISABLE_ICS_ENIP= export ZEEK_DISABLE_ICS_ETHERCAT= -export ZEEK_DISABLE_ICS_GENISYS= +export ZEEK_DISABLE_ICS_GENISYS=true export ZEEK_DISABLE_ICS_OPCUA_BINARY= export ZEEK_DISABLE_ICS_MODBUS= export ZEEK_DISABLE_ICS_PROFINET= @@ -110,6 +111,10 @@ export SURICATA_MANAGED_DIR=/var/lib/suricata export SURICATA_MANAGED_RULES_DIR="$SURICATA_MANAGED_DIR"/rules export SURICATA_REFRESH_CRON_EXPRESSION="15 2 * * *" export SURICATA_UPDATE_ETOPEN=true +export SURICATA_STATS_ENABLED=false +export SURICATA_STATS_EVE_ENABLED=false +export SURICATA_STATS_INTERVAL=30 +export SURICATA_STATS_DECODER_EVENTS=false # affects Arkime only for now: beats values are stored in keystores per-beat export OS_PROTOCOL=https diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh index 0a627c95f..0992717e4 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh @@ -69,8 +69,8 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t sed -r -i "s/(freeSpaceG)\s*=\s*.*/\1=$ARKIME_FREESPACEG/" "$ARKIME_CONFIG_FILE" fi # pcap compression - COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}" - COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}" + COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-zstd}" + COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-3}" sed -r -i "s/(simpleCompression)\s*=\s*.*/\1=$COMPRESSION_TYPE/" "$ARKIME_CONFIG_FILE" if [[ "$COMPRESSION_TYPE" == "zstd" ]]; then sed -r -i "s/(simpleZstdLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "$ARKIME_CONFIG_FILE" diff --git a/sensor-iso/zeek/Dockerfile b/sensor-iso/zeek/Dockerfile deleted file mode 100644 index c4e69ce8e..000000000 --- a/sensor-iso/zeek/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -FROM debian:12-slim - -# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. - -LABEL maintainer="malcolm@inl.gov" - -ENV DEBIAN_FRONTEND noninteractive -ENV TERM xterm - -ENV CMAKE_C_COMPILER clang-14 -ENV CMAKE_CXX_COMPILER clang++-14 -ENV CC clang-14 -ENV CXX clang++-14 -ENV CXXFLAGS "-stdlib=libc++ -lc++abi" - -RUN apt-get -q update && \ - apt-get -y -q --no-install-recommends upgrade && \ - apt-get install -q -y --no-install-recommends \ - bison \ - ca-certificates \ - ccache \ - checkinstall \ - clang \ - cmake \ - curl \ - flex \ - git \ - libc++-dev \ - libc++abi-dev \ - libfl-dev \ - libgoogle-perftools-dev \ - libgoogle-perftools4 \ - libkrb5-3 \ - libkrb5-dev \ - libmaxminddb-dev \ - libpcap-dev \ - libssl-dev \ - libtcmalloc-minimal4 \ - make \ - ninja-build \ - python3 \ - python3-dev \ - python3-git \ - python3-semantic-version \ - sudo \ - swig \ - zlib1g-dev - -ADD build-zeek-deb.sh /usr/local/bin/ - -ENTRYPOINT [ "/bin/bash", "/usr/local/bin/build-zeek-deb.sh" ] \ No newline at end of file diff --git a/sensor-iso/zeek/build-docker-image.sh b/sensor-iso/zeek/build-docker-image.sh deleted file mode 100755 index 3538212ff..000000000 --- a/sensor-iso/zeek/build-docker-image.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. - -# force-navigate to script directory -SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pushd "$SCRIPT_PATH" >/dev/null 2>&1 - -docker build -t zeek-build:latest . - -popd >/dev/null 2>&1 diff --git a/sensor-iso/zeek/build-zeek-deb.sh b/sensor-iso/zeek/build-zeek-deb.sh deleted file mode 100755 index 0ea95d4dc..000000000 --- a/sensor-iso/zeek/build-zeek-deb.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. - -export CCACHE_DIR="/var/spool/ccache" -export CCACHE_COMPRESS=1 -export CMAKE_C_COMPILER="clang-14" -export CC="$CMAKE_C_COMPILER" -export CMAKE_CXX_COMPILER="clang++-14" -export CXX="$CMAKE_CXX_COMPILER" -export CXXFLAGS="-stdlib=libc++ -lc++abi" -export PYTHONDONTWRITEBYTECODE=1 -export PYTHONUNBUFFERED=1 - -ZEEK_URL=https://github.com/zeek/zeek.git -ZEEK_VERSION=6.1.0 -ZEEK_DIR=/opt/zeek -BUILD_JOBS=0 -OUTPUT_DIR=/tmp -unset VERBOSE - -while getopts b:p:o:j:v opts; do - case ${opts} in - b) ZEEK_VERSION=${OPTARG} ;; - p) ZEEK_DIR=${OPTARG} ;; - o) OUTPUT_DIR=${OPTARG} ;; - j) BUILD_JOBS=${OPTARG} ;; - v) VERBOSE=1 ;; - esac -done - -set -e -if [[ -n $VERBOSE ]]; then - set -x -fi - -cd /tmp -mkdir ./"zeek-v${ZEEK_VERSION}" -curl -sSL "https://download.zeek.org/zeek-${ZEEK_VERSION}.tar.gz" | tar xzf - -C ./"zeek-v${ZEEK_VERSION}" --strip-components 1 - -mkdir -p "${CCACHE_DIR}" -pushd /tmp/"zeek-v${ZEEK_VERSION}" >/dev/null 2>&1 -./configure --prefix="${ZEEK_DIR}" --generator=Ninja --ccache --enable-perftools -mkdir -p build -pushd build >/dev/null 2>&1 -ninja -j "${BUILD_JOBS}" -checkinstall -y -D --strip=yes --stripso=yes --install=no --fstrans=no --pkgname="zeek" --pkgversion="$ZEEK_VERSION" --pkgarch="amd64" --pkgsource="$ZEEK_URL" ninja install -ls -l *.deb && mv -v *.deb "$OUTPUT_DIR"/ -popd >/dev/null 2>&1 -popd >/dev/null 2>&1 - -if [[ -n $VERBOSE ]]; then - set +x -fi -set +e diff --git a/sensor-raspi/.gitignore b/sensor-raspi/.gitignore new file mode 100644 index 000000000..ac52d7e71 --- /dev/null +++ b/sensor-raspi/.gitignore @@ -0,0 +1,6 @@ +raspi_*.img +raspi_*.gz +raspi_*.xz +raspi_*.log +raspi_*.yaml +shared/ \ No newline at end of file diff --git a/sensor-raspi/Dockerfile b/sensor-raspi/Dockerfile new file mode 100644 index 000000000..56a7966ce --- /dev/null +++ b/sensor-raspi/Dockerfile @@ -0,0 +1,25 @@ +FROM ghcr.io/mmguero/qemu-live-iso:latest + +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +LABEL maintainer="malcolm@inl.gov" +LABEL org.opencontainers.image.authors='malcolm@inl.gov' +LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm' +LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/sensor-iso/README.md' +LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm' +LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency' +LABEL org.opencontainers.image.title='ghcr.io/cisagov/malcolm/hedgehog-raspi' +LABEL org.opencontainers.image.description='Hedgehog Linux network sensor Raspberry Pi image wrapped in a Docker image' + +ARG QEMU_CPU=4 +ARG QEMU_RAM=4096 +ENV QEMU_CPU $QEMU_CPU +ENV QEMU_RAM $QEMU_RAM + +# just using the base to serve the .img for download, not actually run it in qemu +ENV NOVNC_START false +ENV QEMU_RESTART false +ENV QEMU_START false + +ADD --chown=${DEFAULT_UID}:${DEFAULT_GID} https://raw.githubusercontent.com/cisagov/Malcolm/main/docs/images/hedgehog/logo/favicon.ico /image/favicon.ico +ADD --chown=${DEFAULT_UID}:${DEFAULT_GID} raspi_4_bookworm*.* /image/ diff --git a/sensor-raspi/Makefile b/sensor-raspi/Makefile new file mode 100644 index 000000000..2067f470c --- /dev/null +++ b/sensor-raspi/Makefile @@ -0,0 +1,75 @@ +all: shasums + +# List all the supported and built Pi platforms here. They get expanded +# to names like 'raspi_2_buster.yaml' and 'raspi_3_bullseye.img.xz'. +BUILD_FAMILIES := 1 2 3 4 +BUILD_RELEASES := bullseye bookworm trixie + +platforms := $(foreach plat, $(BUILD_FAMILIES),$(foreach rel, $(BUILD_RELEASES), raspi_$(plat)_$(rel))) + +shasums: $(addsuffix .img.sha256,$(platforms)) $(addsuffix .img.xz.sha256,$(platforms)) +xzimages: $(addsuffix .img.xz,$(platforms)) +images: $(addsuffix .img,$(platforms)) +yaml: $(addsuffix .yaml,$(platforms)) + +ifeq ($(shell id -u),0) +as_root = +else ifneq (, $(shell which fakemachine)) +$(warning "This should normally be run as root, but found 'fakemachine', so using that.") +as_root = fakemachine -v $(abspath $(CURDIR)/..) -- env --chdir $(abspath $(CURDIR)) +else +$(error "This must be run as root") +endif + +target_platforms: + @echo $(platforms) + +# Generate targets based on all family * release combinations: +define dynamic_yaml_target = + raspi_$(1)_$(2).yaml: raspi_master.yaml generate-recipe.py + raspi_$(1)_$(2).yaml: + ./generate-recipe.py $(1) $(2) +endef +$(foreach release,$(BUILD_RELEASES), \ + $(foreach family,$(BUILD_FAMILIES), \ + $(eval $(call dynamic_yaml_target,$(family),$(release))))) + +%.img.sha256: %.img + echo $@ + sha256sum $< > $@ + +%.img.xz.sha256: %.img.xz + echo $@ + sha256sum $< > $@ + +%.img.xz: %.img + xz -f -k -z -9 $< + +%.img.bmap: %.img + bmaptool create -o $@ $< + +%.img: %.yaml + touch $(@:.img=.log) + time nice $(as_root) vmdb2 --verbose --rootfs-tarball=$(subst .img,.tar.gz,$@) --output=$@ $(subst .img,.yaml,$@) --log $(subst .img,.log,$@) + chmod 0644 $@ $(@,.img=.log) + +_ck_root: + [ `whoami` = 'root' ] # Only root can summon vmdb2 ☹ + +_clean_yaml: + rm -f $(addsuffix .yaml,$(platforms)) raspi_base_bullseye.yaml raspi_base_bookworm.yaml raspi_base_trixie.yaml +_clean_images: + rm -f $(addsuffix .img,$(platforms)) +_clean_xzimages: + rm -f $(addsuffix .img.xz,$(platforms)) +_clean_bmaps: + rm -f $(addsuffix .img.bmap,$(platforms)) +_clean_shasums: + rm -f $(addsuffix .img.sha256,$(platforms)) $(addsuffix .img.xz.sha256,$(platforms)) +_clean_logs: + rm -f $(addsuffix .log,$(platforms)) +_clean_tarballs: + rm -f $(addsuffix .tar.gz,$(platforms)) +clean: _clean_xzimages _clean_images _clean_shasums _clean_yaml _clean_tarballs _clean_logs _clean_bmaps + +.PHONY: _ck_root _build_img clean _clean_images _clean_yaml _clean_tarballs _clean_logs diff --git a/sensor-raspi/README.md b/sensor-raspi/README.md new file mode 100644 index 000000000..370e21de6 --- /dev/null +++ b/sensor-raspi/README.md @@ -0,0 +1,119 @@ +# Raspberry Pi Hedgehog Sensor Build + +This build process builds upon the Debian RPi build process located here: +http://salsa.debian.org/raspi-team/image-specs + +## Building an image + +Steps to build a Hedgehog Raspberry Pi image. +If you are reading this document online, you should first +clone this repository: + +```shell +git clone https://github.com/cisagov/Malcolm.git +cd Malcolm/sensor-raspi +``` + +For this you will first need to install the following packages on a +Debian Bullseye (11), Devuan Daedalus (5), or higher system: + +* binfmt-support +* bmap-tools +* debootstrap +* dosfstools +* fakemachine (optional, only available on amd64) +* kpartx +* qemu-utils +* qemu-user-static +* time +* vmdb2 (>= 0.17) +* python3 +* zerofree (because of [#1021341](https://bugs.debian.org/1021341)) + +To install these (as root): +```shell + apt install -y vmdb2 dosfstools qemu-utils qemu-user-static debootstrap binfmt-support time kpartx bmap-tools python3 zerofree + apt install -y fakemachine +``` + +If debootstrap still fails with exec format error, try +running `dpkg-reconfigure qemu-user-static`. This calls +`/var/lib/dpkg/info/qemu-user-static.postinst` which uses binfmt-support +to register the executable format with /usr/bin/qemu-$fmt-static + +This repository includes a master YAML recipe (which is basically a +configuration file) for all of the generated images, diverting as +little as possible in a parametrized way. The master recipe is +[raspi_master.yaml](raspi_master.yaml). + +A Makefile is supplied to drive the build of the recipes into images. +If `fakemachine` is installed, it can be run as an unprivileged user. +Otherwise, because some steps of building the image require root privileges, +you'll need to execute `make` as root. + +The argument to `make` is constructed as follows: +`raspi__.` + +Whereby is one of `1`, `2`, `3` or `4`, is either +`bullseye`, `bookworm`, or `trixie`; and is `img` or `yaml`. + +Model `1` should be used for the Raspberry Pi 0, 0w and 1, models A and +B. Model `2` for the Raspberry Pi 2 models A and B. Model `3` for all +models of the Raspberry Pi 3 and model `4` for all models of the +Raspberry Pi 4. +So if you want to build the default image for a Raspberry Pi 3B+ with +Bullseye, you can just issue: + +```shell + make raspi_4_bookworm.img +``` + +**NOTE:** While this setup will build hedgehog for all raspberry pi variants, it is highly unlikely +that any variant other than RPI 4 (8GB version) will have adequate resources to function effectively as a sensor. + +At this point; it might be wise to go do something else. The build **WILL** take a while. +Initial testing on a 8-core 16GB build machine took approximately 5.5 hours to complete the image. + +## Installing the image onto the Raspberry Pi + +If the build completes properly, it can be tested locally before writing to an SD card if desired. +To do so, simply run (as root): + +```shell + mount -o loop,offset=$((1048576*512)) raspi_4_bookworm.img /mnt && chroot /mnt +``` + +If an error is returned by the mount command, there is a chance the image was corrupted during the build. +It is, unfortunately, advised to run `make clean` and rebuild the image. + +Plug an SD card which you would like to **entirely OVERWRITE** into the Build machine's SD card reader. + +Assuming your SD card reader provides the device `/dev/mmcblk0` +(**Beware** If you choose the wrong device, you might overwrite +important parts of your system. Double check it's the correct +device!), copy the image onto the SD card: + +```shell +bmaptool copy raspi_3_bullseye.img.xz /dev/mmcblk0 +``` + +Alternatively, if you don't have `bmap-tools` installed, you can use +`dd` with the compressed image: + +```shell +xzcat raspi_3_bullseye.img.xz | dd of=/dev/mmcblk0 bs=64k oflag=dsync status=progress +``` + +Or with the uncompressed image: + +```shell +dd if=raspi_3_bullseye.img of=/dev/mmcblk0 bs=64k oflag=dsync status=progress +``` + +Then, plug the SD card into the Raspberry Pi, and power it up. + +The image uses the hostname `Hedgehog-rpi-0w`, `Hedgehog-rpi-2`, `Hedgehog-rpi-3`, or `Hedgehog-rpi-4` depending on the +target build. The provided image will allow you to log in with the +`sensor` account with a default password of `Hedgehog_Linux` or +`root` account with a default password of `Hedgehog_Linux_Root`, but only logging in at the +physical console (be it serial or by USB keyboard and HDMI monitor). diff --git a/sensor-raspi/attribution.txt b/sensor-raspi/attribution.txt new file mode 100644 index 000000000..f60c94c38 --- /dev/null +++ b/sensor-raspi/attribution.txt @@ -0,0 +1,32 @@ +This license document applies to some of the Debian-related build +code found in this directory: + +--- + +Copyright © 2017, Michael Stapelberg and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Michael Stapelberg nor the + names of contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY Michael Stapelberg ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL Michael Stapelberg BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/sensor-raspi/build_via_vagrant.sh b/sensor-raspi/build_via_vagrant.sh new file mode 100755 index 000000000..6141a6aa3 --- /dev/null +++ b/sensor-raspi/build_via_vagrant.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +unset SSH_AUTH_SOCK + +function vm_state() { + vagrant status --machine-readable | grep ",state," | egrep -o '([a-z_]*)$' +} + +function vm_is_running() { + STATE="$(vm_state)" + if [[ "$STATE" == "running" ]] ; then + return 0 + else + return 1 + fi +} + +function vm_execute() { + echo "Running $1" >&2 + vagrant ssh --no-tty --command "$1" +} + +unset FORCE_PROVISION +XZ_EXT= +IMAGE=raspi_4_bookworm.img +while getopts 'fi:z' OPTION; do + case "$OPTION" in + f) + FORCE_PROVISION=0 + ;; + + i) + IMAGE=${OPTARG} + ;; + + z) + XZ_EXT=.xz + ;; + + ?) + echo "script usage: $(basename $0) [-f]" >&2 + exit 1 + ;; + esac +done +shift "$(($OPTIND -1))" + + +function cleanup_shared_and_docs { + # clean up temporary files + rm -rf "$SCRIPT_PATH"/shared \ + "$SCRIPT_PATH"/.fuse_hidden* + # if we created an .xz, delete the original .img + if [[ -f "$SCRIPT_PATH/$IMAGE" ]] && [[ -n "$XZ_EXT" ]] && [[ -f "$SCRIPT_PATH/$IMAGE$XZ_EXT" ]] ; then + rm -f "$SCRIPT_PATH/$IMAGE" + fi +} + + +pushd "$SCRIPT_PATH"/ >/dev/null 2>&1 +make clean >/dev/null 2>&1 +popd >/dev/null 2>&1 + +pushd "$SCRIPT_PATH"/vagrant + +VM_NAME="$(grep "config.vm.box" Vagrantfile | tr -d "[:space:]" | sed "s/.*=//")" + +if [[ -n $FORCE_PROVISION ]]; then + echo "Destroying build machine to force provisioning..." >&2 + vagrant destroy -f + sleep 1 +fi + +# make sure the VM is up and running, or start it otherwise +if ! vm_is_running; then + echo "Starting build machine..." >&2 + vagrant up + NEED_SHUTDOWN=true + sleep 1 +fi +until vm_is_running; do + echo "Waiting for $VM_NAME..." >&2 + sleep 1 +done +echo "$VM_NAME is running!" >&2 + +# make sure we can connect via SSH +echo "Checking SSH availability..." >&2 +until vm_execute 'sudo whoami' | grep -q "root" ; do + echo "Waiting for SSH availability..." >&2 + sleep 1 +done +echo "SSH available." >&2 + +cleanup_shared_and_docs +mkdir "$SCRIPT_PATH"/shared +YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" +[[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt +[[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_LICENSE_KEY" > "$SCRIPT_PATH"/shared/maxmind_license.txt +[[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot +echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot +trap cleanup_shared_and_docs EXIT + +vm_execute "sudo bash -c \"whoami && cd /Malcolm/sensor-raspi && pwd && make ${IMAGE}${XZ_EXT}\"" + +if [[ -n $NEED_SHUTDOWN ]]; then + echo "Shutting down $VM_NAME..." >&2 + vagrant halt + sleep 1 + while vm_is_running; do + echo "Waiting for $VM_NAME to shutdown..." >&2 + sleep 1 + done + echo "$VM_NAME is stopped." >&2 +fi + +popd diff --git a/sensor-raspi/debian/salsa-ci.yml b/sensor-raspi/debian/salsa-ci.yml new file mode 100644 index 000000000..a06e92265 --- /dev/null +++ b/sensor-raspi/debian/salsa-ci.yml @@ -0,0 +1,43 @@ +--- +stages: + # Garbage in is garbage out, so verify our input + - check input + - build + +variables: + DEBIAN_FRONTEND: "noninteractive" + # At https://salsa.debian.org/salsa-ci-team/pipeline/container_registry one can see which images are available + SALSA_CI_IMAGES: registry.salsa.debian.org/salsa-ci-team/pipeline + BASE_CI_IMAGES: ${SALSA_CI_IMAGES}/base + +yamllint: + stage: check input + image: $BASE_CI_IMAGES:unstable + dependencies: [] + script: + - apt-get update && apt-get upgrade -y + - apt-get install -y yamllint + - yamllint -c debian/yamllint.yml . + +shellcheck: + stage: check input + image: $BASE_CI_IMAGES:unstable + dependencies: [] + script: + - apt-get update && apt-get upgrade -y + - apt-get install -y shellcheck + - shellcheck -e SC1090,SC1091 -s dash $(find rootfs/etc/initramfs-tools -type f -executable | xargs grep -l '^#!/bin/sh') + +build yamls: + stage: build + image: $BASE_CI_IMAGES:unstable + dependencies: [] + script: + - apt-get update && apt-get upgrade -y + - apt-get install -y python3 make git + - make yaml + - mkdir build + - cp raspi*.yaml build/ + artifacts: + paths: + - build/ diff --git a/sensor-raspi/debian/yamllint.yml b/sensor-raspi/debian/yamllint.yml new file mode 100644 index 000000000..53974a025 --- /dev/null +++ b/sensor-raspi/debian/yamllint.yml @@ -0,0 +1,6 @@ +--- + +extends: default + +rules: + line-length: disable diff --git a/sensor-raspi/generate-recipe.py b/sensor-raspi/generate-recipe.py new file mode 100755 index 000000000..e40b8a638 --- /dev/null +++ b/sensor-raspi/generate-recipe.py @@ -0,0 +1,197 @@ +#!/usr/bin/python3 + +import re +import sys +import subprocess +import os + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +MALCOLM_DIR = os.path.dirname(SCRIPT_DIR) +SENSOR_DIR = os.path.join(MALCOLM_DIR, 'sensor-iso') + +# pylint: disable=invalid-name + +### Sanity/usage checks + +if len(sys.argv) != 3: + print("E: need 2 arguments", file=sys.stderr) + sys.exit(1) + +version = sys.argv[1] +if version not in ["1", "2", "3", "4"]: + print("E: unsupported version %s" % version, file=sys.stderr) + sys.exit(1) + +suite = sys.argv[2] +if suite not in ['bullseye', 'bookworm', 'trixie']: + print("E: unsupported suite %s" % suite, file=sys.stderr) + sys.exit(1) +target_yaml = 'raspi_%s_%s.yaml' % (version, suite) + + +### Setting variables based on suite and version starts here + +# Arch, kernel, DTB: +if version == '1': + arch = 'armel' + linux = 'linux-image-rpi' + dtb = '/usr/lib/linux-image-*-rpi/bcm*rpi-*.dtb' +elif version == '2': + arch = 'armhf' + linux = 'linux-image-armmp' + dtb = '/usr/lib/linux-image-*-armmp/bcm*rpi*.dtb' +elif version in ['3', '4']: + arch = 'arm64' + linux = 'linux-image-arm64' + dtb = '/usr/lib/linux-image-*-arm64/broadcom/bcm*rpi*.dtb' + +# Bookworm introduced the 'non-free-firmware' component¹; before that, +# raspi-firmware was in 'non-free' +# +# ¹ https://www.debian.org/vote/2022/vote_003 +if suite != 'bullseye': + firmware_component = 'non-free-firmware' + firmware_component_old = 'non-free' +else: + firmware_component = 'non-free' + firmware_component_old = '' + +# wireless firmware: +if version != '2': + wireless_firmware = 'firmware-brcm80211' +else: + wireless_firmware = '' + +# bluetooth firmware: +if version != '2': + bluetooth_firmware = 'bluez-firmware' +else: + bluetooth_firmware = '' + +# Pi 4 on buster required some backports. Let's keep variables around, ready to +# be used whenever we need to pull specific things from backports. +backports_enable = False +backports_suite = '%s-backports' % suite + +# Serial console: +if version in ['1', '2']: + serial = 'ttyAMA0,115200' +elif version in ['3', '4']: + serial = 'ttyS1,115200' + +# CMA fixup: +extra_chroot_shell_cmds = [] +if version == '4': + extra_chroot_shell_cmds = [ + "sed -i 's/cma=64M //' /boot/firmware/cmdline.txt", + ] + +# Hostname: +hostname = 'Hedgehog-rpi-%s' % version + +# Nothing yet! +extra_root_shell_cmds = [ + 'cp sensor_install.sh "${ROOT?}/root/"', + '/bin/bash -c \'mkdir -p "${ROOT?}/opt/"{buildshared,deps,hooks,patches,sensor/sensor_ctl/suricata/rules-default,arkime/etc,zeek/bin}\'', + 'cp "%s/arkime/patch/"* "${ROOT?}/opt/patches/" || true' % MALCOLM_DIR, + 'cp "%s/arkime/etc/"* "${ROOT?}/opt/arkime/etc" || true' % SENSOR_DIR, + 'cp -r "%s/suricata/rules-default/"* "${ROOT?}/opt/sensor/sensor_ctl/suricata/rules-default/" || true' + % MALCOLM_DIR, + 'cp -r shared/* "${ROOT?}/opt/buildshared"', + 'cp -r "%s/interface/"* "${ROOT?}/opt/sensor"' % SENSOR_DIR, + 'cp -r "%s/shared/bin/"* "${ROOT?}/usr/local/bin"' % MALCOLM_DIR, + 'cp "%s/scripts/malcolm_utils.py" "${ROOT?}/usr/local/bin/"' % MALCOLM_DIR, + 'cp "%s/config/archives/beats.list.chroot" "${ROOT?}/etc/apt/sources.list.d/beats.list"' % SENSOR_DIR, + 'cp "%s/config/archives/beats.key.chroot" "${ROOT?}/etc/apt/keyrings/"' % SENSOR_DIR, + 'cp "%s/config/archives/fluentbit.list.chroot" "${ROOT?}/etc/apt/sources.list.d/fluentbit.list"' % SENSOR_DIR, + 'cp "%s/config/archives/fluentbit.key.chroot" "${ROOT?}/etc/apt/keyrings/"' % SENSOR_DIR, + 'cp -r "%s/config/includes.chroot/"* "${ROOT?}/"' % SENSOR_DIR, + 'rm -r "${ROOT?}/etc/live"', + 'cp -r "%s/config/hooks/normal/"* "${ROOT?}/opt/hooks/"' % SENSOR_DIR, + 'cp -r "%s/config/package-lists/"* "${ROOT?}/opt/deps/"' % SENSOR_DIR, + 'cp -r "%s/docs/images/hedgehog/logo/hedgehog-ascii-text.txt"* "${ROOT?}/root/"' % MALCOLM_DIR, +] + +# Extend list just in case version is 4 +extra_chroot_shell_cmds.extend( + [ + 'chmod 755 /root/sensor_install.sh', + '/root/sensor_install.sh 2>&1 | tee -a /root/sensor_install_debug', + ] +) + +### The following prepares substitutions based on variables set earlier + +# Enable backports with a reason, or add commented-out entry: +if backports_enable: + backports_stanza = """ +%s +deb http://deb.debian.org/debian/ %s main %s +""" % ( + backports_enable, + backports_suite, + firmware_component, + ) +else: + # ugh + backports_stanza = """ +# Backports are _not_ enabled by default. +# Enable them by uncommenting the following line: +# deb http://deb.debian.org/debian %s main %s +""" % ( + backports_suite, + firmware_component, + ) + +# gitcommit = subprocess.getoutput("git show -s --pretty='format:%C(auto)%h (%s, %ad)' --date=short ") +buildtime = subprocess.getoutput("date --utc +'%Y-%m-%d %H:%M'") + +### Write results: + + +def align_replace(text, pattern, replacement): + """ + This helper lets us keep the indentation of the matched pattern + with the upcoming replacement, across multiple lines. Naive + implementation, please make it more pythonic! + """ + lines = text.splitlines() + for i, line in enumerate(lines): + m = re.match(r'^(\s+)%s' % pattern, line) + if m: + indent = m.group(1) + del lines[i] + for r in replacement: + lines.insert(i, '%s%s' % (indent, r)) + i = i + 1 + break + return '\n'.join(lines) + '\n' + + +with open('raspi_master.yaml', 'r') as in_file: + with open(target_yaml, 'w') as out_file: + in_text = in_file.read() + out_text = ( + in_text.replace('__RELEASE__', suite) + .replace('__ARCH__', arch) + .replace('__FIRMWARE_COMPONENT__', firmware_component) + .replace('__FIRMWARE_COMPONENT_OLD__', firmware_component_old) + .replace('__LINUX_IMAGE__', linux) + .replace('__DTB__', dtb) + .replace('__WIRELESS_FIRMWARE__', wireless_firmware) + .replace('__BLUETOOTH_FIRMWARE__', bluetooth_firmware) + .replace('__SERIAL_CONSOLE__', serial) + .replace('__HOST__', hostname) + .replace('__BUILDTIME__', buildtime) + ) + # .replace('__GITCOMMIT__', gitcommit) \ + # .replace('__BUILDTIME__', buildtime) + + out_text = align_replace(out_text, '__EXTRA_ROOT_SHELL_CMDS__', extra_root_shell_cmds) + out_text = align_replace(out_text, '__EXTRA_CHROOT_SHELL_CMDS__', extra_chroot_shell_cmds) + out_text = align_replace(out_text, '__BACKPORTS__', backports_stanza.splitlines()) + + # Try not to keep lines where the placeholder was replaced + # with nothing at all (including on a "list item" line): + filtered = [x for x in out_text.splitlines() if not re.match(r'^\s+$', x) and not re.match(r'^\s+-\s*$', x)] + out_file.write('\n'.join(filtered) + "\n") diff --git a/sensor-raspi/raspi_master.yaml b/sensor-raspi/raspi_master.yaml new file mode 100644 index 000000000..38ed3e9f1 --- /dev/null +++ b/sensor-raspi/raspi_master.yaml @@ -0,0 +1,181 @@ +--- +# See https://wiki.debian.org/RaspberryPi3 for known issues and more details. +# image.yml based on revision: __GITCOMMIT__ + +steps: + - mkimg: "{{ output }}" + size: 16384M + + - mklabel: msdos + device: "{{ output }}" + + - mkpart: primary + fs-type: 'fat32' + device: "{{ output }}" + start: 4MiB + end: 512MiB + tag: tag-firmware + + - mkpart: primary + device: "{{ output }}" + start: 512MiB + end: 100% + tag: tag-root + + - kpartx: "{{ output }}" + + - mkfs: vfat + partition: tag-firmware + label: RASPIFIRM + + - mkfs: ext4 + partition: tag-root + label: RASPIROOT + + - mount: tag-root + + - mount: tag-firmware + mount-on: tag-root + dirname: '/boot/firmware' + + - unpack-rootfs: tag-root + + - debootstrap: __RELEASE__ + require_empty_target: false + mirror: http://deb.debian.org/debian + target: tag-root + arch: __ARCH__ + components: + - main + - __FIRMWARE_COMPONENT__ + - __FIRMWARE_COMPONENT_OLD__ + unless: rootfs_unpacked + + - create-file: /etc/apt/sources.list + contents: |+ + deb http://deb.debian.org/debian __RELEASE__ main contrib __FIRMWARE_COMPONENT__ __FIRMWARE_COMPONENT_OLD__ + deb http://deb.debian.org/debian __RELEASE__-updates main contrib __FIRMWARE_COMPONENT__ __FIRMWARE_COMPONENT_OLD__ + deb http://security.debian.org/debian-security __RELEASE__-security main contrib __FIRMWARE_COMPONENT__ __FIRMWARE_COMPONENT_OLD__ + __BACKPORTS__ + + unless: rootfs_unpacked + + - copy-file: /etc/initramfs-tools/hooks/rpi-resizerootfs + src: rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs + perm: 0755 + unless: rootfs_unpacked + + - copy-file: /etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs + src: rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs + perm: 0755 + unless: rootfs_unpacked + + - apt: install + packages: + - curl + - dosfstools + - clamav + - clamav-daemon + - clamav-freshclam + - iw + - gpg + - parted + - ssh + - wpasupplicant + - systemd-timesyncd + - __LINUX_IMAGE__ + - raspi-firmware + - __WIRELESS_FIRMWARE__ + - __BLUETOOTH_FIRMWARE__ + tag: tag-root + unless: rootfs_unpacked + + - cache-rootfs: tag-root + unless: rootfs_unpacked + + - shell: | + echo "__HOST__-$(date +%Y%m%d)" > "${ROOT?}/etc/hostname" + + # Allow root logins locally with no password + sed -i 's,root:[^:]*:,root::,' "${ROOT?}/etc/shadow" + + install -m 644 -o root -g root rootfs/etc/fstab "${ROOT?}/etc/fstab" + + install -m 644 -o root -g root rootfs/etc/network/interfaces.d/eth0 "${ROOT?}/etc/network/interfaces.d/eth0" + install -m 600 -o root -g root rootfs/etc/network/interfaces.d/wlan0 "${ROOT?}/etc/network/interfaces.d/wlan0" + + install -m 755 -o root -g root rootfs/usr/local/sbin/rpi-set-sysconf "${ROOT?}/usr/local/sbin/rpi-set-sysconf" + install -m 644 -o root -g root rootfs/etc/systemd/system/rpi-set-sysconf.service "${ROOT?}/etc/systemd/system/" + install -m 644 -o root -g root rootfs/boot/firmware/sysconf.txt "${ROOT?}/boot/firmware/sysconf.txt" + mkdir -p "${ROOT?}/etc/systemd/system/basic.target.requires/" + ln -s /etc/systemd/system/rpi-set-sysconf.service "${ROOT?}/etc/systemd/system/basic.target.requires/rpi-set-sysconf.service" + + install -m 644 -o root -g root rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service "${ROOT?}/etc/systemd/system/" + mkdir -p "${ROOT?}/etc/systemd/system/multi-user.target.requires/" + ln -s /etc/systemd/system/rpi-reconfigure-raspi-firmware.service "${ROOT?}/etc/systemd/system/multi-user.target.requires/rpi-reconfigure-raspi-firmware.service" + + install -m 644 -o root -g root rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service "${ROOT?}/etc/systemd/system/" + ln -s /etc/systemd/system/rpi-generate-ssh-host-keys.service "${ROOT?}/etc/systemd/system/multi-user.target.requires/rpi-generate-ssh-host-keys.service" + rm -f "${ROOT?}"/etc/ssh/ssh_host_*_key* + + __EXTRA_ROOT_SHELL_CMDS__ + root-fs: tag-root + + # Copy the relevant device tree files to the boot partition + - chroot: tag-root + shell: | + install -m 644 -o root -g root __DTB__ /boot/firmware/ + + # Clean up archive cache (likely not useful) and lists (likely outdated) to + # reduce image size by several hundred megabytes. + - chroot: tag-root + shell: | + apt-get clean + rm -rf /var/lib/apt/lists + + # Modify the kernel commandline we take from the firmware to boot from + # the partition labeled raspiroot instead of forcing it to mmcblk0p2. + # Also insert the serial console right before the root= parameter. + # + # These changes will be overwritten after the hardware is probed + # after dpkg reconfigures raspi-firmware (upon first boot), so make + # sure we don't lose label-based booting. + - chroot: tag-root + shell: | + sed -i 's/root=/console=__SERIAL_CONSOLE__ root=/' /boot/firmware/cmdline.txt + sed -i 's#root=/dev/mmcblk0p2#root=LABEL=RASPIROOT#' /boot/firmware/cmdline.txt + sed -i 's/^#ROOTPART=.*/ROOTPART=LABEL=RASPIROOT/' /etc/default/raspi*-firmware + + __EXTRA_CHROOT_SHELL_CMDS__ + + # TODO(https://github.com/larswirzenius/vmdb2/issues/24): remove once vmdb + # clears /etc/resolv.conf on its own. + - shell: | + rm "${ROOT?}/etc/resolv.conf" + root-fs: tag-root + + # Clear /etc/machine-id and /var/lib/dbus/machine-id, as both should + # be auto-generated upon first boot. From the manpage + # (machine-id(5)): + # + # For normal operating system installations, where a custom image is + # created for a specific machine, /etc/machine-id should be + # populated during installation. + # + # Note this will also trigger ConditionFirstBoot=yes for systemd. + # On Buster, /etc/machine-id should be an emtpy file, not an absent file + # On Bullseye, /etc/machine-id should not exist in an image + - chroot: tag-root + shell: | + rm -f /etc/machine-id /var/lib/dbus/machine-id + echo "uninitialized" > /etc/machine-id + + # Resize script is now in the initrd for first boot; no need to ship it. + # Removing here to avoid any update-initramfs triggers from removing script during build process + rm -f "/etc/initramfs-tools/hooks/rpi-resizerootfs" + rm -f "/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs" + + # Create /etc/raspi-image-id to know, from what commit the image was built + - chroot: tag-root + shell: | + echo "Image built on __BUILDTIME__ (UTC)" > "/etc/raspi-image-id" diff --git a/sensor-raspi/rootfs/boot/firmware/sysconf.txt b/sensor-raspi/rootfs/boot/firmware/sysconf.txt new file mode 100644 index 000000000..4ea5a247a --- /dev/null +++ b/sensor-raspi/rootfs/boot/firmware/sysconf.txt @@ -0,0 +1,35 @@ +# This file will be automatically evaluated and installed at next boot +# time, and regenerated (to avoid leaking passwords and such information). +# +# To force it to be evaluated immediately, you can run (as root): +# +# /usr/sbin/rpi-set-sysconf +# +# You can disable the file evaluation by disabling the rpi-set-sysconf +# service in systemd: +# +# systemctl disable rpi-set-sysconf +# +# Comments (all portions of a line following a '#' character) are +# ignored. This file is read line by line. Valid +# configuration lines are of the form 'key=value'. Whitespace around +# 'key' and 'value' is ignored. This file will be _regenerated_ every +# time it is evaluated. +# +# We follow the convention to indent with one space comments, and +# leave no space to indicate the line is an example that could be +# uncommented. + +# root_pw - Set a password for the root user (by default, it allows +# for a passwordless login) +#root_pw=FooBar + +# root_authorized_key - Set an authorized key for a root ssh login +#root_authorized_key= + +# hostname - Set the system hostname. +#hostname=rpi + +# We found the following unhandled keys - That means, the +# configuration script does not know how to handle them. Please +# double-check them! diff --git a/sensor-raspi/rootfs/etc/fstab b/sensor-raspi/rootfs/etc/fstab new file mode 100644 index 000000000..805b59973 --- /dev/null +++ b/sensor-raspi/rootfs/etc/fstab @@ -0,0 +1,4 @@ +# The root file system has fs_passno=1 as per fstab(5) for automatic fsck. +LABEL=RASPIROOT / ext4 rw 0 1 +# All other file systems have fs_passno=2 as per fstab(5) for automatic fsck. +LABEL=RASPIFIRM /boot/firmware vfat rw 0 2 diff --git a/sensor-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs b/sensor-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs new file mode 100755 index 000000000..35e67fce3 --- /dev/null +++ b/sensor-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs @@ -0,0 +1,52 @@ +#!/bin/sh +set -e + +# +# List the soft prerequisites here. This is a space separated list of +# names, of scripts that are in the same directory as this one, that +# must be run before this one can be. +# +PREREQS="" +case $1 in + prereqs) echo "$PREREQS"; exit 0;; +esac + +. /usr/share/initramfs-tools/hook-functions + +# List ALL the programs we need, because we explicitly call them +# Don't *assume* it will be included! +# The update-initramfs script will figure out any dependencies +# that also need to be included, so lets not do that +# +# Find the path as used by the package itself; usrmerge may not be used + +# from coreutils +copy_exec /usr/bin/realpath +copy_exec /usr/bin/tail +copy_exec /usr/bin/test + +# from dosfstools +copy_exec /sbin/fsck.vfat + +# from e2fsprogs +copy_exec /sbin/resize2fs +copy_exec /sbin/fsck.ext4 + +# from grep +copy_exec /bin/grep + +# from logsave +copy_exec /sbin/logsave + +# from mount +copy_exec /bin/mount +copy_exec /bin/umount + +# from parted +copy_exec /sbin/parted +copy_exec /sbin/partprobe + +# from util-linux +copy_exec /bin/lsblk +copy_exec /sbin/blkid +copy_exec /sbin/fsck diff --git a/sensor-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs b/sensor-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs new file mode 100755 index 000000000..382684776 --- /dev/null +++ b/sensor-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs @@ -0,0 +1,59 @@ +#!/bin/sh +set -e + +# +# List the soft prerequisites here. This is a space separated list of +# names, of scripts that are in the same directory as this one, that +# must be run before this one can be. +# +PREREQS="" +case $1 in + prereqs) echo "$PREREQS"; exit 0;; +esac + +. /scripts/functions + +# Given the root partition, get the underlying device and partition number +rootpart=$(realpath "$ROOT") +rootpart_nr=$(blkid -sPART_ENTRY_NUMBER -o value -p "$rootpart") +rootdev="/dev/$(lsblk -no pkname "$rootpart")" + +# Parted will detect if the GPT label is messed up and fix it +# automatically, we just need to tell it to do so. +parted -s "$rootdev" print 2>&1 | grep -z "fix the GPT" && { + echo "Fix" | parted ---pretend-input-tty "$rootdev" print +} + +# Check if there's free space at the end of the device +free_space="$(parted -m -s "$rootdev" print free | tail -n1 | grep free)" +if test -z "$free_space"; then + # Great, we already resized; nothing left to do! + exit 0 +fi + +log_begin_msg "$0 resizing $ROOT" + +# Unmount for safety; fail if unset or empty (shellcheck SC2154) +umount "${rootmnt:?}" + +# Expand the partition size to fill the entire device +parted -s "$rootdev" resizepart "$rootpart_nr" 100% + +wait_for_udev 5 + +# Now resize the filesystem +partprobe "$rootdev" +resize2fs "$rootpart" + +# After resizing, (re)check the root partition's filesystem +fsck "$rootpart" + +# Remount root +# Don't quote ${ROOTFLAGS} as that results in an extra (empty) argument +# to 'mount', which in turn causes a boot failure +# shellcheck disable=SC2086 +if ! mount -r ${FSTYPE:+-t "${FSTYPE}"} ${ROOTFLAGS} "${ROOT}" "${rootmnt?}"; then + panic "Failed to mount ${ROOT} as root file system." +fi + +log_end_msg diff --git a/sensor-raspi/rootfs/etc/network/interfaces.d/eth0 b/sensor-raspi/rootfs/etc/network/interfaces.d/eth0 new file mode 100644 index 000000000..0caae5ffb --- /dev/null +++ b/sensor-raspi/rootfs/etc/network/interfaces.d/eth0 @@ -0,0 +1,3 @@ +auto eth0 +iface eth0 inet dhcp +iface eth0 inet6 auto diff --git a/sensor-raspi/rootfs/etc/network/interfaces.d/wlan0 b/sensor-raspi/rootfs/etc/network/interfaces.d/wlan0 new file mode 100644 index 000000000..fe24333e4 --- /dev/null +++ b/sensor-raspi/rootfs/etc/network/interfaces.d/wlan0 @@ -0,0 +1,4 @@ +auto wlan0 +allow-hotplug wlan0 +iface wlan0 inet dhcp + wpa-conf /etc/wpa_supplicant/wpa_supplicant.conf diff --git a/sensor-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service b/sensor-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service new file mode 100644 index 000000000..496f2c47f --- /dev/null +++ b/sensor-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service @@ -0,0 +1,10 @@ +[Unit] +Description=generate SSH host keys +ConditionPathExistsGlob=!/etc/ssh/ssh_host_*_key + +[Service] +Type=oneshot +ExecStart=/usr/sbin/dpkg-reconfigure -fnoninteractive openssh-server + +[Install] +RequiredBy=multi-user.target diff --git a/sensor-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service b/sensor-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service new file mode 100644 index 000000000..d8c555826 --- /dev/null +++ b/sensor-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service @@ -0,0 +1,14 @@ +[Unit] +Description=Reconfigure raspi-firmware to regenerate config.txt matching actual hardware +Before=sysinit.target +DefaultDependencies=no +RequiresMountsFor=/boot/firmware + +[Service] +Type=oneshot +TimeoutSec=infinity +ExecStart=/usr/sbin/dpkg-reconfigure raspi-firmware +ExecStart=/bin/systemctl --no-reload disable %n + +[Install] +RequiredBy=sysinit.target diff --git a/sensor-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service b/sensor-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service new file mode 100644 index 000000000..46bddcfd3 --- /dev/null +++ b/sensor-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service @@ -0,0 +1,9 @@ +[Unit] +Description=Set up system configuration + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/rpi-set-sysconf + +[Install] +RequiredBy=basic.target diff --git a/sensor-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf b/sensor-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf new file mode 100644 index 000000000..f4b47f66f --- /dev/null +++ b/sensor-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf @@ -0,0 +1,9 @@ +ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev +update_config=1 +country=US + +# network={ +# ssid="Your network SSID" +# psk="Your WPA/WPA2 security key" +# key_mgmt=WPA-PSK +# } diff --git a/sensor-raspi/rootfs/usr/local/sbin/rpi-set-sysconf b/sensor-raspi/rootfs/usr/local/sbin/rpi-set-sysconf new file mode 100644 index 000000000..f4f1b00c2 --- /dev/null +++ b/sensor-raspi/rootfs/usr/local/sbin/rpi-set-sysconf @@ -0,0 +1,157 @@ +#!/usr/bin/perl +use strict; +use warnings; +use IO::File; +use IO::Pipe; +use feature 'switch'; + +my ($filename, $conf); + +$filename = '/boot/firmware/sysconf.txt'; + +logger('info', "Reading the system configuration settings from $filename"); +$conf = read_conf($filename); + +if (my $pass = delete($conf->{root_pw})) { + my $pipe; + logger('debug', 'Resetting root password'); + unless (open($pipe, '|-', '/usr/sbin/chpasswd')) { + my $err = $!; + logger('error', "Could not run chpasswd: $err"); + die $err; + } + $pipe->print("root:$pass"); + close($pipe); +} + +if (my $root_authorized_key = delete($conf->{root_authorized_key})) { + my $fh; + logger('debug', "Adding key to root's authorized_keys"); + if(! -d "/root/.ssh") { + if(!mkdir("/root/.ssh", 0700)) { + my $err = sprintf("Could not create /root/.ssh directory: %s", $!); + logger('error', $err); + die $err; + } + } + + unless ($fh = IO::File->new('/root/.ssh/authorized_keys', 'w', 0600)) { + my $err = $!; + logger('error', "Could not write /root/.ssh/authorized_keys: $err"); + die $err; + } + $fh->print($root_authorized_key); + $fh->close; +} + +if (my $name = delete($conf->{hostname})) { + my $fh; + logger('debug', "Setting hostname to '$name'"); + unless ($fh = IO::File->new('/etc/hostname', 'w')) { + my $err = $!; + logger('error', "Could not write hostname '$name': $err"); + die $err; + } + $fh->print($name); + $fh->close; + system('hostname', '--file', '/etc/hostname'); +} + +rewrite_conf_file($filename, $conf); + +exit 0; + +sub read_conf { + my ($file, $conf, $fh); + $file = shift; + + $conf = {}; + unless ($fh = IO::File->new($filename, 'r')) { + my $err = $!; + logger('error', "Could not read from configuration file '$filename': $err"); + # Not finding the config file is not fatal: there is just + # nothing to configure! + return $conf; + } + while (my $line = $fh->getline) { + my ($key, $value); + # Allow for comments, and properly ignore them + $line =~ s/#.+//; + if ( ($key, $value) = ($line =~ m/^\s*([^=]+)\s*=\s*(.*)\s*$/)) { + $key = lc($key); + if (exists($conf->{$key})) { + logger('warn', + "Repeated configuration key: $key. " . + "Overwriting with new value ($value)"); + } + $conf->{$key} = $value; + } + } + $fh->close; + + return $conf; +} + +sub logger { + my ($prio, $msg) = @_; + system('logger', '-p', "daemon.$prio", + '-t', 'rpi-set-sysconf', $msg); +} + +sub rewrite_conf_file { + my ($filename, $conf) = @_; + my $fh; + unless ($fh = IO::File->new($filename, 'w')) { + my $err = $!; + logger('error', "Could not write to configuration file '$filename': $err"); + die $err; + } + $fh->print( +q(# This file will be automatically evaluated and installed at next boot +# time, and regenerated (to avoid leaking passwords and such information). +# +# To force it to be evaluated immediately, you can run (as root): +# +# /usr/local/sbin/rpi-set-sysconf +# +# You can disable the file evaluation by disabling the rpi-set-sysconf +# service in systemd: +# +# systemctl disable rpi-set-sysconf +# +# Comments (all portions of a line following a '#' character) are +# ignored. This file is read line by line. Valid +# configuration lines are of the form 'key=value'. Whitespace around +# 'key' and 'value' is ignored. This file will be _regenerated_ every +# time it is evaluated. +# +# We follow the convention to indent with one space comments, and +# leave no space to indicate the line is an example that could be +# uncommented. + +# root_pw - Set a password for the root user (by default, it allows +# for a passwordless login) +#root_pw=FooBar + +# root_authorized_key - Set an authorized key for a root ssh login +#root_authorized_key= + +# hostname - Set the system hostname. +#hostname=rpi +)); + + if (scalar keys %$conf) { + logger('warn', 'Unprocessed keys left in $filename: ' . + join(', ', sort keys %$conf)); + $fh->print( +q( +# We found the following unhandled keys - That means, the +# configuration script does not know how to handle them. Please +# double-check them! +)); + $fh->print(join('', map {sprintf("%s=%s\n", $_, $conf->{$_})} sort keys %$conf)); + } + $fh->close; +} + + diff --git a/sensor-raspi/sensor_install.sh b/sensor-raspi/sensor_install.sh new file mode 100644 index 000000000..627ede05f --- /dev/null +++ b/sensor-raspi/sensor_install.sh @@ -0,0 +1,489 @@ +#!/bin/bash -e +# This script is copied into a chroot'd environment +# Paths will be absolute and will reflect the path on the RPI Sensor + +PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/sbin:/usr/local/bin' +umask 0022 + +if [ "$(id -u)" != "0" ]; then + echo "This script must be run as root" 1>&2 + exit $BUILD_ERROR_CODE +fi + +IMAGE_NAME=hedgehog +IMAGE_PUBLISHER=cisagov +IMAGE_VERSION=1.0.0 +IMAGE_DISTRIBUTION=bookworm + +# Determine number of proc cores available +# Use caution messing with this value; build process may trigger OOM and fail!! +PROC_CNT=$(nproc) +ARCH="$(dpkg --print-architecture)" +export DEBIAN_FRONTEND=noninteractive + +# Used to build RPI without graphical features +# Changing to 1 is mostly unimplemented +BUILD_GUI=0 + +RUN_PATH="(pwd)" +DEBS_DIR="${HOME}/debs" +DEPS_DIR='/opt/deps' +SHARED_DIR='/opt/buildshared' +WORK_DIR="$(mktemp -d -t hedgehog-XXXXXX)" +SENSOR_DIR='/opt/sensor' + +BEATS_VER="8.12.1" +BEATS_OSS="-oss" + +# Option to build from sources if desired +# Building from source will increase build time A LOT (especially Zeek)! +BUILD_ARKIME_FROM_SOURCE=1 +BUILD_YARA_FROM_SOURCE=1 +BUILD_ZEEK_FROM_SOURCE=0 + +# Build time dependencies for arkime, htpdate, capa, and yara +BUILD_DEPS='automake checkinstall libjansson-dev libmagic-dev libnl-genl-3-dev libtool ' +BUILD_DEPS+='meson ninja-build python3-dev re2c ruby ruby-dev ruby-rubygems ' + +# Build dependencies we're leaving in place after installation (for building new Zeek plugins in the wild, mostly) +BUILD_DEPS_KEEP='build-essential ccache cmake flex gcc g++ git libfl-dev libgoogle-perftools-dev ' +BUILD_DEPS_KEEP+='libgoogle-perftools4 libkrb5-3 libkrb5-dev libmaxminddb-dev libpcap-dev libssl-dev libtcmalloc-minimal4 ' +BUILD_DEPS_KEEP+='make patch pkg-config python3-git python3-pip python3-semantic-version python3-setuptools python3-venv swig wget zlib1g-dev ' + +BUILD_ERROR_CODE=1 + +################################ +######### Functions ############ +################################ + +build_arkime(){ + mkdir -p /tmp/arkime-deb + arkime_ver='5.0.0-1' + curl -sSL -o /tmp/arkime-deb/arkime.deb "https://github.com/arkime/arkime/releases/download/v5.0.0/arkime_${arkime_ver}.ubuntu2204_arm64.deb" + dpkg -i /tmp/arkime-deb/*.deb || apt-get -f install -y --no-install-suggests +} + +build_arkime_src(){ + + arkime_repo='https://github.com/arkime/arkime.git' + arkime_ver='5.0.0' + arkime_dir='/opt/arkime' + build_jobs=$((PROC_CNT/2)) + + apt-get install $BUILD_DEPS $BUILD_DEPS_KEEP -y --no-install-suggests + + gem install --no-document fpm + + mkdir -p "${WORK_DIR}/arkime" && cd "$_" + git clone --recurse-submodules --branch="v${arkime_ver}" "$arkime_repo" ./ + + for patch_file in /opt/patches/*.patch; do + patch -p 1 -r - --no-backup-if-mismatch < $patch_file || true + done + + export PATH="${arkime_dir}/bin:${WORK_DIR}/arkime/node_modules/.bin:${PATH}" + + # I'm getting "Client network socket disconnected before secure TLS connection was established" when building Arkime, + # and this workaround seems to address it (see https://github.com/npm/cli/issues/4652) + for FILE in $(grep -rIcH 'npm ci' ./ | grep -v ':0$' | cut -d: -f 1); do sed -i "s/npm ci/npm ci --maxsockets 1/g" "$FILE"; done + + # configure the number of build threads + sed -i "s/MAKE=make/MAKE='make -j${build_jobs}'/" easybutton-build.sh + + ./easybutton-build.sh --dir "$arkime_dir" + + make install -j${build_jobs} + + cp -r ./capture/plugins/lua/samples "${arkime_dir}"/lua + + cat NOTICE release/CAPTURENOTICE > "${arkime_dir}/NOTICE.txt" + + rm -f $arkime_dir/etc/*.systemd.service + + fpm -s dir -t deb -n arkime -x opt/arkime/logs -x opt/arkime/raw \ + -v "$arkime_ver" --iteration 1 --template-scripts --after-install "release/afterinstall.sh" \ + --url "https://arkime.com" --description "Arkime Full Packet System" \ + -d libwww-perl -d libjson-perl -d ethtool -d libyaml-dev \ + -p "${DEBS_DIR}/arkime_${arkime_ver}_${ARCH}.deb" "$arkime_dir" + + cd "${WORK_DIR}" + rm -rf "${WORK_DIR}/arkime" "$arkime_dir" + + dpkg -i "${DEBS_DIR}/arkime_${arkime_ver}_${ARCH}.deb" +} + +build_htpdate() { + + # Htpdate in Debian repos doesn't compile https functionality + + htpdate_url='https://github.com/twekkel/htpdate' + htpdate_vers="$(curl -sqI $htpdate_url/releases/latest | awk -F '/' '/^location/ {print substr($NF,2,length($NF)-2)}')" + htpdate_release=1 + + apt-get install $BUILD_DEPS $BUILD_DEPS_KEEP -y --no-install-suggests + + mkdir -p "${WORK_DIR}"/htpdate && cd "$_" + curl -sSL "$htpdate_url/tarball/v$htpdate_vers" | tar xzf - --strip-components=1 + + sed -i '/.*man8.*/d' Makefile + + make https + + checkinstall -y -D --nodoc --strip=yes --stripso=yes --install=no --fstrans=no \ + --pkgname=htpdate --pkgversion=$htpdate_vers --pkgarch="$ARCH" --pkgsource="$htpdate_url" \ + --pkgrelease="$htpdate_release" --pakdir "$DEBS_DIR" + + # htpdate is installed outside of dpkg with checkinstall + make uninstall + + cd "${WORK_DIR}" + + dpkg -i "${DEBS_DIR}/htpdate_${htpdate_vers}-${htpdate_release}_${ARCH}.deb" +} + +build_interface() { + + interface_dir="${SENSOR_DIR}" + cd "$interface_dir" + + if [[ $BUILD_GUI -eq 1 ]]; then + # Items below required for GUI interface. Requires graphical DE to be useful + sed -i "s@/home/sensor/sensor_interface@${SENSOR_DIR}@g" "${interface_dir}/kiosk.service" + python3 -m pip install --break-system-packages --no-compile --no-cache-dir --force-reinstall \ + --upgrade -r requirements.txt + rm -rf "${interface_dir}/.git" "${interface_dir}/requirements.txt" + else + cd "${interface_dir}" + rm -rf .git requirements.txt init.sh kiosk.service sensor_interface/ + cd "$OLDPWD" + fi + + sed -i 's/CAPTURE_INTERFACE=.*/CAPTURE_INTERFACE=xxxx/g' "${interface_dir}/sensor_ctl/control_vars.conf" + rm -f "${interface_dir}/sensor_ctl/supervisor.d/fluentbit-aide.conf" \ + "${interface_dir}/sensor_ctl/supervisor.d/documentation.conf" + sed -i '/_AIDE/d' "${interface_dir}/sensor_ctl/control_vars.conf" +} + +build_yara_src() { + + # Build Yara from source code + + apt-get install $BUILD_DEPS $BUILD_DEPS_KEEP -y --no-install-suggests + + yara_url="https://github.com/VirusTotal/YARA" + yara_ver="$(curl -sqI ${yara_url}/releases/latest | awk -F '/' '/^location/ {print substr($NF,2,length($NF)-2)}')" + yara_release=1 + + mkdir -p "${WORK_DIR}/yara" && cd "$_" + curl -sSL "${yara_url}/tarball/v${yara_ver}" | tar xzf - --strip-components=1 + ./bootstrap.sh + ./configure --prefix=/usr --with-crypto --enable-magic --enable-cuckoo + make -j $PROC_CNT + + checkinstall -y -D --strip=yes --stripso=yes --nodoc --install=no --fstrans=no --pkgname="yara" \ + --pkgversion="$yara_ver" --pkgrelease="$yara_release" --pkgarch="$ARCH" --pkgsource="$yara_url" --pakdir="$DEBS_DIR" + + # Files are installed by checkinstall outside of DPKG + # Remove them since a deb has been created for later installation + make uninstall + + cd "${WORK_DIR}" + rm -rf "${WORK_DIR}/yara" + + dpkg -i "${DEBS_DIR}/yara_${yara_ver}-${yara_release}_${ARCH}.deb" +} + +build_zeek() { + # install zeek from debs from OpenSUSE + mkdir -p /tmp/zeek-debs + /bin/bash /usr/local/bin/zeek-deb-download.sh -o /tmp/zeek-debs + dpkg -i /tmp/zeek-debs/*.deb +} + +build_zeek_src() { + + # Build Zeek from source code + # Leaving this code here for future use if needed + + export CCACHE_DIR=/var/spool/ccache + export CCACHE_COMPRESS=1 + export PYTHONDONTWRITEBYTECODE=1 + export PYTHONUNBUFFERED=1 + + zeek_url=https://github.com/zeek/zeek.git + zeek_version=6.1.0 + zeek_release=1 + zeek_dir=/opt/zeek + # Zeek's build eats a ton of resources; prevent OOM from the killing build process + build_jobs=$((PROC_CNT/2)) + # Testing was done on a 8 cpu host with 16GB of ram. + # Successful Zeek from source build alone took: 6.5 hours + output_dir=/tmp + unset VERBOSE + + mkdir -p "${WORK_DIR}/zeek" && cd "$_" + curl -sSL "https://download.zeek.org/zeek-${zeek_version}.tar.gz" | tar xzf - --strip-components=1 + + mkdir -p "${CCACHE_DIR}" + ./configure --prefix="${zeek_dir}" --disable-broker-tests --disable-cpp-tests \ + --disable-btest-pcaps --disable-btest --generator=Ninja --ccache --enable-perftools + + mkdir -p build && cd "$_" + ninja -j "$build_jobs" + + checkinstall -y -D --strip=yes --stripso=yes --nodoc --install=no --fstrans=no \ + --pkgname="zeek" --pkgversion="$zeek_version" --pkgarch="$ARCH" --pkgsource="$zeek_url" \ + --pkgrelease="$zeek_release" --pakdir="$DEBS_DIR" ninja install + + # Files are installed by checkinstall outside of DPKG + # Remove them since a deb has been created for later installation + ninja uninstall + + cd "${WORK_DIR}" + rm -rf "${WORK_DIR}/zeek" + + dpkg -i "${DEBS_DIR}/zeek_${zeek_ver}-${zeek_release}_${ARCH}.deb" + +} + +clean_up() { + + # Set Hedgehog banner + mv /root/hedgehog-ascii-text.txt /etc/issue + + # Remove ethernet interface files left by installation + # Sensor setup will create necessary files when user runs setup + rm -f /etc/network/interfaces.d/eth0 + + # Ensure user network conf goes into proper file + touch /etc/network/interfaces.d/sensor + + # Remove this script and any debugging files + # Comment this out in order to troubleshoot the build process in a chroot + # Build process writes to /root/sensor_install_debug by default + rm -f /root/sensor_install* + + # Remove extra installation files + rm -rf $WORK_DIR \ + $SHARED_DIR \ + /opt/deps \ + /opt/hedgehog_install_artifacts \ + /opt/hooks \ + /opt/patches \ + /root/.bash_history \ + /root/.wget-hsts \ + /tmp/* + find /var/log/ -type f -print0 2>/dev/null | \ + xargs -0 -r -I XXX bash -c "file 'XXX' | grep -q text && > 'XXX'" + + # Remove unnecessary build components + apt-get remove $BUILD_DEPS -y + apt-get autoremove -y + apt-get clean + + # Ensure locale and console are setup correctly + locale-gen en_US.UTF-8 en.UTF-8 + update-locale LANG=en_US.UTF-8 LANGUAGE=en.UTF-8 + sed -i -e 's/CHARMAP=.*/CHARMAP="UTF-8"/' -e 's/CODESET=.*/CODESET="Lat15"/' /etc/default/console-setup + dpkg-reconfigure console-setup + + umount -A -f /dev/pts /run /dev /proc /sys + +} + +clean_up_gui_files() { + rm -rf /etc/skel/.config/autostart \ + /etc/skel/.config/xfce4 \ + /etc/skel/.local/share/xfce4 \ + /etc/skel/.config/gtk-3.0 \ + /etc/skel/.config/*dconf* +} + +create_user() { + + # Set defaults but it is STRONGLY recommended that these be changed before deploying Sensor + local user='sensor' + local pass='Hedgehog_Linux' + local root_pass='Hedgehog_Linux_Root' + + groupadd "$user" + useradd -m -g sensor -u 1000 -s /bin/bash "$user" + + echo -n "${user}:${pass}" | chpasswd --crypt-method YESCRYPT + echo -n "root:${root_pass}" | chpasswd --crypt-method YESCRYPT + +} + +install_deps() { + + local deps='' + + if [ $BUILD_GUI -eq 0 ]; then + rm -f "${DEPS_DIR}/"{desktopmanager,live,virtualguest}.list.chroot + rm -f "${DEPS_DIR}/grub.list.binary" + fi + + for file in "${DEPS_DIR}/"*.chroot; do + sed -i '$a\' "$file" + deps+=$(tr '\n' ' ' < "$file") + done + + # Remove Sensor-ISO packages not relevant to RPI + # Rar is excluded because Debian doesn't have an ARM package + # htpdate removed because repo version doesn't support https + # aide is removed as we're not applying the same hardening requirements ot the rpi image + declare -a graphical_deps=( aide aide-common efibootmgr fonts-dejavu fuseext2 fusefat fuseiso gdb ) + graphical_deps+=( gparted gdebi google-perftools gvfs gvfs-daemons gvfs-fuse ghostscript ghostscript-x ) + graphical_deps+=( hfsplus hfsprogs hfsutils htpdate libgtk2.0-bin menu neofetch pmount rar ) + graphical_deps+=( ssh-askpass udisks2 upower user-setup xbitmaps zenity zenity-common ) + graphical_deps+=( libsmbclient samba-common samba-common-bin samba-dsdb-modules samba-libs smbclient ) + + deps=$(echo ${deps} ${graphical_deps[@]} | tr ' ' '\n' | sort | uniq -u | tr '\n' ' ') + + apt-get update + # Hedgehog conf files are copied into env before this runs; keep those config files by default + apt-get -o Dpkg::Options::="--force-confold" install -q $deps -y --no-install-suggests + apt-get clean + +} + +install_files() { + + # Shared Scripts setup + ln -s /usr/local/bin/malcolm_utils.py "/opt/zeek/bin/" + mv /usr/local/bin/zeekdeploy.sh "/opt/zeek/bin/" + rm -rf /usr/local/bin/aide_integrity_check.sh + + # Setup OS information + sensor_ver_file="${SENSOR_DIR}/.os-info" + + if [[ -f "$SHARED_DIR/version.txt" ]]; then + SHARED_IMAGE_VERSION="$(cat "$SHARED_DIR/version.txt" | head -n 1)" + [[ -n $SHARED_IMAGE_VERSION ]] && IMAGE_VERSION="$SHARED_IMAGE_VERSION" + fi + + echo "BUILD_ID=\"$(date +\'%Y-%m-%d\')-${IMAGE_VERSION}\"" > "$sensor_ver_file" + echo "VARIANT=\"Hedgehog Linux (Sensor) v${IMAGE_VERSION}\"" >> "$sensor_ver_file" + echo "VARIANT_ID=\"hedgehog-sensor\"" >> "$sensor_ver_file" + echo "ID_LIKE=\"debian\"" >> "$sensor_ver_file" + echo "HOME_URL=\"https://${IMAGE_PUBLISHER}.github.io/Malcolm\"" >> "$sensor_ver_file" + echo "DOCUMENTATION_URL=\"https://${IMAGE_PUBLISHER}.github.io/Malcolm/docs/hedgehog.html\"" >> "$sensor_ver_file" + echo "SUPPORT_URL=\"https://github.com/${IMAGE_PUBLISHER}\"" >> "$sensor_ver_file" + echo "BUG_REPORT_URL=\"https://github.com/${IMAGE_PUBLISHER}/malcolm/issues\"" >> "$sensor_ver_file" + + # Setup MaxMind Geo IP info + mkdir -p /opt/arkime/etc + pushd /opt/arkime >/dev/null 2>&1 + MAXMIND_GEOIP_DB_LICENSE_KEY="" + + if [[ -f "$SHARED_DIR/maxmind_license.txt" ]]; then + MAXMIND_GEOIP_DB_LICENSE_KEY="$(cat "$SHARED_DIR/maxmind_license.txt" | head -n 1)" + if [[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]]; then + for DB in ASN Country City; do + curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" + if [[ -f "GeoLite2-$DB.mmdb.tar.gz" ]]; then + tar xvf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 --no-same-owner + rm -f "GeoLite2-$DB.mmdb.tar.gz" + fi + done + fi + fi + curl -s -S -L -o ./etc/ipv4-address-space.csv "https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv" + curl -s -S -L -o ./etc/oui.txt "https://www.wireshark.org/download/automated/data/manuf" + popd >/dev/null 2>&1 + + # Prepare Fluentbit and Beats repo GPG keys + local apt_lists='/etc/apt/sources.list.d' + local apt_keys='/etc/apt/keyrings' + local beats_key="${apt_keys}/beats.gpg" + local fluentbit_key="${apt_keys}/fluentbit.gpg" + + gpg --dearmor --batch --yes -o "$beats_key" "${apt_keys}/beats.key.chroot" + gpg --dearmor --batch --yes -o "$fluentbit_key" "${apt_keys}/fluentbit.key.chroot" + + rm "${apt_keys}/beats.key.chroot" "${apt_keys}/fluentbit.key.chroot" + + sed -i -e "s|deb |deb [signed-by=${beats_key}] |" "${apt_lists}/beats.list" + sed -i -e "s|deb |deb [signed-by=${fluentbit_key}] |" "${apt_lists}/fluentbit.list" + + # Prepare debs directory for other packages + mkdir -p "${DEBS_DIR}" + + # Disable ipv6 + echo 'ipv6.disable=1' > /etc/default/raspi-extra-cmdline + + # Add RPI hostname to /etc/hosts + echo "127.0.1.1 $(head -n 1 /etc/hostname)" >> /etc/hosts + + # mark as first run + touch "${SENSOR_DIR}"/firstrun +} + +install_hooks() { + + set -e + + local hooks_dir='/opt/hooks' + + if [[ $BUILD_GUI -eq 0 ]]; then + rm -f "${hooks_dir}"/*login.hook.chroot + rm -f "${hooks_dir}"/*stig-scripts.hook.chroot + fi + + for file in ${hooks_dir}/*.hook.chroot; do + /bin/bash "$file" + done + +} + +################################ +########## Main ################ +################################ + +# Make sure necessary virtual filesystems available in chroot +mount -t proc /proc /proc +mount -t devtmpfs /dev /dev +mount -t devpts /dev/pts /dev/pts +mount -t sysfs /sys /sys +mount -t tmpfs /run /run + + +[[ -f "$SHARED_DIR/environment.chroot" ]] && \ + . "$SHARED_DIR/environment.chroot" + +install_files +install_deps +build_interface + +# Remove GUI related files if not building RPI with a DE +# See comment above about BUILD_GUI usage +if [[ $BUILD_GUI -eq 0 ]]; then + clean_up_gui_files +fi + +if [ $BUILD_ARKIME_FROM_SOURCE -eq 1 ]; then + build_arkime_src +else + build_arkime +fi + +if [ $BUILD_YARA_FROM_SOURCE -eq 1 ]; then + build_yara_src +else + # Not implemented currently + #build_yara + build_yara_src +fi + +if [ $BUILD_ZEEK_FROM_SOURCE -eq 1 ]; then + build_zeek_src +else + build_zeek +fi + +install_hooks +build_htpdate +create_user +clean_up + +exit 0 diff --git a/sensor-raspi/vagrant/Vagrantfile b/sensor-raspi/vagrant/Vagrantfile new file mode 100644 index 000000000..cb46f2255 --- /dev/null +++ b/sensor-raspi/vagrant/Vagrantfile @@ -0,0 +1,59 @@ +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +unless Vagrant.has_plugin?("vagrant-sshfs") + raise 'vagrant-sshfs plugin is not installed!' +end + +# hack: https://github.com/hashicorp/vagrant/issues/8878#issuecomment-345112810 +class VagrantPlugins::ProviderVirtualBox::Action::Network + def dhcp_server_matches_config?(dhcp_server, config) + true + end +end + +Vagrant.configure("2") do |config| + + config.vm.define "vagrant-hedgehog-raspi-build" + config.vm.box = "bento/debian-12" + + config.vm.network "private_network", type: "dhcp" + config.ssh.config = "ssh_config" + + config.vm.synced_folder '.', '/vagrant', disabled: true + config.vm.synced_folder "../..", "/Malcolm", type: "sshfs", disabled: false + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + + config.vm.provider "virtualbox" do |vb| + vb.memory = "8192" + vb.cpus = 4 + end + + config.vm.provider "libvirt" do |lv| + lv.memory = "8192" + lv.cpus = 4 + end + + config.vm.provision "shell", inline: <<-STEP1 + dpkg-reconfigure debconf -f noninteractive -p critical + export DEBIAN_FRONTEND=noninteractive + sed -i "s/main/main contrib non-free non-free-firmware/g" /etc/apt/sources.list + apt-get -qqy update + apt-get -y install \ + binfmt-support \ + bmap-tools \ + ca-certificates \ + debootstrap \ + dosfstools \ + kpartx \ + psmisc \ + python3 \ + qemu-user-static \ + qemu-utils \ + time \ + vmdb2 \ + zerofree + STEP1 +end diff --git a/sensor-raspi/vagrant/ssh_config b/sensor-raspi/vagrant/ssh_config new file mode 100644 index 000000000..10ff23648 --- /dev/null +++ b/sensor-raspi/vagrant/ssh_config @@ -0,0 +1,3 @@ +Host * + IdentitiesOnly yes + GSSAPIAuthentication no diff --git a/shared/bin/agg-init.sh b/shared/bin/agg-init.sh index b0215f97c..b17910c91 100755 --- a/shared/bin/agg-init.sh +++ b/shared/bin/agg-init.sh @@ -12,6 +12,7 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then # remove default accounts/groups we don't want, create/set directories for non-user users for stig to not complain CleanDefaultAccounts + FIRST_RUN=0 MAIN_USER="$(id -nu 1000)" if [[ -n $MAIN_USER ]]; then @@ -24,6 +25,7 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then # if Malcolm's config file has never been touched, configure it now MAIN_USER_HOME="$(getent passwd "$MAIN_USER" | cut -d: -f6)" if [[ -f "$MAIN_USER_HOME"/Malcolm/firstrun ]]; then + FIRST_RUN=1 if [[ -r "$MAIN_USER_HOME"/Malcolm/scripts/install.py ]]; then /usr/bin/env python3 "$MAIN_USER_HOME"/Malcolm/scripts/install.py --configure --defaults --restart-malcolm fi @@ -38,6 +40,9 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then # we're going to let wicd manage networking on the aggregator, so remove physical interfaces from /etc/network/interfaces InitializeAggregatorNetworking + # disable automatic running of some services + [[ "$FIRST_RUN" == 1 ]] && DisableServices + # block some call-homes BadTelemetry diff --git a/shared/bin/common-init.sh b/shared/bin/common-init.sh index 19bbd3f9b..9031044a2 100755 --- a/shared/bin/common-init.sh +++ b/shared/bin/common-init.sh @@ -45,6 +45,36 @@ function CleanDefaultAccounts() { chmod 700 "/etc/cron.hourly" "/etc/cron.daily" "/etc/cron.weekly" "/etc/cron.monthly" "/etc/cron.d" >/dev/null 2>&1 || true } +# disable automatic running of some services +function DisableServices() { + for ACTION in stop disable; do + for SERVICE in apt-daily-upgrade.service \ + apt-daily-upgrade.timer \ + apt-daily.service \ + apt-daily.timer \ + clamav-clamonacc.service \ + clamav-daemon.socket \ + clamav-daemon.service \ + clamav-freshclam.service \ + ctrl-alt-del.target \ + filebeat.service \ + fluent-bit.service \ + htpdate.service \ + ntpsec.service \ + sendmail.service \ + supervisor.service \ + suricata.service; do + systemctl "$ACTION" "$SERVICE" >/dev/null 2>&1 || true + done + done + for ACTION in disable remove; do + for SERVICE in clamav-daemon \ + clamav-freshclam; do + update-rc.d -f "$SERVICE" "$ACTION" >/dev/null 2>&1 || true + done + done +} + # setup initially-created user's directory based on /etc/skel function InjectSkeleton() { if [ -n "$1" ]; then diff --git a/shared/bin/configure-capture.py b/shared/bin/configure-capture.py index d463fa206..44cc1b01c 100755 --- a/shared/bin/configure-capture.py +++ b/shared/bin/configure-capture.py @@ -284,12 +284,12 @@ def input_opensearch_connection_info( protocol=return_dict[Constants.BEAT_OS_PROTOCOL], host=return_dict[Constants.BEAT_OS_HOST], port=return_dict[Constants.BEAT_OS_PORT], - username=return_dict[Constants.BEAT_HTTP_USERNAME] - if (len(return_dict[Constants.BEAT_HTTP_USERNAME]) > 0) - else None, - password=return_dict[Constants.BEAT_HTTP_PASSWORD] - if (len(return_dict[Constants.BEAT_HTTP_PASSWORD]) > 0) - else None, + username=( + return_dict[Constants.BEAT_HTTP_USERNAME] if (len(return_dict[Constants.BEAT_HTTP_USERNAME]) > 0) else None + ), + password=( + return_dict[Constants.BEAT_HTTP_PASSWORD] if (len(return_dict[Constants.BEAT_HTTP_PASSWORD]) > 0) else None + ), ssl_verify=return_dict[Constants.BEAT_OS_SSL_VERIFY], ) if retcode == 200: @@ -875,9 +875,11 @@ def main(): code, arkime_password2 = d.passwordbox( f"{Constants.MSG_CONFIG_ARKIME_VIEWER_PASSWORD} (again)", insecure=True, - init=previous_config_values[Constants.ARKIME_PASSWORD_SECRET] - if (arkime_password == previous_config_values[Constants.ARKIME_PASSWORD_SECRET]) - else "", + init=( + previous_config_values[Constants.ARKIME_PASSWORD_SECRET] + if (arkime_password == previous_config_values[Constants.ARKIME_PASSWORD_SECRET]) + else "" + ), ) if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError @@ -1062,9 +1064,11 @@ def main(): 'Logstash Port', 2, 1, - previous_config_values[Constants.BEAT_LS_PORT] - if Constants.BEAT_LS_PORT in previous_config_values - else "5044", + ( + previous_config_values[Constants.BEAT_LS_PORT] + if Constants.BEAT_LS_PORT in previous_config_values + else "5044" + ), 2, 25, 6, @@ -1133,9 +1137,11 @@ def main(): 'SSL Certificate Authorities File', 1, 1, - previous_config_values[Constants.BEAT_LS_SSL_CA_CRT] - if Constants.BEAT_LS_SSL_CA_CRT in previous_config_values - else f"{Constants.BEAT_LS_CERT_DIR_DEFAULT}/ca.crt", + ( + previous_config_values[Constants.BEAT_LS_SSL_CA_CRT] + if Constants.BEAT_LS_SSL_CA_CRT in previous_config_values + else f"{Constants.BEAT_LS_CERT_DIR_DEFAULT}/ca.crt" + ), 1, 35, 30, @@ -1145,9 +1151,11 @@ def main(): 'SSL Certificate File', 2, 1, - previous_config_values[Constants.BEAT_LS_SSL_CLIENT_CRT] - if Constants.BEAT_LS_SSL_CLIENT_CRT in previous_config_values - else f"{Constants.BEAT_LS_CERT_DIR_DEFAULT}/client.crt", + ( + previous_config_values[Constants.BEAT_LS_SSL_CLIENT_CRT] + if Constants.BEAT_LS_SSL_CLIENT_CRT in previous_config_values + else f"{Constants.BEAT_LS_CERT_DIR_DEFAULT}/client.crt" + ), 2, 35, 30, @@ -1157,9 +1165,11 @@ def main(): 'SSL Key File', 3, 1, - previous_config_values[Constants.BEAT_LS_SSL_CLIENT_KEY] - if Constants.BEAT_LS_SSL_CLIENT_KEY in previous_config_values - else f"{Constants.BEAT_LS_CERT_DIR_DEFAULT}/client.key", + ( + previous_config_values[Constants.BEAT_LS_SSL_CLIENT_KEY] + if Constants.BEAT_LS_SSL_CLIENT_KEY in previous_config_values + else f"{Constants.BEAT_LS_CERT_DIR_DEFAULT}/client.key" + ), 3, 35, 30, diff --git a/shared/bin/configure-interfaces.py b/shared/bin/configure-interfaces.py index aff275b4a..d2d7c5f77 100755 --- a/shared/bin/configure-interfaces.py +++ b/shared/bin/configure-interfaces.py @@ -76,7 +76,7 @@ class Constants: MSG_NETWORK_STOP_ERROR = 'Error occured while bringing down the network interface!\n\n' MSG_NETWORK_STOP_SUCCESS = 'Brought down the network interface successfully!\n\n' MSG_TIME_SYNC_TYPE = 'Select time synchronization method' - MSG_TIME_SYNC_HTPDATE_CONFIG = 'Provide values for HTTP/HTTPS Server' + MSG_TIME_SYNC_HTPDATE_CONFIG = 'Provide URL for HTTP/HTTPS Server Time Sync' MSG_TIME_SYNC_TEST_SUCCESS = 'Server time retrieved successfully!\n\n' MSG_TIME_SYNC_CONFIG_SUCCESS = 'Time synchronization configured successfully!\n\n' MSG_TIME_SYNC_TEST_FAILURE = 'Server time could not be retrieved. Ignore error?\n\n' @@ -298,31 +298,27 @@ def main(): elif time_sync_mode == Constants.TIME_SYNC_HTPDATE: # sync time via htpdate, run via cron - http_host = '' - http_port = '' + http_url = '' while True: - # host/port for htpdate + # http/https URL for for htpdate code, values = d.form( Constants.MSG_TIME_SYNC_HTPDATE_CONFIG, - [('URL', 1, 1, '', 1, 25, 30, 255), ('Port', 2, 1, '443', 2, 25, 6, 5)], + [('URL', 1, 1, 'https://1.1.1.1:443', 1, 25, 30, 255)], ) values = [x.strip() for x in values] if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise CancelledError - elif (len(values[0]) <= 0) or (len(values[1]) <= 0) or (not values[1].isnumeric()): + elif len(values[0]) <= 0: code = d.msgbox(text=Constants.MSG_ERR_BAD_HOST) else: - http_host = values[0] - http_port = values[1] + http_url = values[0] break # test with htpdate to see if we can connect - ecode, test_output = run_subprocess( - f"{Constants.TIME_SYNC_HTPDATE_TEST_COMMAND} {http_host}:{http_port}" - ) + ecode, test_output = run_subprocess(f"{Constants.TIME_SYNC_HTPDATE_TEST_COMMAND} {http_url}") if ecode == 0: emsg_str = '\n'.join(test_output) code = d.msgbox(text=f"{Constants.MSG_TIME_SYNC_TEST_SUCCESS}{emsg_str}") @@ -352,15 +348,11 @@ def main(): f.write('SHELL=/bin/bash\n') f.write('PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin\n') f.write('\n') - f.write( - f'*/{htpdate_interval} * * * * root {Constants.TIME_SYNC_HTPDATE_COMMAND} {http_host}:{http_port}\n' - ) + f.write(f'*/{htpdate_interval} * * * * root {Constants.TIME_SYNC_HTPDATE_COMMAND} {http_url}\n') f.write('\n') # now actually do the sync "for real" one time (so we can get in sync before waiting for the interval) - ecode, sync_output = run_subprocess( - f"{Constants.TIME_SYNC_HTPDATE_COMMAND} {http_host}:{http_port}" - ) + ecode, sync_output = run_subprocess(f"{Constants.TIME_SYNC_HTPDATE_COMMAND} {http_url}") emsg_str = '\n'.join(sync_output) code = d.msgbox(text=f"{Constants.MSG_TIME_SYNC_CONFIG_SUCCESS if (ecode == 0) else ''}{emsg_str}") diff --git a/shared/bin/extracted_files_http_server.py b/shared/bin/extracted_files_http_server.py deleted file mode 100755 index 7d3817a2e..000000000 --- a/shared/bin/extracted_files_http_server.py +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Multithreaded simple HTTP directory server. -# -# The files can optionally be archived in a ZIP file, with or without a password, or -# be aes-256-cbc encrypted in a way that's compatible with: -# openssl enc -aes-256-cbc -d -in encrypted.data -out decrypted.data - -import argparse -import hashlib -import os -import sys -from Crypto.Cipher import AES -from datetime import datetime -from http.server import HTTPServer, SimpleHTTPRequestHandler -from socketserver import ThreadingMixIn -from stat import S_IFREG -from stream_zip import ZIP_32, stream_zip -from threading import Thread - - -from malcolm_utils import ( - str2bool, - eprint, - temporary_filename, - EVP_KEY_SIZE, - PKCS5_SALT_LEN, - OPENSSL_ENC_MAGIC, - EVP_BytesToKey, -) - -################################################################################################### -args = None -debug = False -script_name = os.path.basename(__file__) -script_path = os.path.dirname(os.path.realpath(__file__)) -orig_path = os.getcwd() - - -################################################################################################### -# -def LocalFilesForZip(names): - now = datetime.now() - - def contents(name): - with open(name, 'rb') as f: - while chunk := f.read(65536): - yield chunk - - return ((os.path.join('.', os.path.basename(name)), now, S_IFREG | 0o600, ZIP_32, contents(name)) for name in names) - - -################################################################################################### -# -class HTTPHandler(SimpleHTTPRequestHandler): - # return full path based on server base path and requested path - def translate_path(self, path): - path = SimpleHTTPRequestHandler.translate_path(self, path) - relpath = os.path.relpath(path, os.getcwd()) - fullpath = os.path.join(self.server.base_path, relpath) - return fullpath - - # override do_GET so that files are encrypted, if requested - def do_GET(self): - global debug - global args - - fullpath = self.translate_path(self.path) - fileBaseName = os.path.basename(fullpath) - - if os.path.isdir(fullpath): - # directory listing - SimpleHTTPRequestHandler.do_GET(self) - - else: - if args.recursive and (not os.path.isfile(fullpath)) and (not os.path.islink(fullpath)): - for root, dirs, files in os.walk(os.path.dirname(fullpath)): - if fileBaseName in files: - fullpath = os.path.join(root, fileBaseName) - break - - if os.path.isfile(fullpath) or os.path.islink(fullpath): - if args.zip: - # ZIP file (streamed, AES-encrypted with password or unencrypted) - self.send_response(200) - self.send_header('Content-type', "application/zip") - self.send_header('Content-Disposition', f'attachment; filename={fileBaseName}.zip') - self.end_headers() - - for chunk in stream_zip(LocalFilesForZip([fullpath]), password=args.key if args.key else None): - self.wfile.write(chunk) - - elif args.key: - # openssl-compatible encrypted file - self.send_response(200) - self.send_header('Content-type', 'application/octet-stream') - self.send_header('Content-Disposition', f'attachment; filename={fileBaseName}.encrypted') - self.end_headers() - salt = os.urandom(PKCS5_SALT_LEN) - key, iv = EVP_BytesToKey( - EVP_KEY_SIZE, AES.block_size, hashlib.sha256, salt, args.key.encode('utf-8') - ) - cipher = AES.new(key, AES.MODE_CBC, iv) - encrypted = b"" - encrypted += OPENSSL_ENC_MAGIC - encrypted += salt - self.wfile.write(encrypted) - with open(fullpath, 'rb') as f: - padding = b'' - while True: - chunk = f.read(cipher.block_size) - if len(chunk) < cipher.block_size: - remaining = cipher.block_size - len(chunk) - padding = bytes([remaining] * remaining) - self.wfile.write(cipher.encrypt(chunk + padding)) - if padding: - break - - else: - # original file, unencrypted - SimpleHTTPRequestHandler.do_GET(self) - - else: - self.send_error(404, "Not Found") - - -################################################################################################### -# -class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - def __init__(self, base_path, server_address, RequestHandlerClass=HTTPHandler): - self.base_path = base_path - HTTPServer.__init__(self, server_address, RequestHandlerClass) - - -################################################################################################### -# -def serve_on_port(path: str, port: int): - server = ThreadingHTTPServer(path, ("", port)) - print(f"serving {path} at port {port}") - server.serve_forever() - - -################################################################################################### -# main -def main(): - global args - global debug - global orig_path - - defaultDebug = os.getenv('EXTRACTED_FILE_HTTP_SERVER_DEBUG', 'false') - defaultZip = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ZIP', 'false') - defaultRecursive = os.getenv('EXTRACTED_FILE_HTTP_SERVER_RECURSIVE', 'false') - defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440)) - defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'infected') - defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path) - - parser = argparse.ArgumentParser( - description=script_name, add_help=False, usage='{} '.format(script_name) - ) - parser.add_argument( - '-v', - '--verbose', - dest='debug', - type=str2bool, - nargs='?', - const=True, - default=defaultDebug, - metavar='true|false', - help=f"Verbose/debug output ({defaultDebug})", - ) - parser.add_argument( - '-p', - '--port', - dest='port', - help=f"Server port ({defaultPort})", - metavar='', - type=int, - default=defaultPort, - ) - parser.add_argument( - '-d', - '--directory', - dest='serveDir', - help=f'Directory to serve ({defaultDir})', - metavar='', - type=str, - default=defaultDir, - ) - parser.add_argument( - '-k', - '--key', - dest='key', - help="File encryption key (for ZIP file if -z/--zip, otherwise openssl-compatible encryption", - metavar='', - type=str, - default=defaultKey, - ) - parser.add_argument( - '-z', - '--zip', - dest='zip', - type=str2bool, - nargs='?', - const=True, - default=defaultZip, - metavar='true|false', - help=f"Zip file ({defaultZip})", - ) - parser.add_argument( - '-r', - '--recursive', - dest='recursive', - type=str2bool, - nargs='?', - const=True, - default=defaultRecursive, - metavar='true|false', - help=f"Recursively look for requested file if not found", - ) - try: - parser.error = parser.exit - args = parser.parse_args() - except SystemExit: - parser.print_help() - exit(2) - - debug = args.debug - if debug: - eprint(os.path.join(script_path, script_name)) - eprint("Arguments: {}".format(sys.argv[1:])) - eprint("Arguments: {}".format(args)) - else: - sys.tracebacklimit = 0 - - Thread(target=serve_on_port, args=[args.serveDir, args.port]).start() - - -################################################################################################### -if __name__ == '__main__': - main() diff --git a/shared/bin/sensor-init.sh b/shared/bin/sensor-init.sh index 0f0ddaaa0..009d856a5 100755 --- a/shared/bin/sensor-init.sh +++ b/shared/bin/sensor-init.sh @@ -7,6 +7,7 @@ SCRIPT_PATH="$(dirname $(realpath -e "${BASH_SOURCE[0]}"))" echo "sensor" > /etc/installer MAIN_USER="$(id -nu 1000)" +FIRST_RUN=0 if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then . "$SCRIPT_PATH"/common-init.sh @@ -43,6 +44,8 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then [[ -d /opt/sensor/sensor_ctl/arkime/config.ini ]] && chmod 600 /opt/sensor/sensor_ctl/arkime/config.ini + [[ -f /opt/sensor/firstrun ]] && FIRST_RUN=1 && rm -f /opt/sensor/firstrun + fi dpkg -s fluent-bit >/dev/null 2>&1 && \ @@ -53,8 +56,8 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then if [[ -d /opt/zeek.orig ]]; then # as such, we're going to reset zeek to a "clean" state after each reboot. the config files will get # regenerated when we are about to deploy zeek itself - [[ -d /opt/zeek ]] && rm -rf /opt/zeek - rsync -a /opt/zeek.orig/ /opt/zeek + mkdir -p /opt/zeek/ + rsync --archive --delete --force /opt/zeek.orig/ /opt/zeek/ fi if [[ -d /opt/zeek ]]; then chown -R 1000:1000 /opt/zeek/* @@ -130,6 +133,9 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then FixPermissions "$MAIN_USER" fi + # disable automatic running of some services + [[ "$FIRST_RUN" == 1 ]] && DisableServices + # block some call-homes BadTelemetry diff --git a/shared/bin/suricata_config_populate.py b/shared/bin/suricata_config_populate.py index 50c365304..718d3e19c 100755 --- a/shared/bin/suricata_config_populate.py +++ b/shared/bin/suricata_config_populate.py @@ -187,6 +187,10 @@ def __call__(self, repr, data): 'SSH_EVE_ENABLED': False, 'SSH_HASSH': True, 'SSH_PORTS': 22, + 'STATS_ENABLED': False, + 'STATS_EVE_ENABLED': False, + 'STATS_INTERVAL': 30, + 'STATS_DECODER_EVENTS': False, 'STREAM_CHECKSUM_VALIDATION': False, 'STREAM_INLINE': 'auto', 'STREAM_MEMCAP': '64mb', @@ -743,9 +747,11 @@ def main(): 'cluster-id': clusterId, 'block-size': DEFAULT_VARS['AF_PACKET_BLOCK_SIZE'], 'block-timeout': DEFAULT_VARS['AF_PACKET_BLOCK_TIMEOUT'], - 'bpf-filter': DEFAULT_VARS['CAPTURE_FILTER'] - if DEFAULT_VARS['CAPTURE_FILTER'] is not None - else DEFAULT_VARS['PCAP_FILTER'], + 'bpf-filter': ( + DEFAULT_VARS['CAPTURE_FILTER'] + if DEFAULT_VARS['CAPTURE_FILTER'] is not None + else DEFAULT_VARS['PCAP_FILTER'] + ), 'buffer-size': DEFAULT_VARS['AF_PACKET_BUFFER_SIZE'], 'checksum-checks': DEFAULT_VARS['AF_PACKET_CHECKSUM_CHECKS'], 'cluster-type': DEFAULT_VARS['AF_PACKET_CLUSTER_TYPE'], @@ -891,6 +897,17 @@ def main(): deleteIfNone=True, ) + elif dumperName == 'stats': + # for some reason the "enabled:" key isn't in the default + # yaml so we're forcing it here + for cfgKey in ([dumperName, 'enabled', 'STATS_EVE_ENABLED'],): + deep_set( + cfg['outputs'][outputIdx][name]['types'][dumperIdx], + cfgKey[:-1], + DEFAULT_VARS[cfgKey[-1]], + deleteIfNone=True, + ) + elif dumperName == 'tls': for cfgKey in ( [dumperName, 'extended', 'TLS_EXTENDED'], @@ -1170,7 +1187,9 @@ def main(): logging.error(output) # final tweaks - deep_set(cfg, ['stats', 'enabled'], False) + deep_set(cfg, ['stats', 'enabled'], val2bool(DEFAULT_VARS['STATS_ENABLED'])) + deep_set(cfg, ['stats', 'interval'], DEFAULT_VARS['STATS_INTERVAL']) + deep_set(cfg, ['stats', 'decoder-events'], val2bool(DEFAULT_VARS['STATS_DECODER_EVENTS'])) cfg.pop('rule-files', None) deep_set(cfg, ['rule-files'], GetRuleFiles()) diff --git a/shared/bin/zeek-deb-download.sh b/shared/bin/zeek-deb-download.sh new file mode 100755 index 000000000..d8dc01859 --- /dev/null +++ b/shared/bin/zeek-deb-download.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +unset VERBOSE +command -v dpkg >/dev/null 2>&1 && ARCH="$(dpkg --print-architecture)" || ARCH=amd64 +DISTRO=Debian_12 +OUTPUT_DIR=/tmp +ZEEK_VERSION=6.1.1-0 + +while getopts a:d:o:vz: opts; do + case ${opts} in + a) ARCH=${OPTARG} ;; + d) DISTRO=${OPTARG} ;; + o) OUTPUT_DIR=${OPTARG} ;; + v) VERBOSE=1 ;; + z) ZEEK_VERSION=${OPTARG} ;; + esac +done + +set -e +if [[ -n $VERBOSE ]]; then + set -x +fi + +DEB_URL="https://downloadcontentcdn.opensuse.org/repositories/security:/zeek/${DISTRO}" + +pushd "$OUTPUT_DIR" >/dev/null 2>&1 +curl --fail-early -fsSL --remote-name-all \ + "${DEB_URL}/${ARCH}/libbroker-dev_${ZEEK_VERSION}_${ARCH}.deb" \ + "${DEB_URL}/${ARCH}/zeek-core-dev_${ZEEK_VERSION}_${ARCH}.deb" \ + "${DEB_URL}/${ARCH}/zeek-core_${ZEEK_VERSION}_${ARCH}.deb" \ + "${DEB_URL}/${ARCH}/zeek-spicy-dev_${ZEEK_VERSION}_${ARCH}.deb" \ + "${DEB_URL}/${ARCH}/zeek_${ZEEK_VERSION}_${ARCH}.deb" \ + "${DEB_URL}/${ARCH}/zeekctl_${ZEEK_VERSION}_${ARCH}.deb" \ + "${DEB_URL}/all/zeek-client_${ZEEK_VERSION}_all.deb" \ + "${DEB_URL}/all/zeek-zkg_${ZEEK_VERSION}_all.deb" \ + "${DEB_URL}/all/zeek-btest_${ZEEK_VERSION}_all.deb" \ + "${DEB_URL}/all/zeek-btest-data_${ZEEK_VERSION}_all.deb" +popd >/dev/null 2>&1 + +if [[ -n $VERBOSE ]]; then + set +x +fi +set +e diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index 3236e16de..a610c6c5a 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -4,6 +4,7 @@ ##! https://docs.zeek.org/en/stable/script-reference/scripts.html ##! https://github.com/zeek/zeek/blob/master/scripts/site/local.zeek +global disable_stats = (getenv("ZEEK_DISABLE_STATS") == "") ? F : T; global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == "") ? F : T; global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == "") ? F : T; global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == "") ? F : T; @@ -78,6 +79,10 @@ redef ignore_checksums = T; @if (!disable_hash_all_files) @load frameworks/files/hash-all-files @endif +@if (!disable_stats) + @load policy/misc/stats + @load policy/misc/capture-loss +@endif @load policy/protocols/conn/vlan-logging @load policy/protocols/conn/mac-logging @load policy/protocols/modbus/known-masters-slaves @@ -259,11 +264,29 @@ event zeek_init() &priority=-5 { redef SNIFFPASS::log_password_plaintext = T; redef LDAP::default_capture_password = T; @endif + redef LDAP::default_log_search_attributes = F; redef SNIFFPASS::notice_log_enable = F; redef CVE_2021_44228::log = F; -@if ((!disable_ics_all) && (!disable_ics_synchrophasor) && (synchrophasor_detailed)) - redef SYNCHROPHASOR::log_data_frame = T; - redef SYNCHROPHASOR::log_data_detail = T; - redef SYNCHROPHASOR::log_cfg_detail = T; + +@if ((!disable_ics_all) && (!disable_ics_synchrophasor) && (!synchrophasor_detailed)) + hook SYNCHROPHASOR::log_policy_sychrophasor_data_detail( + rec : SYNCHROPHASOR::Synchrophasor_Data_Detail, + id : Log::ID, + filter : Log::Filter) { + break; + } + hook SYNCHROPHASOR::log_policy_sychrophasor_config_detail( + rec : SYNCHROPHASOR::Synchrophasor_Config_Detail, + id : Log::ID, + filter : Log::Filter) { + break; + } + + hook SYNCHROPHASOR::log_policy_sychrophasor_data( + rec : SYNCHROPHASOR::Synchrophasor_Data, + id : Log::ID, + filter : Log::Filter) { + break; + } @endif