Add storage tool ITs

Signed-off-by: Aleksandr Ivanov <[email protected]>

.github/workflows/build.yaml

@@ -3,7 +3,7 @@ name: Tests
 on:
   push:
     branches:
-      - main
+      - storagetool-it
   pull_request:
     branches:
       - main
@@ -222,6 +222,23 @@ jobs:
           cd src/python
           python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }}
 
+  integration_tests_storagetool_ubuntu:
+    runs-on: ubuntu-latest
+    needs: build_ubuntu
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/cache/@v4
+        with:
+          path: |
+            build/blazingmq
+          key: cache-${{ github.sha }}
+
+      - name: Run Integration Tests
+        run: |
+          pip install -r ${{ github.workspace }}/src/python/requirements.txt
+          ${{ github.workspace }}/src/applications/bmqstoragetool/integration-tests/run-tests
+
   build_macosx:
     name: Build [macosx_${{ matrix.arch }}]
     runs-on: ${{ matrix.os }}

src/applications/bmqstoragetool/integration-tests/README.md

@@ -0,0 +1,11 @@
+# BlazingMQ Storage tool Integration Tests
+
+To run the tests:
+
+* (create and) activate a Python 3.8 (or above) `venv`
+    * `python3 -m venv /path/to/venv`
+    * `source /path/to/venv/bin/activate`
+* install required modules
+    * `pip3 install -r src/python/requirements.txt`
+* run the tests
+    * `./run-tests [extra pytest options]`

src/applications/bmqstoragetool/integration-tests/conftest.py

@@ -0,0 +1,113 @@
+# Copyright 2024 Bloomberg Finance L.P.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import blazingmq.dev.paths as paths
+
+
+@pytest.fixture
+def storagetool():
+    return paths.required_paths.storagetool
+
+
+@pytest.fixture
+def journal_file():
+    return (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/test.bmq_journal"
+    )
+
+
+@pytest.fixture
+def journal_path():
+    return (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/test.*"
+    )
+
+
+@pytest.fixture
+def data_file():
+    return (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/test.bmq_data"
+    )
+
+
+@pytest.fixture
+def csl_file():
+    return (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/test.bmq_csl"
+    )
+
+
+def _load_expected_result_file(file_path):
+    with open(file_path, "r") as f:
+        return f.read().encode()
+
+
+@pytest.fixture
+def expected_short_result():
+    short_res_file = (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/short_result.txt"
+    )
+    return _load_expected_result_file(short_res_file)
+
+
+@pytest.fixture
+def expected_detail_result():
+    detailt_res_file = (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/detail_result.txt"
+    )
+    return _load_expected_result_file(detailt_res_file)
+
+
+@pytest.fixture
+def expected_payload_dump():
+    payload_dump_file = (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/payload_dump.txt"
+    )
+    return _load_expected_result_file(payload_dump_file)
+
+
+@pytest.fixture
+def expected_summary_result():
+    summary_result_file = (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/summary_result.txt"
+    )
+    return _load_expected_result_file(summary_result_file)
+
+
+@pytest.fixture
+def expected_queueop_result():
+    queueop_result_file = (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/queueop_result.txt"
+    )
+    return _load_expected_result_file(queueop_result_file)
+
+
+@pytest.fixture
+def expected_journalop_result():
+    journalop_result_file = (
+        paths.required_paths.repository
+        / "src/applications/bmqstoragetool/integration-tests/fixtures/journalop_result.txt"
+    )
+    return _load_expected_result_file(journalop_result_file)

src/applications/bmqstoragetool/integration-tests/fixtures/detail_result.txt

@@ -0,0 +1,45 @@
+=====================================
+MESSAGE Record, index: 3, offset: 224
+    PrimaryLeaseId   : 1
+    SequenceNumber   : 4
+    Timestamp        : 29OCT2024_14:01:24.000000
+    Epoch            : 1730210484
+    FileKey          : 0000000000
+    QueueKey         : 26DACDC974
+    RefCount         : 1
+    MsgOffsetDwords  : 5
+    GUID             : 400000000002B471F5B3AC11AA7D7DAB
+    Crc32c           : 3381945770
+
+CONFIRM Record, index: 5, offset: 344
+    PrimaryLeaseId  : 1
+    SequenceNumber  : 6
+    Timestamp       : 29OCT2024_14:02:21.000000
+    Epoch           : 1730210541
+    QueueKey        : 26DACDC974
+    AppKey          : 3000000000
+    GUID            : 400000000002B471F5B3AC11AA7D7DAB
+
+DELETE Record, index: 6, offset: 404
+    PrimaryLeaseId  : 1
+    SequenceNumber  : 7
+    Timestamp       : 29OCT2024_14:02:21.000000
+    Epoch           : 1730210541
+    QueueKey        : 26DACDC974
+    DeletionFlag    : NONE
+    GUID            : 400000000002B471F5B3AC11AA7D7DAB
+
+======================================
+MESSAGE Record, index: 10, offset: 644
+    PrimaryLeaseId   : 2
+    SequenceNumber   : 2
+    Timestamp        : 29OCT2024_14:03:23.000000
+    Epoch            : 1730210603
+    FileKey          : 0000000000
+    QueueKey         : 26DACDC974
+    RefCount         : 1
+    MsgOffsetDwords  : 8
+    GUID             : 40000000000215B2967EEDFA1085BA02
+    Crc32c           : 3381945770
+
+2 message GUID(s) found.

src/applications/bmqstoragetool/integration-tests/fixtures/journalop_result.txt

@@ -0,0 +1,9 @@
+[ header = [ type = JOURNAL_OP flags = 0 primaryLeaseId = 1 sequenceNumber = 1 timestamp = 1730210423 ] flags = 0 type = SYNCPOINT syncPointType = REGULAR sequenceNum = 1 primaryNodeId = 0 primaryLeaseId = 1 dataFileOffsetDwords = 5 qlistFileOffsetWords = 9 ]
+[ header = [ type = JOURNAL_OP flags = 0 primaryLeaseId = 1 sequenceNumber = 3 timestamp = 1730210466 ] flags = 0 type = SYNCPOINT syncPointType = REGULAR sequenceNum = 3 primaryNodeId = 0 primaryLeaseId = 1 dataFileOffsetDwords = 5 qlistFileOffsetWords = 32 ]
+[ header = [ type = JOURNAL_OP flags = 0 primaryLeaseId = 1 sequenceNumber = 5 timestamp = 1730210484 ] flags = 0 type = SYNCPOINT syncPointType = REGULAR sequenceNum = 5 primaryNodeId = 0 primaryLeaseId = 1 dataFileOffsetDwords = 8 qlistFileOffsetWords = 32 ]
+[ header = [ type = JOURNAL_OP flags = 0 primaryLeaseId = 1 sequenceNumber = 8 timestamp = 1730210542 ] flags = 0 type = SYNCPOINT syncPointType = REGULAR sequenceNum = 8 primaryNodeId = 0 primaryLeaseId = 1 dataFileOffsetDwords = 8 qlistFileOffsetWords = 32 ]
+[ header = [ type = JOURNAL_OP flags = 0 primaryLeaseId = 1 sequenceNumber = 9 timestamp = 1730210569 ] flags = 0 type = SYNCPOINT syncPointType = REGULAR sequenceNum = 9 primaryNodeId = 0 primaryLeaseId = 1 dataFileOffsetDwords = 8 qlistFileOffsetWords = 32 ]
+[ header = [ type = JOURNAL_OP flags = 0 primaryLeaseId = 2 sequenceNumber = 1 timestamp = 1730210574 ] flags = 0 type = SYNCPOINT syncPointType = REGULAR sequenceNum = 1 primaryNodeId = 0 primaryLeaseId = 2 dataFileOffsetDwords = 8 qlistFileOffsetWords = 32 ]
+[ header = [ type = JOURNAL_OP flags = 0 primaryLeaseId = 2 sequenceNumber = 3 timestamp = 1730210603 ] flags = 0 type = SYNCPOINT syncPointType = REGULAR sequenceNum = 3 primaryNodeId = 0 primaryLeaseId = 2 dataFileOffsetDwords = 11 qlistFileOffsetWords = 32 ]
+[ header = [ type = JOURNAL_OP flags = 0 primaryLeaseId = 2 sequenceNumber = 4 timestamp = 1730210611 ] flags = 0 type = SYNCPOINT syncPointType = REGULAR sequenceNum = 4 primaryNodeId = 0 primaryLeaseId = 2 dataFileOffsetDwords = 11 qlistFileOffsetWords = 32 ]
+8 journalOp record(s) found.

src/applications/bmqstoragetool/integration-tests/fixtures/payload_dump.txt

@@ -0,0 +1,29 @@
+400000000002B471F5B3AC11AA7D7DAB
+
+DataRecord index: 0, offset: 40
+DataHeader: 
+    HeaderWords:  3
+    OptionsWords: 0
+    MessageWords: 6
+    Flags:
+        MESSAGE_PROPERTIES: false
+
+Payload: 
+First 11 bytes of payload:
+     0:   68656C6C 6F20776F 726C64                |hello world     |
+
+40000000000215B2967EEDFA1085BA02
+
+DataRecord index: 1, offset: 64
+DataHeader: 
+    HeaderWords:  3
+    OptionsWords: 0
+    MessageWords: 6
+    Flags:
+        MESSAGE_PROPERTIES: false
+
+Payload: 
+First 11 bytes of payload:
+     0:   68656C6C 6F20776F 726C64                |hello world     |
+
+2 message GUID(s) found.

src/applications/bmqstoragetool/integration-tests/fixtures/queueop_result.txt

@@ -0,0 +1,2 @@
+[ header = [ type = QUEUE_OP flags = 0 primaryLeaseId = 1 sequenceNumber = 2 timestamp = 1730210466 ] flags = 0 queueKey = 26DACDC974 appKey = 0000000000 type = CREATION queueUriRecordOffsetWords = 9 ]
+1 queueOp record(s) found.

src/applications/bmqstoragetool/integration-tests/fixtures/short_result.txt

@@ -0,0 +1,3 @@
+400000000002B471F5B3AC11AA7D7DAB
+40000000000215B2967EEDFA1085BA02
+2 message GUID(s) found.

src/applications/bmqstoragetool/integration-tests/fixtures/summary_result.txt

@@ -0,0 +1,55 @@
+2 message(s) found.
+    Number of partially confirmed messages  : 1
+    Number of confirmed messages            : 0
+    Number of outstanding messages          : 1
+Outstanding ratio: 50% (1/2)
+Total number of records: 4
+Number of records per Queue:
+
+Details of journal file: 
+File descriptor: BlazingMQ File Header
+    Protocol Version  : 1
+    Bitness           : 64
+    FileType          : JOURNAL
+    HeaderWords       : 8
+    PartitionId       : 0
+
+
+Journal File Header: 
+    HeaderWords                           : 3
+    RecordWords                           : 15
+    First SyncPointRecord offset words    : 0
+    First SyncPointRecord type            :  ** NA** 
+    First SyncPointRecord primaryNodeId   :  ** NA ** 
+    First SyncPointRecord primaryLeaseId  :  ** NA** 
+    First SyncPointRecord sequenceNumber  :  ** NA ** 
+    First SyncPointRecord dataFileOffset  :  ** NA** 
+    First SyncPointRecord timestamp       :  ** NA** 
+    First SyncPointRecord epoch           :  ** NA ** 
+
+Journal SyncPoint:
+    Last Valid Record Offset           : 764
+    Record Type                        : JOURNAL_OP
+    Record Timestamp                   : 29OCT2024_14:03:31.000000
+    Record Epoch                       : 1730210611
+    Last Valid SyncPoint Offset        : 764
+    SyncPoint Timestamp                : 29OCT2024_14:03:31.000000
+    SyncPoint Epoch                    : 1730210611
+    SyncPoint SeqNum                   : 4
+    SyncPoint Primary NodeId           : 0
+    SyncPoint Primary LeaseId          : 2
+    SyncPoint DataFileOffset (DWORDS)  : 11
+    SyncPoint QlistFileOffset (WORDS)  : 32
+
+Details of data file: 
+BlazingMQ File Header
+    Protocol Version  : 1
+    Bitness           : 64
+    FileType          : DATA
+    HeaderWords       : 8
+    PartitionId       : 0
+
+ Data File Header: 
+    HeaderWords       : 2
+    FileId (FileKey)  : 0000000000
+

src/applications/bmqstoragetool/integration-tests/run-tests

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+# DESCRIPTION
+# ===========
+# This script runs bmqstoragetool integration tests.
+
+# USAGE
+# =====
+#   ./run-tests
+
+set -e
+
+
+repo_dir=$(realpath "$0")
+repo_dir=${repo_dir%/src/*}
+
+export PYTHONPATH=$repo_dir/src/python:$PYTHONPATH
+
+python3 -m pytest "$@"

src/applications/bmqstoragetool/integration-tests/test_breathing.py

@@ -0,0 +1,54 @@
+# Copyright 2024 Bloomberg Finance L.P.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import re
+
+
+def test_breathing(storagetool):
+    """
+    This test checks that storage tool could be run without arguments and print error.
+    """
+    res = subprocess.run([storagetool], capture_output=True)
+    assert res.returncode == 255
+    assert (
+        re.search(b"Neither journal path nor journal file are specified", res.stderr)
+        is not None
+    )
+
+
+def test_print_help(storagetool):
+    """
+    This test checks that storage tool could print help info.
+    """
+
+    res = subprocess.run([storagetool, "-h"], capture_output=True)
+    assert res.returncode == 255
+    assert re.search(r"--help\s+print usage", res.stderr.decode()) is not None
+
+
+def test_wrong_argument(storagetool):
+    """
+    This test checks that storage tool prints error message if wrong argument passed.
+    """
+    res = subprocess.run([storagetool, "--foo"], capture_output=True)
+    assert res.returncode == 255
+    assert (
+        re.search(
+            r"The string \"foo\" does not match any long tag.",
+            res.stderr.decode("utf-8"),
+        )
+        is not None
+    )