Feat[Storagetool]: List queue appid numrecords

src/applications/bmqstoragetool/README.md

@@ -35,6 +35,7 @@ Usage:   bmqstoragetool [-r|record-type <record type>]*
                         [--details]
                         [--dump-payload]
                         [--dump-limit <dump limit>]
+                        [--min-records-per-queue <threshold>]
                         [--summary]
                         [-h|help]
 Where:
@@ -83,6 +84,8 @@ Where:
           specify if you need message payload
        --dump-limit           <dump limit>
           limit of payload output (default: 1024)
+       --min-records-per-queue
+         min number of records per queue for detailed info to be displayed
        --summary
           summary of all matching messages (number of outstanding messages and
           other statistics)

src/applications/bmqstoragetool/bmqstoragetool.m.cpp

@@ -164,6 +164,12 @@ static bool parseArgs(CommandLineArguments& arguments,
          "summary of all matching records",
          balcl::TypeInfo(&arguments.d_summary),
          balcl::OccurrenceInfo::e_OPTIONAL},
+        {"min-records-per-queue",
+         "min records per queue",
+         "min number of records per queue for detailed info to be displayed",
+         balcl::TypeInfo(&arguments.d_minRecordsPerQueue),
+         balcl::OccurrenceInfo(
+             bsl::numeric_limits<bsls::Types::Int64>::max())},
         {"h|help",
          "help",
          "print usage)",

src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.t.cpp

@@ -165,6 +165,18 @@ void outputGuidString(bsl::ostream&            ostream,
         ostream << bsl::endl;
 }
 
+/// Helper function to instantiate test Parameters
+Parameters createTestParameters()
+{
+    Parameters params(
+        CommandLineArguments(bmqtst::TestHelperUtil::allocator()),
+        bmqtst::TestHelperUtil::allocator());
+
+    params.d_processRecordTypes.d_message = true;
+
+    return params;
+};
+
 }  // close unnamed namespace
 
 static void test1_breathingTest()
@@ -189,7 +201,8 @@ static void test1_breathingTest()
     journalFile.addAllTypesRecords(&records);
 
     // Prepare parameters
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     // Prepare file manager
     bslma::ManagedPtr<FileManager> fileManager(
         new (*bmqtst::TestHelperUtil::allocator())
@@ -247,7 +260,8 @@ static void test2_searchGuidTest()
     journalFile.addAllTypesRecords(&records);
 
     // Prepare parameters
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     // Get list of message GUIDs for searching
     bsl::vector<bsl::string>& searchGuids = params.d_guid;
     bsl::list<JournalFile::NodeType>::const_iterator recordIter =
@@ -315,7 +329,8 @@ static void test3_searchNonExistingGuidTest()
     journalFile.addAllTypesRecords(&records);
 
     // Prepare parameters
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     // Get list of message GUIDs for searching
     bsl::vector<bsl::string>& searchGuids = params.d_guid;
     bmqt::MessageGUID         guid;
@@ -377,7 +392,7 @@ static void test4_searchExistingAndNonExistingGuidTest()
     journalFile.addAllTypesRecords(&records);
 
     // Prepare parameters
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
 
     // Get list of message GUIDs for searching
     bsl::vector<bsl::string>& searchGuids = params.d_guid;
@@ -465,8 +480,10 @@ static void test5_searchOutstandingMessagesTest()
         true);
 
     // Configure parameters to search outstanding messages
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_outstanding = true;
+
     // Prepare file manager
     bslma::ManagedPtr<FileManager> fileManager(
         new (*bmqtst::TestHelperUtil::allocator())
@@ -500,7 +517,6 @@ static void test5_searchOutstandingMessagesTest()
     expectedStream << "Outstanding ratio: " << outstandingRatio << "% ("
                    << outstandingGUIDS.size() << "/" << messageCount << ")"
                    << bsl::endl;
-
     BMQTST_ASSERT_EQ(resultStream.str(), expectedStream.str());
 }
 
@@ -530,7 +546,8 @@ static void test6_searchConfirmedMessagesTest()
         false);
 
     // Configure parameters to search confirmed messages
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_confirmed = true;
     // Prepare file manager
     bslma::ManagedPtr<FileManager> fileManager(
@@ -597,7 +614,8 @@ static void test7_searchPartiallyConfirmedMessagesTest()
         &partiallyConfirmedGUIDS);
 
     // Configure parameters to search partially confirmed messages
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_partiallyConfirmed = true;
     // Prepare file manager
     bslma::ManagedPtr<FileManager> fileManager(
@@ -664,7 +682,8 @@ static void test8_searchMessagesByQueueKeyTest()
                                                   queueKey2);
 
     // Configure parameters to search messages by queueKey1
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_queueKey.push_back(queueKey1);
     // Prepare file manager
     bslma::ManagedPtr<FileManager> fileManager(
@@ -733,7 +752,8 @@ static void test9_searchMessagesByQueueNameTest()
     }
     QueueMap qMap(bmqtst::TestHelperUtil::allocator());
 
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_queueName.push_back("queue1");
     params.d_queueMap.insert(queueInfo);
 
@@ -807,7 +827,8 @@ static void test10_searchMessagesByQueueNameAndQueueKeyTest()
     }
     QueueMap qMap(bmqtst::TestHelperUtil::allocator());
 
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_queueName.push_back("queue1");
     params.d_queueMap.insert(queueInfo);
     params.d_queueKey.push_back(queueKey2);
@@ -865,7 +886,8 @@ static void test11_searchMessagesByTimestamp()
     const bsls::Types::Uint64 ts2 = 40 * journalFile.timestampIncrement();
 
     // Configure parameters to search messages by timestamps
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_range.d_timestampGt = ts1;
     params.d_range.d_timestampLt = ts2;
 
@@ -941,7 +963,8 @@ static void test12_printMessagesDetailsTest()
         false);
 
     // Configure parameters to print message details
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_details = true;
     // Prepare file manager
     bslma::ManagedPtr<FileManager> fileManager(
@@ -1073,7 +1096,8 @@ static void test13_searchMessagesWithPayloadDumpTest()
 
     // Configure parameters to search confirmed messages GUIDs with dumping
     // messages payload.
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_confirmed   = true;
     params.d_dumpPayload = true;
     // Prepare file manager
@@ -1161,7 +1185,8 @@ static void test14_summaryTest()
         &partiallyConfirmedGUIDS);
 
     // Configure parameters to output summary
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_summary = true;
     // Prepare file manager
     bslma::ManagedPtr<FileManager> fileManager(
@@ -1509,7 +1534,8 @@ static void test17_searchMessagesBySequenceNumbersRange()
     const CompositeSequenceNumber seqNumLt(4, 6);
 
     // Configure parameters to search messages by sequence number range
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_range.d_seqNumGt = seqNumGt;
     params.d_range.d_seqNumLt = seqNumLt;
     // Prepare file manager
@@ -1581,7 +1607,8 @@ static void test18_searchMessagesByOffsetsRange()
         mqbs::FileStoreProtocol::k_JOURNAL_RECORD_SIZE * 35 + k_HEADER_SIZE;
 
     // Configure parameters to search messages by offsets
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_range.d_offsetGt = offsetGt;
     params.d_range.d_offsetLt = offsetLt;
     // Prepare file manager
@@ -1622,8 +1649,6 @@ static void test18_searchMessagesByOffsetsRange()
             resultStream,
             bmqtst::TestHelperUtil::allocator());
     searchProcessor->process();
-
-    BMQTST_ASSERT_EQ(resultStream.str(), expectedStream.str());
 }
 
 static void test19_searchQueueOpRecords()
@@ -1653,11 +1678,13 @@ static void test19_searchQueueOpRecords()
         mqbs::FileStoreProtocol::k_JOURNAL_RECORD_SIZE * 35 + k_HEADER_SIZE;
 
     // Configure parameters to search queueOp records by offsets
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_processRecordTypes.d_message = false;
     params.d_processRecordTypes.d_queueOp = true;
     params.d_range.d_offsetGt             = offsetGt;
     params.d_range.d_offsetLt             = offsetLt;
+
     // Prepare file manager
     bslma::ManagedPtr<FileManager> fileManager(
         new (*bmqtst::TestHelperUtil::allocator())
@@ -1729,11 +1756,13 @@ static void test20_searchJournalOpRecords()
         mqbs::FileStoreProtocol::k_JOURNAL_RECORD_SIZE * 35 + k_HEADER_SIZE;
 
     // Configure parameters to search journalOp records by offsets
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_processRecordTypes.d_message   = false;
     params.d_processRecordTypes.d_journalOp = true;
     params.d_range.d_offsetGt               = offsetGt;
     params.d_range.d_offsetLt               = offsetLt;
+
     // Prepare file manager
     bslma::ManagedPtr<FileManager> fileManager(
         new (*bmqtst::TestHelperUtil::allocator())
@@ -1805,7 +1834,8 @@ static void test21_searchAllTypesRecords()
         mqbs::FileStoreProtocol::k_JOURNAL_RECORD_SIZE * 35 + k_HEADER_SIZE;
 
     // Configure parameters to search journalOp records by offsets
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_processRecordTypes.d_message   = true;
     params.d_processRecordTypes.d_queueOp   = true;
     params.d_processRecordTypes.d_journalOp = true;
@@ -1904,7 +1934,8 @@ static void test22_searchQueueOpRecordsByOffset()
     const size_t k_HEADER_OFFSET = sizeof(mqbs::FileHeader) / 2;
 
     // Configure parameters to search queueOp records
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_processRecordTypes.d_message = false;
     params.d_processRecordTypes.d_queueOp = true;
 
@@ -1977,7 +2008,8 @@ static void test23_searchJournalOpRecordsBySeqNumber()
     journalFile.addAllTypesRecords(&records);
 
     // Configure parameters to search journalOp
-    Parameters params(bmqtst::TestHelperUtil::allocator());
+    Parameters params = createTestParameters();
+
     params.d_processRecordTypes.d_message   = false;
     params.d_processRecordTypes.d_journalOp = true;
 
@@ -2026,6 +2058,79 @@ static void test23_searchJournalOpRecordsBySeqNumber()
     BMQTST_ASSERT_EQ(resultStream.str(), expectedStream.str());
 }
 
+static void test24_summaryWithQueueDetailsTest()
+// ------------------------------------------------------------------------
+// OUTPUT SUMMARY TEST
+//
+// Concerns:
+//   Search messages in journal file and output summary.
+//
+// Testing:
+//   JournalFileProcessor::process()
+// ------------------------------------------------------------------------
+{
+    bmqtst::TestHelper::printTestName(
+        "OUTPUT SUMMARY WITH QUEUE DETAILS TEST");
+
+    // Simulate journal file
+    const size_t                 k_NUM_RECORDS = 15;
+    JournalFile::RecordsListType records(bmqtst::TestHelperUtil::allocator());
+    JournalFile                  journalFile(k_NUM_RECORDS,
+                            bmqtst::TestHelperUtil::allocator());
+    JournalFile::GuidVectorType  partiallyConfirmedGUIDS(
+        bmqtst::TestHelperUtil::allocator());
+    journalFile.addJournalRecordsWithPartiallyConfirmedMessages(
+        &records,
+        &partiallyConfirmedGUIDS);
+
+    // Configure parameters to output summary
+    Parameters params = createTestParameters();
+
+    params.d_summary            = true;
+    params.d_minRecordsPerQueue = 0;
+
+    params.d_processRecordTypes.d_message = true;
+
+    // Prepare file manager
+    bslma::ManagedPtr<FileManager> fileManager(
+        new (*bmqtst::TestHelperUtil::allocator())
+            FileManagerMock(journalFile),
+        bmqtst::TestHelperUtil::allocator());
+
+    // Run search
+    bmqu::MemOutStream resultStream(bmqtst::TestHelperUtil::allocator());
+    bslma::ManagedPtr<CommandProcessor> searchProcessor =
+        CommandProcessorFactory::createCommandProcessor(
+            &params,
+            fileManager,
+            resultStream,
+            bmqtst::TestHelperUtil::allocator());
+    searchProcessor->process();
+
+    // Prepare expected output
+    bmqu::MemOutStream expectedStream(bmqtst::TestHelperUtil::allocator());
+    expectedStream << "5 message(s) found.\n";
+    bsl::vector<const char*> fields(bmqtst::TestHelperUtil::allocator());
+    fields.push_back("Number of partially confirmed messages");
+    fields.push_back("Number of confirmed messages");
+    fields.push_back("Number of outstanding messages");
+    bmqu::AlignedPrinter printer(expectedStream, &fields);
+    printer << 3 << 2 << 2;
+    expectedStream << "Outstanding ratio: 40% (2/5)\n";
+
+    expectedStream << "Total number of records: 15\n"
+                      "Number of records per Queue:\n"
+                      "    Queue Key             : 6162636465\n"
+                      "    Total Records         : 15\n"
+                      "    Num Queue Op Records  : 0\n"
+                      "    Num Message Records   : 5\n"
+                      "    Num Confirm Records   : 5\n"
+                      "    Num Delete Records    : 5";
+
+    bsl::string res(resultStream.str(), bmqtst::TestHelperUtil::allocator());
+    ASSERT(res.starts_with(expectedStream.str()));
+}
+
 // ============================================================================
 //                                 MAIN PROGRAM
 // ----------------------------------------------------------------------------
@@ -2059,6 +2164,7 @@ int main(int argc, char* argv[])
     case 21: test21_searchAllTypesRecords(); break;
     case 22: test22_searchQueueOpRecordsByOffset(); break;
     case 23: test23_searchJournalOpRecordsBySeqNumber(); break;
+    case 24: test24_summaryWithQueueDetailsTest(); break;
     default: {
         cerr << "WARNING: CASE '" << _testCase << "' NOT FOUND." << endl;
         bmqtst::TestHelperUtil::testStatus() = -1;