Skip to content

Commit

Permalink
IGNITE-23581 Fix ItMetastorageGroupDisasterRecoveryTest#oldMgLeaderDo…
Browse files Browse the repository at this point in the history
…esNotHijackLeadership() (apache#4661)

* Make sure Metastorage does not diverge in this test (it's an unrelated matter)
* Include stack trace info in a failed assertion description
  • Loading branch information
rpuch authored Oct 31, 2024
1 parent 8682358 commit ac40e77
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.ignite.internal.util.ExceptionUtils;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeMatcher;
Expand Down Expand Up @@ -106,7 +107,7 @@ protected void describeMismatchSafely(CompletableFuture<?> item, Description mis
try {
item.join();
} catch (Exception e) {
mismatchDescription.appendText("was completed exceptionally with ").appendValue(unwrapCause(e));
mismatchDescription.appendText("was completed exceptionally with ").appendValue(describeThrowable(unwrapCause(e)));
}
} else if (item.isDone()) {
mismatchDescription.appendText("was completed successfully");
Expand All @@ -115,6 +116,10 @@ protected void describeMismatchSafely(CompletableFuture<?> item, Description mis
}
}

private static String describeThrowable(Throwable throwable) {
return throwable.toString() + ", stack trace " + ExceptionUtils.getFullStackTrace(throwable);
}

private boolean matchesWithCause(Throwable e) {
for (Throwable current = e; current != null; current = current.getCause()) {
if (matchesException(current) || Arrays.stream(current.getSuppressed()).anyMatch(this::matchesWithCause)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -209,13 +209,12 @@ void oldMgLeaderDoesNotHijackLeadership() throws Exception {
startAndInitCluster(2, new int[]{0}, new int[]{1});
waitTillClusterStateIsSavedToVaultOnConductor(0);

// This makes the MG majority go away.
cluster.stopNode(1);

IgniteImpl igniteImpl0BeforeRestart = igniteImpl(0);

assertThatMgHasNoMajority(igniteImpl0BeforeRestart);
// Copy Metastorage state from old leader (1) to future leader (0) to make sure that 1 is not ahead of 0 and there will be
// no Metastorage divergence when we make 0 new leader and migrate 1 to cluster again.
// This stops both nodes.
makeSure2NodesHaveSameMetastorageState(1, 0);

cluster.startEmbeddedNode(0);
initiateMgRepairVia(0, 1, 0);

IgniteImpl restartedIgniteImpl0 = waitTillNodeRestartsInternally(0);
Expand Down Expand Up @@ -257,14 +256,10 @@ void migratesNodesThatSawNoReparationToNewClusterIfMetastorageDidNotDiverge() th
startAndInitCluster(2, new int[]{0}, new int[]{1});
waitTillClusterStateIsSavedToVaultOnConductor(0);

ComponentWorkingDir msWorkDir0 = igniteImpl(0).metastorageWorkDir();
ComponentWorkingDir msWorkDir1 = igniteImpl(1).metastorageWorkDir();

IntStream.of(0, 1).parallel().forEach(cluster::stopNode);

// Copy Metastorage state from old leader (1) to future leader (0) to make sure that 1 is not ahead of 0 and there will be
// no Metastorage divergence when we make 0 new leader and migrate 1 to cluster again.
copyMetastorageState(msWorkDir1, msWorkDir0);
// This stops both nodes.
makeSure2NodesHaveSameMetastorageState(1, 0);

// Repair MG with just node 0 in CMG.
cluster.startEmbeddedNode(0);
Expand All @@ -279,6 +274,15 @@ void migratesNodesThatSawNoReparationToNewClusterIfMetastorageDidNotDiverge() th
assertTopologyContainsNode(1, topologySnapshot);
}

private void makeSure2NodesHaveSameMetastorageState(int leaderIndex, int followerIndex) throws IOException {
ComponentWorkingDir followerMsWorkDir = igniteImpl(followerIndex).metastorageWorkDir();
ComponentWorkingDir leaderMsWorkDir = igniteImpl(leaderIndex).metastorageWorkDir();

IntStream.of(followerIndex, leaderIndex).parallel().forEach(cluster::stopNode);

copyMetastorageState(leaderMsWorkDir, followerMsWorkDir);
}

private static void copyMetastorageState(ComponentWorkingDir source, ComponentWorkingDir dest) throws IOException {
replaceDir(source.dbPath(), dest.dbPath());
replaceDir(source.raftLogPath(), dest.raftLogPath());
Expand Down

0 comments on commit ac40e77

Please sign in to comment.