Skip to content

Commit

Permalink
Multiplayer tests now fail on network errors during tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@JonasReich
JonasReich committed Mar 3, 2024
1 parent 86780fc commit 4a6e47b
Show file tree
Hide file tree
Showing 6 changed files with 119 additions and 81 deletions.
Binary file modified Content/Tests/Multiplayer/FT_Multiplayer.uasset
View file
Binary file not shown.
24 changes: 20 additions & 4 deletions Source/OUUTestUtilities/Private/Multiplayer/OUUMultiplayerFunctionalTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,17 +88,22 @@ void AOUUMultiplayerFunctionalTest::ServerNotifyClientSyncMarkerReached(
bool AOUUMultiplayerFunctionalTest::RunTest(const TArray<FString>& Params)
{
UOUUMultiplayerTestController::Get().NotifyFunctionalTestStarted();
return Super::RunTest(Params);
const bool bWasStarted = Super::RunTest(Params);
Multicast_OnTestStarted();
return bWasStarted;
}

void AOUUMultiplayerFunctionalTest::FinishTest(EFunctionalTestResult TestResult, const FString& Message)
{
ensureMsgf(
if (ensureMsgf(
HasAuthority(),
TEXT("The FinishTest function should only ever be called on Authority! Use snyc point nodes for all "
"intermediate steps you want to ensure synchronicity."));
UOUUMultiplayerTestController::Get().NotifyFunctionalTestEnded(TestResult);
"intermediate steps you want to ensure synchronicity.")) == false)
{
return;
}
Super::FinishTest(TestResult, Message);
Multicast_OnTestEnded(TestResult, TestIndex, TotalNumTests);
}

void AOUUMultiplayerFunctionalTest::GetLifetimeReplicatedProps(TArray<FLifetimeProperty>& OutLifetimeProps) const
Expand All @@ -107,6 +112,17 @@ void AOUUMultiplayerFunctionalTest::GetLifetimeReplicatedProps(TArray<FLifetimeP
DOREPLIFETIME(AOUUMultiplayerFunctionalTest, LastServerAcknowledgedSyncMarker);
}

void AOUUMultiplayerFunctionalTest::Multicast_OnTestStarted_Implementation()
{
UOUUMultiplayerTestController::Get().NotifyFunctionalTestStarted();
K2_OnTestStarted();
}

void AOUUMultiplayerFunctionalTest::Multicast_OnTestEnded_Implementation(EFunctionalTestResult TestResult, int32 InTestIndex, int32 InTotalNumTests)
{
UOUUMultiplayerTestController::Get().NotifyFunctionalTestEnded(TestResult, InTestIndex, InTotalNumTests);
}

void AOUUMultiplayerFunctionalTest::OnRep_ServerSyncMarker() const
{
ensure(LastServerAcknowledgedSyncMarker == LastLocalSyncMarker);
Expand Down
2 changes: 1 addition & 1 deletion Source/OUUTestUtilities/Private/Multiplayer/OUUMultiplayerTestClientSignal.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ void AOUUMultiplayerTestClientSignal::BeginPlay()
Super::BeginPlay();
if (HasAuthority())
{
// ???
// ???
}
else if (GetOwner())
{
Expand Down
149 changes: 78 additions & 71 deletions Source/OUUTestUtilities/Private/Multiplayer/OUUMultiplayerTestController.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,13 @@
#include "Interfaces/OnlineSessionInterface.h"
#include "Kismet/GameplayStatics.h"
#include "LogOpenUnrealUtilities.h"
#include "Misc/AutomationTest.h"
#include "Multiplayer/OUUMultiplayerFunctionalTest.h"
#include "Multiplayer/OUUMultiplayerTestClientSignal.h"
#include "Online/OnlineSessionNames.h"
#include "OnlineSessionSettings.h"
#include "OnlineSubsystemUtils.h"
#include "Traits/IteratorTraits.h"

UE_DISABLE_OPTIMIZATION

namespace OUU::TestUtilities
{
const FString GTestRole_Server = TEXT("Server");
Expand All @@ -36,37 +33,46 @@ UOUUMultiplayerTestController& UOUUMultiplayerTestController::Get()
void UOUUMultiplayerTestController::NotifyServerPostSignalReplicated()
{
ServerNumSignalsReplicated++;
MarkHeartbeatActive(FString::Printf(TEXT("[SERVER] post signal replicated")));
Heartbeat(FString::Printf(TEXT("Post signal replicated")));
}

void UOUUMultiplayerTestController::NotifyFunctionalTestStarted()
{
MarkHeartbeatActive(TEXT("[SERVER] test started..."));
// #TODO start listening for disconnects?
/*
GEngine->NetworkFailureEvent.AddLambda(
[this](UWorld*, UNetDriver*, ENetworkFailure::Type Failure, const FString& Reason) {
UE_LOG(
LogOpenUnrealUtilities,
Fatal,
TEXT("Network failure (%s): %s"),
ENetworkFailure::ToString(Failure),
*Reason);
});
*/
Heartbeat(TEXT("Test started..."));
if (GEngine->NetworkFailureEvent.IsBoundToObject(this) == false)
{
GEngine->NetworkFailureEvent
.AddWeakLambda(this, [this](UWorld*, UNetDriver*, ENetworkFailure::Type Failure, const FString& Reason) {
UE_LOG(
LogOpenUnrealUtilities,
Fatal,
TEXT("Network failure (%s): %s"),
ENetworkFailure::ToString(Failure),
*Reason);
});
}
}

void UOUUMultiplayerTestController::NotifyFunctionalTestEnded(EFunctionalTestResult TestResult)
void UOUUMultiplayerTestController::NotifyFunctionalTestEnded(EFunctionalTestResult TestResult, int32 TestIndex, int32 TotalNumTests)
{
MarkHeartbeatActive(TEXT("[SERVER] test ended..."));
// #TODO stop listening for disconnects?
ServerNumFinishedTests++;
if (TestResult != EFunctionalTestResult::Succeeded)
if (TestIndex == TotalNumTests - 1)
{
ServerNumFailedTests++;
// stop listening for network failures / disconnects.
// even a successful client disconnect results in an error for me.
GEngine->NetworkFailureEvent.RemoveAll(this);
}

ServerRunNextFunctionalTest();
Heartbeat(TEXT("Test ended..."));
if (bIsServer)
{
ServerNumFinishedTests++;
if (TestResult != EFunctionalTestResult::Succeeded)
{
ServerNumFailedTests++;
}

ServerRunNextFunctionalTest();
}
}

void UOUUMultiplayerTestController::OnInit()
Expand All @@ -75,7 +81,6 @@ void UOUUMultiplayerTestController::OnInit()
Instance = this;

Super::OnInit();
FString TestRole;
FParse::Value(FCommandLine::Get(), TEXT("OUUMPTestRole="), OUT TestRole);
UE_LOG(LogOpenUnrealUtilities, Log, TEXT("OUUMPTestRole=%s"), *TestRole);
bIsServer = TestRole == OUU::TestUtilities::GTestRole_Server;
Expand All @@ -87,7 +92,7 @@ void UOUUMultiplayerTestController::OnInit()

void UOUUMultiplayerTestController::OnPostMapChange(UWorld* World)
{
if (bIsServer)
if (bIsServer && bUseSessionSearch)
{
if (bServerInitialized)
{
Expand All @@ -96,7 +101,7 @@ void UOUUMultiplayerTestController::OnPostMapChange(UWorld* World)
}
bServerInitialized = true;

MarkHeartbeatActive(TEXT("[SERVER] create session..."));
Heartbeat(TEXT("Create session..."));
FOnlineSessionSettings Settings;
Settings.NumPublicConnections = 2;
Settings.bShouldAdvertise = true;
Expand All @@ -121,7 +126,7 @@ void UOUUMultiplayerTestController::OnTick(float TimeDelta)
{
ServerCheckRunFirstTest();
}
else
else if (bUseSessionSearch)
{
// #TODO is there a better way to delay client session search?
constexpr float WaitUntilSessionSearch = 5.f;
Expand All @@ -147,53 +152,58 @@ FUniqueNetIdPtr UOUUMultiplayerTestController::GetLocalNetID() const
return UniqueNetID;
}

void UOUUMultiplayerTestController::Heartbeat(const FString& Message)
{
MarkHeartbeatActive(FString::Printf(TEXT("[%s] %s"), *TestRole, *Message));
}

void UOUUMultiplayerTestController::OnCreateSessionComplete(FName, bool Success)
{
if (Success == false)
{
MarkHeartbeatActive(TEXT("[SERVER] failed to create session"));
Heartbeat(TEXT("Failed to create session"));
EndTest(1);
}

// CRASH???
// as listen server?
MarkHeartbeatActive(TEXT("[SERVER] enable listen server..."));
Heartbeat(TEXT("Enable listen server..."));
GetWorld()->GetGameInstance()->EnableListenServer(true);

MarkHeartbeatActive(TEXT("[SERVER] Collecting tests..."));
for (auto* FTest : TActorRange<AFunctionalTest>(GetWorld()))
{
if (auto* MultiplayerFTest = Cast<AOUUMultiplayerFunctionalTest>(FTest))
{
ServerAllFunctionalTests.Add(MultiplayerFTest);
}
else
{
UE_LOG(
LogOpenUnrealUtilities,
Fatal,
TEXT("Invalid functional test %s on map %s is not a multiplayer test"),
*GetCurrentMap(),
*GetNameSafe(FTest));
}
}

ServerTotalNumTests = ServerAllFunctionalTests.Num();
}

void UOUUMultiplayerTestController::ServerOnPostLogin(AGameModeBase* GameModeBase, APlayerController* PlayerController)
{
auto* Signal = GetWorld()->SpawnActor<AOUUMultiplayerTestClientSignal>();
check(Signal);
Signal->SetOwner(PlayerController);
MarkHeartbeatActive(FString::Printf(TEXT("[SERVER] post login (%s)"), *GetNameSafe(PlayerController)));
Heartbeat(FString::Printf(TEXT("Post login (%s)"), *GetNameSafe(PlayerController)));
ServerCheckRunFirstTest();
}

void UOUUMultiplayerTestController::ServerCheckRunFirstTest()
{
// #TODO replace with parameter
if (bServerStartedFirstTest == false && ServerNumSignalsReplicated == 2 && GetWorld()->GetGameState()->PlayerArray.Num() == 3)
if (bServerStartedFirstTest == false && ServerNumSignalsReplicated == 2
&& GetWorld()->GetGameState()->PlayerArray.Num() == 3)
{
Heartbeat(TEXT("Start condition fulfilled. Collecting tests..."));
for (auto* FTest : TActorRange<AFunctionalTest>(GetWorld()))
{
if (auto* MultiplayerFTest = Cast<AOUUMultiplayerFunctionalTest>(FTest))
{
ServerAllFunctionalTests.Add(MultiplayerFTest);
}
else
{
UE_LOG(
LogOpenUnrealUtilities,
Fatal,
TEXT("Invalid functional test %s on map %s is not a multiplayer test"),
*GetCurrentMap(),
*GetNameSafe(FTest));
}
}

ServerTotalNumTests = ServerAllFunctionalTests.Num();

ServerRunNextFunctionalTest();
}
}
Expand All @@ -204,30 +214,30 @@ void UOUUMultiplayerTestController::ServerRunNextFunctionalTest()
if (ServerAllFunctionalTests.Num() == 0)
{
ensure(ServerNumFinishedTests == ServerTotalNumTests);
MarkHeartbeatActive(TEXT("[SERVER] all tests completed"));
EndTest(ServerNumFailedTests);
Heartbeat(TEXT("All tests completed"));
// Wait 3 more seconds for clients to disconnect.
constexpr float EndTestDelay = 3.f;
FTimerHandle TempHandle;
GetWorld()->GetTimerManager().SetTimer(
IN OUT TempHandle,
FTimerDelegate::CreateLambda([this]() { EndTest(ServerNumFailedTests); }),
EndTestDelay,
false);
}
else
{
const auto NextTest = ServerAllFunctionalTests.Pop();
check(NextTest.IsValid());
MarkHeartbeatActive(TEXT("[SERVER] starting test..."));
Heartbeat(TEXT("Starting test..."));
NextTest->TestIndex = ServerNumFinishedTests;
NextTest->TotalNumTests = ServerTotalNumTests;
IFunctionalTestingModule::Get().RunTestOnMap(NextTest->GetName(), false, false);
}
}

void UOUUMultiplayerTestController::OnFunctionalTestEnd(FAutomationTestBase* AutomationTest)
{
MarkHeartbeatActive(FString(TEXT("test completed: ")) + AutomationTest->GetTestName());
if (AutomationTest->GetLastExecutionSuccessState() == false)
{
ServerNumFailedTests++;
}
}

void UOUUMultiplayerTestController::ClientStartSessionSearch()
{
MarkHeartbeatActive(TEXT("[CLIENT] find sessions..."));
Heartbeat(TEXT("Start session search..."));

ClientSessionSearch = MakeShared<FOnlineSessionSearch>();
ClientSessionSearch->MaxSearchResults = 1;
Expand Down Expand Up @@ -260,11 +270,8 @@ void UOUUMultiplayerTestController::ClientJoinSessionComplete(FName, EOnJoinSess
FString ConnectString;
if (GetSessions()->GetResolvedConnectString(NAME_GameSession, ConnectString))
{
UE_LOG_ONLINE_SESSION(Log, TEXT("Join session: traveling to %s"), *ConnectString);
MarkHeartbeatActive(TEXT("[CLIENT] travel..."));
Heartbeat(TEXT("Client travel..."));
GetWorld()->GetFirstPlayerController()->ClientTravel(ConnectString, TRAVEL_Absolute);
}
}
}

UE_ENABLE_OPTIMIZATION
14 changes: 13 additions & 1 deletion Source/OUUTestUtilities/Public/Multiplayer/OUUMultiplayerFunctionalTest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ class OUUTESTUTILITIES_API AOUUMultiplayerFunctionalTest : public AFunctionalTes

AOUUMultiplayerFunctionalTest();

// For tracking how far we are along the test progression
int32 TestIndex = INDEX_NONE;
int32 TotalNumTests = INDEX_NONE;

// Called on server + all clients if we detected that a sync point was reached.
DECLARE_EVENT_OneParam(AOUUMultiplayerFunctionalTest, FOnSyncMarkerReached, int32);
FOnSyncMarkerReached OnSyncMarkerReached;
Expand All @@ -32,7 +36,15 @@ class OUUTESTUTILITIES_API AOUUMultiplayerFunctionalTest : public AFunctionalTes
// - UObject
void GetLifetimeReplicatedProps(TArray<FLifetimeProperty>& OutLifetimeProps) const override;
// --


public:
UFUNCTION(NetMulticast, Reliable)
void Multicast_OnTestStarted();
UFUNCTION(BlueprintImplementableEvent, DisplayName = "Multiplayer Test Started")
void K2_OnTestStarted();
UFUNCTION(NetMulticast, Reliable)
void Multicast_OnTestEnded(EFunctionalTestResult TestResult, int32 InTestIndex, int32 InTotalNumTests);

private:
UPROPERTY(Transient)
TMap<APlayerController*, int32> ClientSyncMarkerLocations;
Expand Down
11 changes: 7 additions & 4 deletions Source/OUUTestUtilities/Public/Multiplayer/OUUMultiplayerTestController.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#include "OUUMultiplayerTestController.generated.h"

enum class EFunctionalTestResult : uint8;
class AFunctionalTest;
class AOUUMultiplayerFunctionalTest;

UCLASS()
class OUUTESTUTILITIES_API UOUUMultiplayerTestController : public UGauntletTestController
Expand All @@ -25,7 +25,7 @@ class OUUTESTUTILITIES_API UOUUMultiplayerTestController : public UGauntletTestC
void NotifyServerPostSignalReplicated();
// Notify the controller that a functional test started.
void NotifyFunctionalTestStarted();
void NotifyFunctionalTestEnded(EFunctionalTestResult TestResult);
void NotifyFunctionalTestEnded(EFunctionalTestResult TestResult, int32 TestIndex, int32 TotalNumTests);

// - UGauntletTestController
void OnInit() override;
Expand All @@ -37,12 +37,12 @@ class OUUTESTUTILITIES_API UOUUMultiplayerTestController : public UGauntletTestC
// --- SHARED
IOnlineSessionPtr GetSessions() const;
FUniqueNetIdPtr GetLocalNetID() const;
void Heartbeat(const FString& Message);
// ---- SERVER
void OnCreateSessionComplete(FName, bool Success);
void ServerOnPostLogin(AGameModeBase* GameModeBase, APlayerController* PlayerController);
void ServerCheckRunFirstTest();
void ServerRunNextFunctionalTest();
void OnFunctionalTestEnd(FAutomationTestBase* AutomationTest);
// ---- CLIENT
void ClientStartSessionSearch();
void ClientSessionSearchComplete(bool Success);
Expand All @@ -51,17 +51,20 @@ class OUUTESTUTILITIES_API UOUUMultiplayerTestController : public UGauntletTestC
private:
// --- SHARED
static UOUUMultiplayerTestController* Instance;
FString TestRole;
bool bIsServer = false;
float TotalTickTime = 0.f;
bool bUseSessionSearch = true;
// --- SERVER
bool bServerInitialized = false;
bool bServerStartedFirstTest = false;
int32 ServerNumFailedTests = 0;
int32 ServerTotalNumTests = 0;
int32 ServerNumFinishedTests = 0;
int32 ServerNumSignalsReplicated = 0;
TArray<TWeakObjectPtr<AFunctionalTest>> ServerAllFunctionalTests;
TArray<TWeakObjectPtr<AOUUMultiplayerFunctionalTest>> ServerAllFunctionalTests;
// --- CLIENT
bool bClientSessionSearchStarted = false;
TSharedPtr<FOnlineSessionSearch> ClientSessionSearch;
};

0 comments on commit 4a6e47b

Please sign in to comment.