diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerSpectator.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerSpectator.cs
index 1090f1d10e..1ac012c0b5 100644
--- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerSpectator.cs
+++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerSpectator.cs
@@ -199,8 +199,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
 
             // Player 2 should catch up to player 1 after unpausing.
             AddUntilStep("player 2 not catching up", () => !getInstance(56).IsCatchingUp);
-            AddAssert("player 1 time == player 2 time", () => Math.Abs(getGameplayOffset(55, 56)) <= 2 * PlayerInstance.SYNC_TARGET);
-            AddWaitStep("wait a bit", 5);
+            AddWaitStep("wait a bit", 10);
         }
 
         private void loadSpectateScreen()
diff --git a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MultiplayerSpectator.cs b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MultiplayerSpectator.cs
index db5de45f72..636e7b9a49 100644
--- a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MultiplayerSpectator.cs
+++ b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MultiplayerSpectator.cs
@@ -30,7 +30,7 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
         private readonly PlayerInstance[] instances;
         private PlayerGrid grid;
         private MultiplayerSpectatorLeaderboard leaderboard;
-        private double? loadStartTime;
+        private double? loadFinishTime;
 
         public MultiplayerSpectator(int[] userIds)
             : base(userIds.AsSpan().Slice(0, Math.Min(16, userIds.Length)).ToArray())
@@ -76,7 +76,8 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
         {
             base.UpdateAfterChildren();
 
-            loadStartTime ??= Time.Current;
+            if (AllPlayersLoaded)
+                loadFinishTime ??= Time.Current;
 
             updateGameplayPlayingState();
         }
@@ -88,7 +89,7 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
                 // All players have frames...
                 instances.All(i => i.Score.Replay.Frames.Count > 0)
                 // Or any player has frames and the maximum start delay has been exceeded.
-                || (Time.Current - loadStartTime > maximum_start_delay
+                || (Time.Current - loadFinishTime > maximum_start_delay
                     && instances.Any(i => i.Score.Replay.Frames.Count > 0))
             );