diff --git a/osu.Game.Tests/OnlinePlay/TestSceneCatchUpSyncManager.cs b/osu.Game.Tests/OnlinePlay/TestSceneCatchUpSyncManager.cs
index d4e591cf09..6851df3832 100644
--- a/osu.Game.Tests/OnlinePlay/TestSceneCatchUpSyncManager.cs
+++ b/osu.Game.Tests/OnlinePlay/TestSceneCatchUpSyncManager.cs
@@ -31,32 +31,24 @@ namespace osu.Game.Tests.OnlinePlay
         }
 
         [Test]
-        public void TestMasterClockStartsWhenAllPlayerClocksHaveFrames()
+        public void TestPlayerClocksStartWhenAllHaveFrames()
         {
             setWaiting(() => player1, false);
-            assertMasterState(false);
             assertPlayerClockState(() => player1, false);
             assertPlayerClockState(() => player2, false);
 
             setWaiting(() => player2, false);
-            assertMasterState(true);
             assertPlayerClockState(() => player1, true);
             assertPlayerClockState(() => player2, true);
         }
 
         [Test]
-        public void TestMasterClockDoesNotStartWhenNoneReadyForMaximumDelayTime()
-        {
-            AddWaitStep($"wait {CatchUpSyncManager.MAXIMUM_START_DELAY} milliseconds", (int)Math.Ceiling(CatchUpSyncManager.MAXIMUM_START_DELAY / TimePerAction));
-            assertMasterState(false);
-        }
-
-        [Test]
-        public void TestMasterClockStartsWhenAnyReadyForMaximumDelayTime()
+        public void TestReadyPlayersStartWhenReadyForMaximumDelayTime()
         {
             setWaiting(() => player1, false);
             AddWaitStep($"wait {CatchUpSyncManager.MAXIMUM_START_DELAY} milliseconds", (int)Math.Ceiling(CatchUpSyncManager.MAXIMUM_START_DELAY / TimePerAction));
-            assertMasterState(true);
+            assertPlayerClockState(() => player1, true);
+            assertPlayerClockState(() => player2, false);
         }
 
         [Test]
@@ -153,9 +145,6 @@ namespace osu.Game.Tests.OnlinePlay
         private void setPlayerClockTime(Func<TestSpectatorPlayerClock> playerClock, double offsetFromMaster)
             => AddStep($"set player clock {playerClock().Id} = master - {offsetFromMaster}", () => playerClock().Seek(master.CurrentTime - offsetFromMaster));
 
-        private void assertMasterState(bool running)
-            => AddAssert($"master clock {(running ? "is" : "is not")} running", () => master.IsRunning == running);
-
         private void assertCatchingUp(Func<TestSpectatorPlayerClock> playerClock, bool catchingUp) =>
             AddAssert($"player clock {playerClock().Id} {(catchingUp ? "is" : "is not")} catching up", () => playerClock().IsCatchingUp == catchingUp);
 
@@ -201,6 +190,11 @@ namespace osu.Game.Tests.OnlinePlay
 
         private class TestManualClock : ManualClock, IAdjustableClock
         {
+            public TestManualClock()
+            {
+                IsRunning = true;
+            }
+
             public void Start() => IsRunning = true;
 
             public void Stop() => IsRunning = false;
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs
index 7f3056771d..7584d67afe 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs
@@ -39,8 +39,6 @@ namespace osu.Game.Tests.Visual.Gameplay
         [Resolved]
         private OsuGameBase game { get; set; }
 
-        private int nextFrame;
-
         private BeatmapSetInfo importedBeatmap;
 
         private int importedBeatmapId;
@@ -49,8 +47,6 @@ namespace osu.Game.Tests.Visual.Gameplay
         {
             base.SetUpSteps();
 
-            AddStep("reset sent frames", () => nextFrame = 0);
-
             AddStep("import beatmap", () =>
             {
                 importedBeatmap = ImportBeatmapTest.LoadOszIntoOsu(game, virtualTrack: true).Result;
@@ -103,7 +99,8 @@ namespace osu.Game.Tests.Visual.Gameplay
             waitForPlayer();
             checkPaused(true);
 
-            sendFrames(1000); // send enough frames to ensure play won't be paused
+            // send enough frames to ensure play won't be paused
+            sendFrames(100);
 
             checkPaused(false);
         }
@@ -112,12 +109,12 @@ namespace osu.Game.Tests.Visual.Gameplay
         public void TestSpectatingDuringGameplay()
         {
             start();
+            sendFrames(300);
 
             loadSpectatingScreen();
             waitForPlayer();
 
-            AddStep("advance frame count", () => nextFrame = 300);
-            sendFrames();
+            sendFrames(300);
 
             AddUntilStep("playing from correct point in time", () => player.ChildrenOfType<DrawableRuleset>().First().FrameStableClock.CurrentTime > 30000);
         }
@@ -218,11 +215,7 @@ namespace osu.Game.Tests.Visual.Gameplay
 
         private void sendFrames(int count = 10)
         {
-            AddStep("send frames", () =>
-            {
-                testSpectatorClient.SendFrames(streamingUser.Id, nextFrame, count);
-                nextFrame += count;
-            });
+            AddStep("send frames", () => testSpectatorClient.SendFrames(streamingUser.Id, count));
         }
 
         private void loadSpectatingScreen()
diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiSpectatorLeaderboard.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiSpectatorLeaderboard.cs
index 2537198503..e14df62af1 100644
--- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiSpectatorLeaderboard.cs
+++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiSpectatorLeaderboard.cs
@@ -62,10 +62,10 @@ namespace osu.Game.Tests.Visual.Multiplayer
                 // For player 2, send frames in sets of 10.
                 for (int i = 0; i < 100; i++)
                 {
-                    SpectatorClient.SendFrames(PLAYER_1_ID, i, 1);
+                    SpectatorClient.SendFrames(PLAYER_1_ID, 1);
 
                     if (i % 10 == 0)
-                        SpectatorClient.SendFrames(PLAYER_2_ID, i, 10);
+                        SpectatorClient.SendFrames(PLAYER_2_ID, 10);
                 }
             });
 
diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiSpectatorScreen.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiSpectatorScreen.cs
index ea8f7813fd..5112029873 100644
--- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiSpectatorScreen.cs
+++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiSpectatorScreen.cs
@@ -8,6 +8,7 @@ using osu.Framework.Allocation;
 using osu.Framework.Graphics;
 using osu.Framework.Testing;
 using osu.Game.Beatmaps;
+using osu.Game.Rulesets.UI;
 using osu.Game.Screens.OnlinePlay.Multiplayer.Spectate;
 using osu.Game.Screens.Play;
 using osu.Game.Tests.Beatmaps.IO;
@@ -25,7 +26,6 @@ namespace osu.Game.Tests.Visual.Multiplayer
         private MultiSpectatorScreen spectatorScreen;
 
         private readonly List<int> playingUserIds = new List<int>();
-        private readonly Dictionary<int, int> nextFrame = new Dictionary<int, int>();
 
         private BeatmapSetInfo importedSet;
         private BeatmapInfo importedBeatmap;
@@ -40,11 +40,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
         }
 
         [SetUp]
-        public new void Setup() => Schedule(() =>
-        {
-            nextFrame.Clear();
-            playingUserIds.Clear();
-        });
+        public new void Setup() => Schedule(() => playingUserIds.Clear());
 
         [Test]
         public void TestDelayedStart()
@@ -55,8 +51,6 @@ namespace osu.Game.Tests.Visual.Multiplayer
                 Client.CurrentMatchPlayingUserIds.Add(PLAYER_2_ID);
                 playingUserIds.Add(PLAYER_1_ID);
                 playingUserIds.Add(PLAYER_2_ID);
-                nextFrame[PLAYER_1_ID] = 0;
-                nextFrame[PLAYER_2_ID] = 0;
             });
 
             loadSpectateScreen(false);
@@ -82,6 +76,23 @@ namespace osu.Game.Tests.Visual.Multiplayer
             AddWaitStep("wait a bit", 20);
         }
 
+        [Test]
+        public void TestTimeDoesNotProgressWhileAllPlayersPaused()
+        {
+            start(new[] { PLAYER_1_ID, PLAYER_2_ID });
+            loadSpectateScreen();
+
+            sendFrames(PLAYER_1_ID, 40);
+            sendFrames(PLAYER_2_ID, 20);
+
+            checkPaused(PLAYER_2_ID, true);
+            checkPausedInstant(PLAYER_1_ID, false);
+            AddAssert("master clock still running", () => this.ChildrenOfType<MasterGameplayClockContainer>().Single().IsRunning);
+
+            checkPaused(PLAYER_1_ID, true);
+            AddUntilStep("master clock paused", () => !this.ChildrenOfType<MasterGameplayClockContainer>().Single().IsRunning);
+        }
+
         [Test]
         public void TestPlayersMustStartSimultaneously()
         {
@@ -185,8 +196,8 @@ namespace osu.Game.Tests.Visual.Multiplayer
 
             sendFrames(PLAYER_1_ID, 10);
             sendFrames(PLAYER_2_ID, 20);
-            assertMuted(PLAYER_1_ID, false);
-            assertMuted(PLAYER_2_ID, true);
+            checkPaused(PLAYER_1_ID, false);
+            assertOneNotMuted();
 
             checkPaused(PLAYER_1_ID, true);
             assertMuted(PLAYER_1_ID, true);
@@ -204,6 +215,36 @@ namespace osu.Game.Tests.Visual.Multiplayer
             assertMuted(PLAYER_2_ID, true);
         }
 
+        [Test]
+        public void TestSpectatingDuringGameplay()
+        {
+            var players = new[] { PLAYER_1_ID, PLAYER_2_ID };
+
+            start(players);
+            sendFrames(players, 300);
+
+            loadSpectateScreen();
+            sendFrames(players, 300);
+
+            AddUntilStep("playing from correct point in time", () => this.ChildrenOfType<DrawableRuleset>().All(r => r.FrameStableClock.CurrentTime > 30000));
+        }
+
+        [Test]
+        public void TestSpectatingDuringGameplayWithLateFrames()
+        {
+            start(new[] { PLAYER_1_ID, PLAYER_2_ID });
+            sendFrames(new[] { PLAYER_1_ID, PLAYER_2_ID }, 300);
+
+            loadSpectateScreen();
+            sendFrames(PLAYER_1_ID, 300);
+
+            AddWaitStep("wait maximum start delay seconds", (int)(CatchUpSyncManager.MAXIMUM_START_DELAY / TimePerAction));
+            checkPaused(PLAYER_1_ID, false);
+
+            sendFrames(PLAYER_2_ID, 300);
+            AddUntilStep("player 2 playing from correct point in time", () => getPlayer(PLAYER_2_ID).ChildrenOfType<DrawableRuleset>().Single().FrameStableClock.CurrentTime > 30000);
+        }
+
         private void loadSpectateScreen(bool waitForPlayerLoad = true)
         {
             AddStep("load screen", () =>
@@ -226,7 +267,6 @@ namespace osu.Game.Tests.Visual.Multiplayer
                     Client.CurrentMatchPlayingUserIds.Add(id);
                     SpectatorClient.StartPlay(id, beatmapId ?? importedBeatmapId);
                     playingUserIds.Add(id);
-                    nextFrame[id] = 0;
                 }
             });
         }
@@ -238,10 +278,7 @@ namespace osu.Game.Tests.Visual.Multiplayer
             AddStep("send frames", () =>
             {
                 foreach (int id in userIds)
-                {
-                    SpectatorClient.SendFrames(id, nextFrame[id], count);
-                    nextFrame[id] += count;
-                }
+                    SpectatorClient.SendFrames(id, count);
             });
         }
 
@@ -249,7 +286,14 @@ namespace osu.Game.Tests.Visual.Multiplayer
             => AddUntilStep($"{userId} is {(state ? "paused" : "playing")}", () => getPlayer(userId).ChildrenOfType<GameplayClockContainer>().First().GameplayClock.IsRunning != state);
 
         private void checkPausedInstant(int userId, bool state)
-            => AddAssert($"{userId} is {(state ? "paused" : "playing")}", () => getPlayer(userId).ChildrenOfType<GameplayClockContainer>().First().GameplayClock.IsRunning != state);
+        {
+            checkPaused(userId, state);
+
+            // Todo: The following should work, but is broken because SpectatorScreen retrieves the WorkingBeatmap via the BeatmapManager, bypassing the test scene clock and running real-time.
+            // AddAssert($"{userId} is {(state ? "paused" : "playing")}", () => getPlayer(userId).ChildrenOfType<GameplayClockContainer>().First().GameplayClock.IsRunning != state);
+        }
+
+        private void assertOneNotMuted() => AddAssert("one player not muted", () => spectatorScreen.ChildrenOfType<PlayerArea>().Count(p => !p.Mute) == 1);
 
         private void assertMuted(int userId, bool muted)
             => AddAssert($"{userId} {(muted ? "is" : "is not")} muted", () => getInstance(userId).Mute == muted);
diff --git a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/CatchUpSpectatorPlayerClock.cs b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/CatchUpSpectatorPlayerClock.cs
index 9e1a020eca..20d12d62a3 100644
--- a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/CatchUpSpectatorPlayerClock.cs
+++ b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/CatchUpSpectatorPlayerClock.cs
@@ -34,7 +34,11 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
 
         public void Stop() => IsRunning = false;
 
-        public bool Seek(double position) => true;
+        public bool Seek(double position)
+        {
+            CurrentTime = position;
+            return true;
+        }
 
         public void ResetSpeedAdjustments()
         {
diff --git a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/CatchUpSyncManager.cs b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/CatchUpSyncManager.cs
index efc12eaaa5..94278a47b6 100644
--- a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/CatchUpSyncManager.cs
+++ b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/CatchUpSyncManager.cs
@@ -1,8 +1,11 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
+using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Timing;
 
@@ -28,16 +31,22 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
         /// </summary>
         public const double MAXIMUM_START_DELAY = 15000;
 
+        public event Action ReadyToStart;
+
         /// <summary>
         /// The master clock which is used to control the timing of all player clocks clocks.
         /// </summary>
         public IAdjustableClock MasterClock { get; }
 
+        public IBindable<MasterClockState> MasterState => masterState;
+
         /// <summary>
         /// The player clocks.
         /// </summary>
         private readonly List<ISpectatorPlayerClock> playerClocks = new List<ISpectatorPlayerClock>();
 
+        private readonly Bindable<MasterClockState> masterState = new Bindable<MasterClockState>();
+
         private bool hasStarted;
         private double? firstStartAttemptTime;
 
@@ -46,7 +55,11 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
             MasterClock = master;
         }
 
-        public void AddPlayerClock(ISpectatorPlayerClock clock) => playerClocks.Add(clock);
+        public void AddPlayerClock(ISpectatorPlayerClock clock)
+        {
+            Debug.Assert(!playerClocks.Contains(clock));
+            playerClocks.Add(clock);
+        }
 
         public void RemovePlayerClock(ISpectatorPlayerClock clock) => playerClocks.Remove(clock);
 
@@ -62,8 +75,8 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
                 return;
             }
 
-            updateCatchup();
-            updateMasterClock();
+            updatePlayerCatchup();
+            updateMasterState();
         }
 
         /// <summary>
@@ -81,14 +94,20 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
             int readyCount = playerClocks.Count(s => !s.WaitingOnFrames.Value);
 
             if (readyCount == playerClocks.Count)
-                return hasStarted = true;
+                return performStart();
 
             if (readyCount > 0)
             {
                 firstStartAttemptTime ??= Time.Current;
 
                 if (Time.Current - firstStartAttemptTime > MAXIMUM_START_DELAY)
-                    return hasStarted = true;
+                    return performStart();
+            }
+
+            bool performStart()
+            {
+                ReadyToStart?.Invoke();
+                return hasStarted = true;
             }
 
             return false;
@@ -97,7 +116,7 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
         /// <summary>
         /// Updates the catchup states of all player clocks clocks.
         /// </summary>
-        private void updateCatchup()
+        private void updatePlayerCatchup()
         {
             for (int i = 0; i < playerClocks.Count; i++)
             {
@@ -135,19 +154,12 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
         }
 
         /// <summary>
-        /// Updates the master clock's running state.
+        /// Updates the state of the master clock.
         /// </summary>
-        private void updateMasterClock()
+        private void updateMasterState()
         {
             bool anyInSync = playerClocks.Any(s => !s.IsCatchingUp);
-
-            if (MasterClock.IsRunning != anyInSync)
-            {
-                if (anyInSync)
-                    MasterClock.Start();
-                else
-                    MasterClock.Stop();
-            }
+            masterState.Value = anyInSync ? MasterClockState.Synchronised : MasterClockState.TooFarAhead;
         }
     }
 }
diff --git a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/ISyncManager.cs b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/ISyncManager.cs
index bd698108f6..3c644ccb78 100644
--- a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/ISyncManager.cs
+++ b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/ISyncManager.cs
@@ -1,6 +1,8 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System;
+using osu.Framework.Bindables;
 using osu.Framework.Timing;
 
 namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
@@ -10,11 +12,21 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
     /// </summary>
     public interface ISyncManager
     {
+        /// <summary>
+        /// An event which is invoked when gameplay is ready to start.
+        /// </summary>
+        event Action ReadyToStart;
+
         /// <summary>
         /// The master clock which player clocks should synchronise to.
         /// </summary>
         IAdjustableClock MasterClock { get; }
 
+        /// <summary>
+        /// An event which is invoked when the state of <see cref="MasterClock"/> is changed.
+        /// </summary>
+        IBindable<MasterClockState> MasterState { get; }
+
         /// <summary>
         /// Adds an <see cref="ISpectatorPlayerClock"/> to manage.
         /// </summary>
diff --git a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MasterClockState.cs b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MasterClockState.cs
new file mode 100644
index 0000000000..8982d1669d
--- /dev/null
+++ b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MasterClockState.cs
@@ -0,0 +1,18 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
+{
+    public enum MasterClockState
+    {
+        /// <summary>
+        /// The master clock is synchronised with at least one player clock.
+        /// </summary>
+        Synchronised,
+
+        /// <summary>
+        /// The master clock is too far ahead of any player clock and needs to slow down.
+        /// </summary>
+        TooFarAhead
+    }
+}
diff --git a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MultiSpectatorScreen.cs b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MultiSpectatorScreen.cs
index 983daac909..2a2759e0dd 100644
--- a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MultiSpectatorScreen.cs
+++ b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/MultiSpectatorScreen.cs
@@ -5,6 +5,7 @@ using System;
 using System.Linq;
 using JetBrains.Annotations;
 using osu.Framework.Allocation;
+using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Graphics.Containers;
 using osu.Game.Online.Multiplayer;
@@ -42,6 +43,7 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
         private PlayerGrid grid;
         private MultiSpectatorLeaderboard leaderboard;
         private PlayerArea currentAudioSource;
+        private bool canStartMasterClock;
 
         /// <summary>
         /// Creates a new <see cref="MultiSpectatorScreen"/>.
@@ -100,15 +102,24 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
                 Expanded = { Value = true },
                 Anchor = Anchor.CentreLeft,
                 Origin = Anchor.CentreLeft,
-            }, leaderboardContainer.Add);
+            }, l =>
+            {
+                foreach (var instance in instances)
+                    leaderboard.AddClock(instance.UserId, instance.GameplayClock);
+
+                leaderboardContainer.Add(leaderboard);
+            });
         }
 
         protected override void LoadComplete()
         {
             base.LoadComplete();
 
-            masterClockContainer.Stop();
             masterClockContainer.Reset();
+            masterClockContainer.Stop();
+
+            syncManager.ReadyToStart += onReadyToStart;
+            syncManager.MasterState.BindValueChanged(onMasterStateChanged, true);
         }
 
         protected override void Update()
@@ -129,19 +140,45 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
         private bool isCandidateAudioSource([CanBeNull] ISpectatorPlayerClock clock)
             => clock?.IsRunning == true && !clock.IsCatchingUp && !clock.WaitingOnFrames.Value;
 
+        private void onReadyToStart()
+        {
+            // Seek the master clock to the gameplay time.
+            // This is chosen as the first available frame in the players' replays, which matches the seek by each individual SpectatorPlayer.
+            var startTime = instances.Where(i => i.Score != null)
+                                     .SelectMany(i => i.Score.Replay.Frames)
+                                     .Select(f => f.Time)
+                                     .DefaultIfEmpty(0)
+                                     .Min();
+
+            masterClockContainer.Seek(startTime);
+            masterClockContainer.Start();
+
+            // Although the clock has been started, this flag is set to allow for later synchronisation state changes to also be able to start it.
+            canStartMasterClock = true;
+        }
+
+        private void onMasterStateChanged(ValueChangedEvent<MasterClockState> state)
+        {
+            switch (state.NewValue)
+            {
+                case MasterClockState.Synchronised:
+                    if (canStartMasterClock)
+                        masterClockContainer.Start();
+
+                    break;
+
+                case MasterClockState.TooFarAhead:
+                    masterClockContainer.Stop();
+                    break;
+            }
+        }
+
         protected override void OnUserStateChanged(int userId, SpectatorState spectatorState)
         {
         }
 
         protected override void StartGameplay(int userId, GameplayState gameplayState)
-        {
-            var instance = instances.Single(i => i.UserId == userId);
-
-            instance.LoadScore(gameplayState.Score);
-
-            syncManager.AddPlayerClock(instance.GameplayClock);
-            leaderboard.AddClock(instance.UserId, instance.GameplayClock);
-        }
+            => instances.Single(i => i.UserId == userId).LoadScore(gameplayState.Score);
 
         protected override void EndGameplay(int userId)
         {
diff --git a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/PlayerArea.cs b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/PlayerArea.cs
index fe79e5db72..95ccc08608 100644
--- a/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/PlayerArea.cs
+++ b/osu.Game/Screens/OnlinePlay/Multiplayer/Spectate/PlayerArea.cs
@@ -27,7 +27,7 @@ namespace osu.Game.Screens.OnlinePlay.Multiplayer.Spectate
         /// <summary>
         /// Whether a <see cref="Player"/> is loaded in the area.
         /// </summary>
-        public bool PlayerLoaded => stack?.CurrentScreen is Player;
+        public bool PlayerLoaded => (stack?.CurrentScreen as Player)?.IsLoaded == true;
 
         /// <summary>
         /// The user id this <see cref="PlayerArea"/> corresponds to.
diff --git a/osu.Game/Screens/Play/SpectatorPlayer.cs b/osu.Game/Screens/Play/SpectatorPlayer.cs
index 0286b6b8a6..f662a479ec 100644
--- a/osu.Game/Screens/Play/SpectatorPlayer.cs
+++ b/osu.Game/Screens/Play/SpectatorPlayer.cs
@@ -47,8 +47,9 @@ namespace osu.Game.Screens.Play
         {
             base.StartGameplay();
 
+            // Start gameplay along with the very first arrival frame (the latest one).
+            score.Replay.Frames.Clear();
             spectatorClient.OnNewFrames += userSentFrames;
-            seekToGameplay();
         }
 
         private void userSentFrames(int userId, FrameDataBundle bundle)
@@ -62,6 +63,8 @@ namespace osu.Game.Screens.Play
             if (!this.IsCurrentScreen())
                 return;
 
+            bool isFirstBundle = score.Replay.Frames.Count == 0;
+
             foreach (var frame in bundle.Frames)
             {
                 IConvertibleReplayFrame convertibleFrame = GameplayRuleset.CreateConvertibleReplayFrame();
@@ -73,19 +76,8 @@ namespace osu.Game.Screens.Play
                 score.Replay.Frames.Add(convertedFrame);
             }
 
-            seekToGameplay();
-        }
-
-        private bool seekedToGameplay;
-
-        private void seekToGameplay()
-        {
-            if (seekedToGameplay || score.Replay.Frames.Count == 0)
-                return;
-
-            NonFrameStableSeek(score.Replay.Frames[0].Time);
-
-            seekedToGameplay = true;
+            if (isFirstBundle && score.Replay.Frames.Count > 0)
+                NonFrameStableSeek(score.Replay.Frames[0].Time);
         }
 
         protected override Score CreateScore() => score;
diff --git a/osu.Game/Tests/Visual/Spectator/TestSpectatorClient.cs b/osu.Game/Tests/Visual/Spectator/TestSpectatorClient.cs
index c7aa43b377..f206d4f8b0 100644
--- a/osu.Game/Tests/Visual/Spectator/TestSpectatorClient.cs
+++ b/osu.Game/Tests/Visual/Spectator/TestSpectatorClient.cs
@@ -20,9 +20,15 @@ namespace osu.Game.Tests.Visual.Spectator
 {
     public class TestSpectatorClient : SpectatorClient
     {
+        /// <summary>
+        /// Maximum number of frames sent per bundle via <see cref="SendFrames"/>.
+        /// </summary>
+        public const int FRAME_BUNDLE_SIZE = 10;
+
         public override IBindable<bool> IsConnected { get; } = new Bindable<bool>(true);
 
         private readonly Dictionary<int, int> userBeatmapDictionary = new Dictionary<int, int>();
+        private readonly Dictionary<int, int> userNextFrameDictionary = new Dictionary<int, int>();
 
         [Resolved]
         private IAPIProvider api { get; set; } = null!;
@@ -35,6 +41,7 @@ namespace osu.Game.Tests.Visual.Spectator
         public void StartPlay(int userId, int beatmapId)
         {
             userBeatmapDictionary[userId] = beatmapId;
+            userNextFrameDictionary[userId] = 0;
             sendPlayingState(userId);
         }
 
@@ -57,24 +64,41 @@ namespace osu.Game.Tests.Visual.Spectator
         public new void Schedule(Action action) => base.Schedule(action);
 
         /// <summary>
-        /// Sends frames for an arbitrary user.
+        /// Sends frames for an arbitrary user, in bundles containing 10 frames each.
         /// </summary>
         /// <param name="userId">The user to send frames for.</param>
-        /// <param name="index">The frame index.</param>
-        /// <param name="count">The number of frames to send.</param>
-        public void SendFrames(int userId, int index, int count)
+        /// <param name="count">The total number of frames to send.</param>
+        public void SendFrames(int userId, int count)
         {
             var frames = new List<LegacyReplayFrame>();
 
-            for (int i = index; i < index + count; i++)
-            {
-                var buttonState = i == index + count - 1 ? ReplayButtonState.None : ReplayButtonState.Left1;
+            int currentFrameIndex = userNextFrameDictionary[userId];
+            int lastFrameIndex = currentFrameIndex + count - 1;
 
-                frames.Add(new LegacyReplayFrame(i * 100, RNG.Next(0, 512), RNG.Next(0, 512), buttonState));
+            for (; currentFrameIndex <= lastFrameIndex; currentFrameIndex++)
+            {
+                // This is done in the next frame so that currentFrameIndex is updated to the correct value.
+                if (frames.Count == FRAME_BUNDLE_SIZE)
+                    flush();
+
+                var buttonState = currentFrameIndex == lastFrameIndex ? ReplayButtonState.None : ReplayButtonState.Left1;
+                frames.Add(new LegacyReplayFrame(currentFrameIndex * 100, RNG.Next(0, 512), RNG.Next(0, 512), buttonState));
             }
 
-            var bundle = new FrameDataBundle(new ScoreInfo { Combo = index + count }, frames);
-            ((ISpectatorClient)this).UserSentFrames(userId, bundle);
+            flush();
+
+            userNextFrameDictionary[userId] = currentFrameIndex;
+
+            void flush()
+            {
+                if (frames.Count == 0)
+                    return;
+
+                var bundle = new FrameDataBundle(new ScoreInfo { Combo = currentFrameIndex }, frames.ToArray());
+                ((ISpectatorClient)this).UserSentFrames(userId, bundle);
+
+                frames.Clear();
+            }
         }
 
         protected override Task BeginPlayingInternal(SpectatorState state)