diff --git a/osu.Game.Tests/Visual/Gameplay/TestScenePlayerLoader.cs b/osu.Game.Tests/Visual/Gameplay/TestScenePlayerLoader.cs
index daee3a520c..18edaf1b92 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestScenePlayerLoader.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestScenePlayerLoader.cs
@@ -9,6 +9,7 @@ using NUnit.Framework;
 using osu.Framework.Allocation;
 using osu.Framework.Bindables;
 using osu.Framework.Graphics;
+using osu.Framework.MathUtils;
 using osu.Framework.Screens;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Osu;
@@ -16,12 +17,13 @@ using osu.Game.Rulesets.Scoring;
 using osu.Game.Scoring;
 using osu.Game.Screens;
 using osu.Game.Screens.Play;
+using osu.Game.Screens.Play.PlayerSettings;
 
 namespace osu.Game.Tests.Visual.Gameplay
 {
     public class TestScenePlayerLoader : ManualInputManagerTestScene
     {
-        private PlayerLoader loader;
+        private TestPlayerLoader loader;
         private OsuScreenStack stack;
 
         [SetUp]
@@ -31,19 +33,28 @@ namespace osu.Game.Tests.Visual.Gameplay
             Beatmap.Value = CreateWorkingBeatmap(new OsuRuleset().RulesetInfo);
         });
 
+        [Test]
+        public void TestBlockLoadViaMouseMovement()
+        {
+            AddStep("load dummy beatmap", () => stack.Push(loader = new TestPlayerLoader(() => new TestPlayer(false, false))));
+            AddRepeatStep("move mouse", () => InputManager.MoveMouseTo(loader.VisualSettings.ScreenSpaceDrawQuad.TopLeft + (loader.VisualSettings.ScreenSpaceDrawQuad.BottomRight - loader.VisualSettings.ScreenSpaceDrawQuad.TopLeft) * RNG.NextSingle()), 20);
+            AddAssert("loader still active", () => loader.IsCurrentScreen());
+            AddUntilStep("loads after idle", () => !loader.IsCurrentScreen());
+        }
+
         [Test]
         public void TestLoadContinuation()
         {
             Player player = null;
             SlowLoadPlayer slowPlayer = null;
 
-            AddStep("load dummy beatmap", () => stack.Push(loader = new PlayerLoader(() => player = new TestPlayer(false, false))));
+            AddStep("load dummy beatmap", () => stack.Push(loader = new TestPlayerLoader(() => player = new TestPlayer(false, false))));
             AddUntilStep("wait for current", () => loader.IsCurrentScreen());
             AddStep("mouse in centre", () => InputManager.MoveMouseTo(loader.ScreenSpaceDrawQuad.Centre));
             AddUntilStep("wait for player to be current", () => player.IsCurrentScreen());
             AddStep("load slow dummy beatmap", () =>
             {
-                stack.Push(loader = new PlayerLoader(() => slowPlayer = new SlowLoadPlayer(false, false)));
+                stack.Push(loader = new TestPlayerLoader(() => slowPlayer = new SlowLoadPlayer(false, false)));
                 Scheduler.AddDelayed(() => slowPlayer.AllowLoad.Set(), 5000);
             });
 
@@ -61,7 +72,7 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddStep("load player", () =>
             {
                 Mods.Value = new[] { gameMod = new TestMod() };
-                stack.Push(loader = new PlayerLoader(() => player = new TestPlayer()));
+                stack.Push(loader = new TestPlayerLoader(() => player = new TestPlayer()));
             });
 
             AddUntilStep("wait for loader to become current", () => loader.IsCurrentScreen());
@@ -85,6 +96,16 @@ namespace osu.Game.Tests.Visual.Gameplay
             AddAssert("player mods applied", () => playerMod2.Applied);
         }
 
+        private class TestPlayerLoader : PlayerLoader
+        {
+            public new VisualSettings VisualSettings => base.VisualSettings;
+
+            public TestPlayerLoader(Func<Player> createPlayer)
+                : base(createPlayer)
+            {
+            }
+        }
+
         private class TestMod : Mod, IApplicableToScoreProcessor
         {
             public override string Name => string.Empty;
diff --git a/osu.Game/Screens/Play/PlayerLoader.cs b/osu.Game/Screens/Play/PlayerLoader.cs
index 681ce701d0..5396321160 100644
--- a/osu.Game/Screens/Play/PlayerLoader.cs
+++ b/osu.Game/Screens/Play/PlayerLoader.cs
@@ -18,6 +18,7 @@ using osu.Game.Graphics;
 using osu.Game.Graphics.Containers;
 using osu.Game.Graphics.Sprites;
 using osu.Game.Graphics.UserInterface;
+using osu.Game.Input;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Screens.Menu;
 using osu.Game.Screens.Play.HUD;
@@ -53,6 +54,8 @@ namespace osu.Game.Screens.Play
 
         private InputManager inputManager;
 
+        private IdleTracker idleTracker;
+
         public PlayerLoader(Func<Player> createPlayer)
         {
             this.createPlayer = createPlayer;
@@ -93,7 +96,8 @@ namespace osu.Game.Screens.Play
                         VisualSettings = new VisualSettings(),
                         new InputSettings()
                     }
-                }
+                },
+                idleTracker = new IdleTracker(750)
             });
 
             loadNewPlayer();
@@ -193,7 +197,7 @@ namespace osu.Game.Screens.Play
         // Here because IsHovered will not update unless we do so.
         public override bool HandlePositionalInput => true;
 
-        private bool readyForPush => player.LoadState == LoadState.Ready && IsHovered && GetContainingInputManager()?.DraggedDrawable == null;
+        private bool readyForPush => player.LoadState == LoadState.Ready && (IsHovered || idleTracker.IsIdle.Value) && inputManager?.DraggedDrawable == null;
 
         private void pushWhenLoaded()
         {