diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneSongProgress.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneSongProgress.cs
index 0996b5b0c7..ff14846b68 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneSongProgress.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneSongProgress.cs
@@ -6,6 +6,9 @@ using System.Collections.Generic;
 using NUnit.Framework;
 using osu.Framework.Allocation;
 using osu.Framework.Graphics;
+using osu.Framework.Graphics.Containers;
+using osu.Framework.Graphics.Shapes;
+using osu.Framework.Testing;
 using osu.Framework.Utils;
 using osu.Framework.Timing;
 using osu.Game.Graphics;
diff --git a/osu.Game/Screens/Play/HUDOverlay.cs b/osu.Game/Screens/Play/HUDOverlay.cs
index 236bdc8442..a5f8051557 100644
--- a/osu.Game/Screens/Play/HUDOverlay.cs
+++ b/osu.Game/Screens/Play/HUDOverlay.cs
@@ -131,7 +131,6 @@ namespace osu.Game.Screens.Play
                 BindDrawableRuleset(drawableRuleset);
 
                 Progress.Objects = drawableRuleset.Objects;
-                Progress.AllowSeeking = drawableRuleset.HasReplayLoaded.Value;
                 Progress.RequestSeek = time => RequestSeek(time);
                 Progress.ReferenceClock = drawableRuleset.FrameStableClock;
             }
diff --git a/osu.Game/Screens/Play/SongProgress.cs b/osu.Game/Screens/Play/SongProgress.cs
index 1368158938..aa745f5ba2 100644
--- a/osu.Game/Screens/Play/SongProgress.cs
+++ b/osu.Game/Screens/Play/SongProgress.cs
@@ -46,7 +46,6 @@ namespace osu.Game.Screens.Play
         public override bool HandlePositionalInput => AllowSeeking.Value;
 
         private double firstHitTime => objects.First().StartTime;
-        private double lastHitTime => ((objects.Last() as IHasEndTime)?.EndTime ?? objects.Last().StartTime) + 1;
 
         private IEnumerable<HitObject> objects;
 
@@ -92,7 +91,6 @@ namespace osu.Game.Screens.Play
                 },
                 bar = new SongProgressBar(bottom_bar_height, graph_height, handle_size)
                 {
-                    Alpha = 0,
                     Anchor = Anchor.BottomLeft,
                     Origin = Anchor.BottomLeft,
                     OnSeek = time => RequestSeek?.Invoke(time),
@@ -105,6 +103,9 @@ namespace osu.Game.Screens.Play
         {
             base.LoadComplete();
 
+            if (clock != null)
+                gameplayClock = clock;
+
             config.BindWith(OsuSetting.ShowProgressGraph, ShowGraph);
 
             graph.FillColour = bar.FillColour = colours.BlueLighter;
@@ -151,8 +152,7 @@ namespace osu.Game.Screens.Play
 
         private void updateBarVisibility()
         {
-            bar.FadeTo(AllowSeeking.Value ? 1 : 0, transition_duration, Easing.In);
-            this.MoveTo(new Vector2(0, AllowSeeking.Value ? 0 : bottom_bar_height), transition_duration, Easing.In);
+            bar.ShowHandle = AllowSeeking.Value;
 
             updateInfoMargin();
         }
diff --git a/osu.Game/Screens/Play/SongProgressBar.cs b/osu.Game/Screens/Play/SongProgressBar.cs
index 817b703fca..5052b32335 100644
--- a/osu.Game/Screens/Play/SongProgressBar.cs
+++ b/osu.Game/Screens/Play/SongProgressBar.cs
@@ -21,6 +21,22 @@ namespace osu.Game.Screens.Play
         private readonly Container handleBase;
         private readonly Container handleContainer;
 
+        private bool showHandle;
+
+        public bool ShowHandle
+        {
+            get => showHandle;
+            set
+            {
+                if (value == showHandle)
+                    return;
+
+                showHandle = value;
+
+                handleBase.FadeTo(showHandle ? 1 : 0, 200);
+            }
+        }
+
         public Color4 FillColour
         {
             set => fill.Colour = value;
@@ -75,6 +91,7 @@ namespace osu.Game.Screens.Play
                     Origin = Anchor.BottomLeft,
                     Anchor = Anchor.BottomLeft,
                     Width = 2,
+                    Alpha = 0,
                     Colour = Color4.White,
                     Position = new Vector2(2, 0),
                     Children = new Drawable[]
@@ -128,7 +145,11 @@ namespace osu.Game.Screens.Play
         protected override void OnUserChange(double value)
         {
             scheduledSeek?.Cancel();
-            scheduledSeek = Schedule(() => OnSeek?.Invoke(value));
+            scheduledSeek = Schedule(() =>
+            {
+                if (showHandle)
+                    OnSeek?.Invoke(value);
+            });
         }
     }
 }