diff --git a/osu.Game/Rulesets/UI/HitObjectContainer.cs b/osu.Game/Rulesets/UI/HitObjectContainer.cs
index fb91108605..5fbda305c8 100644
--- a/osu.Game/Rulesets/UI/HitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/HitObjectContainer.cs
@@ -48,7 +48,7 @@ namespace osu.Game.Rulesets.UI
         /// <remarks>
         /// If this <see cref="HitObjectContainer"/> uses pooled objects, this represents the time when the <see cref="HitObject"/>s become alive.
         /// </remarks>
-        internal event Action<DrawableHitObject> HitObjectUsageBegan;
+        internal event Action<HitObject> HitObjectUsageBegan;
 
         /// <summary>
         /// Invoked when a <see cref="HitObject"/> becomes unused by a <see cref="DrawableHitObject"/>.
@@ -56,7 +56,7 @@ namespace osu.Game.Rulesets.UI
         /// <remarks>
         /// If this <see cref="HitObjectContainer"/> uses pooled objects, this represents the time when the <see cref="HitObject"/>s become dead.
         /// </remarks>
-        internal event Action<DrawableHitObject> HitObjectUsageFinished;
+        internal event Action<HitObject> HitObjectUsageFinished;
 
         /// <summary>
         /// The amount of time prior to the current time within which <see cref="HitObject"/>s should be considered alive.
@@ -115,7 +115,7 @@ namespace osu.Game.Rulesets.UI
             bindStartTime(drawable);
             AddInternal(drawableMap[entry] = drawable, false);
 
-            HitObjectUsageBegan?.Invoke(drawable);
+            HitObjectUsageBegan?.Invoke(entry.HitObject);
         }
 
         private void removeDrawable(HitObjectLifetimeEntry entry)
@@ -129,10 +129,10 @@ namespace osu.Game.Rulesets.UI
 
             drawableMap.Remove(entry);
 
-            HitObjectUsageFinished?.Invoke(drawable);
-
             unbindStartTime(drawable);
             RemoveInternal(drawable);
+
+            HitObjectUsageFinished?.Invoke(entry.HitObject);
         }
 
         #endregion
diff --git a/osu.Game/Rulesets/UI/Playfield.cs b/osu.Game/Rulesets/UI/Playfield.cs
index 411bda77b8..2f589f4ce9 100644
--- a/osu.Game/Rulesets/UI/Playfield.cs
+++ b/osu.Game/Rulesets/UI/Playfield.cs
@@ -92,8 +92,8 @@ namespace osu.Game.Rulesets.UI
             {
                 h.NewResult += (d, r) => NewResult?.Invoke(d, r);
                 h.RevertResult += (d, r) => RevertResult?.Invoke(d, r);
-                h.HitObjectUsageBegan += o => HitObjectUsageBegan?.Invoke(o.HitObject);
-                h.HitObjectUsageFinished += o => HitObjectUsageFinished?.Invoke(o.HitObject);
+                h.HitObjectUsageBegan += o => HitObjectUsageBegan?.Invoke(o);
+                h.HitObjectUsageFinished += o => HitObjectUsageFinished?.Invoke(o);
             }));
         }
 
diff --git a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
index c8afe76f19..44732f490b 100644
--- a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
@@ -17,10 +17,6 @@ namespace osu.Game.Rulesets.UI.Scrolling
         private readonly IBindable<double> timeRange = new BindableDouble();
         private readonly IBindable<ScrollingDirection> direction = new Bindable<ScrollingDirection>();
 
-        // Tracks all `DrawableHitObject` (nested or not) applied a `HitObject`.
-        // It dynamically changes based on approximate lifetime when a pooling is used.
-        private readonly HashSet<DrawableHitObject> hitObjectApplied = new HashSet<DrawableHitObject>();
-
         // The lifetime of a hit object in this will be computed in next update.
         private readonly HashSet<DrawableHitObject> toComputeLifetime = new HashSet<DrawableHitObject>();
 
@@ -39,9 +35,6 @@ namespace osu.Game.Rulesets.UI.Scrolling
             RelativeSizeAxes = Axes.Both;
 
             AddLayout(layoutCache);
-
-            HitObjectUsageBegan += onHitObjectUsageBegin;
-            HitObjectUsageFinished += onHitObjectUsageFinished;
         }
 
         [BackgroundDependencyLoader]
@@ -58,7 +51,6 @@ namespace osu.Game.Rulesets.UI.Scrolling
         {
             base.Clear(disposeChildren);
 
-            hitObjectApplied.Clear();
             toComputeLifetime.Clear();
             layoutComputed.Clear();
         }
@@ -159,18 +151,10 @@ namespace osu.Game.Rulesets.UI.Scrolling
         {
             // Lifetime computation is delayed until next update because
             // when the hit object is not pooled this container is not loaded here and `scrollLength` cannot be computed.
-            hitObjectApplied.Add(hitObject);
             toComputeLifetime.Add(hitObject);
             layoutComputed.Remove(hitObject);
         }
 
-        private void onHitObjectUsageFinished(DrawableHitObject hitObject)
-        {
-            hitObjectApplied.Remove(hitObject);
-            toComputeLifetime.Remove(hitObject);
-            layoutComputed.Remove(hitObject);
-        }
-
         private float scrollLength;
 
         protected override void Update()
@@ -179,8 +163,12 @@ namespace osu.Game.Rulesets.UI.Scrolling
 
             if (!layoutCache.IsValid)
             {
-                foreach (var hitObject in hitObjectApplied)
-                    toComputeLifetime.Add(hitObject);
+                toComputeLifetime.Clear();
+                foreach (var hitObject in Objects)
+                {
+                    if (hitObject.HitObject != null)
+                        toComputeLifetime.Add(hitObject);
+                }
 
                 layoutComputed.Clear();