diff --git a/osu.Game/Rulesets/UI/HitObjectContainer.cs b/osu.Game/Rulesets/UI/HitObjectContainer.cs
index 5fbda305c8..ac5d281ddc 100644
--- a/osu.Game/Rulesets/UI/HitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/HitObjectContainer.cs
@@ -114,6 +114,7 @@ namespace osu.Game.Rulesets.UI
 
             bindStartTime(drawable);
             AddInternal(drawableMap[entry] = drawable, false);
+            OnAdd(drawable);
 
             HitObjectUsageBegan?.Invoke(entry.HitObject);
         }
@@ -129,6 +130,7 @@ namespace osu.Game.Rulesets.UI
 
             drawableMap.Remove(entry);
 
+            OnRemove(drawable);
             unbindStartTime(drawable);
             RemoveInternal(drawable);
 
@@ -147,10 +149,12 @@ namespace osu.Game.Rulesets.UI
             hitObject.OnRevertResult += onRevertResult;
 
             AddInternal(hitObject);
+            OnAdd(hitObject);
         }
 
         public virtual bool Remove(DrawableHitObject hitObject)
         {
+            OnRemove(hitObject);
             if (!RemoveInternal(hitObject))
                 return false;
 
@@ -178,6 +182,26 @@ namespace osu.Game.Rulesets.UI
 
         #endregion
 
+        /// <summary>
+        /// Invoked when a <see cref="DrawableHitObject"/> is added to this container.
+        /// </summary>
+        /// <remarks>
+        /// This method is not invoked for nested <see cref="DrawableHitObject"/>s.
+        /// </remarks>
+        protected virtual void OnAdd(DrawableHitObject drawableHitObject)
+        {
+        }
+
+        /// <summary>
+        /// Invoked when a <see cref="DrawableHitObject"/> is removed from this container.
+        /// </summary>
+        /// <remarks>
+        /// This method is not invoked for nested <see cref="DrawableHitObject"/>s.
+        /// </remarks>
+        protected virtual void OnRemove(DrawableHitObject drawableHitObject)
+        {
+        }
+
         public virtual void Clear(bool disposeChildren = true)
         {
             lifetimeManager.ClearEntries();
diff --git a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
index 02ee39e1b8..71f8f95300 100644
--- a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
@@ -150,7 +150,7 @@ namespace osu.Game.Rulesets.UI.Scrolling
         /// <summary>
         /// Make this <see cref="DrawableHitObject"/> lifetime and layout computed in next update.
         /// </summary>
-        internal void InvalidateHitObject(DrawableHitObject hitObject)
+        private void invalidateHitObject(DrawableHitObject hitObject)
         {
             // Lifetime computation is delayed until next update because
             // when the hit object is not pooled this container is not loaded here and `scrollLength` cannot be computed.
@@ -158,6 +158,31 @@ namespace osu.Game.Rulesets.UI.Scrolling
             layoutComputed.Remove(hitObject);
         }
 
+        private void onAddRecursive(DrawableHitObject hitObject)
+        {
+            invalidateHitObject(hitObject);
+
+            hitObject.DefaultsApplied += invalidateHitObject;
+
+            foreach (var nested in hitObject.NestedHitObjects)
+                onAddRecursive(nested);
+        }
+
+        protected override void OnAdd(DrawableHitObject drawableHitObject) => onAddRecursive(drawableHitObject);
+
+        private void onRemoveRecursive(DrawableHitObject hitObject)
+        {
+            toComputeLifetime.Remove(hitObject);
+            layoutComputed.Remove(hitObject);
+
+            hitObject.DefaultsApplied -= invalidateHitObject;
+
+            foreach (var nested in hitObject.NestedHitObjects)
+                onRemoveRecursive(nested);
+        }
+
+        protected override void OnRemove(DrawableHitObject drawableHitObject) => onRemoveRecursive(drawableHitObject);
+
         private float scrollLength;
 
         protected override void Update()
diff --git a/osu.Game/Rulesets/UI/Scrolling/ScrollingPlayfield.cs b/osu.Game/Rulesets/UI/Scrolling/ScrollingPlayfield.cs
index 8aba896b34..2b75f93f9e 100644
--- a/osu.Game/Rulesets/UI/Scrolling/ScrollingPlayfield.cs
+++ b/osu.Game/Rulesets/UI/Scrolling/ScrollingPlayfield.cs
@@ -26,11 +26,6 @@ namespace osu.Game.Rulesets.UI.Scrolling
             Direction.BindTo(ScrollingInfo.Direction);
         }
 
-        protected override void OnNewDrawableHitObject(DrawableHitObject drawableHitObject)
-        {
-            drawableHitObject.HitObjectApplied += d => HitObjectContainer.InvalidateHitObject(d);
-        }
-
         /// <summary>
         /// Given a position in screen space, return the time within this column.
         /// </summary>