diff --git a/osu.Game/Rulesets/UI/HitObjectContainer.cs b/osu.Game/Rulesets/UI/HitObjectContainer.cs
index 917aa54673..fee77af0ba 100644
--- a/osu.Game/Rulesets/UI/HitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/HitObjectContainer.cs
@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
 using osu.Framework.Allocation;
 using osu.Framework.Bindables;
@@ -92,26 +91,19 @@ namespace osu.Game.Rulesets.UI
 
         protected override void AddDrawable(HitObjectLifetimeEntry entry, DrawableHitObject drawable)
         {
-            if (!nonPooledDrawableMap.ContainsKey(entry))
-            {
-                addDrawable(drawable);
-                HitObjectUsageBegan?.Invoke(entry.HitObject);
-            }
+            if (nonPooledDrawableMap.ContainsKey(entry)) return;
 
-            OnAdd(drawable);
+            addDrawable(drawable);
+            HitObjectUsageBegan?.Invoke(entry.HitObject);
         }
 
         protected override void RemoveDrawable(HitObjectLifetimeEntry entry, DrawableHitObject drawable)
         {
             drawable.OnKilled();
+            if (nonPooledDrawableMap.ContainsKey(entry)) return;
 
-            if (!nonPooledDrawableMap.ContainsKey(entry))
-            {
-                removeDrawable(drawable);
-                HitObjectUsageFinished?.Invoke(entry.HitObject);
-            }
-
-            OnRemove(drawable);
+            removeDrawable(drawable);
+            HitObjectUsageFinished?.Invoke(entry.HitObject);
         }
 
         private void addDrawable(DrawableHitObject drawable)
@@ -159,21 +151,6 @@ namespace osu.Game.Rulesets.UI
 
         #endregion
 
-        /// <summary>
-        /// Invoked after a <see cref="DrawableHitObject"/> is added to this container.
-        /// </summary>
-        protected virtual void OnAdd(DrawableHitObject drawableHitObject)
-        {
-            Debug.Assert(drawableHitObject.LoadState >= LoadState.Ready);
-        }
-
-        /// <summary>
-        /// Invoked after a <see cref="DrawableHitObject"/> is removed from this container.
-        /// </summary>
-        protected virtual void OnRemove(DrawableHitObject drawableHitObject)
-        {
-        }
-
         private void onNewResult(DrawableHitObject d, JudgementResult r) => NewResult?.Invoke(d, r);
         private void onRevertResult(DrawableHitObject d, JudgementResult r) => RevertResult?.Invoke(d, r);
 
diff --git a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
index b60aabe0e6..f478e37e3e 100644
--- a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
@@ -2,10 +2,12 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
+using System.Diagnostics;
 using osu.Framework.Allocation;
 using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Layout;
+using osu.Game.Rulesets.Objects;
 using osu.Game.Rulesets.Objects.Drawables;
 using osu.Game.Rulesets.Objects.Types;
 using osuTK;
@@ -140,17 +142,20 @@ namespace osu.Game.Rulesets.UI.Scrolling
             }
         }
 
-        protected override void OnAdd(DrawableHitObject drawableHitObject)
+        protected override void AddDrawable(HitObjectLifetimeEntry entry, DrawableHitObject drawable)
         {
-            invalidateHitObject(drawableHitObject);
-            drawableHitObject.DefaultsApplied += invalidateHitObject;
+            base.AddDrawable(entry, drawable);
+
+            invalidateHitObject(drawable);
+            drawable.DefaultsApplied += invalidateHitObject;
         }
 
-        protected override void OnRemove(DrawableHitObject drawableHitObject)
+        protected override void RemoveDrawable(HitObjectLifetimeEntry entry, DrawableHitObject drawable)
         {
-            layoutComputed.Remove(drawableHitObject);
+            base.RemoveDrawable(entry, drawable);
 
-            drawableHitObject.DefaultsApplied -= invalidateHitObject;
+            drawable.DefaultsApplied -= invalidateHitObject;
+            layoutComputed.Remove(drawable);
         }
 
         private void invalidateHitObject(DrawableHitObject hitObject)
@@ -199,6 +204,9 @@ namespace osu.Game.Rulesets.UI.Scrolling
 
         private double computeOriginAdjustedLifetimeStart(DrawableHitObject hitObject)
         {
+            // Origin position may be relative to the parent size
+            Debug.Assert(hitObject.Parent != null);
+
             float originAdjustment = 0.0f;
 
             // calculate the dimension of the part of the hitobject that should already be visible