diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
index 3838e52f9b..ba6571fe1a 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
@@ -25,6 +25,8 @@ namespace osu.Game.Rulesets.Objects.Drawables
     [Cached(typeof(DrawableHitObject))]
     public abstract class DrawableHitObject : SkinReloadableDrawable
     {
+        public event Action<DrawableHitObject> DefaultsApplied;
+
         public readonly HitObject HitObject;
 
         /// <summary>
@@ -178,6 +180,7 @@ namespace osu.Game.Rulesets.Objects.Drawables
         private void onDefaultsApplied(HitObject hitObject)
         {
             apply(hitObject);
+            DefaultsApplied?.Invoke(this);
         }
 
         private void apply(HitObject hitObject)
diff --git a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
index 57f58be55a..3e01bb1d31 100644
--- a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
@@ -16,17 +16,23 @@ namespace osu.Game.Rulesets.UI.Scrolling
     {
         private readonly IBindable<double> timeRange = new BindableDouble();
         private readonly IBindable<ScrollingDirection> direction = new Bindable<ScrollingDirection>();
+        private readonly Dictionary<DrawableHitObject, Cached> hitObjectInitialStateCache = new Dictionary<DrawableHitObject, Cached>();
 
         [Resolved]
         private IScrollingInfo scrollingInfo { get; set; }
 
-        private readonly LayoutValue initialStateCache = new LayoutValue(Invalidation.RequiredParentSizeToFit | Invalidation.DrawInfo);
+        // Responds to changes in the layout. When the layout is changes, all hit object states must be recomputed.
+        private readonly LayoutValue layoutCache = new LayoutValue(Invalidation.RequiredParentSizeToFit | Invalidation.DrawInfo);
+
+        // A combined cache across all hit object states to reduce per-update iterations.
+        // When invalidated, one or more (but not necessarily all) hitobject states must be re-validated.
+        private readonly Cached combinedObjCache = new Cached();
 
         public ScrollingHitObjectContainer()
         {
             RelativeSizeAxes = Axes.Both;
 
-            AddLayout(initialStateCache);
+            AddLayout(layoutCache);
         }
 
         [BackgroundDependencyLoader]
@@ -35,13 +41,14 @@ namespace osu.Game.Rulesets.UI.Scrolling
             direction.BindTo(scrollingInfo.Direction);
             timeRange.BindTo(scrollingInfo.TimeRange);
 
-            direction.ValueChanged += _ => initialStateCache.Invalidate();
-            timeRange.ValueChanged += _ => initialStateCache.Invalidate();
+            direction.ValueChanged += _ => layoutCache.Invalidate();
+            timeRange.ValueChanged += _ => layoutCache.Invalidate();
         }
 
         public override void Add(DrawableHitObject hitObject)
         {
-            initialStateCache.Invalidate();
+            combinedObjCache.Invalidate();
+            hitObject.DefaultsApplied += onDefaultsApplied;
             base.Add(hitObject);
         }
 
@@ -51,8 +58,10 @@ namespace osu.Game.Rulesets.UI.Scrolling
 
             if (result)
             {
-                initialStateCache.Invalidate();
+                combinedObjCache.Invalidate();
                 hitObjectInitialStateCache.Remove(hitObject);
+
+                hitObject.DefaultsApplied -= onDefaultsApplied;
             }
 
             return result;
@@ -60,23 +69,45 @@ namespace osu.Game.Rulesets.UI.Scrolling
 
         public override void Clear(bool disposeChildren = true)
         {
+            foreach (var h in Objects)
+                h.DefaultsApplied -= onDefaultsApplied;
+
             base.Clear(disposeChildren);
 
-            initialStateCache.Invalidate();
+            combinedObjCache.Invalidate();
             hitObjectInitialStateCache.Clear();
         }
 
+        private void onDefaultsApplied(DrawableHitObject drawableObject)
+        {
+            // The cache may not exist if the hitobject state hasn't been computed yet (e.g. if the hitobject was added + defaults applied in the same frame).
+            // In such a case, combinedObjCache will take care of updating the hitobject.
+            if (hitObjectInitialStateCache.TryGetValue(drawableObject, out var objCache))
+            {
+                combinedObjCache.Invalidate();
+                objCache.Invalidate();
+            }
+        }
+
         private float scrollLength;
 
         protected override void Update()
         {
             base.Update();
 
-            if (!initialStateCache.IsValid)
+            if (!layoutCache.IsValid)
             {
                 foreach (var cached in hitObjectInitialStateCache.Values)
                     cached.Invalidate();
+                combinedObjCache.Invalidate();
 
+                scrollingInfo.Algorithm.Reset();
+
+                layoutCache.Validate();
+            }
+
+            if (!combinedObjCache.IsValid)
+            {
                 switch (direction.Value)
                 {
                     case ScrollingDirection.Up:
@@ -89,15 +120,21 @@ namespace osu.Game.Rulesets.UI.Scrolling
                         break;
                 }
 
-                scrollingInfo.Algorithm.Reset();
-
                 foreach (var obj in Objects)
                 {
+                    if (!hitObjectInitialStateCache.TryGetValue(obj, out var objCache))
+                        objCache = hitObjectInitialStateCache[obj] = new Cached();
+
+                    if (objCache.IsValid)
+                        return;
+
                     computeLifetimeStartRecursive(obj);
                     computeInitialStateRecursive(obj);
+
+                    objCache.Validate();
                 }
 
-                initialStateCache.Validate();
+                combinedObjCache.Validate();
             }
         }
 
@@ -109,8 +146,6 @@ namespace osu.Game.Rulesets.UI.Scrolling
                 computeLifetimeStartRecursive(obj);
         }
 
-        private readonly Dictionary<DrawableHitObject, Cached> hitObjectInitialStateCache = new Dictionary<DrawableHitObject, Cached>();
-
         private double computeOriginAdjustedLifetimeStart(DrawableHitObject hitObject)
         {
             float originAdjustment = 0.0f;
@@ -142,12 +177,6 @@ namespace osu.Game.Rulesets.UI.Scrolling
         // Cant use AddOnce() since the delegate is re-constructed every invocation
         private void computeInitialStateRecursive(DrawableHitObject hitObject) => hitObject.Schedule(() =>
         {
-            if (!hitObjectInitialStateCache.TryGetValue(hitObject, out var cached))
-                cached = hitObjectInitialStateCache[hitObject] = new Cached();
-
-            if (cached.IsValid)
-                return;
-
             if (hitObject.HitObject is IHasEndTime e)
             {
                 switch (direction.Value)
@@ -171,8 +200,6 @@ namespace osu.Game.Rulesets.UI.Scrolling
                 // Nested hitobjects don't need to scroll, but they do need accurate positions
                 updatePosition(obj, hitObject.HitObject.StartTime);
             }
-
-            cached.Validate();
         });
 
         protected override void UpdateAfterChildrenLife()