diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneDrawableScrollingRuleset.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneDrawableScrollingRuleset.cs
index cebe0394c7..ff3d152090 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneDrawableScrollingRuleset.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneDrawableScrollingRuleset.cs
@@ -445,7 +445,7 @@ namespace osu.Game.Tests.Visual.Gameplay
             private readonly Container<DrawableHitObject> container;
 
             public DrawableTestParentHitObject([CanBeNull] TestHitObject hitObject)
-                : base(hitObject)
+            : base(hitObject)
             {
                 InternalChildren[0].Colour = Color4.LightYellow;
                 InternalChildren[1].Colour = Color4.Yellow;
diff --git a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
index 11a7665f0f..802204900b 100644
--- a/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/Scrolling/ScrollingHitObjectContainer.cs
@@ -2,10 +2,13 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
+using JetBrains.Annotations;
 using osu.Framework.Allocation;
 using osu.Framework.Bindables;
+using osu.Framework.Caching;
 using osu.Framework.Graphics;
 using osu.Framework.Layout;
+using osu.Framework.Threading;
 using osu.Game.Rulesets.Objects.Drawables;
 using osu.Game.Rulesets.Objects.Types;
 using osuTK;
@@ -16,6 +19,7 @@ namespace osu.Game.Rulesets.UI.Scrolling
     {
         private readonly IBindable<double> timeRange = new BindableDouble();
         private readonly IBindable<ScrollingDirection> direction = new Bindable<ScrollingDirection>();
+        private readonly Dictionary<DrawableHitObject, InitialState> hitObjectInitialStateCache = new Dictionary<DrawableHitObject, InitialState>();
 
         [Resolved]
         private IScrollingInfo scrollingInfo { get; set; }
@@ -23,7 +27,9 @@ namespace osu.Game.Rulesets.UI.Scrolling
         // Responds to changes in the layout. When the layout changes, all hit object states must be recomputed.
         private readonly LayoutValue layoutCache = new LayoutValue(Invalidation.RequiredParentSizeToFit | Invalidation.DrawInfo);
 
-        private readonly HashSet<DrawableHitObject> invalidHitObjects = new HashSet<DrawableHitObject>();
+        // A combined cache across all hit object states to reduce per-update iterations.
+        // When invalidated, one or more (but not necessarily all) hitobject states must be re-validated.
+        private readonly Cached combinedObjCache = new Cached();
 
         public ScrollingHitObjectContainer()
         {
@@ -46,7 +52,8 @@ namespace osu.Game.Rulesets.UI.Scrolling
         {
             base.Clear(disposeChildren);
 
-            invalidHitObjects.Clear();
+            combinedObjCache.Invalidate();
+            hitObjectInitialStateCache.Clear();
         }
 
         /// <summary>
@@ -143,18 +150,17 @@ namespace osu.Game.Rulesets.UI.Scrolling
 
         /// <summary>
         /// Invalidate the cache of the layout of this hit object.
-        /// A hit object should be invalidated after all its nested hit objects are invalidated.
         /// </summary>
         public void InvalidateDrawableHitObject(DrawableHitObject drawableObject)
         {
-            invalidHitObjects.Add(drawableObject);
+            if (hitObjectInitialStateCache.TryGetValue(drawableObject, out var state))
+                state.Cache.Invalidate();
 
-            // Remove children as nested hit objects will be recursively updated.
-            foreach (var nested in drawableObject.NestedHitObjects)
-                invalidHitObjects.Remove(nested);
+            combinedObjCache.Invalidate();
         }
 
-        private float scrollLength;
+        // Use a nonzero value to prevent infinite results
+        private float scrollLength = 1;
 
         protected override void Update()
         {
@@ -162,11 +168,17 @@ namespace osu.Game.Rulesets.UI.Scrolling
 
             if (!layoutCache.IsValid)
             {
-                foreach (var obj in Objects)
-                    invalidHitObjects.Add(obj);
+                foreach (var state in hitObjectInitialStateCache.Values)
+                    state.Cache.Invalidate();
+                combinedObjCache.Invalidate();
 
                 scrollingInfo.Algorithm.Reset();
 
+                layoutCache.Validate();
+            }
+
+            if (!combinedObjCache.IsValid)
+            {
                 switch (direction.Value)
                 {
                     case ScrollingDirection.Up:
@@ -179,16 +191,24 @@ namespace osu.Game.Rulesets.UI.Scrolling
                         break;
                 }
 
-                layoutCache.Validate();
-            }
+                foreach (var obj in Objects)
+                {
+                    if (!hitObjectInitialStateCache.TryGetValue(obj, out var state))
+                        state = hitObjectInitialStateCache[obj] = new InitialState(new Cached());
 
-            foreach (var obj in invalidHitObjects)
-            {
-                computeInitialStateRecursive(obj);
-                computeLifetimeStartRecursive(obj);
-            }
+                    if (state.Cache.IsValid)
+                        continue;
 
-            invalidHitObjects.Clear();
+                    state.ScheduledComputation?.Cancel();
+                    state.ScheduledComputation = computeInitialStateRecursive(obj);
+
+                    computeLifetimeStartRecursive(obj);
+
+                    state.Cache.Validate();
+                }
+
+                combinedObjCache.Validate();
+            }
         }
 
         private void computeLifetimeStartRecursive(DrawableHitObject hitObject)
@@ -227,7 +247,7 @@ namespace osu.Game.Rulesets.UI.Scrolling
             return scrollingInfo.Algorithm.GetDisplayStartTime(hitObject.HitObject.StartTime, originAdjustment, timeRange.Value, scrollLength);
         }
 
-        private void computeInitialStateRecursive(DrawableHitObject hitObject)
+        private ScheduledDelegate computeInitialStateRecursive(DrawableHitObject hitObject) => hitObject.Schedule(() =>
         {
             if (hitObject.HitObject is IHasDuration e)
             {
@@ -252,7 +272,7 @@ namespace osu.Game.Rulesets.UI.Scrolling
                 // Nested hitobjects don't need to scroll, but they do need accurate positions
                 updatePosition(obj, hitObject.HitObject.StartTime);
             }
-        }
+        });
 
         protected override void UpdateAfterChildrenLife()
         {
@@ -284,5 +304,19 @@ namespace osu.Game.Rulesets.UI.Scrolling
                     break;
             }
         }
+
+        private class InitialState
+        {
+            [NotNull]
+            public readonly Cached Cache;
+
+            [CanBeNull]
+            public ScheduledDelegate ScheduledComputation;
+
+            public InitialState(Cached cache)
+            {
+                Cache = cache;
+            }
+        }
     }
 }