diff --git a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableHit.cs b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableHit.cs
index 5fcae7d0f4..6b041dfe25 100644
--- a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableHit.cs
+++ b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableHit.cs
@@ -28,14 +28,6 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
             FillMode = FillMode.Fit;
         }
 
-        protected override void LoadComplete()
-        {
-            base.LoadComplete();
-
-            // We need to set this here because RelativeSizeAxes won't/can't set our size by default with a different RelativeChildSize
-            Width *= Parent.RelativeChildSize.X;
-        }
-
         protected override void CheckJudgement(bool userTriggered)
         {
             if (!userTriggered)
@@ -71,6 +63,13 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
             return UpdateJudgement(true);
         }
 
+        protected override void Update()
+        {
+            base.Update();
+
+            Size = BaseSize * Parent.RelativeChildSize;
+        }
+
         protected override void UpdateState(ArmedState state)
         {
             var circlePiece = MainPiece as CirclePiece;
diff --git a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableSwell.cs b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableSwell.cs
index fc3bc82520..4eb5c46838 100644
--- a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableSwell.cs
+++ b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableSwell.cs
@@ -199,6 +199,8 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
         {
             base.Update();
 
+            Size = BaseSize * Parent.RelativeChildSize;
+
             // Make the swell stop at the hit target
             X = (float)Math.Max(Time.Current, HitObject.StartTime);
 
diff --git a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableTaikoHitObject.cs b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableTaikoHitObject.cs
index 765002fb34..46cdfcc365 100644
--- a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableTaikoHitObject.cs
+++ b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableTaikoHitObject.cs
@@ -16,6 +16,8 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
     {
         public override Vector2 OriginPosition => new Vector2(DrawHeight / 2);
 
+        protected readonly Vector2 BaseSize;
+
         protected readonly TaikoPiece MainPiece;
 
         public new TaikoHitType HitObject;
@@ -29,7 +31,7 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
             Origin = Anchor.Custom;
 
             RelativeSizeAxes = Axes.Both;
-            Size = new Vector2(HitObject.IsStrong ? TaikoHitObject.DEFAULT_STRONG_SIZE : TaikoHitObject.DEFAULT_SIZE);
+            Size = BaseSize = new Vector2(HitObject.IsStrong ? TaikoHitObject.DEFAULT_STRONG_SIZE : TaikoHitObject.DEFAULT_SIZE);
 
             Add(MainPiece = CreateMainPiece());
             MainPiece.KiaiMode = HitObject.Kiai;
diff --git a/osu.Game/Rulesets/Timing/ScrollingContainer.cs b/osu.Game/Rulesets/Timing/ScrollingContainer.cs
index d2a6939dd3..c8da8d7ac9 100644
--- a/osu.Game/Rulesets/Timing/ScrollingContainer.cs
+++ b/osu.Game/Rulesets/Timing/ScrollingContainer.cs
@@ -1,6 +1,7 @@
 // Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
+using System;
 using System.Linq;
 using osu.Framework.Caching;
 using osu.Framework.Configuration;
@@ -34,7 +35,7 @@ namespace osu.Game.Rulesets.Timing
         /// </summary>
         internal MultiplierControlPoint ControlPoint;
 
-        private Cached<double> durationBacking;
+        private Cached<double> durationBacking = new Cached<double>();
 
         /// <summary>
         /// Creates a new <see cref="ScrollingContainer"/>.
@@ -56,35 +57,23 @@ namespace osu.Game.Rulesets.Timing
             return base.Compare(y, x);
         }
 
-        private double computeDuration()
+        public override void Add(DrawableHitObject drawable)
         {
-            if (!Children.Any())
-                return 0;
-
-            double baseDuration = Children.Max(c => (c.HitObject as IHasEndTime)?.EndTime ?? c.HitObject.StartTime) - ControlPoint.StartTime;
-
-            // If we have a singular hit object at the timing section's start time, let's set a sane default duration
-            if (baseDuration == 0)
-                baseDuration = 1;
-
-            // This container needs to resize such that it completely encloses the hit objects to avoid masking optimisations. This is done by converting the largest
-            // absolutely-sized element along the scrolling axes and adding a corresponding duration value. This introduces a bit of error, but will never under-estimate.ion.
-
-            // Find the largest element that is absolutely-sized along ScrollingAxes
-            float maxAbsoluteSize = Children.Select(c => (ScrollingAxes & Axes.X) > 0 ? c.DrawWidth : c.DrawHeight)
-                                            .DefaultIfEmpty().Max();
-
-            float ourAbsoluteSize = (ScrollingAxes & Axes.X) > 0 ? DrawWidth : DrawHeight;
-
-            // Add the extra duration to account for the absolute size
-            baseDuration *= 1 + maxAbsoluteSize / ourAbsoluteSize;
-
-            return baseDuration;
+            durationBacking.Invalidate();
+            base.Add(drawable);
         }
 
+        public override bool Remove(DrawableHitObject drawable)
+        {
+            durationBacking.Invalidate();
+            return base.Remove(drawable);
+        }
+
+        // Todo: This may underestimate the size of the hit object in some cases, but won't be too much of a problem for now
+        private double computeDuration() => Math.Max(0, Children.Select(c => (c.HitObject as IHasEndTime)?.EndTime ?? c.HitObject.StartTime).DefaultIfEmpty().Max() - ControlPoint.StartTime) + 1000;
+
         /// <summary>
-        /// The maximum duration of any one hit object inside this <see cref="ScrollingContainer"/>. This is calculated as the maximum
-        /// duration of all hit objects relative to this <see cref="ScrollingContainer"/>'s <see cref="MultiplierControlPoint.StartTime"/>.
+        /// An approximate total duration of this scrolling container.
         /// </summary>
         public double Duration => durationBacking.IsValid ? durationBacking : (durationBacking.Value = computeDuration());