diff --git a/osu.Desktop.Tests/Visual/TestCaseScrollingHitObjects.cs b/osu.Desktop.Tests/Visual/TestCaseScrollingHitObjects.cs
index 6b30cc4f32..30ad5871f2 100644
--- a/osu.Desktop.Tests/Visual/TestCaseScrollingHitObjects.cs
+++ b/osu.Desktop.Tests/Visual/TestCaseScrollingHitObjects.cs
@@ -31,7 +31,6 @@ namespace osu.Desktop.Tests.Visual
         public TestCaseScrollingHitObjects()
         {
             OsuSpriteText timeRangeText;
-            ScrollingPlayfield<HitObject, Judgement>.ScrollingHitObjectContainer scrollingHitObjectContainer;
 
             timeRangeBindable = new BindableDouble(2000)
             {
@@ -71,12 +70,6 @@ namespace osu.Desktop.Tests.Visual
                             RelativeSizeAxes = Axes.Both,
                             Alpha = 0.25f
                         },
-                        scrollingHitObjectContainer = new ScrollingPlayfield<HitObject, Judgement>.ScrollingHitObjectContainer(Axes.Y)
-                        {
-                            RelativeSizeAxes = Axes.Both,
-                            VisibleTimeRange = timeRangeBindable,
-                            Masking = true,
-                        },
                         new OsuSpriteText
                         {
                             Text = "t minus 0",
@@ -111,17 +104,12 @@ namespace osu.Desktop.Tests.Visual
             });
 
             timeRangeBindable.TriggerChange();
-
-            scrollingHitObjectContainer.AddSpeedAdjustment(new TestSpeedAdjustmentContainer(new MultiplierControlPoint()));
-
-            AddStep("Add hit object", () => scrollingHitObjectContainer.Add(new TestDrawableHitObject(new HitObject { StartTime = Time.Current + 2000 })));
         }
 
         protected override void Update()
         {
             base.Update();
 
-            topTime.Text = Time.Current.ToString("#,#");
             bottomTime.Text = (Time.Current + timeRangeBindable.Value).ToString("#,#");
         }
 
@@ -154,13 +142,11 @@ namespace osu.Desktop.Tests.Visual
             }
         }
 
-        private class TestDrawableHitObject : DrawableHitObject<HitObject, Judgement>, IScrollingHitObject
+        private class TestDrawableHitObject : DrawableScrollingHitObject<HitObject, Judgement>
         {
             private readonly Box background;
             private const float height = 14;
 
-            public BindableDouble LifetimeOffset { get; } = new BindableDouble();
-
             public TestDrawableHitObject(HitObject hitObject)
                 : base(hitObject)
             {
diff --git a/osu.Game.Rulesets.Mania/Objects/Drawables/DrawableManiaHitObject.cs b/osu.Game.Rulesets.Mania/Objects/Drawables/DrawableManiaHitObject.cs
index cb1352fc4a..c1b2df8898 100644
--- a/osu.Game.Rulesets.Mania/Objects/Drawables/DrawableManiaHitObject.cs
+++ b/osu.Game.Rulesets.Mania/Objects/Drawables/DrawableManiaHitObject.cs
@@ -27,9 +27,6 @@ namespace osu.Game.Rulesets.Mania.Objects.Drawables
 
             if (key != null)
                 Key.BindTo(key);
-
-            RelativePositionAxes = Axes.Y;
-            Y = (float)HitObject.StartTime;
         }
 
         public override Color4 AccentColour
diff --git a/osu.Game.Rulesets.Mania/UI/Column.cs b/osu.Game.Rulesets.Mania/UI/Column.cs
index 424ed77783..694e8c8126 100644
--- a/osu.Game.Rulesets.Mania/UI/Column.cs
+++ b/osu.Game.Rulesets.Mania/UI/Column.cs
@@ -192,8 +192,6 @@ namespace osu.Game.Rulesets.Mania.UI
             }
         }
 
-        public void Add(SpeedAdjustmentContainer speedAdjustment) => HitObjects.AddSpeedAdjustment(speedAdjustment);
-
         /// <summary>
         /// Adds a DrawableHitObject to this Playfield.
         /// </summary>
diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableScrollingHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableScrollingHitObject.cs
index 52b8d3e12d..ec206928b2 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableScrollingHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableScrollingHitObject.cs
@@ -3,6 +3,7 @@
 
 using System;
 using osu.Framework.Configuration;
+using osu.Framework.Graphics;
 using osu.Game.Rulesets.Judgements;
 using osu.Game.Rulesets.Objects.Types;
 
@@ -17,6 +18,19 @@ namespace osu.Game.Rulesets.Objects.Drawables
     {
         public BindableDouble LifetimeOffset { get; } = new BindableDouble();
 
+        Axes IScrollingHitObject.ScrollingAxes
+        {
+            set
+            {
+                RelativePositionAxes = value;
+
+                if ((value & Axes.X) > 0)
+                    X = (float)HitObject.StartTime;
+                if ((value & Axes.Y) > 0)
+                    Y = (float)HitObject.StartTime;
+            }
+        }
+
         protected DrawableScrollingHitObject(TObject hitObject)
             : base(hitObject)
         {
diff --git a/osu.Game/Rulesets/Objects/Drawables/IScrollingHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/IScrollingHitObject.cs
index 560f15f133..30b65237a4 100644
--- a/osu.Game/Rulesets/Objects/Drawables/IScrollingHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/IScrollingHitObject.cs
@@ -9,7 +9,7 @@ namespace osu.Game.Rulesets.Objects.Drawables
     /// <summary>
     /// An interface that exposes properties required for scrolling hit objects to be properly displayed.
     /// </summary>
-    public interface IScrollingHitObject : IDrawable
+    internal interface IScrollingHitObject : IDrawable
     {
         /// <summary>
         /// Time offset before the hit object start time at which this <see cref="IScrollingHitObject"/> becomes visible and the time offset
@@ -21,5 +21,11 @@ namespace osu.Game.Rulesets.Objects.Drawables
         /// </para>
         /// </summary>
         BindableDouble LifetimeOffset { get; }
+
+        /// <summary>
+        /// Axes which this <see cref="IScrollingHitObject"/> will scroll through.
+        /// This is set by the container which this scrolls through.
+        /// </summary>
+        Axes ScrollingAxes { set; }
     }
 }
\ No newline at end of file
diff --git a/osu.Game/Rulesets/Timing/ScrollingContainer.cs b/osu.Game/Rulesets/Timing/ScrollingContainer.cs
index daeca5a910..dcedf102aa 100644
--- a/osu.Game/Rulesets/Timing/ScrollingContainer.cs
+++ b/osu.Game/Rulesets/Timing/ScrollingContainer.cs
@@ -53,6 +53,8 @@ namespace osu.Game.Rulesets.Timing
         /// </summary>
         internal MultiplierControlPoint ControlPoint;
 
+        private Cached<double> durationBacking;
+
         protected override int Compare(Drawable x, Drawable y)
         {
             var xHitObject = x as DrawableHitObject;
@@ -93,8 +95,6 @@ namespace osu.Game.Rulesets.Timing
             base.InvalidateFromChild(invalidation);
         }
 
-        private Cached<double> durationBacking;
-
         private double computeDuration()
         {
             if (!Children.Any())
diff --git a/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs b/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs
index c7249d5b84..f3475621c7 100644
--- a/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs
+++ b/osu.Game/Rulesets/Timing/SpeedAdjustmentContainer.cs
@@ -37,7 +37,6 @@ namespace osu.Game.Rulesets.Timing
         /// <summary>
         /// Axes which the content of this container will scroll through.
         /// </summary>
-        /// <returns></returns>
         public Axes ScrollingAxes { get; internal set; }
 
         public readonly MultiplierControlPoint ControlPoint;
@@ -83,7 +82,12 @@ namespace osu.Game.Rulesets.Timing
         public override void Add(DrawableHitObject drawable)
         {
             var scrollingHitObject = drawable as IScrollingHitObject;
-            scrollingHitObject?.LifetimeOffset.BindTo(VisibleTimeRange);
+
+            if (scrollingHitObject != null)
+            {
+                scrollingHitObject.LifetimeOffset.BindTo(VisibleTimeRange);
+                scrollingHitObject.ScrollingAxes = ScrollingAxes;
+            }
 
             base.Add(drawable);
         }
diff --git a/osu.Game/Rulesets/UI/ScrollingPlayfield.cs b/osu.Game/Rulesets/UI/ScrollingPlayfield.cs
index 8850bd6b96..1dd4ac508b 100644
--- a/osu.Game/Rulesets/UI/ScrollingPlayfield.cs
+++ b/osu.Game/Rulesets/UI/ScrollingPlayfield.cs
@@ -41,7 +41,7 @@ namespace osu.Game.Rulesets.UI
             set { visibleTimeRange.BindTo(value); }
         }
 
-        public new readonly ScrollingHitObjectContainer HitObjects;
+        internal new readonly ScrollingHitObjectContainer HitObjects;
 
         protected ScrollingPlayfield(Axes scrollingAxes, float? customWidth = null)
             : base(customWidth)
@@ -112,7 +112,7 @@ namespace osu.Game.Rulesets.UI
         /// necessary <see cref="VisibleTimeRange"/> for the contained <see cref="SpeedAdjustmentContainer"/>s.
         /// </para>
         /// </summary>
-        public class ScrollingHitObjectContainer : HitObjectContainer<DrawableHitObject<TObject, TJudgement>>
+        internal class ScrollingHitObjectContainer : HitObjectContainer<DrawableHitObject<TObject, TJudgement>>
         {
             private readonly BindableDouble visibleTimeRange = new BindableDouble { Default = 1000 };
             /// <summary>