diff --git a/osu.Game.Tests/Visual/UserInterface/TestSceneToolbarRulesetSelector.cs b/osu.Game.Tests/Visual/UserInterface/TestSceneToolbarRulesetSelector.cs
index 582303024b..0da256855a 100644
--- a/osu.Game.Tests/Visual/UserInterface/TestSceneToolbarRulesetSelector.cs
+++ b/osu.Game.Tests/Visual/UserInterface/TestSceneToolbarRulesetSelector.cs
@@ -7,7 +7,10 @@ using System;
 using System.Collections.Generic;
 using osu.Framework.Graphics;
 using System.Linq;
+using NUnit.Framework;
+using osu.Framework.Allocation;
 using osu.Framework.MathUtils;
+using osu.Game.Rulesets;
 
 namespace osu.Game.Tests.Visual.UserInterface
 {
@@ -19,17 +22,24 @@ namespace osu.Game.Tests.Visual.UserInterface
             typeof(ToolbarRulesetTabButton),
         };
 
-        public TestSceneToolbarRulesetSelector()
-        {
-            ToolbarRulesetSelector selector;
+        [Resolved]
+        private RulesetStore rulesets { get; set; }
 
-            Add(new Container
+        [Test]
+        public void TestDisplay()
+        {
+            ToolbarRulesetSelector selector = null;
+
+            AddStep("create selector", () =>
             {
-                Anchor = Anchor.Centre,
-                Origin = Anchor.Centre,
-                AutoSizeAxes = Axes.X,
-                Height = Toolbar.HEIGHT,
-                Child = selector = new ToolbarRulesetSelector()
+                Child = new Container
+                {
+                    Anchor = Anchor.Centre,
+                    Origin = Anchor.Centre,
+                    AutoSizeAxes = Axes.X,
+                    Height = Toolbar.HEIGHT,
+                    Child = selector = new ToolbarRulesetSelector()
+                };
             });
 
             AddStep("Select random", () =>
@@ -38,5 +48,32 @@ namespace osu.Game.Tests.Visual.UserInterface
             });
             AddStep("Toggle disabled state", () => selector.Current.Disabled = !selector.Current.Disabled);
         }
+
+        [Test]
+        public void TestNonFirstRulesetInitialState()
+        {
+            TestSelector selector = null;
+
+            AddStep("create selector", () =>
+            {
+                Child = new Container
+                {
+                    Anchor = Anchor.Centre,
+                    Origin = Anchor.Centre,
+                    AutoSizeAxes = Axes.X,
+                    Height = Toolbar.HEIGHT,
+                    Child = selector = new TestSelector()
+                };
+
+                selector.Current.Value = rulesets.GetRuleset(2);
+            });
+
+            AddAssert("mode line has moved", () => selector.ModeButtonLine.DrawPosition.X > 0);
+        }
+
+        private class TestSelector : ToolbarRulesetSelector
+        {
+            public new Drawable ModeButtonLine => base.ModeButtonLine;
+        }
     }
 }
diff --git a/osu.Game/Overlays/Toolbar/ToolbarRulesetSelector.cs b/osu.Game/Overlays/Toolbar/ToolbarRulesetSelector.cs
index 95d7d3f73a..f4272ab15c 100644
--- a/osu.Game/Overlays/Toolbar/ToolbarRulesetSelector.cs
+++ b/osu.Game/Overlays/Toolbar/ToolbarRulesetSelector.cs
@@ -20,7 +20,7 @@ namespace osu.Game.Overlays.Toolbar
     {
         private const float padding = 10;
 
-        private Drawable modeButtonLine;
+        protected Drawable ModeButtonLine { get; private set; }
 
         public ToolbarRulesetSelector()
         {
@@ -37,7 +37,7 @@ namespace osu.Game.Overlays.Toolbar
                 {
                     Depth = 1,
                 },
-                modeButtonLine = new Container
+                ModeButtonLine = new Container
                 {
                     Size = new Vector2(padding * 2 + ToolbarButton.WIDTH, 3),
                     Anchor = Anchor.BottomLeft,
@@ -66,17 +66,22 @@ namespace osu.Game.Overlays.Toolbar
             Current.BindValueChanged(_ => moveLineToCurrent(), true);
         }
 
-        private void moveLineToCurrent()
+        private bool hasInitialPosition;
+
+        // Scheduled to allow the flow layout to be computed before the line position is updated
+        private void moveLineToCurrent() => ScheduleAfterChildren(() =>
         {
-            foreach (TabItem<RulesetInfo> tabItem in TabContainer)
+            foreach (var tabItem in TabContainer)
             {
                 if (tabItem.Value == Current.Value)
                 {
-                    modeButtonLine.MoveToX(tabItem.DrawPosition.X, 200, Easing.OutQuint);
+                    ModeButtonLine.MoveToX(tabItem.DrawPosition.X, !hasInitialPosition ? 0 : 200, Easing.OutQuint);
                     break;
                 }
             }
-        }
+
+            hasInitialPosition = true;
+        });
 
         public override bool HandleNonPositionalInput => !Current.Disabled && base.HandleNonPositionalInput;