diff --git a/osu.Game/Overlays/Toolbar/ToolbarMusicButton.cs b/osu.Game/Overlays/Toolbar/ToolbarMusicButton.cs
index d59127d61f..9590c1aa3b 100644
--- a/osu.Game/Overlays/Toolbar/ToolbarMusicButton.cs
+++ b/osu.Game/Overlays/Toolbar/ToolbarMusicButton.cs
@@ -81,6 +81,9 @@ namespace osu.Game.Overlays.Toolbar
 
         protected override bool OnKeyDown(KeyDownEvent e)
         {
+            if (!IsHovered)
+                return false;
+
             switch (e.Key)
             {
                 case Key.Up: