diff --git a/osu.Game/Beatmaps/WorkingBeatmap.cs b/osu.Game/Beatmaps/WorkingBeatmap.cs
index b4046a4e95..c9a60d7948 100644
--- a/osu.Game/Beatmaps/WorkingBeatmap.cs
+++ b/osu.Game/Beatmaps/WorkingBeatmap.cs
@@ -259,6 +259,13 @@ namespace osu.Game.Beatmaps
         [NotNull]
         public Track LoadTrack() => loadedTrack = GetBeatmapTrack() ?? GetVirtualTrack(1000);
 
+        /// <summary>
+        /// Transfer a valid audio track into this working beatmap. Used as an optimisation to avoid reload / track swap
+        /// across difficulties in the same beatmap set.
+        /// </summary>
+        /// <param name="track">The track to transfer.</param>
+        public void TransferTrack([NotNull] Track track) => loadedTrack = track ?? throw new ArgumentNullException(nameof(track));
+
         /// <summary>
         /// Get the loaded audio track instance. <see cref="LoadTrack"/> must have first been called.
         /// This generally happens via MusicController when changing the global beatmap.
diff --git a/osu.Game/Overlays/MusicController.cs b/osu.Game/Overlays/MusicController.cs
index 8bbae33811..c1116ff651 100644
--- a/osu.Game/Overlays/MusicController.cs
+++ b/osu.Game/Overlays/MusicController.cs
@@ -282,10 +282,10 @@ namespace osu.Game.Overlays
         {
             TrackChangeDirection direction = TrackChangeDirection.None;
 
+            bool audioEquals = beatmap.NewValue?.BeatmapInfo?.AudioEquals(current?.BeatmapInfo) ?? false;
+
             if (current != null)
             {
-                bool audioEquals = beatmap.NewValue?.BeatmapInfo?.AudioEquals(current.BeatmapInfo) ?? false;
-
                 if (audioEquals)
                     direction = TrackChangeDirection.None;
                 else if (queuedDirection.HasValue)
@@ -305,8 +305,15 @@ namespace osu.Game.Overlays
 
             current = beatmap.NewValue;
 
-            if (CurrentTrack.IsDummyDevice || !beatmap.OldValue.BeatmapInfo.AudioEquals(current?.BeatmapInfo))
+            if (!audioEquals || CurrentTrack.IsDummyDevice)
+            {
                 changeTrack();
+            }
+            else
+            {
+                // transfer still valid track to new working beatmap
+                current.TransferTrack(beatmap.OldValue.Track);
+            }
 
             TrackChanged?.Invoke(current, direction);