diff --git a/osu.Game/Screens/Select/BeatmapCarousel.cs b/osu.Game/Screens/Select/BeatmapCarousel.cs
index 7bb0b95b9a..4dbfe65ebf 100644
--- a/osu.Game/Screens/Select/BeatmapCarousel.cs
+++ b/osu.Game/Screens/Select/BeatmapCarousel.cs
@@ -424,7 +424,7 @@ namespace osu.Game.Screens.Select
             float drawHeight = DrawHeight;
 
             // Remove all items that should no longer be on-screen
-            scrollableContent.RemoveAll(p => p.Y < Current - p.DrawHeight || p.Y > Current + drawHeight || !p.IsPresent);
+            scrollableContent.RemoveAll(p => p.CanBeRemoved && (p.Y < Current - p.DrawHeight || p.Y > Current + drawHeight || !p.IsPresent));
 
             // Find index range of all items that should be on-screen
             Trace.Assert(Items.Count == yPositions.Count);
@@ -486,6 +486,15 @@ namespace osu.Game.Screens.Select
                 updateItem(p, halfHeight);
         }
 
+        protected override void Dispose(bool isDisposing)
+        {
+            base.Dispose(isDisposing);
+
+            // aggressively dispose "off-screen" items to reduce GC pressure.
+            foreach (var i in Items)
+                i.Dispose();
+        }
+
         private CarouselBeatmapSet createCarouselSet(BeatmapSetInfo beatmapSet)
         {
             if (beatmapSet.Beatmaps.All(b => b.Hidden))
diff --git a/osu.Game/Screens/Select/Carousel/DrawableCarouselBeatmapSet.cs b/osu.Game/Screens/Select/Carousel/DrawableCarouselBeatmapSet.cs
index d554a22735..3fcea908a0 100644
--- a/osu.Game/Screens/Select/Carousel/DrawableCarouselBeatmapSet.cs
+++ b/osu.Game/Screens/Select/Carousel/DrawableCarouselBeatmapSet.cs
@@ -30,6 +30,7 @@ namespace osu.Game.Screens.Select.Carousel
 
         private DialogOverlay dialogOverlay;
         private readonly BeatmapSetInfo beatmapSet;
+        private DelayedLoadUnloadWrapper delayed;
 
         public DrawableCarouselBeatmapSet(CarouselBeatmapSet set)
             : base(set)
@@ -37,6 +38,10 @@ namespace osu.Game.Screens.Select.Carousel
             beatmapSet = set.BeatmapSet;
         }
 
+        public override bool CanBeRemoved => delayed?.DelayedLoadCompleted != true;
+
+        protected override bool RequiresChildrenUpdate => true;
+
         [BackgroundDependencyLoader(true)]
         private void load(LocalisationEngine localisation, BeatmapManager manager, BeatmapSetOverlay beatmapOverlay, DialogOverlay overlay)
         {
@@ -50,12 +55,12 @@ namespace osu.Game.Screens.Select.Carousel
 
             Children = new Drawable[]
             {
-                new DelayedLoadWrapper(
+                delayed = new DelayedLoadUnloadWrapper(() =>
                     new PanelBackground(manager.GetWorkingBeatmap(beatmapSet.Beatmaps.FirstOrDefault()))
                     {
                         RelativeSizeAxes = Axes.Both,
                         OnLoadComplete = d => d.FadeInFromZero(1000, Easing.OutQuint),
-                    }, 300
+                    }, 300, 5000
                 ),
                 new FillFlowContainer
                 {
diff --git a/osu.Game/Screens/Select/Carousel/DrawableCarouselItem.cs b/osu.Game/Screens/Select/Carousel/DrawableCarouselItem.cs
index 8a0052559e..5bcf4e6441 100644
--- a/osu.Game/Screens/Select/Carousel/DrawableCarouselItem.cs
+++ b/osu.Game/Screens/Select/Carousel/DrawableCarouselItem.cs
@@ -24,6 +24,11 @@ namespace osu.Game.Screens.Select.Carousel
 
         public override bool IsPresent => base.IsPresent || Item.Visible;
 
+        /// <summary>
+        /// Whether this item can be removed from the scroll container (usually due to being off-screen).
+        /// </summary>
+        public virtual bool CanBeRemoved => true;
+
         public readonly CarouselItem Item;
 
         private Container nestedContainer;
@@ -86,7 +91,13 @@ namespace osu.Game.Screens.Select.Carousel
             base.OnHoverLost(state);
         }
 
-        public void SetMultiplicativeAlpha(float alpha) => borderContainer.Alpha = alpha;
+        protected bool VisibleInCarousel;
+
+        public void SetMultiplicativeAlpha(float alpha)
+        {
+            borderContainer.Alpha = alpha;
+            VisibleInCarousel = alpha > 0;
+        }
 
         protected override void LoadComplete()
         {