diff --git a/osu.Desktop.VisualTests/Tests/TestCaseGamefield.cs b/osu.Desktop.VisualTests/Tests/TestCaseGamefield.cs
index fb5bd8cde4..a44780d673 100644
--- a/osu.Desktop.VisualTests/Tests/TestCaseGamefield.cs
+++ b/osu.Desktop.VisualTests/Tests/TestCaseGamefield.cs
@@ -55,7 +55,7 @@ namespace osu.Desktop.VisualTests.Tests
             }
 
             var controlPointInfo = new ControlPointInfo();
-            controlPointInfo.ControlPoints.Add(new TimingControlPoint
+            controlPointInfo.TimingPoints.Add(new TimingControlPoint
             {
                 BeatLength = 200
             });
diff --git a/osu.Game.Rulesets.Mania/UI/ManiaHitRenderer.cs b/osu.Game.Rulesets.Mania/UI/ManiaHitRenderer.cs
index 8d5e22933c..8b3623cbab 100644
--- a/osu.Game.Rulesets.Mania/UI/ManiaHitRenderer.cs
+++ b/osu.Game.Rulesets.Mania/UI/ManiaHitRenderer.cs
@@ -2,11 +2,13 @@
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using OpenTK;
 using OpenTK.Input;
 using osu.Framework.Configuration;
 using osu.Framework.Graphics;
+using osu.Framework.Lists;
 using osu.Game.Beatmaps;
 using osu.Game.Beatmaps.ControlPoints;
 using osu.Game.Beatmaps.Timing;
@@ -38,11 +40,14 @@ namespace osu.Game.Rulesets.Mania.UI
             double lastSpeedMultiplier = 1;
             double lastBeatLength = 500;
 
-            // Generate the timing points, making non-timing changes use the previous timing change
-            var timingChanges = Beatmap.ControlPointInfo.ControlPoints.Where(c => c is TimingControlPoint || c is DifficultyControlPoint).Select(c =>
-            {
-                var change = new TimingChange();
+            // Merge timing + difficulty points
+            var allPoints = new SortedList<ControlPoint>(Comparer<ControlPoint>.Default);
+            allPoints.AddRange(Beatmap.ControlPointInfo.TimingPoints);
+            allPoints.AddRange(Beatmap.ControlPointInfo.DifficultyPoints);
 
+            // Generate the timing points, making non-timing changes use the previous timing change
+            var timingChanges = allPoints.Select(c =>
+            {
                 var timingPoint = c as TimingControlPoint;
                 var difficultyPoint = c as DifficultyControlPoint;
 
diff --git a/osu.Game.Rulesets.Taiko/UI/TaikoHitRenderer.cs b/osu.Game.Rulesets.Taiko/UI/TaikoHitRenderer.cs
index 7b0a0b085d..f4d411e15f 100644
--- a/osu.Game.Rulesets.Taiko/UI/TaikoHitRenderer.cs
+++ b/osu.Game.Rulesets.Taiko/UI/TaikoHitRenderer.cs
@@ -45,7 +45,7 @@ namespace osu.Game.Rulesets.Taiko.UI
             TaikoHitObject lastObject = Beatmap.HitObjects[Beatmap.HitObjects.Count - 1];
             double lastHitTime = 1 + (lastObject as IHasEndTime)?.EndTime ?? lastObject.StartTime;
 
-            var timingPoints = Beatmap.ControlPointInfo.ControlPoints.OfType<TimingControlPoint>().ToList();
+            var timingPoints = Beatmap.ControlPointInfo.TimingPoints.ToList();
 
             if (timingPoints.Count == 0)
                 return;
diff --git a/osu.Game/Beatmaps/ControlPoints/ControlPoint.cs b/osu.Game/Beatmaps/ControlPoints/ControlPoint.cs
index 5707caa235..0d1dc21e96 100644
--- a/osu.Game/Beatmaps/ControlPoints/ControlPoint.cs
+++ b/osu.Game/Beatmaps/ControlPoints/ControlPoint.cs
@@ -1,13 +1,17 @@
 // Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
+using System;
+
 namespace osu.Game.Beatmaps.ControlPoints
 {
-    public class ControlPoint
+    public class ControlPoint : IComparable<ControlPoint>
     {
         /// <summary>
         /// The time at which the control point takes effect.
         /// </summary>
         public double Time;
+
+        public int CompareTo(ControlPoint other) => Time.CompareTo(other.Time);
     }
 }
diff --git a/osu.Game/Beatmaps/ControlPoints/ControlPointInfo.cs b/osu.Game/Beatmaps/ControlPoints/ControlPointInfo.cs
index 90c50a3e05..84b6dcfc20 100644
--- a/osu.Game/Beatmaps/ControlPoints/ControlPointInfo.cs
+++ b/osu.Game/Beatmaps/ControlPoints/ControlPointInfo.cs
@@ -1,63 +1,82 @@
 using System.Collections.Generic;
 using System.Linq;
+using osu.Framework.Lists;
 
 namespace osu.Game.Beatmaps.ControlPoints
 {
     public class ControlPointInfo
     {
-        /// <summary>
-        /// All the control points.
-        /// </summary>
-        public readonly List<ControlPoint> ControlPoints = new List<ControlPoint>();
+        public readonly SortedList<TimingControlPoint> TimingPoints = new SortedList<TimingControlPoint>(Comparer<TimingControlPoint>.Default);
+        public readonly SortedList<DifficultyControlPoint> DifficultyPoints = new SortedList<DifficultyControlPoint>(Comparer<DifficultyControlPoint>.Default);
+        public readonly SortedList<SoundControlPoint> SoundPoints = new SortedList<SoundControlPoint>(Comparer<SoundControlPoint>.Default);
+        public readonly SortedList<EffectControlPoint> EffectPoints = new SortedList<EffectControlPoint>(Comparer<EffectControlPoint>.Default);
 
         /// <summary>
         /// Finds the difficulty control point that is active at <paramref name="time"/>.
         /// </summary>
         /// <param name="time">The time to find the difficulty control point at.</param>
         /// <returns>The difficulty control point.</returns>
-        public DifficultyControlPoint DifficultyPointAt(double time) =>
-            ControlPoints.OfType<DifficultyControlPoint>().LastOrDefault(t => t.Time <= time) ?? new DifficultyControlPoint();
+        public DifficultyControlPoint DifficultyPointAt(double time) => binarySearch(DifficultyPoints, time);
 
         /// <summary>
         /// Finds the effect control point that is active at <paramref name="time"/>.
         /// </summary>
         /// <param name="time">The time to find the effect control point at.</param>
         /// <returns>The effect control point.</returns>
-        public EffectControlPoint EffectPointAt(double time) =>
-            ControlPoints.OfType<EffectControlPoint>().LastOrDefault(t => t.Time <= time) ?? new EffectControlPoint();
+        public EffectControlPoint EffectPointAt(double time) => binarySearch(EffectPoints, time);
 
         /// <summary>
         /// Finds the sound control point that is active at <paramref name="time"/>.
         /// </summary>
         /// <param name="time">The time to find the sound control point at.</param>
         /// <returns>The sound control point.</returns>
-        public SoundControlPoint SoundPointAt(double time) =>
-            ControlPoints.OfType<SoundControlPoint>().LastOrDefault(t => t.Time <= time) ?? new SoundControlPoint();
+        public SoundControlPoint SoundPointAt(double time) => binarySearch(SoundPoints, time);
 
         /// <summary>
         /// Finds the timing control point that is active at <paramref name="time"/>.
         /// </summary>
         /// <param name="time">The time to find the timing control point at.</param>
         /// <returns>The timing control point.</returns>
-        public TimingControlPoint TimingPointAt(double time) =>
-            ControlPoints.OfType<TimingControlPoint>().LastOrDefault(t => t.Time <= time) ?? new TimingControlPoint();
+        public TimingControlPoint TimingPointAt(double time) => binarySearch(TimingPoints, time);
 
         /// <summary>
         /// Finds the maximum BPM represented by any timing control point.
         /// </summary>
         public double BPMMaximum =>
-            60000 / (ControlPoints.OfType<TimingControlPoint>().OrderBy(c => c.BeatLength).FirstOrDefault() ?? new TimingControlPoint()).BeatLength;
+            60000 / (TimingPoints.OrderBy(c => c.BeatLength).FirstOrDefault() ?? new TimingControlPoint()).BeatLength;
 
         /// <summary>
         /// Finds the minimum BPM represented by any timing control point.
         /// </summary>
         public double BPMMinimum =>
-            60000 / (ControlPoints.OfType<TimingControlPoint>().OrderByDescending(c => c.BeatLength).FirstOrDefault() ?? new TimingControlPoint()).BeatLength;
+            60000 / (TimingPoints.OrderByDescending(c => c.BeatLength).FirstOrDefault() ?? new TimingControlPoint()).BeatLength;
 
         /// <summary>
         /// Finds the mode BPM (most common BPM) represented by the control points.
         /// </summary>
         public double BPMMode =>
-            60000 / (ControlPoints.OfType<TimingControlPoint>().GroupBy(c => c.BeatLength).OrderByDescending(grp => grp.Count()).FirstOrDefault()?.FirstOrDefault() ?? new TimingControlPoint()).BeatLength;
+            60000 / (TimingPoints.GroupBy(c => c.BeatLength).OrderByDescending(grp => grp.Count()).FirstOrDefault()?.FirstOrDefault() ?? new TimingControlPoint()).BeatLength;
+
+        private T binarySearch<T>(SortedList<T> list, double time)
+            where T : ControlPoint, new()
+        {
+            if (list.Count == 0)
+                return new T();
+
+            if (time < list[0].Time)
+                return new T();
+
+            int index = list.BinarySearch(new T() { Time = time });
+
+            // Check if we've found an exact match (t == time)
+            if (index >= 0)
+                return list[index];
+
+            index = ~index;
+
+            if (index == list.Count)
+                return list[list.Count - 1];
+            return list[index - 1];
+        }
     }
 }
\ No newline at end of file
diff --git a/osu.Game/Beatmaps/Formats/OsuLegacyDecoder.cs b/osu.Game/Beatmaps/Formats/OsuLegacyDecoder.cs
index e140e2282c..875efab5f9 100644
--- a/osu.Game/Beatmaps/Formats/OsuLegacyDecoder.cs
+++ b/osu.Game/Beatmaps/Formats/OsuLegacyDecoder.cs
@@ -285,7 +285,7 @@ namespace osu.Game.Beatmaps.Formats
 
             if (timingChange && (beatLength != timingPoint.BeatLength || timeSignature != timingPoint.TimeSignature))
             {
-                beatmap.ControlPointInfo.ControlPoints.Add(new TimingControlPoint
+                beatmap.ControlPointInfo.TimingPoints.Add(new TimingControlPoint
                 {
                     Time = time,
                     BeatLength = beatLength,
@@ -295,7 +295,7 @@ namespace osu.Game.Beatmaps.Formats
 
             if (speedMultiplier != difficultyPoint.SpeedMultiplier)
             {
-                beatmap.ControlPointInfo.ControlPoints.Add(new DifficultyControlPoint
+                beatmap.ControlPointInfo.DifficultyPoints.Add(new DifficultyControlPoint
                 {
                     Time = time,
                     SpeedMultiplier = speedMultiplier
@@ -304,7 +304,7 @@ namespace osu.Game.Beatmaps.Formats
 
             if (stringSampleSet != soundPoint.SampleBank || sampleVolume != soundPoint.SampleVolume)
             {
-                beatmap.ControlPointInfo.ControlPoints.Add(new SoundControlPoint
+                beatmap.ControlPointInfo.SoundPoints.Add(new SoundControlPoint
                 {
                     Time = time,
                     SampleBank = stringSampleSet,
@@ -314,7 +314,7 @@ namespace osu.Game.Beatmaps.Formats
 
             if (kiaiMode != effectPoint.KiaiMode || omitFirstBarSignature != effectPoint.OmitFirstBarLine)
             {
-                beatmap.ControlPointInfo.ControlPoints.Add(new EffectControlPoint
+                beatmap.ControlPointInfo.EffectPoints.Add(new EffectControlPoint
                 {
                     Time = time,
                     KiaiMode = kiaiMode,