diff --git a/osu.Game.Rulesets.Catch.Tests/TestSceneCatcher.cs b/osu.Game.Rulesets.Catch.Tests/TestSceneCatcher.cs
index e8231b07ad..11e3a5be57 100644
--- a/osu.Game.Rulesets.Catch.Tests/TestSceneCatcher.cs
+++ b/osu.Game.Rulesets.Catch.Tests/TestSceneCatcher.cs
@@ -74,12 +74,12 @@ namespace osu.Game.Rulesets.Catch.Tests
             });
             AddStep("revert second result", () =>
             {
-                catcher.OnRevertResult(drawableObject2, result2);
+                catcher.OnRevertResult(result2);
             });
             checkHyperDash(true);
             AddStep("revert first result", () =>
             {
-                catcher.OnRevertResult(drawableObject1, result1);
+                catcher.OnRevertResult(result1);
             });
             checkHyperDash(false);
         }
@@ -96,7 +96,7 @@ namespace osu.Game.Rulesets.Catch.Tests
             checkState(CatcherAnimationState.Kiai);
             AddStep("revert result", () =>
             {
-                catcher.OnRevertResult(drawableObject, result);
+                catcher.OnRevertResult(result);
             });
             checkState(CatcherAnimationState.Idle);
         }
diff --git a/osu.Game.Rulesets.Catch/UI/CatchComboDisplay.cs b/osu.Game.Rulesets.Catch/UI/CatchComboDisplay.cs
index dbbe905879..3d0062d32f 100644
--- a/osu.Game.Rulesets.Catch/UI/CatchComboDisplay.cs
+++ b/osu.Game.Rulesets.Catch/UI/CatchComboDisplay.cs
@@ -63,12 +63,12 @@ namespace osu.Game.Rulesets.Catch.UI
             updateCombo(result.ComboAtJudgement + 1, judgedObject.AccentColour.Value);
         }
 
-        public void OnRevertResult(DrawableCatchHitObject judgedObject, JudgementResult result)
+        public void OnRevertResult(JudgementResult result)
         {
             if (!result.Type.AffectsCombo() || !result.HasResult)
                 return;
 
-            updateCombo(result.ComboAtJudgement, judgedObject.AccentColour.Value);
+            updateCombo(result.ComboAtJudgement, null);
         }
 
         private void updateCombo(int newCombo, Color4? hitObjectColour)
diff --git a/osu.Game.Rulesets.Catch/UI/CatchPlayfield.cs b/osu.Game.Rulesets.Catch/UI/CatchPlayfield.cs
index c33d021876..cf7337fd0d 100644
--- a/osu.Game.Rulesets.Catch/UI/CatchPlayfield.cs
+++ b/osu.Game.Rulesets.Catch/UI/CatchPlayfield.cs
@@ -103,7 +103,7 @@ namespace osu.Game.Rulesets.Catch.UI
         private void onNewResult(DrawableHitObject judgedObject, JudgementResult result)
             => CatcherArea.OnNewResult((DrawableCatchHitObject)judgedObject, result);
 
-        private void onRevertResult(DrawableHitObject judgedObject, JudgementResult result)
-            => CatcherArea.OnRevertResult((DrawableCatchHitObject)judgedObject, result);
+        private void onRevertResult(JudgementResult result)
+            => CatcherArea.OnRevertResult(result);
     }
 }
diff --git a/osu.Game.Rulesets.Catch/UI/Catcher.cs b/osu.Game.Rulesets.Catch/UI/Catcher.cs
index 411330f6fc..1c52c092ec 100644
--- a/osu.Game.Rulesets.Catch/UI/Catcher.cs
+++ b/osu.Game.Rulesets.Catch/UI/Catcher.cs
@@ -254,7 +254,7 @@ namespace osu.Game.Rulesets.Catch.UI
             }
         }
 
-        public void OnRevertResult(DrawableCatchHitObject drawableObject, JudgementResult result)
+        public void OnRevertResult(JudgementResult result)
         {
             var catchResult = (CatchJudgementResult)result;
 
@@ -268,8 +268,8 @@ namespace osu.Game.Rulesets.Catch.UI
                     SetHyperDashState();
             }
 
-            caughtObjectContainer.RemoveAll(d => d.HitObject == drawableObject.HitObject, false);
-            droppedObjectTarget.RemoveAll(d => d.HitObject == drawableObject.HitObject, false);
+            caughtObjectContainer.RemoveAll(d => d.HitObject == result.HitObject, false);
+            droppedObjectTarget.RemoveAll(d => d.HitObject == result.HitObject, false);
         }
 
         /// <summary>
diff --git a/osu.Game.Rulesets.Catch/UI/CatcherArea.cs b/osu.Game.Rulesets.Catch/UI/CatcherArea.cs
index 4f7535d13a..1b99270b65 100644
--- a/osu.Game.Rulesets.Catch/UI/CatcherArea.cs
+++ b/osu.Game.Rulesets.Catch/UI/CatcherArea.cs
@@ -73,10 +73,10 @@ namespace osu.Game.Rulesets.Catch.UI
             comboDisplay.OnNewResult(hitObject, result);
         }
 
-        public void OnRevertResult(DrawableCatchHitObject hitObject, JudgementResult result)
+        public void OnRevertResult(JudgementResult result)
         {
-            comboDisplay.OnRevertResult(hitObject, result);
-            Catcher.OnRevertResult(hitObject, result);
+            comboDisplay.OnRevertResult(result);
+            Catcher.OnRevertResult(result);
         }
 
         protected override void Update()
diff --git a/osu.Game/Rulesets/Judgements/JudgementResultEntry.cs b/osu.Game/Rulesets/Judgements/JudgementResultEntry.cs
new file mode 100644
index 0000000000..c3f44804c3
--- /dev/null
+++ b/osu.Game/Rulesets/Judgements/JudgementResultEntry.cs
@@ -0,0 +1,26 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using osu.Game.Rulesets.Objects;
+
+namespace osu.Game.Rulesets.Judgements
+{
+    internal class JudgementResultEntry : IComparable<JudgementResultEntry>
+    {
+        public readonly double Time;
+
+        public readonly HitObjectLifetimeEntry HitObjectEntry;
+
+        public readonly JudgementResult Result;
+
+        public JudgementResultEntry(double time, HitObjectLifetimeEntry hitObjectEntry, JudgementResult result)
+        {
+            Time = time;
+            HitObjectEntry = hitObjectEntry;
+            Result = result;
+        }
+
+        public int CompareTo(JudgementResultEntry? other) => Time.CompareTo(other?.Time);
+    }
+}
\ No newline at end of file
diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
index be5a7f71e7..02fc5637d8 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
@@ -82,6 +82,9 @@ namespace osu.Game.Rulesets.Objects.Drawables
         /// <summary>
         /// Invoked by this or a nested <see cref="DrawableHitObject"/> prior to a <see cref="JudgementResult"/> being reverted.
         /// </summary>
+        /// <remarks>
+        /// This is only invoked if this <see cref="DrawableHitObject"/> is alive when the result is reverted.
+        /// </remarks>
         public event Action<DrawableHitObject, JudgementResult> OnRevertResult;
 
         /// <summary>
@@ -222,6 +225,8 @@ namespace osu.Game.Rulesets.Objects.Drawables
 
             ensureEntryHasResult();
 
+            entry.RevertResult += onRevertResult;
+
             foreach (var h in HitObject.NestedHitObjects)
             {
                 var pooledDrawableNested = pooledObjectProvider?.GetPooledDrawableRepresentation(h, this);
@@ -234,7 +239,6 @@ namespace osu.Game.Rulesets.Objects.Drawables
                     OnNestedDrawableCreated?.Invoke(drawableNested);
 
                 drawableNested.OnNewResult += onNewResult;
-                drawableNested.OnRevertResult += onRevertResult;
                 drawableNested.ApplyCustomUpdateState += onApplyCustomUpdateState;
 
                 // This is only necessary for non-pooled DHOs. For pooled DHOs, this is handled inside GetPooledDrawableRepresentation().
@@ -309,7 +313,6 @@ namespace osu.Game.Rulesets.Objects.Drawables
             foreach (var obj in nestedHitObjects)
             {
                 obj.OnNewResult -= onNewResult;
-                obj.OnRevertResult -= onRevertResult;
                 obj.ApplyCustomUpdateState -= onApplyCustomUpdateState;
             }
 
@@ -318,6 +321,8 @@ namespace osu.Game.Rulesets.Objects.Drawables
 
             HitObject.DefaultsApplied -= onDefaultsApplied;
 
+            entry.RevertResult -= onRevertResult;
+
             OnFree();
 
             ParentHitObject = null;
@@ -366,7 +371,11 @@ namespace osu.Game.Rulesets.Objects.Drawables
 
         private void onNewResult(DrawableHitObject drawableHitObject, JudgementResult result) => OnNewResult?.Invoke(drawableHitObject, result);
 
-        private void onRevertResult(DrawableHitObject drawableHitObject, JudgementResult result) => OnRevertResult?.Invoke(drawableHitObject, result);
+        private void onRevertResult()
+        {
+            updateState(ArmedState.Idle);
+            OnRevertResult?.Invoke(this, Result);
+        }
 
         private void onApplyCustomUpdateState(DrawableHitObject drawableHitObject, ArmedState state) => ApplyCustomUpdateState?.Invoke(drawableHitObject, state);
 
@@ -578,26 +587,6 @@ namespace osu.Game.Rulesets.Objects.Drawables
 
         #endregion
 
-        protected override void Update()
-        {
-            base.Update();
-
-            if (Result != null && Result.HasResult)
-            {
-                double endTime = HitObject.GetEndTime();
-
-                if (Result.TimeOffset + endTime > Time.Current)
-                {
-                    OnRevertResult?.Invoke(this, Result);
-
-                    Result.TimeOffset = 0;
-                    Result.Type = HitResult.None;
-
-                    updateState(ArmedState.Idle);
-                }
-            }
-        }
-
         public override bool UpdateSubTreeMasking(Drawable source, RectangleF maskingBounds) => false;
 
         protected override void UpdateAfterChildren()
diff --git a/osu.Game/Rulesets/Objects/HitObjectLifetimeEntry.cs b/osu.Game/Rulesets/Objects/HitObjectLifetimeEntry.cs
index fedf419973..b517f6b9e6 100644
--- a/osu.Game/Rulesets/Objects/HitObjectLifetimeEntry.cs
+++ b/osu.Game/Rulesets/Objects/HitObjectLifetimeEntry.cs
@@ -1,6 +1,7 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System;
 using osu.Framework.Bindables;
 using osu.Framework.Graphics.Performance;
 using osu.Game.Rulesets.Judgements;
@@ -26,6 +27,8 @@ namespace osu.Game.Rulesets.Objects
 
         private readonly IBindable<double> startTimeBindable = new BindableDouble();
 
+        internal event Action? RevertResult;
+
         /// <summary>
         /// Creates a new <see cref="HitObjectLifetimeEntry"/>.
         /// </summary>
@@ -95,5 +98,7 @@ namespace osu.Game.Rulesets.Objects
         /// Set <see cref="LifetimeEntry.LifetimeStart"/> using <see cref="InitialLifetimeOffset"/>.
         /// </summary>
         internal void SetInitialLifetime() => LifetimeStart = HitObject.StartTime - InitialLifetimeOffset;
+
+        internal void OnRevertResult() => RevertResult?.Invoke();
     }
 }
diff --git a/osu.Game/Rulesets/UI/DrawableRuleset.cs b/osu.Game/Rulesets/UI/DrawableRuleset.cs
index 71b452c309..8edc5517cb 100644
--- a/osu.Game/Rulesets/UI/DrawableRuleset.cs
+++ b/osu.Game/Rulesets/UI/DrawableRuleset.cs
@@ -134,7 +134,7 @@ namespace osu.Game.Rulesets.UI
             playfield = new Lazy<Playfield>(() => CreatePlayfield().With(p =>
             {
                 p.NewResult += (_, r) => NewResult?.Invoke(r);
-                p.RevertResult += (_, r) => RevertResult?.Invoke(r);
+                p.RevertResult += r => RevertResult?.Invoke(r);
             }));
         }
 
diff --git a/osu.Game/Rulesets/UI/HitObjectContainer.cs b/osu.Game/Rulesets/UI/HitObjectContainer.cs
index 7cbf49aa31..099be486b3 100644
--- a/osu.Game/Rulesets/UI/HitObjectContainer.cs
+++ b/osu.Game/Rulesets/UI/HitObjectContainer.cs
@@ -28,11 +28,6 @@ namespace osu.Game.Rulesets.UI
         /// </summary>
         public event Action<DrawableHitObject, JudgementResult> NewResult;
 
-        /// <summary>
-        /// Invoked when a <see cref="DrawableHitObject"/> judgement is reverted.
-        /// </summary>
-        public event Action<DrawableHitObject, JudgementResult> RevertResult;
-
         /// <summary>
         /// Invoked when a <see cref="HitObject"/> becomes used by a <see cref="DrawableHitObject"/>.
         /// </summary>
@@ -111,7 +106,6 @@ namespace osu.Game.Rulesets.UI
         private void addDrawable(DrawableHitObject drawable)
         {
             drawable.OnNewResult += onNewResult;
-            drawable.OnRevertResult += onRevertResult;
 
             bindStartTime(drawable);
             AddInternal(drawable);
@@ -120,7 +114,6 @@ namespace osu.Game.Rulesets.UI
         private void removeDrawable(DrawableHitObject drawable)
         {
             drawable.OnNewResult -= onNewResult;
-            drawable.OnRevertResult -= onRevertResult;
 
             unbindStartTime(drawable);
 
@@ -154,7 +147,6 @@ namespace osu.Game.Rulesets.UI
         #endregion
 
         private void onNewResult(DrawableHitObject d, JudgementResult r) => NewResult?.Invoke(d, r);
-        private void onRevertResult(DrawableHitObject d, JudgementResult r) => RevertResult?.Invoke(d, r);
 
         #region Comparator + StartTime tracking
 
diff --git a/osu.Game/Rulesets/UI/Playfield.cs b/osu.Game/Rulesets/UI/Playfield.cs
index a7881678f1..9535ebb9ed 100644
--- a/osu.Game/Rulesets/UI/Playfield.cs
+++ b/osu.Game/Rulesets/UI/Playfield.cs
@@ -22,6 +22,8 @@ using osu.Game.Rulesets.Objects.Drawables;
 using osu.Game.Skinning;
 using osuTK;
 using osu.Game.Rulesets.Objects.Pooling;
+using osu.Game.Rulesets.Scoring;
+using osu.Framework.Extensions.ObjectExtensions;
 
 namespace osu.Game.Rulesets.UI
 {
@@ -35,9 +37,9 @@ namespace osu.Game.Rulesets.UI
         public event Action<DrawableHitObject, JudgementResult> NewResult;
 
         /// <summary>
-        /// Invoked when a <see cref="DrawableHitObject"/> judgement is reverted.
+        /// Invoked when a judgement result is reverted.
         /// </summary>
-        public event Action<DrawableHitObject, JudgementResult> RevertResult;
+        public event Action<JudgementResult> RevertResult;
 
         /// <summary>
         /// The <see cref="DrawableHitObject"/> contained in this Playfield.
@@ -98,6 +100,8 @@ namespace osu.Game.Rulesets.UI
 
         private readonly HitObjectEntryManager entryManager = new HitObjectEntryManager();
 
+        private readonly Stack<JudgementResultEntry> judgementResults;
+
         /// <summary>
         /// Creates a new <see cref="Playfield"/>.
         /// </summary>
@@ -107,14 +111,15 @@ namespace osu.Game.Rulesets.UI
 
             hitObjectContainerLazy = new Lazy<HitObjectContainer>(() => CreateHitObjectContainer().With(h =>
             {
-                h.NewResult += (d, r) => NewResult?.Invoke(d, r);
-                h.RevertResult += (d, r) => RevertResult?.Invoke(d, r);
+                h.NewResult += onNewResult;
                 h.HitObjectUsageBegan += o => HitObjectUsageBegan?.Invoke(o);
                 h.HitObjectUsageFinished += o => HitObjectUsageFinished?.Invoke(o);
             }));
 
             entryManager.OnEntryAdded += onEntryAdded;
             entryManager.OnEntryRemoved += onEntryRemoved;
+
+            judgementResults = new Stack<JudgementResultEntry>();
         }
 
         [BackgroundDependencyLoader]
@@ -224,7 +229,7 @@ namespace osu.Game.Rulesets.UI
             otherPlayfield.DisplayJudgements.BindTo(DisplayJudgements);
 
             otherPlayfield.NewResult += (d, r) => NewResult?.Invoke(d, r);
-            otherPlayfield.RevertResult += (d, r) => RevertResult?.Invoke(d, r);
+            otherPlayfield.RevertResult += r => RevertResult?.Invoke(r);
             otherPlayfield.HitObjectUsageBegan += h => HitObjectUsageBegan?.Invoke(h);
             otherPlayfield.HitObjectUsageFinished += h => HitObjectUsageFinished?.Invoke(h);
 
@@ -252,6 +257,10 @@ namespace osu.Game.Rulesets.UI
                         updatable.Update(this);
                 }
             }
+
+            // When rewinding, revert future judgements in the reverse order.
+            while (judgementResults.Count > 0 && Time.Current < judgementResults.Peek().Time)
+                revertResult(judgementResults.Pop());
         }
 
         /// <summary>
@@ -443,6 +452,25 @@ namespace osu.Game.Rulesets.UI
 
         #endregion
 
+        private void onNewResult(DrawableHitObject drawable, JudgementResult result)
+        {
+            // Not using result.TimeAbsolute because that might change and also there is a potential precision issue.
+            judgementResults.Push(new JudgementResultEntry(Time.Current, drawable.Entry.AsNonNull(), result));
+
+            NewResult?.Invoke(drawable, result);
+        }
+
+        private void revertResult(JudgementResultEntry entry)
+        {
+            var result = entry.Result;
+            RevertResult?.Invoke(result);
+
+            result.TimeOffset = 0;
+            result.Type = HitResult.None;
+
+            entry.HitObjectEntry.OnRevertResult();
+        }
+
         #region Editor logic
 
         /// <summary>