diff --git a/osu.Game/Rulesets/Judgements/JudgementResult.cs b/osu.Game/Rulesets/Judgements/JudgementResult.cs
index 6749fd7932..bf29919e34 100644
--- a/osu.Game/Rulesets/Judgements/JudgementResult.cs
+++ b/osu.Game/Rulesets/Judgements/JudgementResult.cs
@@ -3,6 +3,7 @@
 
 #nullable disable
 
+using System;
 using JetBrains.Annotations;
 using osu.Game.Rulesets.Objects;
 using osu.Game.Rulesets.Objects.Drawables;
@@ -33,19 +34,30 @@ namespace osu.Game.Rulesets.Judgements
         public readonly Judgement Judgement;
 
         /// <summary>
-        /// The offset of <see cref="TimeAbsolute"/> from the end time of <see cref="HitObject"/>, clamped by <see cref="HitObject.MaximumJudgementOffset"/>.
-        /// Populated when this <see cref="JudgementResult"/> is applied via <see cref="DrawableHitObject.ApplyResult"/>.
-        /// </summary>
-        public double TimeOffset { get; internal set; }
-
-        /// <summary>
-        /// The absolute time at which this <see cref="JudgementResult"/> occurred.
+        /// The time at which this <see cref="JudgementResult"/> occurred.
         /// Populated when this <see cref="JudgementResult"/> is applied via <see cref="DrawableHitObject.ApplyResult"/>.
         /// </summary>
         /// <remarks>
-        /// This is initially set to the end time of <see cref="HitObject"/>.
+        /// This is used instead of <see cref="TimeAbsolute"/> to check whether this <see cref="JudgementResult"/> should be reverted.
         /// </remarks>
-        public double TimeAbsolute { get; internal set; }
+        internal double? RawTime { get; set; }
+
+        /// <summary>
+        /// The offset of <see cref="TimeAbsolute"/> from the end time of <see cref="HitObject"/>, clamped by <see cref="osu.Game.Rulesets.Objects.HitObject.MaximumJudgementOffset"/>.
+        /// </summary>
+        public double TimeOffset
+        {
+            get => RawTime != null ? Math.Min(RawTime.Value - HitObject.GetEndTime(), HitObject.MaximumJudgementOffset) : 0;
+            internal set => RawTime = HitObject.GetEndTime() + value;
+        }
+
+        /// <summary>
+        /// The absolute time at which this <see cref="JudgementResult"/> occurred, clamped by the end time of <see cref="HitObject"/> plus <see cref="osu.Game.Rulesets.Objects.HitObject.MaximumJudgementOffset"/>.
+        /// </summary>
+        /// <remarks>
+        /// The end time of <see cref="HitObject"/> is returned if this result is not populated yet.
+        /// </remarks>
+        public double TimeAbsolute => RawTime != null ? Math.Min(RawTime.Value, HitObject.GetEndTime() + HitObject.MaximumJudgementOffset) : HitObject.GetEndTime();
 
         /// <summary>
         /// The combo prior to this <see cref="JudgementResult"/> occurring.
@@ -92,8 +104,7 @@ namespace osu.Game.Rulesets.Judgements
         internal void Reset()
         {
             Type = HitResult.None;
-            TimeOffset = 0;
-            TimeAbsolute = HitObject.GetEndTime();
+            RawTime = null;
         }
 
         public override string ToString() => $"{Type} (Score:{Judgement.NumericResultFor(this)} HP:{Judgement.HealthIncreaseFor(this)} {Judgement})";
diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
index d30fc00bfc..f7c6340419 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
@@ -661,7 +661,7 @@ namespace osu.Game.Rulesets.Objects.Drawables
                     $"{GetType().ReadableName()} applied an invalid hit result (was: {Result.Type}, expected: [{Result.Judgement.MinResult} ... {Result.Judgement.MaxResult}]).");
             }
 
-            Result.TimeOffset = Math.Min(HitObject.MaximumJudgementOffset, Time.Current - HitObject.GetEndTime());
+            Result.RawTime = Time.Current;
 
             if (Result.HasResult)
                 updateState(Result.IsHit ? ArmedState.Hit : ArmedState.Miss);
diff --git a/osu.Game/Rulesets/UI/Playfield.cs b/osu.Game/Rulesets/UI/Playfield.cs
index 5d6a8de33c..b1c3b78e67 100644
--- a/osu.Game/Rulesets/UI/Playfield.cs
+++ b/osu.Game/Rulesets/UI/Playfield.cs
@@ -258,8 +258,16 @@ namespace osu.Game.Rulesets.UI
             }
 
             // When rewinding, revert future judgements in the reverse order.
-            while (judgedEntries.Count > 0 && Time.Current < judgedEntries.Peek().Result.AsNonNull().TimeAbsolute)
+            while (judgedEntries.Count > 0)
+            {
+                var result = judgedEntries.Peek().Result;
+                Debug.Assert(result?.RawTime != null);
+
+                if (Time.Current >= result.RawTime.Value)
+                    break;
+
                 revertResult(judgedEntries.Pop());
+            }
         }
 
         /// <summary>
@@ -453,7 +461,7 @@ namespace osu.Game.Rulesets.UI
 
         private void onNewResult(DrawableHitObject drawable, JudgementResult result)
         {
-            Debug.Assert(result != null && drawable.Entry?.Result == result);
+            Debug.Assert(result != null && drawable.Entry?.Result == result && result.RawTime != null);
             judgedEntries.Push(drawable.Entry.AsNonNull());
 
             NewResult?.Invoke(drawable, result);