diff --git a/osu.Game.Rulesets.Mania/Objects/Drawables/DrawableHoldNoteTick.cs b/osu.Game.Rulesets.Mania/Objects/Drawables/DrawableHoldNoteTick.cs
index 324f4e4e99..557fbf6ea8 100644
--- a/osu.Game.Rulesets.Mania/Objects/Drawables/DrawableHoldNoteTick.cs
+++ b/osu.Game.Rulesets.Mania/Objects/Drawables/DrawableHoldNoteTick.cs
@@ -94,7 +94,7 @@ namespace osu.Game.Rulesets.Mania.Objects.Drawables
 
         protected override void UpdateState(ArmedState state)
         {
-            switch (State)
+            switch (State.Value)
             {
                 case ArmedState.Hit:
                     AccentColour = Color4.Green;
diff --git a/osu.Game.Rulesets.Osu/Tests/TestCaseHitObjects.cs b/osu.Game.Rulesets.Osu/Tests/TestCaseHitObjects.cs
index 2ac15c55a7..99526b64ee 100644
--- a/osu.Game.Rulesets.Osu/Tests/TestCaseHitObjects.cs
+++ b/osu.Game.Rulesets.Osu/Tests/TestCaseHitObjects.cs
@@ -111,7 +111,7 @@ namespace osu.Game.Rulesets.Osu.Tests
             h.Depth = depth++;
 
             if (auto)
-                h.State = ArmedState.Hit;
+                h.State.Value = ArmedState.Hit;
 
             playfieldContainer.Add(h);
             var proxyable = h as IDrawableHitObjectWithProxiedApproach;
diff --git a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableHit.cs b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableHit.cs
index 55eaa8dbb8..6c14a71a4c 100644
--- a/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableHit.cs
+++ b/osu.Game.Rulesets.Taiko/Objects/Drawables/DrawableHit.cs
@@ -72,7 +72,7 @@ namespace osu.Game.Rulesets.Taiko.Objects.Drawables
             var offset = !AllJudged ? 0 : Time.Current - HitObject.StartTime;
             using (BeginDelayedSequence(HitObject.StartTime - Time.Current + offset, true))
             {
-                switch (State)
+                switch (State.Value)
                 {
                     case ArmedState.Idle:
                         this.Delay(HitObject.HitWindowMiss).Expire();
diff --git a/osu.Game/Rulesets/Judgements/DrawableJudgement.cs b/osu.Game/Rulesets/Judgements/DrawableJudgement.cs
index 12edfd802a..5ab4b7636b 100644
--- a/osu.Game/Rulesets/Judgements/DrawableJudgement.cs
+++ b/osu.Game/Rulesets/Judgements/DrawableJudgement.cs
@@ -83,7 +83,7 @@ namespace osu.Game.Rulesets.Judgements
                     break;
             }
 
-            Expire();
+            Expire(true);
         }
     }
 }
diff --git a/osu.Game/Rulesets/Judgements/Judgement.cs b/osu.Game/Rulesets/Judgements/Judgement.cs
index 0ae33272a7..a1920097d3 100644
--- a/osu.Game/Rulesets/Judgements/Judgement.cs
+++ b/osu.Game/Rulesets/Judgements/Judgement.cs
@@ -17,6 +17,19 @@ namespace osu.Game.Rulesets.Judgements
         /// </summary>
         public virtual HitResult MaxResult => HitResult.Perfect;
 
+        /// <summary>
+        /// The combo prior to this judgement occurring.
+        /// </summary>
+        internal int ComboAtJudgement { get; set; }
+
+        /// <summary>
+        /// The highest combo achieved prior to this judgement occurring.
+        /// </summary>
+        internal int HighestComboAtJudgement { get; set; }
+
+        /// <summary>
+        /// Whether a successful hit occurred.
+        /// </summary>
         public bool IsHit => Result > HitResult.Miss;
 
         /// <summary>
diff --git a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
index bcd6734af6..9b4f7e7fc7 100644
--- a/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
+++ b/osu.Game/Rulesets/Objects/Drawables/DrawableHitObject.cs
@@ -13,6 +13,7 @@ using OpenTK.Graphics;
 using osu.Game.Audio;
 using System.Linq;
 using osu.Game.Graphics;
+using osu.Framework.Configuration;
 
 namespace osu.Game.Rulesets.Objects.Drawables
 {
@@ -30,6 +31,9 @@ namespace osu.Game.Rulesets.Objects.Drawables
         /// </summary>
         public virtual bool DisplayJudgement => true;
 
+        public override bool RemoveCompletedTransforms => false;
+        public override bool RemoveWhenNotAlive => false;
+
         protected DrawableHitObject(HitObject hitObject)
         {
             HitObject = hitObject;
@@ -40,6 +44,7 @@ namespace osu.Game.Rulesets.Objects.Drawables
         where TObject : HitObject
     {
         public event Action<DrawableHitObject, Judgement> OnJudgement;
+        public event Action<DrawableHitObject, Judgement> OnJudgementRemoved;
 
         public new readonly TObject HitObject;
 
@@ -56,31 +61,42 @@ namespace osu.Game.Rulesets.Objects.Drawables
 
         protected List<SampleChannel> Samples = new List<SampleChannel>();
 
+        public readonly Bindable<ArmedState> State = new Bindable<ArmedState>();
+
         protected DrawableHitObject(TObject hitObject)
             : base(hitObject)
         {
             HitObject = hitObject;
         }
 
-        private ArmedState state;
-        public ArmedState State
+        [BackgroundDependencyLoader]
+        private void load(AudioManager audio)
         {
-            get { return state; }
-
-            set
+            foreach (SampleInfo sample in HitObject.Samples)
             {
-                if (state == value)
-                    return;
-                state = value;
+                SampleChannel channel = audio.Sample.Get($@"Gameplay/{sample.Bank}-{sample.Name}");
 
-                if (!IsLoaded)
-                    return;
+                if (channel == null)
+                    continue;
 
+                channel.Volume.Value = sample.Volume;
+                Samples.Add(channel);
+            }
+        }
+
+        protected override void LoadComplete()
+        {
+            base.LoadComplete();
+
+            State.ValueChanged += state =>
+            {
                 UpdateState(state);
 
                 if (State == ArmedState.Hit)
                     PlaySamples();
-            }
+            };
+
+            State.TriggerChange();
         }
 
         protected void PlaySamples()
@@ -88,16 +104,8 @@ namespace osu.Game.Rulesets.Objects.Drawables
             Samples.ForEach(s => s?.Play());
         }
 
-        protected override void LoadComplete()
-        {
-            base.LoadComplete();
-
-            //force application of the state that was set before we loaded.
-            UpdateState(State);
-        }
-
-        private bool hasJudgementResult;
         private bool judgementOccurred;
+        private bool hasJudgementResult => Judgements.LastOrDefault()?.Result >= HitResult.Miss;
 
         /// <summary>
         /// Whether this <see cref="DrawableHitObject"/> and all of its nested <see cref="DrawableHitObject"/>s have been judged.
@@ -110,7 +118,6 @@ namespace osu.Game.Rulesets.Objects.Drawables
         /// <param name="judgement">The <see cref="Judgement"/>.</param>
         protected void AddJudgement(Judgement judgement)
         {
-            hasJudgementResult = judgement.Result >= HitResult.Miss;
             judgementOccurred = true;
 
             // Ensure that the judgement is given a valid time offset, because this may not get set by the caller
@@ -124,10 +131,10 @@ namespace osu.Game.Rulesets.Objects.Drawables
                 case HitResult.None:
                     break;
                 case HitResult.Miss:
-                    State = ArmedState.Miss;
+                    State.Value = ArmedState.Miss;
                     break;
                 default:
-                    State = ArmedState.Hit;
+                    State.Value = ArmedState.Hit;
                     break;
             }
 
@@ -170,6 +177,25 @@ namespace osu.Game.Rulesets.Objects.Drawables
         /// implies that this check occurred after the end time of <see cref="HitObject"/>. </param>
         protected virtual void CheckForJudgements(bool userTriggered, double timeOffset) { }
 
+        protected override void Update()
+        {
+            base.Update();
+
+            var endTime = (HitObject as IHasEndTime)?.EndTime ?? HitObject.StartTime;
+
+            while (judgements.Count > 0)
+            {
+                var lastJudgement = judgements[judgements.Count - 1];
+                if (lastJudgement.TimeOffset + endTime <= Time.Current)
+                    break;
+
+                judgements.RemoveAt(judgements.Count - 1);
+                State.Value = ArmedState.Idle;
+
+                OnJudgementRemoved?.Invoke(this, lastJudgement);
+            }
+        }
+
         protected override void UpdateAfterChildren()
         {
             base.UpdateAfterChildren();
@@ -177,21 +203,6 @@ namespace osu.Game.Rulesets.Objects.Drawables
             UpdateJudgement(false);
         }
 
-        [BackgroundDependencyLoader]
-        private void load(AudioManager audio)
-        {
-            foreach (SampleInfo sample in HitObject.Samples)
-            {
-                SampleChannel channel = audio.Sample.Get($@"Gameplay/{sample.Bank}-{sample.Name}");
-
-                if (channel == null)
-                    continue;
-
-                channel.Volume.Value = sample.Volume;
-                Samples.Add(channel);
-            }
-        }
-
         private List<DrawableHitObject<TObject>> nestedHitObjects;
         protected IEnumerable<DrawableHitObject<TObject>> NestedHitObjects => nestedHitObjects;
 
diff --git a/osu.Game/Rulesets/Scoring/ScoreProcessor.cs b/osu.Game/Rulesets/Scoring/ScoreProcessor.cs
index ec5d47c7c7..4dd88600b2 100644
--- a/osu.Game/Rulesets/Scoring/ScoreProcessor.cs
+++ b/osu.Game/Rulesets/Scoring/ScoreProcessor.cs
@@ -185,6 +185,7 @@ namespace osu.Game.Rulesets.Scoring
             Debug.Assert(base_portion + combo_portion == 1.0);
 
             rulesetContainer.OnJudgement += AddJudgement;
+            rulesetContainer.OnJudgementRemoved += RemoveJudgement;
 
             SimulateAutoplay(rulesetContainer.Beatmap);
             Reset(true);
@@ -213,13 +214,26 @@ namespace osu.Game.Rulesets.Scoring
         protected void AddJudgement(Judgement judgement)
         {
             OnNewJudgement(judgement);
-            NotifyNewJudgement(judgement);
+            updateScore();
 
+            NotifyNewJudgement(judgement);
             UpdateFailed();
         }
 
+        protected void RemoveJudgement(Judgement judgement)
+        {
+            OnJudgementRemoved(judgement);
+            updateScore();
+        }
+
+        /// <summary>
+        /// Applies a judgement.
+        /// </summary>
+        /// <param name="judgement">The judgement to apply/</param>
         protected virtual void OnNewJudgement(Judgement judgement)
         {
+            judgement.ComboAtJudgement = Combo;
+            judgement.HighestComboAtJudgement = HighestCombo;
 
             if (judgement.AffectsCombo)
             {
@@ -242,7 +256,30 @@ namespace osu.Game.Rulesets.Scoring
             }
             else if (judgement.IsHit)
                 bonusScore += judgement.NumericResult;
+        }
 
+        /// <summary>
+        /// Removes a judgement. This should reverse everything in <see cref="OnNewJudgement(Judgement)"/>.
+        /// </summary>
+        /// <param name="judgement">The judgement to remove.</param>
+        protected virtual void OnJudgementRemoved(Judgement judgement)
+        {
+            Combo.Value = judgement.ComboAtJudgement;
+            HighestCombo.Value = judgement.HighestComboAtJudgement;
+
+            if (judgement.AffectsCombo)
+            {
+                baseScore -= judgement.NumericResult;
+                rollingMaxBaseScore -= judgement.MaxNumericResult;
+
+                Hits--;
+            }
+            else if (judgement.IsHit)
+                bonusScore -= judgement.NumericResult;
+        }
+
+        private void updateScore()
+        {
             if (rollingMaxBaseScore != 0)
                 Accuracy.Value = baseScore / rollingMaxBaseScore;
 
diff --git a/osu.Game/Rulesets/UI/RulesetContainer.cs b/osu.Game/Rulesets/UI/RulesetContainer.cs
index 6f53b76031..36dce7218d 100644
--- a/osu.Game/Rulesets/UI/RulesetContainer.cs
+++ b/osu.Game/Rulesets/UI/RulesetContainer.cs
@@ -104,6 +104,7 @@ namespace osu.Game.Rulesets.UI
         where TObject : HitObject
     {
         public event Action<Judgement> OnJudgement;
+        public event Action<Judgement> OnJudgementRemoved;
 
         /// <summary>
         /// The Beatmap
@@ -241,6 +242,8 @@ namespace osu.Game.Rulesets.UI
                     OnJudgement?.Invoke(j);
                 };
 
+                drawableObject.OnJudgementRemoved += (d, j) => { OnJudgementRemoved?.Invoke(j); };
+
                 Playfield.Add(drawableObject);
             }