Ensure duplicate mods cannot be defined

osu.Game.Tests/Mods/ModUtilsTest.cs

@@ -14,6 +14,14 @@ namespace osu.Game.Tests.Mods
     [TestFixture]
     public class ModUtilsTest
     {
+        [Test]
+        public void TestModIsNotCompatibleWithItself()
+        {
+            var mod = new Mock<CustomMod1>();
+            Assert.That(ModUtils.CheckCompatibleSet(new[] { mod.Object, mod.Object }, out var invalid), Is.False);
+            Assert.That(invalid, Is.EquivalentTo(new[] { mod.Object }));
+        }
+
         [Test]
         public void TestModIsCompatibleByItself()
         {

osu.Game/Utils/ModUtils.cs

@@ -51,14 +51,31 @@ namespace osu.Game.Utils
         /// <returns>Whether all <see cref="Mod"/>s in the combination are compatible with each-other.</returns>
         public static bool CheckCompatibleSet(IEnumerable<Mod> combination, [NotNullWhen(false)] out List<Mod>? invalidMods)
         {
-            combination = FlattenMods(combination).ToArray();
+            var mods = FlattenMods(combination).ToArray();
             invalidMods = null;
 
-            foreach (var mod in combination)
+            // ensure there are no duplicate mod definitions.
+            for (int i = 0; i < mods.Length; i++)
+            {
+                var candidate = mods[i];
+
+                for (int j = i + 1; j < mods.Length; j++)
+                {
+                    var m = mods[j];
+
+                    if (candidate.Equals(m))
+                    {
+                        invalidMods ??= new List<Mod>();
+                        invalidMods.Add(m);
+                    }
+                }
+            }
+
+            foreach (var mod in mods)
             {
                 foreach (var type in mod.IncompatibleMods)
                 {
-                    foreach (var invalid in combination.Where(m => type.IsInstanceOfType(m)))
+                    foreach (var invalid in mods.Where(m => type.IsInstanceOfType(m)))
                     {
                         if (invalid == mod)
                             continue;