From c3d3c03f0fad8b994bf99d8ee3c587654eff3a66 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 12:15:40 +0900 Subject: [PATCH 01/26] Fix `TestScenePlaylistsScreen` crashing on entering room --- osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs b/osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs index e52f823f0b..63bd7c8068 100644 --- a/osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs +++ b/osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs @@ -2,8 +2,6 @@ // See the LICENCE file in the repository root for full licence text. using NUnit.Framework; -using osu.Framework.Allocation; -using osu.Game.Overlays; namespace osu.Game.Tests.Visual.Playlists { @@ -12,9 +10,6 @@ namespace osu.Game.Tests.Visual.Playlists { protected override bool UseOnlineAPI => true; - [Cached] - private MusicController musicController { get; set; } = new MusicController(); - public TestScenePlaylistsScreen() { var multi = new Screens.OnlinePlay.Playlists.Playlists(); From a714941f932cd07c0f1315a3a78a534a9e1101ae Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:06:22 +0900 Subject: [PATCH 02/26] Rename EF variable to make reading code easier --- osu.Game/Database/EFToRealmMigrator.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 7683accc5c..98037bb8a3 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -30,20 +30,20 @@ namespace osu.Game.Database public void Run() { - using (var db = efContextFactory.GetForWrite()) + using (var ef = efContextFactory.GetForWrite()) { - migrateSettings(db); - migrateSkins(db); + migrateSettings(ef); + migrateSkins(ef); - migrateBeatmaps(db); - migrateScores(db); + migrateBeatmaps(ef); + migrateScores(ef); } } - private void migrateBeatmaps(DatabaseWriteUsage db) + private void migrateBeatmaps(DatabaseWriteUsage ef) { // can be removed 20220730. - var existingBeatmapSets = db.Context.EFBeatmapSetInfo + var existingBeatmapSets = ef.Context.EFBeatmapSetInfo .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) @@ -117,7 +117,7 @@ namespace osu.Game.Database } } - db.Context.RemoveRange(existingBeatmapSets); + ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. transaction.Commit(); From b1a75ce48098116ee14a6eb9db39cd1339cdf4dc Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:17:43 +0900 Subject: [PATCH 03/26] Permanently delete `client.db` after migration completes --- osu.Game/Database/EFToRealmMigrator.cs | 4 ++++ osu.Game/OsuGameBase.cs | 1 + 2 files changed, 5 insertions(+) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 98037bb8a3..7ad8e507bb 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -38,6 +38,10 @@ namespace osu.Game.Database migrateBeatmaps(ef); migrateScores(ef); } + + // Delete the database permanently. + // Will cause future startups to not attempt migration. + efContextFactory.ResetDatabase(); } private void migrateBeatmaps(DatabaseWriteUsage ef) diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index a9538c1e8c..18b22588a6 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -193,6 +193,7 @@ namespace osu.Game dependencies.Cache(RulesetStore = new RulesetStore(realmFactory, Storage)); dependencies.CacheAs(RulesetStore); + // A non-null context factory means there's still content to migrate. if (efContextFactory != null) new EFToRealmMigrator(efContextFactory, realmFactory, LocalConfig).Run(); From 798482c94122eb9d23221e90e33ccc390a52380a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:19:25 +0900 Subject: [PATCH 04/26] Create backups before deleting scores and beatmaps from EF database --- osu.Game/Database/DatabaseContextFactory.cs | 8 +++++++ osu.Game/Database/EFToRealmMigrator.cs | 25 +++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index f79505d7c5..c2a60122eb 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -1,6 +1,7 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; @@ -144,6 +145,13 @@ namespace osu.Game.Database Database = { AutoTransactionsEnabled = false } }; + public void CreateBackup(string filename) + { + using (var source = storage.GetStream(DATABASE_NAME)) + using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + source.CopyTo(destination); + } + public void ResetDatabase() { lock (writeLock) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 7ad8e507bb..3ecbd50b3e 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -1,6 +1,8 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System; +using System.Collections.Generic; using System.Linq; using Microsoft.EntityFrameworkCore; using osu.Game.Beatmaps; @@ -31,13 +33,16 @@ namespace osu.Game.Database public void Run() { using (var ef = efContextFactory.GetForWrite()) - { migrateSettings(ef); + + using (var ef = efContextFactory.GetForWrite()) migrateSkins(ef); + using (var ef = efContextFactory.GetForWrite()) migrateBeatmaps(ef); + + using (var ef = efContextFactory.GetForWrite()) migrateScores(ef); - } // Delete the database permanently. // Will cause future startups to not attempt migration. @@ -47,13 +52,13 @@ namespace osu.Game.Database private void migrateBeatmaps(DatabaseWriteUsage ef) { // can be removed 20220730. - var existingBeatmapSets = ef.Context.EFBeatmapSetInfo - .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) - .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) - .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) - .Include(s => s.Files).ThenInclude(f => f.FileInfo) - .Include(s => s.Metadata) - .ToList(); + List existingBeatmapSets = ef.Context.EFBeatmapSetInfo + .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) + .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) + .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) + .Include(s => s.Files).ThenInclude(f => f.FileInfo) + .Include(s => s.Metadata) + .ToList(); // previous entries in EF are removed post migration. if (!existingBeatmapSets.Any()) @@ -121,6 +126,7 @@ namespace osu.Game.Database } } + efContextFactory.CreateBackup($"client.before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. @@ -207,6 +213,7 @@ namespace osu.Game.Database } } + efContextFactory.CreateBackup($"client.before_scores_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. From cf30d48721242567eb44d7d0e31fb6cb9e2d1b69 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:19:31 +0900 Subject: [PATCH 05/26] Add more logging during migration process --- osu.Game/Database/EFToRealmMigrator.cs | 29 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 3ecbd50b3e..d8a3b66c7c 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Linq; using Microsoft.EntityFrameworkCore; +using osu.Framework.Logging; using osu.Game.Beatmaps; using osu.Game.Configuration; using osu.Game.Models; @@ -64,6 +65,8 @@ namespace osu.Game.Database if (!existingBeatmapSets.Any()) return; + Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { @@ -71,6 +74,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any(s => !s.Protected)) { + Logger.Log($"Migrating {existingBeatmapSets.Count} beatmaps", LoggingTarget.Database); + foreach (var beatmapSet in existingBeatmapSets) { var realmBeatmapSet = new BeatmapSetInfo @@ -161,17 +166,19 @@ namespace osu.Game.Database private void migrateScores(DatabaseWriteUsage db) { // can be removed 20220730. - var existingScores = db.Context.ScoreInfo - .Include(s => s.Ruleset) - .Include(s => s.BeatmapInfo) - .Include(s => s.Files) - .ThenInclude(f => f.FileInfo) - .ToList(); + List existingScores = db.Context.ScoreInfo + .Include(s => s.Ruleset) + .Include(s => s.BeatmapInfo) + .Include(s => s.Files) + .ThenInclude(f => f.FileInfo) + .ToList(); // previous entries in EF are removed post migration. if (!existingScores.Any()) return; + Logger.Log("Beginning scores migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { @@ -179,6 +186,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any()) { + Logger.Log($"Migrating {existingScores.Count} scores", LoggingTarget.Database); + foreach (var score in existingScores) { var realmScore = new ScoreInfo @@ -254,6 +263,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any(s => !s.Protected)) { + Logger.Log($"Migrating {existingSkins.Count} skins", LoggingTarget.Database); + foreach (var skin in existingSkins) { var realmSkin = new SkinInfo @@ -297,18 +308,22 @@ namespace osu.Game.Database private void migrateSettings(DatabaseWriteUsage db) { // migrate ruleset settings. can be removed 20220315. - var existingSettings = db.Context.DatabasedSetting; + var existingSettings = db.Context.DatabasedSetting.ToList(); // previous entries in EF are removed post migration. if (!existingSettings.Any()) return; + Logger.Log("Beginning settings migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { // only migrate data if the realm database is empty. if (!realm.All().Any()) { + Logger.Log($"Migrating {existingSettings.Count} settings", LoggingTarget.Database); + foreach (var dkb in existingSettings) { if (dkb.RulesetID == null) From 2b1c15b6cc1b7a6f51157cb598e63b835322ea3c Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:30:32 +0900 Subject: [PATCH 06/26] Allow `BlockAllOperations` to be called from a non-update thread if update has never run --- osu.Game/Database/RealmContextFactory.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 9307e06be0..01c54d6ee2 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -366,17 +366,17 @@ namespace osu.Game.Database if (isDisposed) throw new ObjectDisposedException(nameof(RealmContextFactory)); - if (!ThreadSafety.IsUpdateThread) - throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread."); - - Logger.Log(@"Blocking realm operations.", LoggingTarget.Database); - try { contextCreationLock.Wait(); lock (contextLock) { + if (!ThreadSafety.IsUpdateThread && context != null) + throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread."); + + Logger.Log(@"Blocking realm operations.", LoggingTarget.Database); + context?.Dispose(); context = null; } From bf50a9b8f8d92d0a23a498822513d65d9fade133 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:30:41 +0900 Subject: [PATCH 07/26] Also backup the realm database before migration --- osu.Game/Database/EFToRealmMigrator.cs | 8 ++++++-- osu.Game/Database/RealmContextFactory.cs | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index d8a3b66c7c..9fad29b3f4 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -75,6 +75,9 @@ namespace osu.Game.Database if (!realm.All().Any(s => !s.Protected)) { Logger.Log($"Migrating {existingBeatmapSets.Count} beatmaps", LoggingTarget.Database); + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + realmContextFactory.CreateBackup($"client.{migration}.realm"); + efContextFactory.CreateBackup($"client.{migration}.db"); foreach (var beatmapSet in existingBeatmapSets) { @@ -131,7 +134,6 @@ namespace osu.Game.Database } } - efContextFactory.CreateBackup($"client.before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. @@ -187,6 +189,9 @@ namespace osu.Game.Database if (!realm.All().Any()) { Logger.Log($"Migrating {existingScores.Count} scores", LoggingTarget.Database); + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + realmContextFactory.CreateBackup($"client.{migration}.realm"); + efContextFactory.CreateBackup($"client.{migration}.db"); foreach (var score in existingScores) { @@ -222,7 +227,6 @@ namespace osu.Game.Database } } - efContextFactory.CreateBackup($"client.before_scores_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 01c54d6ee2..c86a9e54e1 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -2,6 +2,7 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.IO; using System.Linq; using System.Reflection; using System.Threading; @@ -353,6 +354,16 @@ namespace osu.Game.Database private string? getRulesetShortNameFromLegacyID(long rulesetId) => efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName; + public void CreateBackup(string filename) + { + using (BlockAllOperations()) + { + using (var source = storage.GetStream(Filename)) + using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + source.CopyTo(destination); + } + } + /// /// Flush any active contexts and block any further writes. /// From 3429fd87681b928a52954b4a3b32ae38fd2bd794 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:41:02 +0900 Subject: [PATCH 08/26] Fix transaction scope and add even more logging --- osu.Game/Database/DatabaseContextFactory.cs | 2 + osu.Game/Database/EFToRealmMigrator.cs | 215 +++++++++++--------- osu.Game/Database/RealmContextFactory.cs | 1 + 3 files changed, 122 insertions(+), 96 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index c2a60122eb..cc690a9fda 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; +using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Statistics; @@ -147,6 +148,7 @@ namespace osu.Game.Database public void CreateBackup(string filename) { + Logger.Log($"Creating full EF database backup at {filename}", LoggingTarget.Database); using (var source = storage.GetStream(DATABASE_NAME)) using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 9fad29b3f4..8f5de80e3b 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -47,6 +47,7 @@ namespace osu.Game.Database // Delete the database permanently. // Will cause future startups to not attempt migration. + Logger.Log("Migration successful, deleting EF database", LoggingTarget.Database); efContextFactory.ResetDatabase(); } @@ -61,83 +62,94 @@ namespace osu.Game.Database .Include(s => s.Metadata) .ToList(); - // previous entries in EF are removed post migration. - if (!existingBeatmapSets.Any()) - return; - Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database); - using (var realm = realmContextFactory.CreateContext()) - using (var transaction = realm.BeginWrite()) + // previous entries in EF are removed post migration. + if (!existingBeatmapSets.Any()) { + Logger.Log("No beatmaps found to migrate", LoggingTarget.Database); + return; + } + + using (var realm = realmContextFactory.CreateContext()) + { + Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database); + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + efContextFactory.CreateBackup($"client.{migration}.db"); + // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. - if (!realm.All().Any(s => !s.Protected)) + if (realm.All().Any(s => !s.Protected)) + { + Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database); + } + else { - Logger.Log($"Migrating {existingBeatmapSets.Count} beatmaps", LoggingTarget.Database); - string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; realmContextFactory.CreateBackup($"client.{migration}.realm"); - efContextFactory.CreateBackup($"client.{migration}.db"); - foreach (var beatmapSet in existingBeatmapSets) + using (var transaction = realm.BeginWrite()) { - var realmBeatmapSet = new BeatmapSetInfo + foreach (var beatmapSet in existingBeatmapSets) { - OnlineID = beatmapSet.OnlineID ?? -1, - DateAdded = beatmapSet.DateAdded, - Status = beatmapSet.Status, - DeletePending = beatmapSet.DeletePending, - Hash = beatmapSet.Hash, - Protected = beatmapSet.Protected, - }; - - migrateFiles(beatmapSet, realm, realmBeatmapSet); - - foreach (var beatmap in beatmapSet.Beatmaps) - { - var realmBeatmap = new BeatmapInfo + var realmBeatmapSet = new BeatmapSetInfo { - DifficultyName = beatmap.DifficultyName, - Status = beatmap.Status, - OnlineID = beatmap.OnlineID ?? -1, - Length = beatmap.Length, - BPM = beatmap.BPM, - Hash = beatmap.Hash, - StarRating = beatmap.StarRating, - MD5Hash = beatmap.MD5Hash, - Hidden = beatmap.Hidden, - AudioLeadIn = beatmap.AudioLeadIn, - StackLeniency = beatmap.StackLeniency, - SpecialStyle = beatmap.SpecialStyle, - LetterboxInBreaks = beatmap.LetterboxInBreaks, - WidescreenStoryboard = beatmap.WidescreenStoryboard, - EpilepsyWarning = beatmap.EpilepsyWarning, - SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate, - DistanceSpacing = beatmap.DistanceSpacing, - BeatDivisor = beatmap.BeatDivisor, - GridSize = beatmap.GridSize, - TimelineZoom = beatmap.TimelineZoom, - Countdown = beatmap.Countdown, - CountdownOffset = beatmap.CountdownOffset, - MaxCombo = beatmap.MaxCombo, - Bookmarks = beatmap.Bookmarks, - Ruleset = realm.Find(beatmap.RulesetInfo.ShortName), - Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty), - Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata), - BeatmapSet = realmBeatmapSet, + OnlineID = beatmapSet.OnlineID ?? -1, + DateAdded = beatmapSet.DateAdded, + Status = beatmapSet.Status, + DeletePending = beatmapSet.DeletePending, + Hash = beatmapSet.Hash, + Protected = beatmapSet.Protected, }; - realmBeatmapSet.Beatmaps.Add(realmBeatmap); + migrateFiles(beatmapSet, realm, realmBeatmapSet); + + foreach (var beatmap in beatmapSet.Beatmaps) + { + var realmBeatmap = new BeatmapInfo + { + DifficultyName = beatmap.DifficultyName, + Status = beatmap.Status, + OnlineID = beatmap.OnlineID ?? -1, + Length = beatmap.Length, + BPM = beatmap.BPM, + Hash = beatmap.Hash, + StarRating = beatmap.StarRating, + MD5Hash = beatmap.MD5Hash, + Hidden = beatmap.Hidden, + AudioLeadIn = beatmap.AudioLeadIn, + StackLeniency = beatmap.StackLeniency, + SpecialStyle = beatmap.SpecialStyle, + LetterboxInBreaks = beatmap.LetterboxInBreaks, + WidescreenStoryboard = beatmap.WidescreenStoryboard, + EpilepsyWarning = beatmap.EpilepsyWarning, + SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate, + DistanceSpacing = beatmap.DistanceSpacing, + BeatDivisor = beatmap.BeatDivisor, + GridSize = beatmap.GridSize, + TimelineZoom = beatmap.TimelineZoom, + Countdown = beatmap.Countdown, + CountdownOffset = beatmap.CountdownOffset, + MaxCombo = beatmap.MaxCombo, + Bookmarks = beatmap.Bookmarks, + Ruleset = realm.Find(beatmap.RulesetInfo.ShortName), + Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty), + Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata), + BeatmapSet = realmBeatmapSet, + }; + + realmBeatmapSet.Beatmaps.Add(realmBeatmap); + } + + realm.Add(realmBeatmapSet); } - realm.Add(realmBeatmapSet); + transaction.Commit(); } } + Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. - - transaction.Commit(); } } @@ -175,62 +187,73 @@ namespace osu.Game.Database .ThenInclude(f => f.FileInfo) .ToList(); - // previous entries in EF are removed post migration. - if (!existingScores.Any()) - return; - Logger.Log("Beginning scores migration to realm", LoggingTarget.Database); - using (var realm = realmContextFactory.CreateContext()) - using (var transaction = realm.BeginWrite()) + // previous entries in EF are removed post migration. + if (!existingScores.Any()) { + Logger.Log("No scores found to migrate", LoggingTarget.Database); + return; + } + + using (var realm = realmContextFactory.CreateContext()) + { + Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database); + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + efContextFactory.CreateBackup($"client.{migration}.db"); + // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. - if (!realm.All().Any()) + if (realm.All().Any()) + { + Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database); + } + else { - Logger.Log($"Migrating {existingScores.Count} scores", LoggingTarget.Database); - string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; realmContextFactory.CreateBackup($"client.{migration}.realm"); - efContextFactory.CreateBackup($"client.{migration}.db"); - foreach (var score in existingScores) + using (var transaction = realm.BeginWrite()) { - var realmScore = new ScoreInfo + foreach (var score in existingScores) { - Hash = score.Hash, - DeletePending = score.DeletePending, - OnlineID = score.OnlineID ?? -1, - ModsJson = score.ModsJson, - StatisticsJson = score.StatisticsJson, - User = score.User, - TotalScore = score.TotalScore, - MaxCombo = score.MaxCombo, - Accuracy = score.Accuracy, - HasReplay = ((IScoreInfo)score).HasReplay, - Date = score.Date, - PP = score.PP, - BeatmapInfo = realm.All().First(b => b.Hash == score.BeatmapInfo.Hash), - Ruleset = realm.Find(score.Ruleset.ShortName), - Rank = score.Rank, - HitEvents = score.HitEvents, - Passed = score.Passed, - Combo = score.Combo, - Position = score.Position, - Statistics = score.Statistics, - Mods = score.Mods, - APIMods = score.APIMods, - }; + var realmScore = new ScoreInfo + { + Hash = score.Hash, + DeletePending = score.DeletePending, + OnlineID = score.OnlineID ?? -1, + ModsJson = score.ModsJson, + StatisticsJson = score.StatisticsJson, + User = score.User, + TotalScore = score.TotalScore, + MaxCombo = score.MaxCombo, + Accuracy = score.Accuracy, + HasReplay = ((IScoreInfo)score).HasReplay, + Date = score.Date, + PP = score.PP, + BeatmapInfo = realm.All().First(b => b.Hash == score.BeatmapInfo.Hash), + Ruleset = realm.Find(score.Ruleset.ShortName), + Rank = score.Rank, + HitEvents = score.HitEvents, + Passed = score.Passed, + Combo = score.Combo, + Position = score.Position, + Statistics = score.Statistics, + Mods = score.Mods, + APIMods = score.APIMods, + }; - migrateFiles(score, realm, realmScore); + migrateFiles(score, realm, realmScore); - realm.Add(realmScore); + realm.Add(realmScore); + } + + transaction.Commit(); } } + Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. - - transaction.Commit(); } } diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index c86a9e54e1..8548e63e94 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -358,6 +358,7 @@ namespace osu.Game.Database { using (BlockAllOperations()) { + Logger.Log($"Creating full realm database backup at {filename}", LoggingTarget.Database); using (var source = storage.GetStream(Filename)) using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); From 246a4a4bfe3abb8219965c55a9bd56c0317953d0 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 16:05:12 +0900 Subject: [PATCH 09/26] Add support for starting with a fresh realm database if the existing one is not usable The most common scenario is switching between schema versions when testing. This should alleviate the manual overhead of that for the majority of cases. For users, this will show a notification on startup if their database was purged, similar to what we had with EF. --- osu.Game/Database/RealmContextFactory.cs | 37 +++++++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 8548e63e94..e6fe5dfc54 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -105,8 +105,20 @@ namespace osu.Game.Database if (!Filename.EndsWith(realm_extension, StringComparison.Ordinal)) Filename += realm_extension; - // This method triggers the first `CreateContext` call, which will implicitly run realm migrations and bring the schema up-to-date. - cleanupPendingDeletions(); + try + { + // This method triggers the first `CreateContext` call, which will implicitly run realm migrations and bring the schema up-to-date. + cleanupPendingDeletions(); + } + catch (Exception e) + { + Logger.Error(e, "Realm startup failed with unrecoverable error; starting with a fresh database. A backup of your database has been made."); + + CreateBackup($"{Filename.Replace(realm_extension, string.Empty)}_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_corrupt{realm_extension}"); + storage.Delete(Filename); + + cleanupPendingDeletions(); + } } private void cleanupPendingDeletions() @@ -396,14 +408,23 @@ namespace osu.Game.Database const int sleep_length = 200; int timeout = 5000; - // see https://github.com/realm/realm-dotnet/discussions/2657 - while (!Compact()) + try { - Thread.Sleep(sleep_length); - timeout -= sleep_length; + // see https://github.com/realm/realm-dotnet/discussions/2657 + while (!Compact()) + { + Thread.Sleep(sleep_length); + timeout -= sleep_length; - if (timeout < 0) - throw new TimeoutException(@"Took too long to acquire lock"); + if (timeout < 0) + throw new TimeoutException(@"Took too long to acquire lock"); + } + } + catch (Exception e) + { + // Compact may fail if the realm is in a bad state. + // We still want to continue with the blocking operation, though. + Logger.Log($"Realm compact failed with error {e}", LoggingTarget.Database); } } catch From 6b0bf38c93cb994f2c8d144e26d4001e5ad01060 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 20:47:53 +0900 Subject: [PATCH 10/26] Use a single EF context to avoid scores getting cascade deleted along the way --- osu.Game/Database/EFToRealmMigrator.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 8f5de80e3b..d479f81ead 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -34,16 +34,12 @@ namespace osu.Game.Database public void Run() { using (var ef = efContextFactory.GetForWrite()) + { migrateSettings(ef); - - using (var ef = efContextFactory.GetForWrite()) migrateSkins(ef); - - using (var ef = efContextFactory.GetForWrite()) migrateBeatmaps(ef); - - using (var ef = efContextFactory.GetForWrite()) migrateScores(ef); + } // Delete the database permanently. // Will cause future startups to not attempt migration. From 7f65f3a47f954ee85e2172c35178450ace95b677 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 22:57:39 +0900 Subject: [PATCH 11/26] Remove all usage of `BaseDifficulty` (and access `Difficulty` instead) --- osu.Game.Rulesets.Catch.Tests/TestSceneAutoJuiceStream.cs | 2 +- osu.Game.Rulesets.Catch.Tests/TestSceneBananaShower.cs | 2 +- osu.Game.Rulesets.Catch.Tests/TestSceneCatchStacker.cs | 2 +- .../TestSceneDrawableHitObjects.cs | 2 +- osu.Game.Rulesets.Catch.Tests/TestSceneHyperDash.cs | 2 +- osu.Game.Rulesets.Catch.Tests/TestSceneJuiceStream.cs | 2 +- osu.Game.Rulesets.Mania.Tests/TestSceneHoldNoteInput.cs | 6 +++--- osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs | 2 +- .../Editor/Checks/CheckTooShortSpinnersTest.cs | 2 +- .../Mods/TestSceneOsuModDifficultyAdjust.cs | 2 +- .../TestSceneHitCircleLongCombo.cs | 2 +- osu.Game.Rulesets.Osu.Tests/TestSceneNoSpinnerStacking.cs | 2 +- osu.Game.Rulesets.Osu.Tests/TestSceneSliderInput.cs | 2 +- .../TestSceneStartTimeOrderedHitPolicy.cs | 2 +- .../DrawableTaikoRulesetTestScene.cs | 2 +- .../Skinning/TestSceneDrawableTaikoMascot.cs | 2 +- osu.Game.Tests/Beatmaps/Formats/LegacyScoreDecoderTest.cs | 2 +- osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs | 2 +- osu.Game.Tests/Resources/TestResources.cs | 2 +- .../Gameplay/TestSceneGameplaySampleTriggerSource.cs | 2 +- .../Multiplayer/TestSceneMultiplayerMatchSongSelect.cs | 2 +- .../Visual/Navigation/TestScenePresentBeatmap.cs | 4 ++-- osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs | 4 ++-- .../Visual/SongSelect/TestSceneAdvancedStats.cs | 4 ++-- .../Visual/SongSelect/TestSceneBeatmapInfoWedge.cs | 2 +- .../UserInterface/TestSceneModDifficultyAdjustSettings.cs | 2 +- osu.Game/Beatmaps/Beatmap.cs | 8 ++++---- osu.Game/Beatmaps/BeatmapInfo.cs | 7 ------- osu.Game/Beatmaps/BeatmapManager.cs | 2 +- osu.Game/Beatmaps/BeatmapModelManager.cs | 2 +- osu.Game/Beatmaps/DummyWorkingBeatmap.cs | 2 +- osu.Game/Rulesets/Mods/DifficultyAdjustSettingsControl.cs | 2 +- osu.Game/Screens/Select/Carousel/CarouselBeatmap.cs | 8 ++++---- osu.Game/Stores/BeatmapImporter.cs | 2 +- 34 files changed, 44 insertions(+), 51 deletions(-) diff --git a/osu.Game.Rulesets.Catch.Tests/TestSceneAutoJuiceStream.cs b/osu.Game.Rulesets.Catch.Tests/TestSceneAutoJuiceStream.cs index f552c3c27b..1014158fc1 100644 --- a/osu.Game.Rulesets.Catch.Tests/TestSceneAutoJuiceStream.cs +++ b/osu.Game.Rulesets.Catch.Tests/TestSceneAutoJuiceStream.cs @@ -22,7 +22,7 @@ namespace osu.Game.Rulesets.Catch.Tests { BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty { CircleSize = 6, SliderMultiplier = 3 }, + Difficulty = new BeatmapDifficulty { CircleSize = 6, SliderMultiplier = 3 }, Ruleset = ruleset } }; diff --git a/osu.Game.Rulesets.Catch.Tests/TestSceneBananaShower.cs b/osu.Game.Rulesets.Catch.Tests/TestSceneBananaShower.cs index e89a95ae37..96ac5c4bf2 100644 --- a/osu.Game.Rulesets.Catch.Tests/TestSceneBananaShower.cs +++ b/osu.Game.Rulesets.Catch.Tests/TestSceneBananaShower.cs @@ -22,7 +22,7 @@ namespace osu.Game.Rulesets.Catch.Tests { BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty { CircleSize = 6 }, + Difficulty = new BeatmapDifficulty { CircleSize = 6 }, Ruleset = ruleset } }; diff --git a/osu.Game.Rulesets.Catch.Tests/TestSceneCatchStacker.cs b/osu.Game.Rulesets.Catch.Tests/TestSceneCatchStacker.cs index 1ff31697b8..0a4ef49e19 100644 --- a/osu.Game.Rulesets.Catch.Tests/TestSceneCatchStacker.cs +++ b/osu.Game.Rulesets.Catch.Tests/TestSceneCatchStacker.cs @@ -17,7 +17,7 @@ namespace osu.Game.Rulesets.Catch.Tests { BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty { CircleSize = 6 }, + Difficulty = new BeatmapDifficulty { CircleSize = 6 }, Ruleset = ruleset } }; diff --git a/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs b/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs index 23f6222eb6..27196bc30c 100644 --- a/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs +++ b/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs @@ -35,7 +35,7 @@ namespace osu.Game.Rulesets.Catch.Tests HitObjects = new List { new Fruit() }, BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty(), + Difficulty = new BeatmapDifficulty(), Metadata = new BeatmapMetadata { Artist = @"Unknown", diff --git a/osu.Game.Rulesets.Catch.Tests/TestSceneHyperDash.cs b/osu.Game.Rulesets.Catch.Tests/TestSceneHyperDash.cs index 163fee49fb..a5b44dc605 100644 --- a/osu.Game.Rulesets.Catch.Tests/TestSceneHyperDash.cs +++ b/osu.Game.Rulesets.Catch.Tests/TestSceneHyperDash.cs @@ -63,7 +63,7 @@ namespace osu.Game.Rulesets.Catch.Tests BeatmapInfo = { Ruleset = ruleset, - BaseDifficulty = new BeatmapDifficulty { CircleSize = 3.6f } + Difficulty = new BeatmapDifficulty { CircleSize = 3.6f } } }; diff --git a/osu.Game.Rulesets.Catch.Tests/TestSceneJuiceStream.cs b/osu.Game.Rulesets.Catch.Tests/TestSceneJuiceStream.cs index 269e783899..4601234669 100644 --- a/osu.Game.Rulesets.Catch.Tests/TestSceneJuiceStream.cs +++ b/osu.Game.Rulesets.Catch.Tests/TestSceneJuiceStream.cs @@ -24,7 +24,7 @@ namespace osu.Game.Rulesets.Catch.Tests { BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty { CircleSize = 5, SliderMultiplier = 2 }, + Difficulty = new BeatmapDifficulty { CircleSize = 5, SliderMultiplier = 2 }, Ruleset = ruleset }, HitObjects = new List diff --git a/osu.Game.Rulesets.Mania.Tests/TestSceneHoldNoteInput.cs b/osu.Game.Rulesets.Mania.Tests/TestSceneHoldNoteInput.cs index 4387bc6b3b..f973cb5ed3 100644 --- a/osu.Game.Rulesets.Mania.Tests/TestSceneHoldNoteInput.cs +++ b/osu.Game.Rulesets.Mania.Tests/TestSceneHoldNoteInput.cs @@ -264,7 +264,7 @@ namespace osu.Game.Rulesets.Mania.Tests }, BeatmapInfo = { - BaseDifficulty = new BeatmapDifficulty + Difficulty = new BeatmapDifficulty { SliderTickRate = 4, OverallDifficulty = 10, @@ -306,7 +306,7 @@ namespace osu.Game.Rulesets.Mania.Tests }, BeatmapInfo = { - BaseDifficulty = new BeatmapDifficulty { SliderTickRate = tick_rate }, + Difficulty = new BeatmapDifficulty { SliderTickRate = tick_rate }, Ruleset = new ManiaRuleset().RulesetInfo }, }; @@ -383,7 +383,7 @@ namespace osu.Game.Rulesets.Mania.Tests }, BeatmapInfo = { - BaseDifficulty = new BeatmapDifficulty { SliderTickRate = 4 }, + Difficulty = new BeatmapDifficulty { SliderTickRate = 4 }, Ruleset = new ManiaRuleset().RulesetInfo }, }; diff --git a/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs b/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs index 9d0aaec2ba..47e0e6d7b1 100644 --- a/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs +++ b/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs @@ -73,7 +73,7 @@ namespace osu.Game.Rulesets.Mania.Beatmaps public static int GetColumnCountForNonConvert(BeatmapInfo beatmapInfo) { - double roundedCircleSize = Math.Round(beatmapInfo.BaseDifficulty.CircleSize); + double roundedCircleSize = Math.Round(beatmapInfo.Difficulty.CircleSize); return (int)Math.Max(1, roundedCircleSize); } diff --git a/osu.Game.Rulesets.Osu.Tests/Editor/Checks/CheckTooShortSpinnersTest.cs b/osu.Game.Rulesets.Osu.Tests/Editor/Checks/CheckTooShortSpinnersTest.cs index 787807a8ea..1f3d4297f1 100644 --- a/osu.Game.Rulesets.Osu.Tests/Editor/Checks/CheckTooShortSpinnersTest.cs +++ b/osu.Game.Rulesets.Osu.Tests/Editor/Checks/CheckTooShortSpinnersTest.cs @@ -107,7 +107,7 @@ namespace osu.Game.Rulesets.Osu.Tests.Editor.Checks var beatmap = new Beatmap { HitObjects = hitObjects, - BeatmapInfo = new BeatmapInfo { BaseDifficulty = new BeatmapDifficulty(beatmapDifficulty) } + BeatmapInfo = new BeatmapInfo { Difficulty = new BeatmapDifficulty(beatmapDifficulty) } }; return new BeatmapVerifierContext(beatmap, new TestWorkingBeatmap(beatmap)); diff --git a/osu.Game.Rulesets.Osu.Tests/Mods/TestSceneOsuModDifficultyAdjust.cs b/osu.Game.Rulesets.Osu.Tests/Mods/TestSceneOsuModDifficultyAdjust.cs index db8546c71b..9d06ff5801 100644 --- a/osu.Game.Rulesets.Osu.Tests/Mods/TestSceneOsuModDifficultyAdjust.cs +++ b/osu.Game.Rulesets.Osu.Tests/Mods/TestSceneOsuModDifficultyAdjust.cs @@ -26,7 +26,7 @@ namespace osu.Game.Rulesets.Osu.Tests.Mods { BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty + Difficulty = new BeatmapDifficulty { CircleSize = 8 } diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneHitCircleLongCombo.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneHitCircleLongCombo.cs index 8cf29ddfbf..4e17c4c363 100644 --- a/osu.Game.Rulesets.Osu.Tests/TestSceneHitCircleLongCombo.cs +++ b/osu.Game.Rulesets.Osu.Tests/TestSceneHitCircleLongCombo.cs @@ -17,7 +17,7 @@ namespace osu.Game.Rulesets.Osu.Tests { BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty { CircleSize = 6 }, + Difficulty = new BeatmapDifficulty { CircleSize = 6 }, Ruleset = ruleset } }; diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneNoSpinnerStacking.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneNoSpinnerStacking.cs index ef05bcd320..5e92bac986 100644 --- a/osu.Game.Rulesets.Osu.Tests/TestSceneNoSpinnerStacking.cs +++ b/osu.Game.Rulesets.Osu.Tests/TestSceneNoSpinnerStacking.cs @@ -17,7 +17,7 @@ namespace osu.Game.Rulesets.Osu.Tests { BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty { OverallDifficulty = 10 }, + Difficulty = new BeatmapDifficulty { OverallDifficulty = 10 }, Ruleset = ruleset } }; diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneSliderInput.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneSliderInput.cs index f3392724ec..2368cc7365 100644 --- a/osu.Game.Rulesets.Osu.Tests/TestSceneSliderInput.cs +++ b/osu.Game.Rulesets.Osu.Tests/TestSceneSliderInput.cs @@ -358,7 +358,7 @@ namespace osu.Game.Rulesets.Osu.Tests }, BeatmapInfo = { - BaseDifficulty = new BeatmapDifficulty { SliderTickRate = 3 }, + Difficulty = new BeatmapDifficulty { SliderTickRate = 3 }, Ruleset = new OsuRuleset().RulesetInfo }, }); diff --git a/osu.Game.Rulesets.Osu.Tests/TestSceneStartTimeOrderedHitPolicy.cs b/osu.Game.Rulesets.Osu.Tests/TestSceneStartTimeOrderedHitPolicy.cs index 2d43e1b95e..53fa3624b8 100644 --- a/osu.Game.Rulesets.Osu.Tests/TestSceneStartTimeOrderedHitPolicy.cs +++ b/osu.Game.Rulesets.Osu.Tests/TestSceneStartTimeOrderedHitPolicy.cs @@ -364,7 +364,7 @@ namespace osu.Game.Rulesets.Osu.Tests HitObjects = hitObjects, BeatmapInfo = { - BaseDifficulty = new BeatmapDifficulty { SliderTickRate = 3 }, + Difficulty = new BeatmapDifficulty { SliderTickRate = 3 }, Ruleset = new OsuRuleset().RulesetInfo }, }); diff --git a/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs b/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs index 4bdb85ba60..85d5d6419c 100644 --- a/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs +++ b/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs @@ -32,7 +32,7 @@ namespace osu.Game.Rulesets.Taiko.Tests HitObjects = new List { new Hit { Type = HitType.Centre } }, BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty(), + Difficulty = new BeatmapDifficulty(), Metadata = new BeatmapMetadata { Artist = @"Unknown", diff --git a/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs b/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs index b976735223..ffdf0f725d 100644 --- a/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs +++ b/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs @@ -158,7 +158,7 @@ namespace osu.Game.Rulesets.Taiko.Tests.Skinning HitObjects = new List { new Hit { Type = HitType.Centre } }, BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty(), + Difficulty = new BeatmapDifficulty(), Metadata = new BeatmapMetadata { Artist = "Unknown", diff --git a/osu.Game.Tests/Beatmaps/Formats/LegacyScoreDecoderTest.cs b/osu.Game.Tests/Beatmaps/Formats/LegacyScoreDecoderTest.cs index 9ac7838821..3d4b05b52b 100644 --- a/osu.Game.Tests/Beatmaps/Formats/LegacyScoreDecoderTest.cs +++ b/osu.Game.Tests/Beatmaps/Formats/LegacyScoreDecoderTest.cs @@ -128,7 +128,7 @@ namespace osu.Game.Tests.Beatmaps.Formats { MD5Hash = md5Hash, Ruleset = new OsuRuleset().RulesetInfo, - BaseDifficulty = new BeatmapDifficulty() + Difficulty = new BeatmapDifficulty() } }); } diff --git a/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs b/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs index 3d78043c73..fa4c4bee8e 100644 --- a/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs +++ b/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs @@ -18,7 +18,7 @@ namespace osu.Game.Tests.NonVisual.Filtering { Ruleset = new RulesetInfo { OnlineID = 0 }, StarRating = 4.0d, - BaseDifficulty = new BeatmapDifficulty + Difficulty = new BeatmapDifficulty { ApproachRate = 5.0f, DrainRate = 3.0f, diff --git a/osu.Game.Tests/Resources/TestResources.cs b/osu.Game.Tests/Resources/TestResources.cs index 1d99a5c20d..4c8b943279 100644 --- a/osu.Game.Tests/Resources/TestResources.cs +++ b/osu.Game.Tests/Resources/TestResources.cs @@ -133,7 +133,7 @@ namespace osu.Game.Tests.Resources Hash = Guid.NewGuid().ToString().ComputeMD5Hash(), Ruleset = rulesetInfo, Metadata = metadata, - BaseDifficulty = new BeatmapDifficulty + Difficulty = new BeatmapDifficulty { OverallDifficulty = diff, } diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneGameplaySampleTriggerSource.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneGameplaySampleTriggerSource.cs index fccc1a377c..ac39395567 100644 --- a/osu.Game.Tests/Visual/Gameplay/TestSceneGameplaySampleTriggerSource.cs +++ b/osu.Game.Tests/Visual/Gameplay/TestSceneGameplaySampleTriggerSource.cs @@ -29,7 +29,7 @@ namespace osu.Game.Tests.Visual.Gameplay { BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty { CircleSize = 6, SliderMultiplier = 3 }, + Difficulty = new BeatmapDifficulty { CircleSize = 6, SliderMultiplier = 3 }, Ruleset = ruleset } }; diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs index 181b0c71f2..34b9485002 100644 --- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs +++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs @@ -76,7 +76,7 @@ namespace osu.Game.Tests.Visual.Multiplayer Length = length, BPM = bpm, Metadata = metadata, - BaseDifficulty = new BeatmapDifficulty() + Difficulty = new BeatmapDifficulty() }; beatmaps.Add(beatmap); diff --git a/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs b/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs index 69b30ec6a0..606d3f929a 100644 --- a/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs +++ b/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs @@ -114,14 +114,14 @@ namespace osu.Game.Tests.Visual.Navigation { OnlineID = i * 1024, Metadata = metadata, - BaseDifficulty = new BeatmapDifficulty(), + Difficulty = new BeatmapDifficulty(), Ruleset = ruleset ?? new OsuRuleset().RulesetInfo }, new BeatmapInfo { OnlineID = i * 2048, Metadata = metadata, - BaseDifficulty = new BeatmapDifficulty(), + Difficulty = new BeatmapDifficulty(), Ruleset = ruleset ?? new OsuRuleset().RulesetInfo }, } diff --git a/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs b/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs index f9649db135..3e354d5188 100644 --- a/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs +++ b/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs @@ -44,7 +44,7 @@ namespace osu.Game.Tests.Visual.Navigation AuthorString = "SomeAuthor", Title = "import" }, - BaseDifficulty = new BeatmapDifficulty(), + Difficulty = new BeatmapDifficulty(), Ruleset = new OsuRuleset().RulesetInfo }, new BeatmapInfo @@ -56,7 +56,7 @@ namespace osu.Game.Tests.Visual.Navigation AuthorString = "SomeAuthor", Title = "import" }, - BaseDifficulty = new BeatmapDifficulty(), + Difficulty = new BeatmapDifficulty(), Ruleset = new OsuRuleset().RulesetInfo }, } diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneAdvancedStats.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneAdvancedStats.cs index bc3e8553f7..7ceae0a69b 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestSceneAdvancedStats.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestSceneAdvancedStats.cs @@ -38,7 +38,7 @@ namespace osu.Game.Tests.Visual.SongSelect private BeatmapInfo exampleBeatmapInfo => new BeatmapInfo { Ruleset = rulesets.AvailableRulesets.First(), - BaseDifficulty = new BeatmapDifficulty + Difficulty = new BeatmapDifficulty { CircleSize = 7.2f, DrainRate = 3, @@ -68,7 +68,7 @@ namespace osu.Game.Tests.Visual.SongSelect AddStep("set beatmap", () => advancedStats.BeatmapInfo = new BeatmapInfo { Ruleset = rulesets.GetRuleset(3) ?? throw new InvalidOperationException(), - BaseDifficulty = new BeatmapDifficulty + Difficulty = new BeatmapDifficulty { CircleSize = 5, DrainRate = 4.3f, diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs index 9ad5242df4..3d65efc059 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs @@ -216,7 +216,7 @@ namespace osu.Game.Tests.Visual.SongSelect Ruleset = ruleset, StarRating = 6, DifficultyName = $"{ruleset.ShortName}Version", - BaseDifficulty = new BeatmapDifficulty() + Difficulty = new BeatmapDifficulty() }, HitObjects = objects }; diff --git a/osu.Game.Tests/Visual/UserInterface/TestSceneModDifficultyAdjustSettings.cs b/osu.Game.Tests/Visual/UserInterface/TestSceneModDifficultyAdjustSettings.cs index f8652573f4..8e1f426f7b 100644 --- a/osu.Game.Tests/Visual/UserInterface/TestSceneModDifficultyAdjustSettings.cs +++ b/osu.Game.Tests/Visual/UserInterface/TestSceneModDifficultyAdjustSettings.cs @@ -267,7 +267,7 @@ namespace osu.Game.Tests.Visual.UserInterface { BeatmapInfo = new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty + Difficulty = new BeatmapDifficulty { OverallDifficulty = value, CircleSize = value, diff --git a/osu.Game/Beatmaps/Beatmap.cs b/osu.Game/Beatmaps/Beatmap.cs index ccecd69d21..69a86f4928 100644 --- a/osu.Game/Beatmaps/Beatmap.cs +++ b/osu.Game/Beatmaps/Beatmap.cs @@ -28,7 +28,7 @@ namespace osu.Game.Beatmaps difficulty = value; if (beatmapInfo != null) - beatmapInfo.BaseDifficulty = difficulty.Clone(); + beatmapInfo.Difficulty = difficulty.Clone(); } } @@ -41,8 +41,8 @@ namespace osu.Game.Beatmaps { beatmapInfo = value; - if (beatmapInfo?.BaseDifficulty != null) - Difficulty = beatmapInfo.BaseDifficulty.Clone(); + if (beatmapInfo?.Difficulty != null) + Difficulty = beatmapInfo.Difficulty.Clone(); } } @@ -57,7 +57,7 @@ namespace osu.Game.Beatmaps AuthorString = @"Unknown Creator", }, DifficultyName = @"Normal", - BaseDifficulty = Difficulty, + Difficulty = Difficulty, }; } diff --git a/osu.Game/Beatmaps/BeatmapInfo.cs b/osu.Game/Beatmaps/BeatmapInfo.cs index b0e10c2c38..b91f05cd11 100644 --- a/osu.Game/Beatmaps/BeatmapInfo.cs +++ b/osu.Game/Beatmaps/BeatmapInfo.cs @@ -165,13 +165,6 @@ namespace osu.Game.Beatmaps } } - [Ignored] - public BeatmapDifficulty BaseDifficulty - { - get => Difficulty; - set => Difficulty = value; - } - [Ignored] public string? Path => File?.Filename; diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index f901b0fbc1..564f15361d 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -93,7 +93,7 @@ namespace osu.Game.Beatmaps { new BeatmapInfo { - BaseDifficulty = new BeatmapDifficulty(), + Difficulty = new BeatmapDifficulty(), Ruleset = ruleset, Metadata = metadata, WidescreenStoryboard = true, diff --git a/osu.Game/Beatmaps/BeatmapModelManager.cs b/osu.Game/Beatmaps/BeatmapModelManager.cs index 73ada09ef6..3822c6e121 100644 --- a/osu.Game/Beatmaps/BeatmapModelManager.cs +++ b/osu.Game/Beatmaps/BeatmapModelManager.cs @@ -58,7 +58,7 @@ namespace osu.Game.Beatmaps // Warning: The directionality here is important. Changes have to be copied *from* beatmapContent (which comes from editor and is being saved) // *to* the beatmapInfo (which is a database model and needs to receive values without the taiko slider velocity multiplier for correct operation). // CopyTo() will undo such adjustments, while CopyFrom() will not. - beatmapContent.Difficulty.CopyTo(beatmapInfo.BaseDifficulty); + beatmapContent.Difficulty.CopyTo(beatmapInfo.Difficulty); // All changes to metadata are made in the provided beatmapInfo, so this should be copied to the `IBeatmap` before encoding. beatmapContent.BeatmapInfo = beatmapInfo; diff --git a/osu.Game/Beatmaps/DummyWorkingBeatmap.cs b/osu.Game/Beatmaps/DummyWorkingBeatmap.cs index 9ea8517764..6e879d09d5 100644 --- a/osu.Game/Beatmaps/DummyWorkingBeatmap.cs +++ b/osu.Game/Beatmaps/DummyWorkingBeatmap.cs @@ -32,7 +32,7 @@ namespace osu.Game.Beatmaps Title = "no beatmaps available!" }, BeatmapSet = new BeatmapSetInfo(), - BaseDifficulty = new BeatmapDifficulty + Difficulty = new BeatmapDifficulty { DrainRate = 0, CircleSize = 0, diff --git a/osu.Game/Rulesets/Mods/DifficultyAdjustSettingsControl.cs b/osu.Game/Rulesets/Mods/DifficultyAdjustSettingsControl.cs index 5d7565e56b..45873a321a 100644 --- a/osu.Game/Rulesets/Mods/DifficultyAdjustSettingsControl.cs +++ b/osu.Game/Rulesets/Mods/DifficultyAdjustSettingsControl.cs @@ -74,7 +74,7 @@ namespace osu.Game.Rulesets.Mods return; } - var difficulty = beatmap.Value.BeatmapInfo.BaseDifficulty; + var difficulty = beatmap.Value.BeatmapInfo.Difficulty; // generally should always be implemented, else the slider will have a zero default. if (difficultyBindable.ReadCurrentFromDifficulty == null) diff --git a/osu.Game/Screens/Select/Carousel/CarouselBeatmap.cs b/osu.Game/Screens/Select/Carousel/CarouselBeatmap.cs index d2c7c75da8..d54a3bb54e 100644 --- a/osu.Game/Screens/Select/Carousel/CarouselBeatmap.cs +++ b/osu.Game/Screens/Select/Carousel/CarouselBeatmap.cs @@ -39,10 +39,10 @@ namespace osu.Game.Screens.Select.Carousel } match &= !criteria.StarDifficulty.HasFilter || criteria.StarDifficulty.IsInRange(BeatmapInfo.StarRating); - match &= !criteria.ApproachRate.HasFilter || criteria.ApproachRate.IsInRange(BeatmapInfo.BaseDifficulty.ApproachRate); - match &= !criteria.DrainRate.HasFilter || criteria.DrainRate.IsInRange(BeatmapInfo.BaseDifficulty.DrainRate); - match &= !criteria.CircleSize.HasFilter || criteria.CircleSize.IsInRange(BeatmapInfo.BaseDifficulty.CircleSize); - match &= !criteria.OverallDifficulty.HasFilter || criteria.OverallDifficulty.IsInRange(BeatmapInfo.BaseDifficulty.OverallDifficulty); + match &= !criteria.ApproachRate.HasFilter || criteria.ApproachRate.IsInRange(BeatmapInfo.Difficulty.ApproachRate); + match &= !criteria.DrainRate.HasFilter || criteria.DrainRate.IsInRange(BeatmapInfo.Difficulty.DrainRate); + match &= !criteria.CircleSize.HasFilter || criteria.CircleSize.IsInRange(BeatmapInfo.Difficulty.CircleSize); + match &= !criteria.OverallDifficulty.HasFilter || criteria.OverallDifficulty.IsInRange(BeatmapInfo.Difficulty.OverallDifficulty); match &= !criteria.Length.HasFilter || criteria.Length.IsInRange(BeatmapInfo.Length); match &= !criteria.BPM.HasFilter || criteria.BPM.IsInRange(BeatmapInfo.BPM); diff --git a/osu.Game/Stores/BeatmapImporter.cs b/osu.Game/Stores/BeatmapImporter.cs index 5303bd6fba..d285a6b61c 100644 --- a/osu.Game/Stores/BeatmapImporter.cs +++ b/osu.Game/Stores/BeatmapImporter.cs @@ -218,7 +218,7 @@ namespace osu.Game.Stores } var decodedInfo = decoded.BeatmapInfo; - var decodedDifficulty = decodedInfo.BaseDifficulty; + var decodedDifficulty = decodedInfo.Difficulty; var ruleset = realm.All().FirstOrDefault(r => r.OnlineID == decodedInfo.RulesetID); From 3596c6ed5d59707c08296ff0ddbc08200a557659 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 23:25:30 +0900 Subject: [PATCH 12/26] Add some missing `IgnoredAttributes` to reduce automapper overhead --- osu.Game/Beatmaps/BeatmapInfo.cs | 1 + osu.Game/Beatmaps/BeatmapMetadata.cs | 1 + osu.Game/Beatmaps/BeatmapSetInfo.cs | 1 + 3 files changed, 3 insertions(+) diff --git a/osu.Game/Beatmaps/BeatmapInfo.cs b/osu.Game/Beatmaps/BeatmapInfo.cs index b0e10c2c38..9ad9e9b1c9 100644 --- a/osu.Game/Beatmaps/BeatmapInfo.cs +++ b/osu.Game/Beatmaps/BeatmapInfo.cs @@ -64,6 +64,7 @@ namespace osu.Game.Beatmaps [Ignored] public RealmNamedFileUsage? File => BeatmapSet?.Files.FirstOrDefault(f => f.File.Hash == Hash); + [Ignored] public BeatmapOnlineStatus Status { get => (BeatmapOnlineStatus)StatusInt; diff --git a/osu.Game/Beatmaps/BeatmapMetadata.cs b/osu.Game/Beatmaps/BeatmapMetadata.cs index 6349476550..89f507581b 100644 --- a/osu.Game/Beatmaps/BeatmapMetadata.cs +++ b/osu.Game/Beatmaps/BeatmapMetadata.cs @@ -47,6 +47,7 @@ namespace osu.Game.Beatmaps #region Compatibility properties + [Ignored] public string AuthorString { get => Author.Username; diff --git a/osu.Game/Beatmaps/BeatmapSetInfo.cs b/osu.Game/Beatmaps/BeatmapSetInfo.cs index f6ca184ea3..a934d1a2e3 100644 --- a/osu.Game/Beatmaps/BeatmapSetInfo.cs +++ b/osu.Game/Beatmaps/BeatmapSetInfo.cs @@ -32,6 +32,7 @@ namespace osu.Game.Beatmaps public IList Files { get; } = null!; + [Ignored] public BeatmapOnlineStatus Status { get => (BeatmapOnlineStatus)StatusInt; From 67bf95bc91d8cda84cd53b1f521ebc8b1e78c7e1 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 23:30:40 +0900 Subject: [PATCH 13/26] Remove all usage of `AuthorString` --- .../TestSceneDrawableHitObjects.cs | 2 +- .../DrawableTaikoRulesetTestScene.cs | 2 +- .../Skinning/TestSceneDrawableTaikoMascot.cs | 2 +- .../NonVisual/Filtering/FilterMatchingTest.cs | 2 +- osu.Game.Tests/Resources/TestResources.cs | 2 +- .../TestSceneMultiplayerMatchSongSelect.cs | 2 +- .../Visual/Navigation/TestScenePresentBeatmap.cs | 2 +- .../Visual/Navigation/TestScenePresentScore.cs | 4 ++-- .../Visual/SongSelect/TestSceneBeatmapCarousel.cs | 2 +- .../Visual/SongSelect/TestSceneBeatmapInfoWedge.cs | 4 ++-- osu.Game/Beatmaps/Beatmap.cs | 2 +- osu.Game/Beatmaps/BeatmapMetadata.cs | 11 ----------- osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs | 2 +- osu.Game/Database/EFToRealmMigrator.cs | 1 - osu.Game/Screens/Edit/Setup/MetadataSection.cs | 2 +- 15 files changed, 15 insertions(+), 27 deletions(-) diff --git a/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs b/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs index 23f6222eb6..f078a4353d 100644 --- a/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs +++ b/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs @@ -40,7 +40,7 @@ namespace osu.Game.Rulesets.Catch.Tests { Artist = @"Unknown", Title = @"You're breathtaking", - AuthorString = @"Everyone", + Author = { Username = @"Everyone" }, }, Ruleset = new CatchRuleset().RulesetInfo }, diff --git a/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs b/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs index 4bdb85ba60..54bc6f1912 100644 --- a/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs +++ b/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs @@ -37,7 +37,7 @@ namespace osu.Game.Rulesets.Taiko.Tests { Artist = @"Unknown", Title = @"Sample Beatmap", - AuthorString = @"peppy", + Author = { Username = @"peppy" }, }, Ruleset = new TaikoRuleset().RulesetInfo }, diff --git a/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs b/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs index b976735223..e73fb6bb1f 100644 --- a/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs +++ b/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs @@ -163,7 +163,7 @@ namespace osu.Game.Rulesets.Taiko.Tests.Skinning { Artist = "Unknown", Title = "Sample Beatmap", - AuthorString = "Craftplacer", + Author = { Username = "Craftplacer" }, }, Ruleset = new TaikoRuleset().RulesetInfo }, diff --git a/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs b/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs index 3d78043c73..7ac26699a5 100644 --- a/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs +++ b/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs @@ -30,7 +30,7 @@ namespace osu.Game.Tests.NonVisual.Filtering ArtistUnicode = "check unicode too", Title = "Title goes here", TitleUnicode = "Title goes here", - AuthorString = "The Author", + Author = { Username = "The Author" }, Source = "unit tests", Tags = "look for tags too", }, diff --git a/osu.Game.Tests/Resources/TestResources.cs b/osu.Game.Tests/Resources/TestResources.cs index 1d99a5c20d..e1da31abcb 100644 --- a/osu.Game.Tests/Resources/TestResources.cs +++ b/osu.Game.Tests/Resources/TestResources.cs @@ -89,7 +89,7 @@ namespace osu.Game.Tests.Resources // Create random metadata, then we can check if sorting works based on these Artist = "Some Artist " + RNG.Next(0, 9), Title = $"Some Song (set id {setId}) {Guid.NewGuid()}", - AuthorString = "Some Guy " + RNG.Next(0, 9), + Author = { Username = "Some Guy " + RNG.Next(0, 9) }, }; var beatmapSet = new BeatmapSetInfo diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs index 181b0c71f2..ae7705b97d 100644 --- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs +++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs @@ -52,7 +52,7 @@ namespace osu.Game.Tests.Visual.Multiplayer { Artist = "Some Artist", Title = "Some Beatmap", - AuthorString = "Some Author" + Author = { Username = "Some Author" }, }; var beatmapSetInfo = new BeatmapSetInfo diff --git a/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs b/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs index 69b30ec6a0..1b82094603 100644 --- a/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs +++ b/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs @@ -100,7 +100,7 @@ namespace osu.Game.Tests.Visual.Navigation var metadata = new BeatmapMetadata { Artist = "SomeArtist", - AuthorString = "SomeAuthor", + Author = { Username = "SomeAuthor" }, Title = $"import {i}" }; diff --git a/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs b/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs index f9649db135..b8e49d155e 100644 --- a/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs +++ b/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs @@ -41,7 +41,7 @@ namespace osu.Game.Tests.Visual.Navigation Metadata = new BeatmapMetadata { Artist = "SomeArtist", - AuthorString = "SomeAuthor", + Author = { Username = "SomeAuthor" }, Title = "import" }, BaseDifficulty = new BeatmapDifficulty(), @@ -53,7 +53,7 @@ namespace osu.Game.Tests.Visual.Navigation Metadata = new BeatmapMetadata { Artist = "SomeArtist", - AuthorString = "SomeAuthor", + Author = { Username = "SomeAuthor" }, Title = "import" }, BaseDifficulty = new BeatmapDifficulty(), diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs index 0f5bea10e8..0298c3bea9 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs @@ -409,7 +409,7 @@ namespace osu.Game.Tests.Visual.SongSelect set.Beatmaps.ForEach(b => b.Metadata.Artist = zzz_string); if (i == 16) - set.Beatmaps.ForEach(b => b.Metadata.AuthorString = zzz_string); + set.Beatmaps.ForEach(b => b.Metadata.Author.Username = zzz_string); sets.Add(set); } diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs index 9ad5242df4..efacc395f1 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs @@ -208,7 +208,7 @@ namespace osu.Game.Tests.Visual.SongSelect { Metadata = new BeatmapMetadata { - AuthorString = $"{ruleset.ShortName}Author", + Author = { Username = $"{ruleset.ShortName}Author" }, Artist = $"{ruleset.ShortName}Artist", Source = $"{ruleset.ShortName}Source", Title = $"{ruleset.ShortName}Title" @@ -230,7 +230,7 @@ namespace osu.Game.Tests.Visual.SongSelect { Metadata = new BeatmapMetadata { - AuthorString = "WWWWWWWWWWWWWWW", + Author = { Username = "WWWWWWWWWWWWWWW" }, Artist = "Verrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrry long Artist", Source = "Verrrrry long Source", Title = "Verrrrry long Title" diff --git a/osu.Game/Beatmaps/Beatmap.cs b/osu.Game/Beatmaps/Beatmap.cs index ccecd69d21..a9eb7da5c9 100644 --- a/osu.Game/Beatmaps/Beatmap.cs +++ b/osu.Game/Beatmaps/Beatmap.cs @@ -54,7 +54,7 @@ namespace osu.Game.Beatmaps { Artist = @"Unknown", Title = @"Unknown", - AuthorString = @"Unknown Creator", + Author = { Username = @"Unknown Creator" }, }, DifficultyName = @"Normal", BaseDifficulty = Difficulty, diff --git a/osu.Game/Beatmaps/BeatmapMetadata.cs b/osu.Game/Beatmaps/BeatmapMetadata.cs index 89f507581b..a3385e3abe 100644 --- a/osu.Game/Beatmaps/BeatmapMetadata.cs +++ b/osu.Game/Beatmaps/BeatmapMetadata.cs @@ -45,17 +45,6 @@ namespace osu.Game.Beatmaps IUser IBeatmapMetadataInfo.Author => Author; - #region Compatibility properties - - [Ignored] - public string AuthorString - { - get => Author.Username; - set => Author.Username = value; - } - - #endregion - public override string ToString() => this.GetDisplayTitle(); } } diff --git a/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs b/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs index e5db9d045a..893eb8ab78 100644 --- a/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs +++ b/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs @@ -251,7 +251,7 @@ namespace osu.Game.Beatmaps.Formats break; case @"Creator": - metadata.AuthorString = pair.Value; + metadata.Author.Username = pair.Value; break; case @"Version": diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 7683accc5c..bd7a7677f2 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -144,7 +144,6 @@ namespace osu.Game.Database PreviewTime = metadata.PreviewTime, AudioFile = metadata.AudioFile, BackgroundFile = metadata.BackgroundFile, - AuthorString = metadata.AuthorString, }; } diff --git a/osu.Game/Screens/Edit/Setup/MetadataSection.cs b/osu.Game/Screens/Edit/Setup/MetadataSection.cs index 0d2b093a2e..f0ca3e1bbc 100644 --- a/osu.Game/Screens/Edit/Setup/MetadataSection.cs +++ b/osu.Game/Screens/Edit/Setup/MetadataSection.cs @@ -110,7 +110,7 @@ namespace osu.Game.Screens.Edit.Setup Beatmap.Metadata.TitleUnicode = TitleTextBox.Current.Value; Beatmap.Metadata.Title = RomanisedTitleTextBox.Current.Value; - Beatmap.Metadata.AuthorString = creatorTextBox.Current.Value; + Beatmap.Metadata.Author.Username = creatorTextBox.Current.Value; Beatmap.BeatmapInfo.DifficultyName = difficultyTextBox.Current.Value; Beatmap.Metadata.Source = sourceTextBox.Current.Value; Beatmap.Metadata.Tags = tagsTextBox.Current.Value; From f2b151023eb32294fb00171e45a13ab332bd08f2 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 00:03:19 +0900 Subject: [PATCH 14/26] Create separate automapper configurations to reduce cyclic detach overhead This optimises the `BeatmapSetInfo` detach operation by avoiding detaching `BeatmapSetInfo.Beatmaps[].BeatmapSetInfo` a second time over. --- osu.Game/Database/RealmObjectExtensions.cs | 67 +++++++++++++++------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/osu.Game/Database/RealmObjectExtensions.cs b/osu.Game/Database/RealmObjectExtensions.cs index 4ddf0077ca..5ab7154486 100644 --- a/osu.Game/Database/RealmObjectExtensions.cs +++ b/osu.Game/Database/RealmObjectExtensions.cs @@ -74,21 +74,10 @@ namespace osu.Game.Database private static readonly IMapper mapper = new MapperConfiguration(c => { - c.ShouldMapField = fi => false; + applyCommonConfiguration(c); - // This is specifically to avoid mapping explicit interface implementations. - // If we want to limit this further, we can avoid mapping properties with no setter that are not IList<>. - // Takes a bit of effort to determine whether this is the case though, see https://stackoverflow.com/questions/951536/how-do-i-tell-whether-a-type-implements-ilist - c.ShouldMapProperty = pi => pi.GetMethod?.IsPublic == true; - - c.CreateMap(); - c.CreateMap(); - c.CreateMap(); - c.CreateMap(); - c.CreateMap(); - c.CreateMap(); - c.CreateMap(); - c.CreateMap(); + // This can be further optimised to reduce cyclic retrievals, similar to the optimised set mapper below. + // Only hasn't been done yet as we detach at the point of BeatmapInfo less often. c.CreateMap() .MaxDepth(2) .AfterMap((s, d) => @@ -102,13 +91,28 @@ namespace osu.Game.Database } } }); - c.CreateMap() - .MaxDepth(2) - .AfterMap((s, d) => - { - foreach (var beatmap in d.Beatmaps) - beatmap.BeatmapSet = d; - }); + }).CreateMapper(); + + /// + /// A slightly optimised mapper that avoids double-fetches in cyclic reference. + /// + private static readonly IMapper beatmap_set_mapper = new MapperConfiguration(c => + { + applyCommonConfiguration(c); + + c.CreateMap() + .MaxDepth(1) + .ForMember(b => b.BeatmapSet, cc => cc.Ignore()); + }).CreateMapper(); + + private static void applyCommonConfiguration(IMapperConfigurationExpression c) + { + c.ShouldMapField = fi => false; + + // This is specifically to avoid mapping explicit interface implementations. + // If we want to limit this further, we can avoid mapping properties with no setter that are not IList<>. + // Takes a bit of effort to determine whether this is the case though, see https://stackoverflow.com/questions/951536/how-do-i-tell-whether-a-type-implements-ilist + c.ShouldMapProperty = pi => pi.GetMethod?.IsPublic == true; c.Internal().ForAllMaps((typeMap, expression) => { @@ -118,7 +122,23 @@ namespace osu.Game.Database m.Ignore(); }); }); - }).CreateMapper(); + + c.CreateMap(); + c.CreateMap(); + c.CreateMap(); + c.CreateMap(); + c.CreateMap(); + c.CreateMap(); + c.CreateMap(); + c.CreateMap(); + c.CreateMap() + .MaxDepth(2) + .AfterMap((s, d) => + { + foreach (var beatmap in d.Beatmaps) + beatmap.BeatmapSet = d; + }); + } /// /// Create a detached copy of the each item in the collection. @@ -153,6 +173,9 @@ namespace osu.Game.Database if (!item.IsManaged) return item; + if (item is BeatmapSetInfo) + return beatmap_set_mapper.Map(item); + return mapper.Map(item); } From 60b80c88b60b356abc0ab3e27ef9b09817b34b30 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 00:49:16 +0900 Subject: [PATCH 15/26] Avoid file retrieval overhead when detaching `BeatmapSetInfo` It seems that no usages of `BeatmapSetInfo` detaches require files - a `WorkingBeatmap` is always obtained before doing further lookups. Therefore we can omit this include unless the detaching object is a `BeatmapInfo`. A refetch is performed when retrieving a `WorkingBeatmap` to complete the equation. --- osu.Game/Beatmaps/BeatmapManager.cs | 18 +++++++++++++++- osu.Game/Database/RealmObjectExtensions.cs | 24 +++++++++++++++------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index f901b0fbc1..d983c087a0 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -299,7 +299,23 @@ namespace osu.Game.Beatmaps #region Implementation of IWorkingBeatmapCache - public WorkingBeatmap GetWorkingBeatmap(BeatmapInfo? importedBeatmap) => workingBeatmapCache.GetWorkingBeatmap(importedBeatmap); + public WorkingBeatmap GetWorkingBeatmap(BeatmapInfo? importedBeatmap) + { + // Detached sets don't come with files. + // If we seem to be missing files, now is a good time to re-fetch. + if (importedBeatmap?.BeatmapSet?.Files.Count == 0) + { + using (var realm = contextFactory.CreateContext()) + { + var refetch = realm.Find(importedBeatmap.ID)?.Detach(); + + if (refetch != null) + importedBeatmap = refetch; + } + } + + return workingBeatmapCache.GetWorkingBeatmap(importedBeatmap); + } public WorkingBeatmap GetWorkingBeatmap(ILive? importedBeatmap) { diff --git a/osu.Game/Database/RealmObjectExtensions.cs b/osu.Game/Database/RealmObjectExtensions.cs index 5ab7154486..140f41c5d8 100644 --- a/osu.Game/Database/RealmObjectExtensions.cs +++ b/osu.Game/Database/RealmObjectExtensions.cs @@ -76,6 +76,14 @@ namespace osu.Game.Database { applyCommonConfiguration(c); + c.CreateMap() + .MaxDepth(2) + .AfterMap((s, d) => + { + foreach (var beatmap in d.Beatmaps) + beatmap.BeatmapSet = d; + }); + // This can be further optimised to reduce cyclic retrievals, similar to the optimised set mapper below. // Only hasn't been done yet as we detach at the point of BeatmapInfo less often. c.CreateMap() @@ -100,6 +108,15 @@ namespace osu.Game.Database { applyCommonConfiguration(c); + c.CreateMap() + .MaxDepth(2) + .ForMember(b => b.Files, cc => cc.Ignore()) + .AfterMap((s, d) => + { + foreach (var beatmap in d.Beatmaps) + beatmap.BeatmapSet = d; + }); + c.CreateMap() .MaxDepth(1) .ForMember(b => b.BeatmapSet, cc => cc.Ignore()); @@ -131,13 +148,6 @@ namespace osu.Game.Database c.CreateMap(); c.CreateMap(); c.CreateMap(); - c.CreateMap() - .MaxDepth(2) - .AfterMap((s, d) => - { - foreach (var beatmap in d.Beatmaps) - beatmap.BeatmapSet = d; - }); } /// From 1db2135d7022f16c58f4131c1ffaec6494002be0 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 01:05:22 +0900 Subject: [PATCH 16/26] Update tests to match new behaviour --- osu.Game.Tests/Database/BeatmapImporterTests.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/osu.Game.Tests/Database/BeatmapImporterTests.cs b/osu.Game.Tests/Database/BeatmapImporterTests.cs index 1bba73bea0..227314cffd 100644 --- a/osu.Game.Tests/Database/BeatmapImporterTests.cs +++ b/osu.Game.Tests/Database/BeatmapImporterTests.cs @@ -57,8 +57,9 @@ namespace osu.Game.Tests.Database { detachedBeatmapSet = live.Detach(); - Assert.AreEqual(live.Files.Count, detachedBeatmapSet.Files.Count); - Assert.AreEqual(live.Files.Select(f => f.File).Count(), detachedBeatmapSet.Files.Select(f => f.File).Count()); + // files are omitted + Assert.AreEqual(0, detachedBeatmapSet.Files.Count); + Assert.AreEqual(live.Beatmaps.Count, detachedBeatmapSet.Beatmaps.Count); Assert.AreEqual(live.Beatmaps.Select(f => f.Difficulty).Count(), detachedBeatmapSet.Beatmaps.Select(f => f.Difficulty).Count()); Assert.AreEqual(live.Metadata, detachedBeatmapSet.Metadata); @@ -67,11 +68,9 @@ namespace osu.Game.Tests.Database Debug.Assert(detachedBeatmapSet != null); // Check detached instances can all be accessed without throwing. - Assert.NotNull(detachedBeatmapSet.Files.Count); - Assert.NotZero(detachedBeatmapSet.Files.Select(f => f.File).Count()); + Assert.AreEqual(0, detachedBeatmapSet.Files.Count); Assert.NotNull(detachedBeatmapSet.Beatmaps.Count); Assert.NotZero(detachedBeatmapSet.Beatmaps.Select(f => f.Difficulty).Count()); - Assert.NotNull(detachedBeatmapSet.Beatmaps.First().Path); Assert.NotNull(detachedBeatmapSet.Metadata); // Check cyclic reference to beatmap set @@ -96,9 +95,12 @@ namespace osu.Game.Tests.Database Assert.NotNull(beatmapSet); Debug.Assert(beatmapSet != null); - BeatmapSetInfo? detachedBeatmapSet = null; + // Detach at the BeatmapInfo point, similar to what GetWorkingBeatmap does. + BeatmapInfo? detachedBeatmap = null; - beatmapSet.PerformRead(s => detachedBeatmapSet = s.Detach()); + beatmapSet.PerformRead(s => detachedBeatmap = s.Beatmaps.First().Detach()); + + BeatmapSetInfo? detachedBeatmapSet = detachedBeatmap?.BeatmapSet; Debug.Assert(detachedBeatmapSet != null); From 96d07e20edad425dd505fc70ec216f7c8160b2ab Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 02:17:43 +0900 Subject: [PATCH 17/26] Revert nunit test adaptor version bump until console output bug is resolved Tests have started to output too much log content, causing viewing CI failures to be painfully impossible. Roll back for now. Fix may be related to https://github.com/nunit/nunit3-vs-adapter/issues/941, although we don't use filter. --- .../osu.Game.Rulesets.EmptyFreeform.Tests.csproj | 2 +- .../osu.Game.Rulesets.Pippidon.Tests.csproj | 2 +- .../osu.Game.Rulesets.EmptyScrolling.Tests.csproj | 2 +- .../osu.Game.Rulesets.Pippidon.Tests.csproj | 2 +- osu.Game.Benchmarks/osu.Game.Benchmarks.csproj | 2 +- .../osu.Game.Rulesets.Catch.Tests.csproj | 2 +- .../osu.Game.Rulesets.Mania.Tests.csproj | 2 +- osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj | 2 +- .../osu.Game.Rulesets.Taiko.Tests.csproj | 2 +- osu.Game.Tests/osu.Game.Tests.csproj | 2 +- osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform.Tests/osu.Game.Rulesets.EmptyFreeform.Tests.csproj b/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform.Tests/osu.Game.Rulesets.EmptyFreeform.Tests.csproj index a7078f1c09..3c6aaa39ca 100644 --- a/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform.Tests/osu.Game.Rulesets.EmptyFreeform.Tests.csproj +++ b/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform.Tests/osu.Game.Rulesets.EmptyFreeform.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj b/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj index c133b0e3f8..0719dd30df 100644 --- a/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj +++ b/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling.Tests/osu.Game.Rulesets.EmptyScrolling.Tests.csproj b/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling.Tests/osu.Game.Rulesets.EmptyScrolling.Tests.csproj index 92b48470e8..d0db43cc81 100644 --- a/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling.Tests/osu.Game.Rulesets.EmptyScrolling.Tests.csproj +++ b/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling.Tests/osu.Game.Rulesets.EmptyScrolling.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj b/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj index c133b0e3f8..0719dd30df 100644 --- a/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj +++ b/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/osu.Game.Benchmarks/osu.Game.Benchmarks.csproj b/osu.Game.Benchmarks/osu.Game.Benchmarks.csproj index a8599f2cb6..57b914bee6 100644 --- a/osu.Game.Benchmarks/osu.Game.Benchmarks.csproj +++ b/osu.Game.Benchmarks/osu.Game.Benchmarks.csproj @@ -9,7 +9,7 @@ - + diff --git a/osu.Game.Rulesets.Catch.Tests/osu.Game.Rulesets.Catch.Tests.csproj b/osu.Game.Rulesets.Catch.Tests/osu.Game.Rulesets.Catch.Tests.csproj index d5496b7479..13f2e25f05 100644 --- a/osu.Game.Rulesets.Catch.Tests/osu.Game.Rulesets.Catch.Tests.csproj +++ b/osu.Game.Rulesets.Catch.Tests/osu.Game.Rulesets.Catch.Tests.csproj @@ -4,7 +4,7 @@ - + diff --git a/osu.Game.Rulesets.Mania.Tests/osu.Game.Rulesets.Mania.Tests.csproj b/osu.Game.Rulesets.Mania.Tests/osu.Game.Rulesets.Mania.Tests.csproj index 2f12b1535e..d51a6da4f9 100644 --- a/osu.Game.Rulesets.Mania.Tests/osu.Game.Rulesets.Mania.Tests.csproj +++ b/osu.Game.Rulesets.Mania.Tests/osu.Game.Rulesets.Mania.Tests.csproj @@ -4,7 +4,7 @@ - + diff --git a/osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj b/osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj index e5b2e070d8..fea2e408f6 100644 --- a/osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj +++ b/osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj @@ -5,7 +5,7 @@ - + diff --git a/osu.Game.Rulesets.Taiko.Tests/osu.Game.Rulesets.Taiko.Tests.csproj b/osu.Game.Rulesets.Taiko.Tests/osu.Game.Rulesets.Taiko.Tests.csproj index e48d80323a..ad3713e047 100644 --- a/osu.Game.Rulesets.Taiko.Tests/osu.Game.Rulesets.Taiko.Tests.csproj +++ b/osu.Game.Rulesets.Taiko.Tests/osu.Game.Rulesets.Taiko.Tests.csproj @@ -4,7 +4,7 @@ - + diff --git a/osu.Game.Tests/osu.Game.Tests.csproj b/osu.Game.Tests/osu.Game.Tests.csproj index c64ef918e3..3b115d43e5 100644 --- a/osu.Game.Tests/osu.Game.Tests.csproj +++ b/osu.Game.Tests/osu.Game.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj b/osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj index fb09a7be1e..130fcfaca1 100644 --- a/osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj +++ b/osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj @@ -7,7 +7,7 @@ - + WinExe From 761e161eec0d774aaae10ed4e3c5c042003385a9 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 09:44:21 +0900 Subject: [PATCH 18/26] Add comment explaining ignore rule --- osu.Game/Database/RealmObjectExtensions.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/osu.Game/Database/RealmObjectExtensions.cs b/osu.Game/Database/RealmObjectExtensions.cs index 5ab7154486..097e620e12 100644 --- a/osu.Game/Database/RealmObjectExtensions.cs +++ b/osu.Game/Database/RealmObjectExtensions.cs @@ -102,6 +102,7 @@ namespace osu.Game.Database c.CreateMap() .MaxDepth(1) + // This is not required as it will be populated in the `AfterMap` call from the `BeatmapInfo`'s parent. .ForMember(b => b.BeatmapSet, cc => cc.Ignore()); }).CreateMapper(); From 3bc091fe6d33cda1291e908b4818b0ddcf8ead64 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 09:46:45 +0900 Subject: [PATCH 19/26] Add ignore rules on more helper properties --- osu.Game/Input/Bindings/RealmKeyBinding.cs | 2 ++ osu.Game/Scoring/ScoreInfo.cs | 1 + 2 files changed, 3 insertions(+) diff --git a/osu.Game/Input/Bindings/RealmKeyBinding.cs b/osu.Game/Input/Bindings/RealmKeyBinding.cs index 6a408847fe..32813ada16 100644 --- a/osu.Game/Input/Bindings/RealmKeyBinding.cs +++ b/osu.Game/Input/Bindings/RealmKeyBinding.cs @@ -20,12 +20,14 @@ namespace osu.Game.Input.Bindings public int? Variant { get; set; } + [Ignored] public KeyCombination KeyCombination { get => KeyCombinationString; set => KeyCombinationString = value.ToString(); } + [Ignored] public object Action { get => ActionInt; diff --git a/osu.Game/Scoring/ScoreInfo.cs b/osu.Game/Scoring/ScoreInfo.cs index e1328d8a06..b268e2cd91 100644 --- a/osu.Game/Scoring/ScoreInfo.cs +++ b/osu.Game/Scoring/ScoreInfo.cs @@ -104,6 +104,7 @@ namespace osu.Game.Scoring } } + [Ignored] public ScoreRank Rank { get => (ScoreRank)RankInt; From 64a023665e9944f93615de503de190a64144bcf2 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:16:54 +0900 Subject: [PATCH 20/26] Avoid taking more than one backup per migration run --- osu.Game/Database/EFToRealmMigrator.cs | 30 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index d479f81ead..b7f3eebe7a 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -24,6 +24,8 @@ namespace osu.Game.Database private readonly RealmContextFactory realmContextFactory; private readonly OsuConfigManager config; + private bool hasTakenBackup; + public EFToRealmMigrator(DatabaseContextFactory efContextFactory, RealmContextFactory realmContextFactory, OsuConfigManager config) { this.efContextFactory = efContextFactory; @@ -70,8 +72,16 @@ namespace osu.Game.Database using (var realm = realmContextFactory.CreateContext()) { Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database); - string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; - efContextFactory.CreateBackup($"client.{migration}.db"); + + if (!hasTakenBackup) + { + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + + efContextFactory.CreateBackup($"client.{migration}.db"); + realmContextFactory.CreateBackup($"client.{migration}.realm"); + + hasTakenBackup = true; + } // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. @@ -81,8 +91,6 @@ namespace osu.Game.Database } else { - realmContextFactory.CreateBackup($"client.{migration}.realm"); - using (var transaction = realm.BeginWrite()) { foreach (var beatmapSet in existingBeatmapSets) @@ -195,8 +203,16 @@ namespace osu.Game.Database using (var realm = realmContextFactory.CreateContext()) { Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database); - string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; - efContextFactory.CreateBackup($"client.{migration}.db"); + + if (!hasTakenBackup) + { + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + + efContextFactory.CreateBackup($"client.{migration}.db"); + realmContextFactory.CreateBackup($"client.{migration}.realm"); + + hasTakenBackup = true; + } // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. @@ -206,8 +222,6 @@ namespace osu.Game.Database } else { - realmContextFactory.CreateBackup($"client.{migration}.realm"); - using (var transaction = realm.BeginWrite()) { foreach (var score in existingScores) From 04e9ffa966afd6b95698591de468006302328441 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:20:43 +0900 Subject: [PATCH 21/26] Freshen some comments --- osu.Game/Database/EFToRealmMigrator.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index b7f3eebe7a..6b911ebad0 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -84,7 +84,7 @@ namespace osu.Game.Database } // only migrate data if the realm database is empty. - // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. + // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (realm.All().Any(s => !s.Protected)) { Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database); @@ -215,7 +215,6 @@ namespace osu.Game.Database } // only migrate data if the realm database is empty. - // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (realm.All().Any()) { Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database); From e1a35714be3720178f2a56ef88bcdb038946f04a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:30:17 +0900 Subject: [PATCH 22/26] Add notification for debug builds when database migration occurs --- osu.Game/Database/DatabaseContextFactory.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index cc690a9fda..635c4373cd 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; +using osu.Framework.Development; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Statistics; @@ -146,11 +147,15 @@ namespace osu.Game.Database Database = { AutoTransactionsEnabled = false } }; - public void CreateBackup(string filename) + public void CreateBackup(string backupFilename) { - Logger.Log($"Creating full EF database backup at {filename}", LoggingTarget.Database); + Logger.Log($"Creating full EF database backup at {backupFilename}", LoggingTarget.Database); + + if (DebugUtils.IsDebugBuild) + Logger.Log("Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.", level: LogLevel.Important); + using (var source = storage.GetStream(DATABASE_NAME)) - using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); } From 195534a1d2e0a890be7b16feb3e936ed2fe426c4 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:31:09 +0900 Subject: [PATCH 23/26] Only output "successful" messages when copy actually occurred --- osu.Game/Database/EFToRealmMigrator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 6b911ebad0..76e5219f87 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -148,10 +148,10 @@ namespace osu.Game.Database } transaction.Commit(); + Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); } } - Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. } @@ -257,10 +257,10 @@ namespace osu.Game.Database } transaction.Commit(); + Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); } } - Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. } From 261b4988f9552b63a951efe3041e34a0b1f377a8 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:58:59 +0900 Subject: [PATCH 24/26] Only catch `RealmExceptions` to avoid blocking the nested `TimeoutException` --- osu.Game/Database/RealmContextFactory.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 52f515a4bb..901f28a449 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -21,6 +21,7 @@ using osu.Game.Stores; using osu.Game.Rulesets; using osu.Game.Scoring; using Realms; +using Realms.Exceptions; #nullable enable @@ -427,7 +428,7 @@ namespace osu.Game.Database throw new TimeoutException(@"Took too long to acquire lock"); } } - catch (Exception e) + catch (RealmException e) { // Compact may fail if the realm is in a bad state. // We still want to continue with the blocking operation, though. From c52899b1fb0f6a6e52d73ee93d74247b931bc6f4 Mon Sep 17 00:00:00 2001 From: Dan Balasescu Date: Wed, 19 Jan 2022 11:56:44 +0900 Subject: [PATCH 25/26] Rename property --- osu.Game/Database/RealmContextFactory.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 10c2438df9..8be8eab567 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -361,13 +361,13 @@ namespace osu.Game.Database private string? getRulesetShortNameFromLegacyID(long rulesetId) => efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName; - public void CreateBackup(string filename) + public void CreateBackup(string backupFilename) { using (BlockAllOperations()) { - Logger.Log($"Creating full realm database backup at {filename}", LoggingTarget.Database); + Logger.Log($"Creating full realm database backup at {backupFilename}", LoggingTarget.Database); using (var source = storage.GetStream(Filename)) - using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); } } From 2363130f8b2d791f98d3d7023e75b964b6240a49 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 12:33:56 +0900 Subject: [PATCH 26/26] Add back `BaseDifficulty` with obsoletion counter to account for custom rulesets --- osu.Game/Beatmaps/BeatmapInfo.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/osu.Game/Beatmaps/BeatmapInfo.cs b/osu.Game/Beatmaps/BeatmapInfo.cs index b91f05cd11..f4d8697042 100644 --- a/osu.Game/Beatmaps/BeatmapInfo.cs +++ b/osu.Game/Beatmaps/BeatmapInfo.cs @@ -165,6 +165,14 @@ namespace osu.Game.Beatmaps } } + [Ignored] + [Obsolete("Use BeatmapInfo.Difficulty instead.")] // can be removed 20220719 + public BeatmapDifficulty BaseDifficulty + { + get => Difficulty; + set => Difficulty = value; + } + [Ignored] public string? Path => File?.Filename;