From c3d3c03f0fad8b994bf99d8ee3c587654eff3a66 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 12:15:40 +0900 Subject: [PATCH 01/19] Fix `TestScenePlaylistsScreen` crashing on entering room --- osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs b/osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs index e52f823f0b..63bd7c8068 100644 --- a/osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs +++ b/osu.Game.Tests/Visual/Playlists/TestScenePlaylistsScreen.cs @@ -2,8 +2,6 @@ // See the LICENCE file in the repository root for full licence text. using NUnit.Framework; -using osu.Framework.Allocation; -using osu.Game.Overlays; namespace osu.Game.Tests.Visual.Playlists { @@ -12,9 +10,6 @@ namespace osu.Game.Tests.Visual.Playlists { protected override bool UseOnlineAPI => true; - [Cached] - private MusicController musicController { get; set; } = new MusicController(); - public TestScenePlaylistsScreen() { var multi = new Screens.OnlinePlay.Playlists.Playlists(); From a714941f932cd07c0f1315a3a78a534a9e1101ae Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:06:22 +0900 Subject: [PATCH 02/19] Rename EF variable to make reading code easier --- osu.Game/Database/EFToRealmMigrator.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 7683accc5c..98037bb8a3 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -30,20 +30,20 @@ namespace osu.Game.Database public void Run() { - using (var db = efContextFactory.GetForWrite()) + using (var ef = efContextFactory.GetForWrite()) { - migrateSettings(db); - migrateSkins(db); + migrateSettings(ef); + migrateSkins(ef); - migrateBeatmaps(db); - migrateScores(db); + migrateBeatmaps(ef); + migrateScores(ef); } } - private void migrateBeatmaps(DatabaseWriteUsage db) + private void migrateBeatmaps(DatabaseWriteUsage ef) { // can be removed 20220730. - var existingBeatmapSets = db.Context.EFBeatmapSetInfo + var existingBeatmapSets = ef.Context.EFBeatmapSetInfo .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) @@ -117,7 +117,7 @@ namespace osu.Game.Database } } - db.Context.RemoveRange(existingBeatmapSets); + ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. transaction.Commit(); From b1a75ce48098116ee14a6eb9db39cd1339cdf4dc Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:17:43 +0900 Subject: [PATCH 03/19] Permanently delete `client.db` after migration completes --- osu.Game/Database/EFToRealmMigrator.cs | 4 ++++ osu.Game/OsuGameBase.cs | 1 + 2 files changed, 5 insertions(+) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 98037bb8a3..7ad8e507bb 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -38,6 +38,10 @@ namespace osu.Game.Database migrateBeatmaps(ef); migrateScores(ef); } + + // Delete the database permanently. + // Will cause future startups to not attempt migration. + efContextFactory.ResetDatabase(); } private void migrateBeatmaps(DatabaseWriteUsage ef) diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index a9538c1e8c..18b22588a6 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -193,6 +193,7 @@ namespace osu.Game dependencies.Cache(RulesetStore = new RulesetStore(realmFactory, Storage)); dependencies.CacheAs(RulesetStore); + // A non-null context factory means there's still content to migrate. if (efContextFactory != null) new EFToRealmMigrator(efContextFactory, realmFactory, LocalConfig).Run(); From 798482c94122eb9d23221e90e33ccc390a52380a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:19:25 +0900 Subject: [PATCH 04/19] Create backups before deleting scores and beatmaps from EF database --- osu.Game/Database/DatabaseContextFactory.cs | 8 +++++++ osu.Game/Database/EFToRealmMigrator.cs | 25 +++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index f79505d7c5..c2a60122eb 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -1,6 +1,7 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; @@ -144,6 +145,13 @@ namespace osu.Game.Database Database = { AutoTransactionsEnabled = false } }; + public void CreateBackup(string filename) + { + using (var source = storage.GetStream(DATABASE_NAME)) + using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + source.CopyTo(destination); + } + public void ResetDatabase() { lock (writeLock) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 7ad8e507bb..3ecbd50b3e 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -1,6 +1,8 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System; +using System.Collections.Generic; using System.Linq; using Microsoft.EntityFrameworkCore; using osu.Game.Beatmaps; @@ -31,13 +33,16 @@ namespace osu.Game.Database public void Run() { using (var ef = efContextFactory.GetForWrite()) - { migrateSettings(ef); + + using (var ef = efContextFactory.GetForWrite()) migrateSkins(ef); + using (var ef = efContextFactory.GetForWrite()) migrateBeatmaps(ef); + + using (var ef = efContextFactory.GetForWrite()) migrateScores(ef); - } // Delete the database permanently. // Will cause future startups to not attempt migration. @@ -47,13 +52,13 @@ namespace osu.Game.Database private void migrateBeatmaps(DatabaseWriteUsage ef) { // can be removed 20220730. - var existingBeatmapSets = ef.Context.EFBeatmapSetInfo - .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) - .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) - .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) - .Include(s => s.Files).ThenInclude(f => f.FileInfo) - .Include(s => s.Metadata) - .ToList(); + List existingBeatmapSets = ef.Context.EFBeatmapSetInfo + .Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo) + .Include(s => s.Beatmaps).ThenInclude(b => b.Metadata) + .Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty) + .Include(s => s.Files).ThenInclude(f => f.FileInfo) + .Include(s => s.Metadata) + .ToList(); // previous entries in EF are removed post migration. if (!existingBeatmapSets.Any()) @@ -121,6 +126,7 @@ namespace osu.Game.Database } } + efContextFactory.CreateBackup($"client.before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. @@ -207,6 +213,7 @@ namespace osu.Game.Database } } + efContextFactory.CreateBackup($"client.before_scores_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. From cf30d48721242567eb44d7d0e31fb6cb9e2d1b69 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:19:31 +0900 Subject: [PATCH 05/19] Add more logging during migration process --- osu.Game/Database/EFToRealmMigrator.cs | 29 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 3ecbd50b3e..d8a3b66c7c 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Linq; using Microsoft.EntityFrameworkCore; +using osu.Framework.Logging; using osu.Game.Beatmaps; using osu.Game.Configuration; using osu.Game.Models; @@ -64,6 +65,8 @@ namespace osu.Game.Database if (!existingBeatmapSets.Any()) return; + Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { @@ -71,6 +74,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any(s => !s.Protected)) { + Logger.Log($"Migrating {existingBeatmapSets.Count} beatmaps", LoggingTarget.Database); + foreach (var beatmapSet in existingBeatmapSets) { var realmBeatmapSet = new BeatmapSetInfo @@ -161,17 +166,19 @@ namespace osu.Game.Database private void migrateScores(DatabaseWriteUsage db) { // can be removed 20220730. - var existingScores = db.Context.ScoreInfo - .Include(s => s.Ruleset) - .Include(s => s.BeatmapInfo) - .Include(s => s.Files) - .ThenInclude(f => f.FileInfo) - .ToList(); + List existingScores = db.Context.ScoreInfo + .Include(s => s.Ruleset) + .Include(s => s.BeatmapInfo) + .Include(s => s.Files) + .ThenInclude(f => f.FileInfo) + .ToList(); // previous entries in EF are removed post migration. if (!existingScores.Any()) return; + Logger.Log("Beginning scores migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { @@ -179,6 +186,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any()) { + Logger.Log($"Migrating {existingScores.Count} scores", LoggingTarget.Database); + foreach (var score in existingScores) { var realmScore = new ScoreInfo @@ -254,6 +263,8 @@ namespace osu.Game.Database // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (!realm.All().Any(s => !s.Protected)) { + Logger.Log($"Migrating {existingSkins.Count} skins", LoggingTarget.Database); + foreach (var skin in existingSkins) { var realmSkin = new SkinInfo @@ -297,18 +308,22 @@ namespace osu.Game.Database private void migrateSettings(DatabaseWriteUsage db) { // migrate ruleset settings. can be removed 20220315. - var existingSettings = db.Context.DatabasedSetting; + var existingSettings = db.Context.DatabasedSetting.ToList(); // previous entries in EF are removed post migration. if (!existingSettings.Any()) return; + Logger.Log("Beginning settings migration to realm", LoggingTarget.Database); + using (var realm = realmContextFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { // only migrate data if the realm database is empty. if (!realm.All().Any()) { + Logger.Log($"Migrating {existingSettings.Count} settings", LoggingTarget.Database); + foreach (var dkb in existingSettings) { if (dkb.RulesetID == null) From 2b1c15b6cc1b7a6f51157cb598e63b835322ea3c Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:30:32 +0900 Subject: [PATCH 06/19] Allow `BlockAllOperations` to be called from a non-update thread if update has never run --- osu.Game/Database/RealmContextFactory.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 9307e06be0..01c54d6ee2 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -366,17 +366,17 @@ namespace osu.Game.Database if (isDisposed) throw new ObjectDisposedException(nameof(RealmContextFactory)); - if (!ThreadSafety.IsUpdateThread) - throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread."); - - Logger.Log(@"Blocking realm operations.", LoggingTarget.Database); - try { contextCreationLock.Wait(); lock (contextLock) { + if (!ThreadSafety.IsUpdateThread && context != null) + throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread."); + + Logger.Log(@"Blocking realm operations.", LoggingTarget.Database); + context?.Dispose(); context = null; } From bf50a9b8f8d92d0a23a498822513d65d9fade133 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:30:41 +0900 Subject: [PATCH 07/19] Also backup the realm database before migration --- osu.Game/Database/EFToRealmMigrator.cs | 8 ++++++-- osu.Game/Database/RealmContextFactory.cs | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index d8a3b66c7c..9fad29b3f4 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -75,6 +75,9 @@ namespace osu.Game.Database if (!realm.All().Any(s => !s.Protected)) { Logger.Log($"Migrating {existingBeatmapSets.Count} beatmaps", LoggingTarget.Database); + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + realmContextFactory.CreateBackup($"client.{migration}.realm"); + efContextFactory.CreateBackup($"client.{migration}.db"); foreach (var beatmapSet in existingBeatmapSets) { @@ -131,7 +134,6 @@ namespace osu.Game.Database } } - efContextFactory.CreateBackup($"client.before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. @@ -187,6 +189,9 @@ namespace osu.Game.Database if (!realm.All().Any()) { Logger.Log($"Migrating {existingScores.Count} scores", LoggingTarget.Database); + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + realmContextFactory.CreateBackup($"client.{migration}.realm"); + efContextFactory.CreateBackup($"client.{migration}.db"); foreach (var score in existingScores) { @@ -222,7 +227,6 @@ namespace osu.Game.Database } } - efContextFactory.CreateBackup($"client.before_scores_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}.db"); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 01c54d6ee2..c86a9e54e1 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -2,6 +2,7 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.IO; using System.Linq; using System.Reflection; using System.Threading; @@ -353,6 +354,16 @@ namespace osu.Game.Database private string? getRulesetShortNameFromLegacyID(long rulesetId) => efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName; + public void CreateBackup(string filename) + { + using (BlockAllOperations()) + { + using (var source = storage.GetStream(Filename)) + using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + source.CopyTo(destination); + } + } + /// /// Flush any active contexts and block any further writes. /// From 3429fd87681b928a52954b4a3b32ae38fd2bd794 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 14:41:02 +0900 Subject: [PATCH 08/19] Fix transaction scope and add even more logging --- osu.Game/Database/DatabaseContextFactory.cs | 2 + osu.Game/Database/EFToRealmMigrator.cs | 215 +++++++++++--------- osu.Game/Database/RealmContextFactory.cs | 1 + 3 files changed, 122 insertions(+), 96 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index c2a60122eb..cc690a9fda 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; +using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Statistics; @@ -147,6 +148,7 @@ namespace osu.Game.Database public void CreateBackup(string filename) { + Logger.Log($"Creating full EF database backup at {filename}", LoggingTarget.Database); using (var source = storage.GetStream(DATABASE_NAME)) using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 9fad29b3f4..8f5de80e3b 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -47,6 +47,7 @@ namespace osu.Game.Database // Delete the database permanently. // Will cause future startups to not attempt migration. + Logger.Log("Migration successful, deleting EF database", LoggingTarget.Database); efContextFactory.ResetDatabase(); } @@ -61,83 +62,94 @@ namespace osu.Game.Database .Include(s => s.Metadata) .ToList(); - // previous entries in EF are removed post migration. - if (!existingBeatmapSets.Any()) - return; - Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database); - using (var realm = realmContextFactory.CreateContext()) - using (var transaction = realm.BeginWrite()) + // previous entries in EF are removed post migration. + if (!existingBeatmapSets.Any()) { + Logger.Log("No beatmaps found to migrate", LoggingTarget.Database); + return; + } + + using (var realm = realmContextFactory.CreateContext()) + { + Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database); + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + efContextFactory.CreateBackup($"client.{migration}.db"); + // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. - if (!realm.All().Any(s => !s.Protected)) + if (realm.All().Any(s => !s.Protected)) + { + Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database); + } + else { - Logger.Log($"Migrating {existingBeatmapSets.Count} beatmaps", LoggingTarget.Database); - string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; realmContextFactory.CreateBackup($"client.{migration}.realm"); - efContextFactory.CreateBackup($"client.{migration}.db"); - foreach (var beatmapSet in existingBeatmapSets) + using (var transaction = realm.BeginWrite()) { - var realmBeatmapSet = new BeatmapSetInfo + foreach (var beatmapSet in existingBeatmapSets) { - OnlineID = beatmapSet.OnlineID ?? -1, - DateAdded = beatmapSet.DateAdded, - Status = beatmapSet.Status, - DeletePending = beatmapSet.DeletePending, - Hash = beatmapSet.Hash, - Protected = beatmapSet.Protected, - }; - - migrateFiles(beatmapSet, realm, realmBeatmapSet); - - foreach (var beatmap in beatmapSet.Beatmaps) - { - var realmBeatmap = new BeatmapInfo + var realmBeatmapSet = new BeatmapSetInfo { - DifficultyName = beatmap.DifficultyName, - Status = beatmap.Status, - OnlineID = beatmap.OnlineID ?? -1, - Length = beatmap.Length, - BPM = beatmap.BPM, - Hash = beatmap.Hash, - StarRating = beatmap.StarRating, - MD5Hash = beatmap.MD5Hash, - Hidden = beatmap.Hidden, - AudioLeadIn = beatmap.AudioLeadIn, - StackLeniency = beatmap.StackLeniency, - SpecialStyle = beatmap.SpecialStyle, - LetterboxInBreaks = beatmap.LetterboxInBreaks, - WidescreenStoryboard = beatmap.WidescreenStoryboard, - EpilepsyWarning = beatmap.EpilepsyWarning, - SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate, - DistanceSpacing = beatmap.DistanceSpacing, - BeatDivisor = beatmap.BeatDivisor, - GridSize = beatmap.GridSize, - TimelineZoom = beatmap.TimelineZoom, - Countdown = beatmap.Countdown, - CountdownOffset = beatmap.CountdownOffset, - MaxCombo = beatmap.MaxCombo, - Bookmarks = beatmap.Bookmarks, - Ruleset = realm.Find(beatmap.RulesetInfo.ShortName), - Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty), - Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata), - BeatmapSet = realmBeatmapSet, + OnlineID = beatmapSet.OnlineID ?? -1, + DateAdded = beatmapSet.DateAdded, + Status = beatmapSet.Status, + DeletePending = beatmapSet.DeletePending, + Hash = beatmapSet.Hash, + Protected = beatmapSet.Protected, }; - realmBeatmapSet.Beatmaps.Add(realmBeatmap); + migrateFiles(beatmapSet, realm, realmBeatmapSet); + + foreach (var beatmap in beatmapSet.Beatmaps) + { + var realmBeatmap = new BeatmapInfo + { + DifficultyName = beatmap.DifficultyName, + Status = beatmap.Status, + OnlineID = beatmap.OnlineID ?? -1, + Length = beatmap.Length, + BPM = beatmap.BPM, + Hash = beatmap.Hash, + StarRating = beatmap.StarRating, + MD5Hash = beatmap.MD5Hash, + Hidden = beatmap.Hidden, + AudioLeadIn = beatmap.AudioLeadIn, + StackLeniency = beatmap.StackLeniency, + SpecialStyle = beatmap.SpecialStyle, + LetterboxInBreaks = beatmap.LetterboxInBreaks, + WidescreenStoryboard = beatmap.WidescreenStoryboard, + EpilepsyWarning = beatmap.EpilepsyWarning, + SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate, + DistanceSpacing = beatmap.DistanceSpacing, + BeatDivisor = beatmap.BeatDivisor, + GridSize = beatmap.GridSize, + TimelineZoom = beatmap.TimelineZoom, + Countdown = beatmap.Countdown, + CountdownOffset = beatmap.CountdownOffset, + MaxCombo = beatmap.MaxCombo, + Bookmarks = beatmap.Bookmarks, + Ruleset = realm.Find(beatmap.RulesetInfo.ShortName), + Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty), + Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata), + BeatmapSet = realmBeatmapSet, + }; + + realmBeatmapSet.Beatmaps.Add(realmBeatmap); + } + + realm.Add(realmBeatmapSet); } - realm.Add(realmBeatmapSet); + transaction.Commit(); } } + Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. - - transaction.Commit(); } } @@ -175,62 +187,73 @@ namespace osu.Game.Database .ThenInclude(f => f.FileInfo) .ToList(); - // previous entries in EF are removed post migration. - if (!existingScores.Any()) - return; - Logger.Log("Beginning scores migration to realm", LoggingTarget.Database); - using (var realm = realmContextFactory.CreateContext()) - using (var transaction = realm.BeginWrite()) + // previous entries in EF are removed post migration. + if (!existingScores.Any()) { + Logger.Log("No scores found to migrate", LoggingTarget.Database); + return; + } + + using (var realm = realmContextFactory.CreateContext()) + { + Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database); + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + efContextFactory.CreateBackup($"client.{migration}.db"); + // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. - if (!realm.All().Any()) + if (realm.All().Any()) + { + Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database); + } + else { - Logger.Log($"Migrating {existingScores.Count} scores", LoggingTarget.Database); - string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; realmContextFactory.CreateBackup($"client.{migration}.realm"); - efContextFactory.CreateBackup($"client.{migration}.db"); - foreach (var score in existingScores) + using (var transaction = realm.BeginWrite()) { - var realmScore = new ScoreInfo + foreach (var score in existingScores) { - Hash = score.Hash, - DeletePending = score.DeletePending, - OnlineID = score.OnlineID ?? -1, - ModsJson = score.ModsJson, - StatisticsJson = score.StatisticsJson, - User = score.User, - TotalScore = score.TotalScore, - MaxCombo = score.MaxCombo, - Accuracy = score.Accuracy, - HasReplay = ((IScoreInfo)score).HasReplay, - Date = score.Date, - PP = score.PP, - BeatmapInfo = realm.All().First(b => b.Hash == score.BeatmapInfo.Hash), - Ruleset = realm.Find(score.Ruleset.ShortName), - Rank = score.Rank, - HitEvents = score.HitEvents, - Passed = score.Passed, - Combo = score.Combo, - Position = score.Position, - Statistics = score.Statistics, - Mods = score.Mods, - APIMods = score.APIMods, - }; + var realmScore = new ScoreInfo + { + Hash = score.Hash, + DeletePending = score.DeletePending, + OnlineID = score.OnlineID ?? -1, + ModsJson = score.ModsJson, + StatisticsJson = score.StatisticsJson, + User = score.User, + TotalScore = score.TotalScore, + MaxCombo = score.MaxCombo, + Accuracy = score.Accuracy, + HasReplay = ((IScoreInfo)score).HasReplay, + Date = score.Date, + PP = score.PP, + BeatmapInfo = realm.All().First(b => b.Hash == score.BeatmapInfo.Hash), + Ruleset = realm.Find(score.Ruleset.ShortName), + Rank = score.Rank, + HitEvents = score.HitEvents, + Passed = score.Passed, + Combo = score.Combo, + Position = score.Position, + Statistics = score.Statistics, + Mods = score.Mods, + APIMods = score.APIMods, + }; - migrateFiles(score, realm, realmScore); + migrateFiles(score, realm, realmScore); - realm.Add(realmScore); + realm.Add(realmScore); + } + + transaction.Commit(); } } + Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. - - transaction.Commit(); } } diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index c86a9e54e1..8548e63e94 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -358,6 +358,7 @@ namespace osu.Game.Database { using (BlockAllOperations()) { + Logger.Log($"Creating full realm database backup at {filename}", LoggingTarget.Database); using (var source = storage.GetStream(Filename)) using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); From 6b0bf38c93cb994f2c8d144e26d4001e5ad01060 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 20:47:53 +0900 Subject: [PATCH 09/19] Use a single EF context to avoid scores getting cascade deleted along the way --- osu.Game/Database/EFToRealmMigrator.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 8f5de80e3b..d479f81ead 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -34,16 +34,12 @@ namespace osu.Game.Database public void Run() { using (var ef = efContextFactory.GetForWrite()) + { migrateSettings(ef); - - using (var ef = efContextFactory.GetForWrite()) migrateSkins(ef); - - using (var ef = efContextFactory.GetForWrite()) migrateBeatmaps(ef); - - using (var ef = efContextFactory.GetForWrite()) migrateScores(ef); + } // Delete the database permanently. // Will cause future startups to not attempt migration. From 3596c6ed5d59707c08296ff0ddbc08200a557659 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 23:25:30 +0900 Subject: [PATCH 10/19] Add some missing `IgnoredAttributes` to reduce automapper overhead --- osu.Game/Beatmaps/BeatmapInfo.cs | 1 + osu.Game/Beatmaps/BeatmapMetadata.cs | 1 + osu.Game/Beatmaps/BeatmapSetInfo.cs | 1 + 3 files changed, 3 insertions(+) diff --git a/osu.Game/Beatmaps/BeatmapInfo.cs b/osu.Game/Beatmaps/BeatmapInfo.cs index b0e10c2c38..9ad9e9b1c9 100644 --- a/osu.Game/Beatmaps/BeatmapInfo.cs +++ b/osu.Game/Beatmaps/BeatmapInfo.cs @@ -64,6 +64,7 @@ namespace osu.Game.Beatmaps [Ignored] public RealmNamedFileUsage? File => BeatmapSet?.Files.FirstOrDefault(f => f.File.Hash == Hash); + [Ignored] public BeatmapOnlineStatus Status { get => (BeatmapOnlineStatus)StatusInt; diff --git a/osu.Game/Beatmaps/BeatmapMetadata.cs b/osu.Game/Beatmaps/BeatmapMetadata.cs index 6349476550..89f507581b 100644 --- a/osu.Game/Beatmaps/BeatmapMetadata.cs +++ b/osu.Game/Beatmaps/BeatmapMetadata.cs @@ -47,6 +47,7 @@ namespace osu.Game.Beatmaps #region Compatibility properties + [Ignored] public string AuthorString { get => Author.Username; diff --git a/osu.Game/Beatmaps/BeatmapSetInfo.cs b/osu.Game/Beatmaps/BeatmapSetInfo.cs index f6ca184ea3..a934d1a2e3 100644 --- a/osu.Game/Beatmaps/BeatmapSetInfo.cs +++ b/osu.Game/Beatmaps/BeatmapSetInfo.cs @@ -32,6 +32,7 @@ namespace osu.Game.Beatmaps public IList Files { get; } = null!; + [Ignored] public BeatmapOnlineStatus Status { get => (BeatmapOnlineStatus)StatusInt; From 67bf95bc91d8cda84cd53b1f521ebc8b1e78c7e1 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 18 Jan 2022 23:30:40 +0900 Subject: [PATCH 11/19] Remove all usage of `AuthorString` --- .../TestSceneDrawableHitObjects.cs | 2 +- .../DrawableTaikoRulesetTestScene.cs | 2 +- .../Skinning/TestSceneDrawableTaikoMascot.cs | 2 +- .../NonVisual/Filtering/FilterMatchingTest.cs | 2 +- osu.Game.Tests/Resources/TestResources.cs | 2 +- .../TestSceneMultiplayerMatchSongSelect.cs | 2 +- .../Visual/Navigation/TestScenePresentBeatmap.cs | 2 +- .../Visual/Navigation/TestScenePresentScore.cs | 4 ++-- .../Visual/SongSelect/TestSceneBeatmapCarousel.cs | 2 +- .../Visual/SongSelect/TestSceneBeatmapInfoWedge.cs | 4 ++-- osu.Game/Beatmaps/Beatmap.cs | 2 +- osu.Game/Beatmaps/BeatmapMetadata.cs | 11 ----------- osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs | 2 +- osu.Game/Database/EFToRealmMigrator.cs | 1 - osu.Game/Screens/Edit/Setup/MetadataSection.cs | 2 +- 15 files changed, 15 insertions(+), 27 deletions(-) diff --git a/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs b/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs index 23f6222eb6..f078a4353d 100644 --- a/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs +++ b/osu.Game.Rulesets.Catch.Tests/TestSceneDrawableHitObjects.cs @@ -40,7 +40,7 @@ namespace osu.Game.Rulesets.Catch.Tests { Artist = @"Unknown", Title = @"You're breathtaking", - AuthorString = @"Everyone", + Author = { Username = @"Everyone" }, }, Ruleset = new CatchRuleset().RulesetInfo }, diff --git a/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs b/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs index 4bdb85ba60..54bc6f1912 100644 --- a/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs +++ b/osu.Game.Rulesets.Taiko.Tests/DrawableTaikoRulesetTestScene.cs @@ -37,7 +37,7 @@ namespace osu.Game.Rulesets.Taiko.Tests { Artist = @"Unknown", Title = @"Sample Beatmap", - AuthorString = @"peppy", + Author = { Username = @"peppy" }, }, Ruleset = new TaikoRuleset().RulesetInfo }, diff --git a/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs b/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs index b976735223..e73fb6bb1f 100644 --- a/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs +++ b/osu.Game.Rulesets.Taiko.Tests/Skinning/TestSceneDrawableTaikoMascot.cs @@ -163,7 +163,7 @@ namespace osu.Game.Rulesets.Taiko.Tests.Skinning { Artist = "Unknown", Title = "Sample Beatmap", - AuthorString = "Craftplacer", + Author = { Username = "Craftplacer" }, }, Ruleset = new TaikoRuleset().RulesetInfo }, diff --git a/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs b/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs index 3d78043c73..7ac26699a5 100644 --- a/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs +++ b/osu.Game.Tests/NonVisual/Filtering/FilterMatchingTest.cs @@ -30,7 +30,7 @@ namespace osu.Game.Tests.NonVisual.Filtering ArtistUnicode = "check unicode too", Title = "Title goes here", TitleUnicode = "Title goes here", - AuthorString = "The Author", + Author = { Username = "The Author" }, Source = "unit tests", Tags = "look for tags too", }, diff --git a/osu.Game.Tests/Resources/TestResources.cs b/osu.Game.Tests/Resources/TestResources.cs index 1d99a5c20d..e1da31abcb 100644 --- a/osu.Game.Tests/Resources/TestResources.cs +++ b/osu.Game.Tests/Resources/TestResources.cs @@ -89,7 +89,7 @@ namespace osu.Game.Tests.Resources // Create random metadata, then we can check if sorting works based on these Artist = "Some Artist " + RNG.Next(0, 9), Title = $"Some Song (set id {setId}) {Guid.NewGuid()}", - AuthorString = "Some Guy " + RNG.Next(0, 9), + Author = { Username = "Some Guy " + RNG.Next(0, 9) }, }; var beatmapSet = new BeatmapSetInfo diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs index 181b0c71f2..ae7705b97d 100644 --- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs +++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMultiplayerMatchSongSelect.cs @@ -52,7 +52,7 @@ namespace osu.Game.Tests.Visual.Multiplayer { Artist = "Some Artist", Title = "Some Beatmap", - AuthorString = "Some Author" + Author = { Username = "Some Author" }, }; var beatmapSetInfo = new BeatmapSetInfo diff --git a/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs b/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs index 69b30ec6a0..1b82094603 100644 --- a/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs +++ b/osu.Game.Tests/Visual/Navigation/TestScenePresentBeatmap.cs @@ -100,7 +100,7 @@ namespace osu.Game.Tests.Visual.Navigation var metadata = new BeatmapMetadata { Artist = "SomeArtist", - AuthorString = "SomeAuthor", + Author = { Username = "SomeAuthor" }, Title = $"import {i}" }; diff --git a/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs b/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs index f9649db135..b8e49d155e 100644 --- a/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs +++ b/osu.Game.Tests/Visual/Navigation/TestScenePresentScore.cs @@ -41,7 +41,7 @@ namespace osu.Game.Tests.Visual.Navigation Metadata = new BeatmapMetadata { Artist = "SomeArtist", - AuthorString = "SomeAuthor", + Author = { Username = "SomeAuthor" }, Title = "import" }, BaseDifficulty = new BeatmapDifficulty(), @@ -53,7 +53,7 @@ namespace osu.Game.Tests.Visual.Navigation Metadata = new BeatmapMetadata { Artist = "SomeArtist", - AuthorString = "SomeAuthor", + Author = { Username = "SomeAuthor" }, Title = "import" }, BaseDifficulty = new BeatmapDifficulty(), diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs index 0f5bea10e8..0298c3bea9 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs @@ -409,7 +409,7 @@ namespace osu.Game.Tests.Visual.SongSelect set.Beatmaps.ForEach(b => b.Metadata.Artist = zzz_string); if (i == 16) - set.Beatmaps.ForEach(b => b.Metadata.AuthorString = zzz_string); + set.Beatmaps.ForEach(b => b.Metadata.Author.Username = zzz_string); sets.Add(set); } diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs index 9ad5242df4..efacc395f1 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapInfoWedge.cs @@ -208,7 +208,7 @@ namespace osu.Game.Tests.Visual.SongSelect { Metadata = new BeatmapMetadata { - AuthorString = $"{ruleset.ShortName}Author", + Author = { Username = $"{ruleset.ShortName}Author" }, Artist = $"{ruleset.ShortName}Artist", Source = $"{ruleset.ShortName}Source", Title = $"{ruleset.ShortName}Title" @@ -230,7 +230,7 @@ namespace osu.Game.Tests.Visual.SongSelect { Metadata = new BeatmapMetadata { - AuthorString = "WWWWWWWWWWWWWWW", + Author = { Username = "WWWWWWWWWWWWWWW" }, Artist = "Verrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrry long Artist", Source = "Verrrrry long Source", Title = "Verrrrry long Title" diff --git a/osu.Game/Beatmaps/Beatmap.cs b/osu.Game/Beatmaps/Beatmap.cs index ccecd69d21..a9eb7da5c9 100644 --- a/osu.Game/Beatmaps/Beatmap.cs +++ b/osu.Game/Beatmaps/Beatmap.cs @@ -54,7 +54,7 @@ namespace osu.Game.Beatmaps { Artist = @"Unknown", Title = @"Unknown", - AuthorString = @"Unknown Creator", + Author = { Username = @"Unknown Creator" }, }, DifficultyName = @"Normal", BaseDifficulty = Difficulty, diff --git a/osu.Game/Beatmaps/BeatmapMetadata.cs b/osu.Game/Beatmaps/BeatmapMetadata.cs index 89f507581b..a3385e3abe 100644 --- a/osu.Game/Beatmaps/BeatmapMetadata.cs +++ b/osu.Game/Beatmaps/BeatmapMetadata.cs @@ -45,17 +45,6 @@ namespace osu.Game.Beatmaps IUser IBeatmapMetadataInfo.Author => Author; - #region Compatibility properties - - [Ignored] - public string AuthorString - { - get => Author.Username; - set => Author.Username = value; - } - - #endregion - public override string ToString() => this.GetDisplayTitle(); } } diff --git a/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs b/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs index e5db9d045a..893eb8ab78 100644 --- a/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs +++ b/osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs @@ -251,7 +251,7 @@ namespace osu.Game.Beatmaps.Formats break; case @"Creator": - metadata.AuthorString = pair.Value; + metadata.Author.Username = pair.Value; break; case @"Version": diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 7683accc5c..bd7a7677f2 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -144,7 +144,6 @@ namespace osu.Game.Database PreviewTime = metadata.PreviewTime, AudioFile = metadata.AudioFile, BackgroundFile = metadata.BackgroundFile, - AuthorString = metadata.AuthorString, }; } diff --git a/osu.Game/Screens/Edit/Setup/MetadataSection.cs b/osu.Game/Screens/Edit/Setup/MetadataSection.cs index 0d2b093a2e..f0ca3e1bbc 100644 --- a/osu.Game/Screens/Edit/Setup/MetadataSection.cs +++ b/osu.Game/Screens/Edit/Setup/MetadataSection.cs @@ -110,7 +110,7 @@ namespace osu.Game.Screens.Edit.Setup Beatmap.Metadata.TitleUnicode = TitleTextBox.Current.Value; Beatmap.Metadata.Title = RomanisedTitleTextBox.Current.Value; - Beatmap.Metadata.AuthorString = creatorTextBox.Current.Value; + Beatmap.Metadata.Author.Username = creatorTextBox.Current.Value; Beatmap.BeatmapInfo.DifficultyName = difficultyTextBox.Current.Value; Beatmap.Metadata.Source = sourceTextBox.Current.Value; Beatmap.Metadata.Tags = tagsTextBox.Current.Value; From 96d07e20edad425dd505fc70ec216f7c8160b2ab Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 02:17:43 +0900 Subject: [PATCH 12/19] Revert nunit test adaptor version bump until console output bug is resolved Tests have started to output too much log content, causing viewing CI failures to be painfully impossible. Roll back for now. Fix may be related to https://github.com/nunit/nunit3-vs-adapter/issues/941, although we don't use filter. --- .../osu.Game.Rulesets.EmptyFreeform.Tests.csproj | 2 +- .../osu.Game.Rulesets.Pippidon.Tests.csproj | 2 +- .../osu.Game.Rulesets.EmptyScrolling.Tests.csproj | 2 +- .../osu.Game.Rulesets.Pippidon.Tests.csproj | 2 +- osu.Game.Benchmarks/osu.Game.Benchmarks.csproj | 2 +- .../osu.Game.Rulesets.Catch.Tests.csproj | 2 +- .../osu.Game.Rulesets.Mania.Tests.csproj | 2 +- osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj | 2 +- .../osu.Game.Rulesets.Taiko.Tests.csproj | 2 +- osu.Game.Tests/osu.Game.Tests.csproj | 2 +- osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform.Tests/osu.Game.Rulesets.EmptyFreeform.Tests.csproj b/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform.Tests/osu.Game.Rulesets.EmptyFreeform.Tests.csproj index a7078f1c09..3c6aaa39ca 100644 --- a/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform.Tests/osu.Game.Rulesets.EmptyFreeform.Tests.csproj +++ b/Templates/Rulesets/ruleset-empty/osu.Game.Rulesets.EmptyFreeform.Tests/osu.Game.Rulesets.EmptyFreeform.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj b/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj index c133b0e3f8..0719dd30df 100644 --- a/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj +++ b/Templates/Rulesets/ruleset-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling.Tests/osu.Game.Rulesets.EmptyScrolling.Tests.csproj b/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling.Tests/osu.Game.Rulesets.EmptyScrolling.Tests.csproj index 92b48470e8..d0db43cc81 100644 --- a/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling.Tests/osu.Game.Rulesets.EmptyScrolling.Tests.csproj +++ b/Templates/Rulesets/ruleset-scrolling-empty/osu.Game.Rulesets.EmptyScrolling.Tests/osu.Game.Rulesets.EmptyScrolling.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj b/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj index c133b0e3f8..0719dd30df 100644 --- a/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj +++ b/Templates/Rulesets/ruleset-scrolling-example/osu.Game.Rulesets.Pippidon.Tests/osu.Game.Rulesets.Pippidon.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/osu.Game.Benchmarks/osu.Game.Benchmarks.csproj b/osu.Game.Benchmarks/osu.Game.Benchmarks.csproj index a8599f2cb6..57b914bee6 100644 --- a/osu.Game.Benchmarks/osu.Game.Benchmarks.csproj +++ b/osu.Game.Benchmarks/osu.Game.Benchmarks.csproj @@ -9,7 +9,7 @@ - + diff --git a/osu.Game.Rulesets.Catch.Tests/osu.Game.Rulesets.Catch.Tests.csproj b/osu.Game.Rulesets.Catch.Tests/osu.Game.Rulesets.Catch.Tests.csproj index d5496b7479..13f2e25f05 100644 --- a/osu.Game.Rulesets.Catch.Tests/osu.Game.Rulesets.Catch.Tests.csproj +++ b/osu.Game.Rulesets.Catch.Tests/osu.Game.Rulesets.Catch.Tests.csproj @@ -4,7 +4,7 @@ - + diff --git a/osu.Game.Rulesets.Mania.Tests/osu.Game.Rulesets.Mania.Tests.csproj b/osu.Game.Rulesets.Mania.Tests/osu.Game.Rulesets.Mania.Tests.csproj index 2f12b1535e..d51a6da4f9 100644 --- a/osu.Game.Rulesets.Mania.Tests/osu.Game.Rulesets.Mania.Tests.csproj +++ b/osu.Game.Rulesets.Mania.Tests/osu.Game.Rulesets.Mania.Tests.csproj @@ -4,7 +4,7 @@ - + diff --git a/osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj b/osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj index e5b2e070d8..fea2e408f6 100644 --- a/osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj +++ b/osu.Game.Rulesets.Osu.Tests/osu.Game.Rulesets.Osu.Tests.csproj @@ -5,7 +5,7 @@ - + diff --git a/osu.Game.Rulesets.Taiko.Tests/osu.Game.Rulesets.Taiko.Tests.csproj b/osu.Game.Rulesets.Taiko.Tests/osu.Game.Rulesets.Taiko.Tests.csproj index e48d80323a..ad3713e047 100644 --- a/osu.Game.Rulesets.Taiko.Tests/osu.Game.Rulesets.Taiko.Tests.csproj +++ b/osu.Game.Rulesets.Taiko.Tests/osu.Game.Rulesets.Taiko.Tests.csproj @@ -4,7 +4,7 @@ - + diff --git a/osu.Game.Tests/osu.Game.Tests.csproj b/osu.Game.Tests/osu.Game.Tests.csproj index c64ef918e3..3b115d43e5 100644 --- a/osu.Game.Tests/osu.Game.Tests.csproj +++ b/osu.Game.Tests/osu.Game.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj b/osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj index fb09a7be1e..130fcfaca1 100644 --- a/osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj +++ b/osu.Game.Tournament.Tests/osu.Game.Tournament.Tests.csproj @@ -7,7 +7,7 @@ - + WinExe From 761e161eec0d774aaae10ed4e3c5c042003385a9 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 09:44:21 +0900 Subject: [PATCH 13/19] Add comment explaining ignore rule --- osu.Game/Database/RealmObjectExtensions.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/osu.Game/Database/RealmObjectExtensions.cs b/osu.Game/Database/RealmObjectExtensions.cs index 5ab7154486..097e620e12 100644 --- a/osu.Game/Database/RealmObjectExtensions.cs +++ b/osu.Game/Database/RealmObjectExtensions.cs @@ -102,6 +102,7 @@ namespace osu.Game.Database c.CreateMap() .MaxDepth(1) + // This is not required as it will be populated in the `AfterMap` call from the `BeatmapInfo`'s parent. .ForMember(b => b.BeatmapSet, cc => cc.Ignore()); }).CreateMapper(); From 3bc091fe6d33cda1291e908b4818b0ddcf8ead64 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 09:46:45 +0900 Subject: [PATCH 14/19] Add ignore rules on more helper properties --- osu.Game/Input/Bindings/RealmKeyBinding.cs | 2 ++ osu.Game/Scoring/ScoreInfo.cs | 1 + 2 files changed, 3 insertions(+) diff --git a/osu.Game/Input/Bindings/RealmKeyBinding.cs b/osu.Game/Input/Bindings/RealmKeyBinding.cs index 6a408847fe..32813ada16 100644 --- a/osu.Game/Input/Bindings/RealmKeyBinding.cs +++ b/osu.Game/Input/Bindings/RealmKeyBinding.cs @@ -20,12 +20,14 @@ namespace osu.Game.Input.Bindings public int? Variant { get; set; } + [Ignored] public KeyCombination KeyCombination { get => KeyCombinationString; set => KeyCombinationString = value.ToString(); } + [Ignored] public object Action { get => ActionInt; diff --git a/osu.Game/Scoring/ScoreInfo.cs b/osu.Game/Scoring/ScoreInfo.cs index e1328d8a06..b268e2cd91 100644 --- a/osu.Game/Scoring/ScoreInfo.cs +++ b/osu.Game/Scoring/ScoreInfo.cs @@ -104,6 +104,7 @@ namespace osu.Game.Scoring } } + [Ignored] public ScoreRank Rank { get => (ScoreRank)RankInt; From 64a023665e9944f93615de503de190a64144bcf2 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:16:54 +0900 Subject: [PATCH 15/19] Avoid taking more than one backup per migration run --- osu.Game/Database/EFToRealmMigrator.cs | 30 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index d479f81ead..b7f3eebe7a 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -24,6 +24,8 @@ namespace osu.Game.Database private readonly RealmContextFactory realmContextFactory; private readonly OsuConfigManager config; + private bool hasTakenBackup; + public EFToRealmMigrator(DatabaseContextFactory efContextFactory, RealmContextFactory realmContextFactory, OsuConfigManager config) { this.efContextFactory = efContextFactory; @@ -70,8 +72,16 @@ namespace osu.Game.Database using (var realm = realmContextFactory.CreateContext()) { Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database); - string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; - efContextFactory.CreateBackup($"client.{migration}.db"); + + if (!hasTakenBackup) + { + string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + + efContextFactory.CreateBackup($"client.{migration}.db"); + realmContextFactory.CreateBackup($"client.{migration}.realm"); + + hasTakenBackup = true; + } // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. @@ -81,8 +91,6 @@ namespace osu.Game.Database } else { - realmContextFactory.CreateBackup($"client.{migration}.realm"); - using (var transaction = realm.BeginWrite()) { foreach (var beatmapSet in existingBeatmapSets) @@ -195,8 +203,16 @@ namespace osu.Game.Database using (var realm = realmContextFactory.CreateContext()) { Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database); - string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; - efContextFactory.CreateBackup($"client.{migration}.db"); + + if (!hasTakenBackup) + { + string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}"; + + efContextFactory.CreateBackup($"client.{migration}.db"); + realmContextFactory.CreateBackup($"client.{migration}.realm"); + + hasTakenBackup = true; + } // only migrate data if the realm database is empty. // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. @@ -206,8 +222,6 @@ namespace osu.Game.Database } else { - realmContextFactory.CreateBackup($"client.{migration}.realm"); - using (var transaction = realm.BeginWrite()) { foreach (var score in existingScores) From 04e9ffa966afd6b95698591de468006302328441 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:20:43 +0900 Subject: [PATCH 16/19] Freshen some comments --- osu.Game/Database/EFToRealmMigrator.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index b7f3eebe7a..6b911ebad0 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -84,7 +84,7 @@ namespace osu.Game.Database } // only migrate data if the realm database is empty. - // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. + // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (realm.All().Any(s => !s.Protected)) { Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database); @@ -215,7 +215,6 @@ namespace osu.Game.Database } // only migrate data if the realm database is empty. - // note that this cannot be written as: `realm.All().All(s => s.Protected)`, because realm does not support `.All()`. if (realm.All().Any()) { Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database); From e1a35714be3720178f2a56ef88bcdb038946f04a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:30:17 +0900 Subject: [PATCH 17/19] Add notification for debug builds when database migration occurs --- osu.Game/Database/DatabaseContextFactory.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index cc690a9fda..635c4373cd 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Threading; using Microsoft.EntityFrameworkCore.Storage; +using osu.Framework.Development; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Statistics; @@ -146,11 +147,15 @@ namespace osu.Game.Database Database = { AutoTransactionsEnabled = false } }; - public void CreateBackup(string filename) + public void CreateBackup(string backupFilename) { - Logger.Log($"Creating full EF database backup at {filename}", LoggingTarget.Database); + Logger.Log($"Creating full EF database backup at {backupFilename}", LoggingTarget.Database); + + if (DebugUtils.IsDebugBuild) + Logger.Log("Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.", level: LogLevel.Important); + using (var source = storage.GetStream(DATABASE_NAME)) - using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); } From 195534a1d2e0a890be7b16feb3e936ed2fe426c4 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 19 Jan 2022 10:31:09 +0900 Subject: [PATCH 18/19] Only output "successful" messages when copy actually occurred --- osu.Game/Database/EFToRealmMigrator.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/EFToRealmMigrator.cs b/osu.Game/Database/EFToRealmMigrator.cs index 6b911ebad0..76e5219f87 100644 --- a/osu.Game/Database/EFToRealmMigrator.cs +++ b/osu.Game/Database/EFToRealmMigrator.cs @@ -148,10 +148,10 @@ namespace osu.Game.Database } transaction.Commit(); + Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); } } - Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database); ef.Context.RemoveRange(existingBeatmapSets); // Intentionally don't clean up the files, so they don't get purged by EF. } @@ -257,10 +257,10 @@ namespace osu.Game.Database } transaction.Commit(); + Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); } } - Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database); db.Context.RemoveRange(existingScores); // Intentionally don't clean up the files, so they don't get purged by EF. } From c52899b1fb0f6a6e52d73ee93d74247b931bc6f4 Mon Sep 17 00:00:00 2001 From: Dan Balasescu Date: Wed, 19 Jan 2022 11:56:44 +0900 Subject: [PATCH 19/19] Rename property --- osu.Game/Database/RealmContextFactory.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 10c2438df9..8be8eab567 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -361,13 +361,13 @@ namespace osu.Game.Database private string? getRulesetShortNameFromLegacyID(long rulesetId) => efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName; - public void CreateBackup(string filename) + public void CreateBackup(string backupFilename) { using (BlockAllOperations()) { - Logger.Log($"Creating full realm database backup at {filename}", LoggingTarget.Database); + Logger.Log($"Creating full realm database backup at {backupFilename}", LoggingTarget.Database); using (var source = storage.GetStream(Filename)) - using (var destination = storage.GetStream(filename, FileAccess.Write, FileMode.CreateNew)) + using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew)) source.CopyTo(destination); } }