sim1222/osukey
1
1
Fork 0
mirror of https://github.com/osukey/osukey.git synced 2025-08-03 22:56:36 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into realm/remove-ef-migrations

Browse Source
This commit is contained in:
Dan Balasescu
2022-01-19 14:01:34 +09:00
committed by GitHub
parent 6ddd2d59d3 3e5d29ed00
commit 3407b34b11
59 changed files with 380 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
osu.Game/Database/DatabaseContextFactory.cs
View File

@ -1,9 +1,12 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System.IO;
using System.Linq;
using System.Threading;
using Microsoft.EntityFrameworkCore.Storage;
using osu.Framework.Development;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Framework.Statistics;
@ -144,6 +147,18 @@ namespace osu.Game.Database
Database = { AutoTransactionsEnabled = false }
};
public void CreateBackup(string backupFilename)
{
Logger.Log($"Creating full EF database backup at {backupFilename}", LoggingTarget.Database);
if (DebugUtils.IsDebugBuild)
Logger.Log("Your development database has been fully migrated to realm. If you switch back to a pre-realm branch and need your previous database, rename the backup file back to \"client.db\".\n\nNote that doing this can potentially leave your file store in a bad state.", level: LogLevel.Important);
using (var source = storage.GetStream(DATABASE_NAME))
using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew))
source.CopyTo(destination);
}
public void ResetDatabase()
{
lock (writeLock)

275
osu.Game/Database/EFToRealmMigrator.cs
View File

@ -1,8 +1,11 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.EntityFrameworkCore;
using osu.Framework.Logging;
using osu.Game.Beatmaps;
using osu.Game.Configuration;
using osu.Game.Models;
@ -21,6 +24,8 @@ namespace osu.Game.Database
private readonly RealmContextFactory realmContextFactory;
private readonly OsuConfigManager config;
private bool hasTakenBackup;
public EFToRealmMigrator(DatabaseContextFactory efContextFactory, RealmContextFactory realmContextFactory, OsuConfigManager config)
{
this.efContextFactory = efContextFactory;
@ -30,97 +35,125 @@ namespace osu.Game.Database
public void Run()
{
using (var db = efContextFactory.GetForWrite())
using (var ef = efContextFactory.GetForWrite())
{
migrateSettings(db);
migrateSkins(db);
migrateBeatmaps(db);
migrateScores(db);
migrateSettings(ef);
migrateSkins(ef);
migrateBeatmaps(ef);
migrateScores(ef);
}
// Delete the database permanently.
// Will cause future startups to not attempt migration.
Logger.Log("Migration successful, deleting EF database", LoggingTarget.Database);
efContextFactory.ResetDatabase();
}
private void migrateBeatmaps(DatabaseWriteUsage db)
private void migrateBeatmaps(DatabaseWriteUsage ef)
{
// can be removed 20220730.
var existingBeatmapSets = db.Context.EFBeatmapSetInfo
.Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo)
.Include(s => s.Beatmaps).ThenInclude(b => b.Metadata)
.Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty)
.Include(s => s.Files).ThenInclude(f => f.FileInfo)
.Include(s => s.Metadata)
.ToList();
List<EFBeatmapSetInfo> existingBeatmapSets = ef.Context.EFBeatmapSetInfo
.Include(s => s.Beatmaps).ThenInclude(b => b.RulesetInfo)
.Include(s => s.Beatmaps).ThenInclude(b => b.Metadata)
.Include(s => s.Beatmaps).ThenInclude(b => b.BaseDifficulty)
.Include(s => s.Files).ThenInclude(f => f.FileInfo)
.Include(s => s.Metadata)
.ToList();
Logger.Log("Beginning beatmaps migration to realm", LoggingTarget.Database);
// previous entries in EF are removed post migration.
if (!existingBeatmapSets.Any())
{
Logger.Log("No beatmaps found to migrate", LoggingTarget.Database);
return;
}
using (var realm = realmContextFactory.CreateContext())
using (var transaction = realm.BeginWrite())
{
// only migrate data if the realm database is empty.
// note that this cannot be written as: `realm.All<BeatmapInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (!realm.All<BeatmapSetInfo>().Any(s => !s.Protected))
Logger.Log($"Found {existingBeatmapSets.Count} beatmaps in EF", LoggingTarget.Database);
if (!hasTakenBackup)
{
foreach (var beatmapSet in existingBeatmapSets)
string migration = $"before_beatmap_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
efContextFactory.CreateBackup($"client.{migration}.db");
realmContextFactory.CreateBackup($"client.{migration}.realm");
hasTakenBackup = true;
}
// only migrate data if the realm database is empty.
// note that this cannot be written as: `realm.All<BeatmapSetInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (realm.All<BeatmapSetInfo>().Any(s => !s.Protected))
{
Logger.Log("Skipping migration as realm already has beatmaps loaded", LoggingTarget.Database);
}
else
{
using (var transaction = realm.BeginWrite())
{
var realmBeatmapSet = new BeatmapSetInfo
foreach (var beatmapSet in existingBeatmapSets)
{
OnlineID = beatmapSet.OnlineID ?? -1,
DateAdded = beatmapSet.DateAdded,
Status = beatmapSet.Status,
DeletePending = beatmapSet.DeletePending,
Hash = beatmapSet.Hash,
Protected = beatmapSet.Protected,
};
migrateFiles(beatmapSet, realm, realmBeatmapSet);
foreach (var beatmap in beatmapSet.Beatmaps)
{
var realmBeatmap = new BeatmapInfo
var realmBeatmapSet = new BeatmapSetInfo
{
DifficultyName = beatmap.DifficultyName,
Status = beatmap.Status,
OnlineID = beatmap.OnlineID ?? -1,
Length = beatmap.Length,
BPM = beatmap.BPM,
Hash = beatmap.Hash,
StarRating = beatmap.StarRating,
MD5Hash = beatmap.MD5Hash,
Hidden = beatmap.Hidden,
AudioLeadIn = beatmap.AudioLeadIn,
StackLeniency = beatmap.StackLeniency,
SpecialStyle = beatmap.SpecialStyle,
LetterboxInBreaks = beatmap.LetterboxInBreaks,
WidescreenStoryboard = beatmap.WidescreenStoryboard,
EpilepsyWarning = beatmap.EpilepsyWarning,
SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate,
DistanceSpacing = beatmap.DistanceSpacing,
BeatDivisor = beatmap.BeatDivisor,
GridSize = beatmap.GridSize,
TimelineZoom = beatmap.TimelineZoom,
Countdown = beatmap.Countdown,
CountdownOffset = beatmap.CountdownOffset,
MaxCombo = beatmap.MaxCombo,
Bookmarks = beatmap.Bookmarks,
Ruleset = realm.Find<RulesetInfo>(beatmap.RulesetInfo.ShortName),
Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty),
Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata),
BeatmapSet = realmBeatmapSet,
OnlineID = beatmapSet.OnlineID ?? -1,
DateAdded = beatmapSet.DateAdded,
Status = beatmapSet.Status,
DeletePending = beatmapSet.DeletePending,
Hash = beatmapSet.Hash,
Protected = beatmapSet.Protected,
};
realmBeatmapSet.Beatmaps.Add(realmBeatmap);
migrateFiles(beatmapSet, realm, realmBeatmapSet);
foreach (var beatmap in beatmapSet.Beatmaps)
{
var realmBeatmap = new BeatmapInfo
{
DifficultyName = beatmap.DifficultyName,
Status = beatmap.Status,
OnlineID = beatmap.OnlineID ?? -1,
Length = beatmap.Length,
BPM = beatmap.BPM,
Hash = beatmap.Hash,
StarRating = beatmap.StarRating,
MD5Hash = beatmap.MD5Hash,
Hidden = beatmap.Hidden,
AudioLeadIn = beatmap.AudioLeadIn,
StackLeniency = beatmap.StackLeniency,
SpecialStyle = beatmap.SpecialStyle,
LetterboxInBreaks = beatmap.LetterboxInBreaks,
WidescreenStoryboard = beatmap.WidescreenStoryboard,
EpilepsyWarning = beatmap.EpilepsyWarning,
SamplesMatchPlaybackRate = beatmap.SamplesMatchPlaybackRate,
DistanceSpacing = beatmap.DistanceSpacing,
BeatDivisor = beatmap.BeatDivisor,
GridSize = beatmap.GridSize,
TimelineZoom = beatmap.TimelineZoom,
Countdown = beatmap.Countdown,
CountdownOffset = beatmap.CountdownOffset,
MaxCombo = beatmap.MaxCombo,
Bookmarks = beatmap.Bookmarks,
Ruleset = realm.Find<RulesetInfo>(beatmap.RulesetInfo.ShortName),
Difficulty = new BeatmapDifficulty(beatmap.BaseDifficulty),
Metadata = getBestMetadata(beatmap.Metadata, beatmapSet.Metadata),
BeatmapSet = realmBeatmapSet,
};
realmBeatmapSet.Beatmaps.Add(realmBeatmap);
}
realm.Add(realmBeatmapSet);
}
realm.Add(realmBeatmapSet);
transaction.Commit();
Logger.Log($"Successfully migrated {existingBeatmapSets.Count} beatmaps to realm", LoggingTarget.Database);
}
}
db.Context.RemoveRange(existingBeatmapSets);
ef.Context.RemoveRange(existingBeatmapSets);
// Intentionally don't clean up the files, so they don't get purged by EF.
transaction.Commit();
}
}
@ -144,69 +177,91 @@ namespace osu.Game.Database
PreviewTime = metadata.PreviewTime,
AudioFile = metadata.AudioFile,
BackgroundFile = metadata.BackgroundFile,
AuthorString = metadata.AuthorString,
};
}
private void migrateScores(DatabaseWriteUsage db)
{
// can be removed 20220730.
var existingScores = db.Context.ScoreInfo
.Include(s => s.Ruleset)
.Include(s => s.BeatmapInfo)
.Include(s => s.Files)
.ThenInclude(f => f.FileInfo)
.ToList();
List<EFScoreInfo> existingScores = db.Context.ScoreInfo
.Include(s => s.Ruleset)
.Include(s => s.BeatmapInfo)
.Include(s => s.Files)
.ThenInclude(f => f.FileInfo)
.ToList();
Logger.Log("Beginning scores migration to realm", LoggingTarget.Database);
// previous entries in EF are removed post migration.
if (!existingScores.Any())
{
Logger.Log("No scores found to migrate", LoggingTarget.Database);
return;
}
using (var realm = realmContextFactory.CreateContext())
using (var transaction = realm.BeginWrite())
{
// only migrate data if the realm database is empty.
// note that this cannot be written as: `realm.All<ScoreInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (!realm.All<ScoreInfo>().Any())
Logger.Log($"Found {existingScores.Count} scores in EF", LoggingTarget.Database);
if (!hasTakenBackup)
{
foreach (var score in existingScores)
string migration = $"before_score_migration_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
efContextFactory.CreateBackup($"client.{migration}.db");
realmContextFactory.CreateBackup($"client.{migration}.realm");
hasTakenBackup = true;
}
// only migrate data if the realm database is empty.
if (realm.All<ScoreInfo>().Any())
{
Logger.Log("Skipping migration as realm already has scores loaded", LoggingTarget.Database);
}
else
{
using (var transaction = realm.BeginWrite())
{
var realmScore = new ScoreInfo
foreach (var score in existingScores)
{
Hash = score.Hash,
DeletePending = score.DeletePending,
OnlineID = score.OnlineID ?? -1,
ModsJson = score.ModsJson,
StatisticsJson = score.StatisticsJson,
User = score.User,
TotalScore = score.TotalScore,
MaxCombo = score.MaxCombo,
Accuracy = score.Accuracy,
HasReplay = ((IScoreInfo)score).HasReplay,
Date = score.Date,
PP = score.PP,
BeatmapInfo = realm.All<BeatmapInfo>().First(b => b.Hash == score.BeatmapInfo.Hash),
Ruleset = realm.Find<RulesetInfo>(score.Ruleset.ShortName),
Rank = score.Rank,
HitEvents = score.HitEvents,
Passed = score.Passed,
Combo = score.Combo,
Position = score.Position,
Statistics = score.Statistics,
Mods = score.Mods,
APIMods = score.APIMods,
};
var realmScore = new ScoreInfo
{
Hash = score.Hash,
DeletePending = score.DeletePending,
OnlineID = score.OnlineID ?? -1,
ModsJson = score.ModsJson,
StatisticsJson = score.StatisticsJson,
User = score.User,
TotalScore = score.TotalScore,
MaxCombo = score.MaxCombo,
Accuracy = score.Accuracy,
HasReplay = ((IScoreInfo)score).HasReplay,
Date = score.Date,
PP = score.PP,
BeatmapInfo = realm.All<BeatmapInfo>().First(b => b.Hash == score.BeatmapInfo.Hash),
Ruleset = realm.Find<RulesetInfo>(score.Ruleset.ShortName),
Rank = score.Rank,
HitEvents = score.HitEvents,
Passed = score.Passed,
Combo = score.Combo,
Position = score.Position,
Statistics = score.Statistics,
Mods = score.Mods,
APIMods = score.APIMods,
};
migrateFiles(score, realm, realmScore);
migrateFiles(score, realm, realmScore);
realm.Add(realmScore);
realm.Add(realmScore);
}
transaction.Commit();
Logger.Log($"Successfully migrated {existingScores.Count} scores to realm", LoggingTarget.Database);
}
}
db.Context.RemoveRange(existingScores);
// Intentionally don't clean up the files, so they don't get purged by EF.
transaction.Commit();
}
}
@ -243,6 +298,8 @@ namespace osu.Game.Database
// note that this cannot be written as: `realm.All<SkinInfo>().All(s => s.Protected)`, because realm does not support `.All()`.
if (!realm.All<SkinInfo>().Any(s => !s.Protected))
{
Logger.Log($"Migrating {existingSkins.Count} skins", LoggingTarget.Database);
foreach (var skin in existingSkins)
{
var realmSkin = new SkinInfo
@ -286,18 +343,22 @@ namespace osu.Game.Database
private void migrateSettings(DatabaseWriteUsage db)
{
// migrate ruleset settings. can be removed 20220315.
var existingSettings = db.Context.DatabasedSetting;
var existingSettings = db.Context.DatabasedSetting.ToList();
// previous entries in EF are removed post migration.
if (!existingSettings.Any())
return;
Logger.Log("Beginning settings migration to realm", LoggingTarget.Database);
using (var realm = realmContextFactory.CreateContext())
using (var transaction = realm.BeginWrite())
{
// only migrate data if the realm database is empty.
if (!realm.All<RealmRulesetSetting>().Any())
{
Logger.Log($"Migrating {existingSettings.Count} settings", LoggingTarget.Database);
foreach (var dkb in existingSettings)
{
if (dkb.RulesetID == null)

59
osu.Game/Database/RealmContextFactory.cs
View File

@ -21,6 +21,7 @@ using osu.Game.Stores;
using osu.Game.Rulesets;
using osu.Game.Scoring;
using Realms;
using Realms.Exceptions;
#nullable enable
@ -105,8 +106,20 @@ namespace osu.Game.Database
if (!Filename.EndsWith(realm_extension, StringComparison.Ordinal))
Filename += realm_extension;
// This method triggers the first `CreateContext` call, which will implicitly run realm migrations and bring the schema up-to-date.
cleanupPendingDeletions();
try
{
// This method triggers the first `CreateContext` call, which will implicitly run realm migrations and bring the schema up-to-date.
cleanupPendingDeletions();
}
catch (Exception e)
{
Logger.Error(e, "Realm startup failed with unrecoverable error; starting with a fresh database. A backup of your database has been made.");
CreateBackup($"{Filename.Replace(realm_extension, string.Empty)}_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_corrupt{realm_extension}");
storage.Delete(Filename);
cleanupPendingDeletions();
}
}
private void cleanupPendingDeletions()
@ -361,6 +374,17 @@ namespace osu.Game.Database
private string? getRulesetShortNameFromLegacyID(long rulesetId) =>
efContextFactory?.Get().RulesetInfo.FirstOrDefault(r => r.ID == rulesetId)?.ShortName;
public void CreateBackup(string backupFilename)
{
using (BlockAllOperations())
{
Logger.Log($"Creating full realm database backup at {backupFilename}", LoggingTarget.Database);
using (var source = storage.GetStream(Filename))
using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew))
source.CopyTo(destination);
}
}
/// <summary>
/// Flush any active contexts and block any further writes.
/// </summary>
@ -374,17 +398,17 @@ namespace osu.Game.Database
if (isDisposed)
throw new ObjectDisposedException(nameof(RealmContextFactory));
if (!ThreadSafety.IsUpdateThread)
throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread.");
Logger.Log(@"Blocking realm operations.", LoggingTarget.Database);
try
{
contextCreationLock.Wait();
lock (contextLock)
{
if (!ThreadSafety.IsUpdateThread && context != null)
throw new InvalidOperationException(@$"{nameof(BlockAllOperations)} must be called from the update thread.");
Logger.Log(@"Blocking realm operations.", LoggingTarget.Database);
context?.Dispose();
context = null;
}
@ -392,14 +416,23 @@ namespace osu.Game.Database
const int sleep_length = 200;
int timeout = 5000;
// see https://github.com/realm/realm-dotnet/discussions/2657
while (!Compact())
try
{
Thread.Sleep(sleep_length);
timeout -= sleep_length;
// see https://github.com/realm/realm-dotnet/discussions/2657
while (!Compact())
{
Thread.Sleep(sleep_length);
timeout -= sleep_length;
if (timeout < 0)
throw new TimeoutException(@"Took too long to acquire lock");
if (timeout < 0)
throw new TimeoutException(@"Took too long to acquire lock");
}
}
catch (RealmException e)
{
// Compact may fail if the realm is in a bad state.
// We still want to continue with the blocking operation, though.
Logger.Log($"Realm compact failed with error {e}", LoggingTarget.Database);
}
}
catch

62
osu.Game/Database/RealmObjectExtensions.cs
View File

@ -74,21 +74,18 @@ namespace osu.Game.Database
private static readonly IMapper mapper = new MapperConfiguration(c =>
{
c.ShouldMapField = fi => false;
applyCommonConfiguration(c);
// This is specifically to avoid mapping explicit interface implementations.
// If we want to limit this further, we can avoid mapping properties with no setter that are not IList<>.
// Takes a bit of effort to determine whether this is the case though, see https://stackoverflow.com/questions/951536/how-do-i-tell-whether-a-type-implements-ilist
c.ShouldMapProperty = pi => pi.GetMethod?.IsPublic == true;
c.CreateMap<BeatmapSetInfo, BeatmapSetInfo>()
.MaxDepth(2)
.AfterMap((s, d) =>
{
foreach (var beatmap in d.Beatmaps)
beatmap.BeatmapSet = d;
});
c.CreateMap<RealmKeyBinding, RealmKeyBinding>();
c.CreateMap<BeatmapMetadata, BeatmapMetadata>();
c.CreateMap<BeatmapDifficulty, BeatmapDifficulty>();
c.CreateMap<RulesetInfo, RulesetInfo>();
c.CreateMap<ScoreInfo, ScoreInfo>();
c.CreateMap<RealmUser, RealmUser>();
c.CreateMap<RealmFile, RealmFile>();
c.CreateMap<RealmNamedFileUsage, RealmNamedFileUsage>();
// This can be further optimised to reduce cyclic retrievals, similar to the optimised set mapper below.
// Only hasn't been done yet as we detach at the point of BeatmapInfo less often.
c.CreateMap<BeatmapInfo, BeatmapInfo>()
.MaxDepth(2)
.AfterMap((s, d) =>
@ -102,14 +99,39 @@ namespace osu.Game.Database
}
}
});
}).CreateMapper();
/// <summary>
/// A slightly optimised mapper that avoids double-fetches in cyclic reference.
/// </summary>
private static readonly IMapper beatmap_set_mapper = new MapperConfiguration(c =>
{
applyCommonConfiguration(c);
c.CreateMap<BeatmapSetInfo, BeatmapSetInfo>()
.MaxDepth(2)
.ForMember(b => b.Files, cc => cc.Ignore())
.AfterMap((s, d) =>
{
foreach (var beatmap in d.Beatmaps)
beatmap.BeatmapSet = d;
});
c.CreateMap<BeatmapInfo, BeatmapInfo>()
.MaxDepth(1)
// This is not required as it will be populated in the `AfterMap` call from the `BeatmapInfo`'s parent.
.ForMember(b => b.BeatmapSet, cc => cc.Ignore());
}).CreateMapper();
private static void applyCommonConfiguration(IMapperConfigurationExpression c)
{
c.ShouldMapField = fi => false;
// This is specifically to avoid mapping explicit interface implementations.
// If we want to limit this further, we can avoid mapping properties with no setter that are not IList<>.
// Takes a bit of effort to determine whether this is the case though, see https://stackoverflow.com/questions/951536/how-do-i-tell-whether-a-type-implements-ilist
c.ShouldMapProperty = pi => pi.GetMethod?.IsPublic == true;
c.Internal().ForAllMaps((typeMap, expression) =>
{
expression.ForAllMembers(m =>
@ -118,7 +140,16 @@ namespace osu.Game.Database
m.Ignore();
});
});
}).CreateMapper();
c.CreateMap<RealmKeyBinding, RealmKeyBinding>();
c.CreateMap<BeatmapMetadata, BeatmapMetadata>();
c.CreateMap<BeatmapDifficulty, BeatmapDifficulty>();
c.CreateMap<RulesetInfo, RulesetInfo>();
c.CreateMap<ScoreInfo, ScoreInfo>();
c.CreateMap<RealmUser, RealmUser>();
c.CreateMap<RealmFile, RealmFile>();
c.CreateMap<RealmNamedFileUsage, RealmNamedFileUsage>();
}
/// <summary>
/// Create a detached copy of the each item in the collection.
@ -153,6 +184,9 @@ namespace osu.Game.Database
if (!item.IsManaged)
return item;
if (item is BeatmapSetInfo)
return beatmap_set_mapper.Map<T>(item);
return mapper.Map<T>(item);
}