From f090e292c974c2147f7f34b3cc79a5461e7df817 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Tue, 28 May 2019 18:59:21 +0900 Subject: [PATCH 01/27] Move ArchiveModelManager import process to async flow --- .../Beatmaps/IO/ImportBeatmapTest.cs | 50 +++--- osu.Game.Tests/Scores/IO/ImportScoreTest.cs | 30 ++-- .../TestSceneBackgroundScreenBeatmap.cs | 2 +- ...tSceneUpdateableBeatmapBackgroundSprite.cs | 2 +- osu.Game.Tests/osu.Game.Tests.csproj | 5 + osu.Game/Beatmaps/BeatmapManager.cs | 163 +++++++++++++----- osu.Game/Database/ArchiveModelManager.cs | 158 +++++++++-------- osu.Game/Database/ICanAcceptFiles.cs | 4 +- osu.Game/IPC/ArchiveImportIPCChannel.cs | 2 +- osu.Game/OsuGameBase.cs | 5 +- .../Notifications/ProgressNotification.cs | 7 + osu.Game/Screens/Menu/Intro.cs | 2 +- osu.Game/Screens/Play/Player.cs | 2 +- osu.Game/Skinning/SkinManager.cs | 6 +- 14 files changed, 273 insertions(+), 165 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index f020c2a805..4c9260f640 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -21,14 +21,14 @@ namespace osu.Game.Tests.Beatmaps.IO public class ImportBeatmapTest { [Test] - public void TestImportWhenClosed() + public async Task TestImportWhenClosed() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportWhenClosed")) { try { - LoadOszIntoOsu(loadOsu(host)); + await LoadOszIntoOsu(loadOsu(host)); } finally { @@ -38,7 +38,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportThenDelete() + public async Task TestImportThenDelete() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenDelete")) @@ -47,7 +47,7 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); deleteBeatmapSet(imported, osu); } @@ -59,7 +59,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportThenImport() + public async Task TestImportThenImport() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenImport")) @@ -68,8 +68,8 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); - var imported = LoadOszIntoOsu(osu); - var importedSecondTime = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); + var importedSecondTime = await LoadOszIntoOsu(osu); // check the newly "imported" beatmap is actually just the restored previous import. since it matches hash. Assert.IsTrue(imported.ID == importedSecondTime.ID); @@ -88,7 +88,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestRollbackOnFailure() + public async Task TestRollbackOnFailure() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestRollbackOnFailure")) @@ -104,7 +104,7 @@ namespace osu.Game.Tests.Beatmaps.IO manager.ItemAdded += (_, __) => fireCount++; manager.ItemRemoved += _ => fireCount++; - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); Assert.AreEqual(0, fireCount -= 1); @@ -132,7 +132,7 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); // this will trigger purging of the existing beatmap (online set id match) but should rollback due to broken osu. - manager.Import(breakTemp); + await manager.Import(breakTemp); // no events should be fired in the case of a rollback. Assert.AreEqual(0, fireCount); @@ -149,7 +149,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportThenImportDifferentHash() + public async Task TestImportThenImportDifferentHash() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenImportDifferentHash")) @@ -159,12 +159,12 @@ namespace osu.Game.Tests.Beatmaps.IO var osu = loadOsu(host); var manager = osu.Dependencies.Get(); - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); imported.Hash += "-changed"; manager.Update(imported); - var importedSecondTime = LoadOszIntoOsu(osu); + var importedSecondTime = await LoadOszIntoOsu(osu); Assert.IsTrue(imported.ID != importedSecondTime.ID); Assert.IsTrue(imported.Beatmaps.First().ID < importedSecondTime.Beatmaps.First().ID); @@ -181,7 +181,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportThenDeleteThenImport() + public async Task TestImportThenDeleteThenImport() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenDeleteThenImport")) @@ -190,11 +190,11 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); deleteBeatmapSet(imported, osu); - var importedSecondTime = LoadOszIntoOsu(osu); + var importedSecondTime = await LoadOszIntoOsu(osu); // check the newly "imported" beatmap is actually just the restored previous import. since it matches hash. Assert.IsTrue(imported.ID == importedSecondTime.ID); @@ -209,7 +209,7 @@ namespace osu.Game.Tests.Beatmaps.IO [TestCase(true)] [TestCase(false)] - public void TestImportThenDeleteThenImportWithOnlineIDMismatch(bool set) + public async Task TestImportThenDeleteThenImportWithOnlineIDMismatch(bool set) { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost($"TestImportThenDeleteThenImport-{set}")) @@ -218,7 +218,7 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); if (set) imported.OnlineBeatmapSetID = 1234; @@ -229,7 +229,7 @@ namespace osu.Game.Tests.Beatmaps.IO deleteBeatmapSet(imported, osu); - var importedSecondTime = LoadOszIntoOsu(osu); + var importedSecondTime = await LoadOszIntoOsu(osu); // check the newly "imported" beatmap has been reimported due to mismatch (even though hashes matched) Assert.IsTrue(imported.ID != importedSecondTime.ID); @@ -243,7 +243,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportWithDuplicateBeatmapIDs() + public async Task TestImportWithDuplicateBeatmapIDs() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportWithDuplicateBeatmapID")) @@ -284,7 +284,7 @@ namespace osu.Game.Tests.Beatmaps.IO var manager = osu.Dependencies.Get(); - var imported = manager.Import(toImport); + var imported = await manager.Import(toImport); Assert.NotNull(imported); Assert.AreEqual(null, imported.Beatmaps[0].OnlineBeatmapID); @@ -330,7 +330,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportWhenFileOpen() + public async Task TestImportWhenFileOpen() { using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportWhenFileOpen")) { @@ -339,7 +339,7 @@ namespace osu.Game.Tests.Beatmaps.IO var osu = loadOsu(host); var temp = TestResources.GetTestBeatmapForImport(); using (File.OpenRead(temp)) - osu.Dependencies.Get().Import(temp); + await osu.Dependencies.Get().Import(temp); ensureLoaded(osu); File.Delete(temp); Assert.IsFalse(File.Exists(temp), "We likely held a read lock on the file when we shouldn't"); @@ -351,13 +351,13 @@ namespace osu.Game.Tests.Beatmaps.IO } } - public static BeatmapSetInfo LoadOszIntoOsu(OsuGameBase osu, string path = null) + public static async Task LoadOszIntoOsu(OsuGameBase osu, string path = null) { var temp = path ?? TestResources.GetTestBeatmapForImport(); var manager = osu.Dependencies.Get(); - manager.Import(temp); + await manager.Import(temp); var imported = manager.GetAllUsableBeatmapSets(); diff --git a/osu.Game.Tests/Scores/IO/ImportScoreTest.cs b/osu.Game.Tests/Scores/IO/ImportScoreTest.cs index e39f18c3cd..4babb07213 100644 --- a/osu.Game.Tests/Scores/IO/ImportScoreTest.cs +++ b/osu.Game.Tests/Scores/IO/ImportScoreTest.cs @@ -23,13 +23,13 @@ namespace osu.Game.Tests.Scores.IO public class ImportScoreTest { [Test] - public void TestBasicImport() + public async Task TestBasicImport() { using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestBasicImport")) { try { - var osu = loadOsu(host); + var osu = await loadOsu(host); var toImport = new ScoreInfo { @@ -43,7 +43,7 @@ namespace osu.Game.Tests.Scores.IO OnlineScoreID = 12345, }; - var imported = loadIntoOsu(osu, toImport); + var imported = await loadIntoOsu(osu, toImport); Assert.AreEqual(toImport.Rank, imported.Rank); Assert.AreEqual(toImport.TotalScore, imported.TotalScore); @@ -62,20 +62,20 @@ namespace osu.Game.Tests.Scores.IO } [Test] - public void TestImportMods() + public async Task TestImportMods() { using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportMods")) { try { - var osu = loadOsu(host); + var osu = await loadOsu(host); var toImport = new ScoreInfo { Mods = new Mod[] { new OsuModHardRock(), new OsuModDoubleTime() }, }; - var imported = loadIntoOsu(osu, toImport); + var imported = await loadIntoOsu(osu, toImport); Assert.IsTrue(imported.Mods.Any(m => m is OsuModHardRock)); Assert.IsTrue(imported.Mods.Any(m => m is OsuModDoubleTime)); @@ -88,13 +88,13 @@ namespace osu.Game.Tests.Scores.IO } [Test] - public void TestImportStatistics() + public async Task TestImportStatistics() { using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportStatistics")) { try { - var osu = loadOsu(host); + var osu = await loadOsu(host); var toImport = new ScoreInfo { @@ -105,7 +105,7 @@ namespace osu.Game.Tests.Scores.IO } }; - var imported = loadIntoOsu(osu, toImport); + var imported = await loadIntoOsu(osu, toImport); Assert.AreEqual(toImport.Statistics[HitResult.Perfect], imported.Statistics[HitResult.Perfect]); Assert.AreEqual(toImport.Statistics[HitResult.Miss], imported.Statistics[HitResult.Miss]); @@ -117,7 +117,7 @@ namespace osu.Game.Tests.Scores.IO } } - private ScoreInfo loadIntoOsu(OsuGameBase osu, ScoreInfo score) + private async Task loadIntoOsu(OsuGameBase osu, ScoreInfo score) { var beatmapManager = osu.Dependencies.Get(); @@ -125,20 +125,24 @@ namespace osu.Game.Tests.Scores.IO score.Ruleset = new OsuRuleset().RulesetInfo; var scoreManager = osu.Dependencies.Get(); - scoreManager.Import(score); + await scoreManager.Import(score); return scoreManager.GetAllUsableScores().First(); } - private OsuGameBase loadOsu(GameHost host) + private async Task loadOsu(GameHost host) { var osu = new OsuGameBase(); + +#pragma warning disable 4014 Task.Run(() => host.Run(osu)); +#pragma warning restore 4014 + waitForOrAssert(() => osu.IsLoaded, @"osu! failed to start in a reasonable amount of time"); var beatmapFile = TestResources.GetTestBeatmapForImport(); var beatmapManager = osu.Dependencies.Get(); - beatmapManager.Import(beatmapFile); + await beatmapManager.Import(beatmapFile); return osu; } diff --git a/osu.Game.Tests/Visual/Background/TestSceneBackgroundScreenBeatmap.cs b/osu.Game.Tests/Visual/Background/TestSceneBackgroundScreenBeatmap.cs index 7104a420a3..8b941e4633 100644 --- a/osu.Game.Tests/Visual/Background/TestSceneBackgroundScreenBeatmap.cs +++ b/osu.Game.Tests/Visual/Background/TestSceneBackgroundScreenBeatmap.cs @@ -72,7 +72,7 @@ namespace osu.Game.Tests.Visual.Background Dependencies.Cache(manager = new BeatmapManager(LocalStorage, factory, rulesets, null, audio, host, Beatmap.Default)); Dependencies.Cache(new OsuConfigManager(LocalStorage)); - manager.Import(TestResources.GetTestBeatmapForImport()); + manager.Import(TestResources.GetTestBeatmapForImport()).Wait(); Beatmap.SetDefault(); } diff --git a/osu.Game.Tests/Visual/UserInterface/TestSceneUpdateableBeatmapBackgroundSprite.cs b/osu.Game.Tests/Visual/UserInterface/TestSceneUpdateableBeatmapBackgroundSprite.cs index f59458ef8d..c361598354 100644 --- a/osu.Game.Tests/Visual/UserInterface/TestSceneUpdateableBeatmapBackgroundSprite.cs +++ b/osu.Game.Tests/Visual/UserInterface/TestSceneUpdateableBeatmapBackgroundSprite.cs @@ -32,7 +32,7 @@ namespace osu.Game.Tests.Visual.UserInterface this.api = api; this.rulesets = rulesets; - testBeatmap = ImportBeatmapTest.LoadOszIntoOsu(osu); + testBeatmap = ImportBeatmapTest.LoadOszIntoOsu(osu).Result; } [Test] diff --git a/osu.Game.Tests/osu.Game.Tests.csproj b/osu.Game.Tests/osu.Game.Tests.csproj index 11d70ee7be..403ae16885 100644 --- a/osu.Game.Tests/osu.Game.Tests.csproj +++ b/osu.Game.Tests/osu.Game.Tests.csproj @@ -18,4 +18,9 @@ + + + ..\..\..\..\..\usr\local\share\dotnet\sdk\NuGetFallbackFolder\microsoft.win32.registry\4.5.0\ref\netstandard2.0\Microsoft.Win32.Registry.dll + + \ No newline at end of file diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index b6fe7f88fa..f5ef52a93b 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -2,10 +2,12 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Linq.Expressions; +using System.Threading; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using osu.Framework.Audio; @@ -14,6 +16,7 @@ using osu.Framework.Extensions; using osu.Framework.Graphics.Textures; using osu.Framework.Logging; using osu.Framework.Platform; +using osu.Framework.Threading; using osu.Game.Beatmaps.Formats; using osu.Game.Database; using osu.Game.IO.Archives; @@ -72,6 +75,8 @@ namespace osu.Game.Beatmaps private readonly List currentDownloads = new List(); + private readonly BeatmapUpdateQueue updateQueue; + public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, AudioManager audioManager, GameHost host = null, WorkingBeatmap defaultBeatmap = null) : base(storage, contextFactory, new BeatmapStore(contextFactory), host) @@ -86,9 +91,11 @@ namespace osu.Game.Beatmaps beatmaps = (BeatmapStore)ModelStore; beatmaps.BeatmapHidden += b => BeatmapHidden?.Invoke(b); beatmaps.BeatmapRestored += b => BeatmapRestored?.Invoke(b); + + updateQueue = new BeatmapUpdateQueue(api); } - protected override void Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive) + protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) { if (archive != null) beatmapSet.Beatmaps = createBeatmapDifficulties(archive); @@ -104,8 +111,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - foreach (BeatmapInfo b in beatmapSet.Beatmaps) - fetchAndPopulateOnlineValues(b); + await Task.WhenAll(beatmapSet.Beatmaps.Select(b => updateQueue.Enqueue(new UpdateItem(b, cancellationToken)).Task).ToArray()); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -181,10 +187,10 @@ namespace osu.Game.Beatmaps request.Success += filename => { - Task.Factory.StartNew(() => + Task.Factory.StartNew(async () => { // This gets scheduled back to the update thread, but we want the import to run in the background. - Import(downloadNotification, filename); + await Import(downloadNotification, filename); currentDownloads.Remove(request); }, TaskCreationOptions.LongRunning); }; @@ -381,47 +387,6 @@ namespace osu.Game.Beatmaps return beatmapInfos; } - /// - /// Query the API to populate missing values like OnlineBeatmapID / OnlineBeatmapSetID or (Rank-)Status. - /// - /// The beatmap to populate. - /// Whether to re-query if the provided beatmap already has populated values. - /// True if population was successful. - private bool fetchAndPopulateOnlineValues(BeatmapInfo beatmap, bool force = false) - { - if (api?.State != APIState.Online) - return false; - - if (!force && beatmap.OnlineBeatmapID != null && beatmap.BeatmapSet.OnlineBeatmapSetID != null - && beatmap.Status != BeatmapSetOnlineStatus.None && beatmap.BeatmapSet.Status != BeatmapSetOnlineStatus.None) - return true; - - Logger.Log("Attempting online lookup for the missing values...", LoggingTarget.Database); - - try - { - var req = new GetBeatmapRequest(beatmap); - - req.Perform(api); - - var res = req.Result; - - Logger.Log($"Successfully mapped to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}.", LoggingTarget.Database); - - beatmap.Status = res.Status; - beatmap.BeatmapSet.Status = res.BeatmapSet.Status; - beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID; - beatmap.OnlineBeatmapID = res.OnlineBeatmapID; - - return true; - } - catch (Exception e) - { - Logger.Log($"Failed ({e})", LoggingTarget.Database); - return false; - } - } - /// /// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation. /// @@ -455,5 +420,111 @@ namespace osu.Game.Beatmaps public override bool IsImportant => false; } } + + private class BeatmapUpdateQueue + { + private readonly IAPIProvider api; + private readonly Queue queue = new Queue(); + + private int activeThreads; + + public BeatmapUpdateQueue(IAPIProvider api) + { + this.api = api; + } + + public UpdateItem Enqueue(UpdateItem item) + { + lock (queue) + { + queue.Enqueue(item); + + if (activeThreads >= 16) + return item; + + new Thread(runWork) { IsBackground = true }.Start(); + activeThreads++; + } + + return item; + } + + private void runWork() + { + while (true) + { + UpdateItem toProcess; + + lock (queue) + { + if (queue.Count == 0) + break; + + toProcess = queue.Dequeue(); + } + + toProcess.PerformUpdate(api); + } + + lock (queue) + activeThreads--; + } + } + + private class UpdateItem + { + public Task Task => tcs.Task; + + private readonly BeatmapInfo beatmap; + private readonly CancellationToken cancellationToken; + + private readonly TaskCompletionSource tcs = new TaskCompletionSource(); + + public UpdateItem(BeatmapInfo beatmap, CancellationToken cancellationToken) + { + this.beatmap = beatmap; + this.cancellationToken = cancellationToken; + } + + public void PerformUpdate(IAPIProvider api) + { + if (cancellationToken.IsCancellationRequested) + { + tcs.SetCanceled(); + return; + } + + if (api?.State != APIState.Online) + { + tcs.SetResult(false); + return; + } + + Logger.Log("Attempting online lookup for the missing values...", LoggingTarget.Database); + + var req = new GetBeatmapRequest(beatmap); + + req.Success += res => + { + Logger.Log($"Successfully mapped to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}.", LoggingTarget.Database); + + beatmap.Status = res.Status; + beatmap.BeatmapSet.Status = res.BeatmapSet.Status; + beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID; + beatmap.OnlineBeatmapID = res.OnlineBeatmapID; + + tcs.SetResult(true); + }; + + req.Failure += e => + { + Logger.Log($"Failed ({e})", LoggingTarget.Database); + + tcs.SetResult(false); + }; + + req.Perform(api); + } + } } } diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 54dbae9ddc..afc614772e 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -5,14 +5,17 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; using Microsoft.EntityFrameworkCore; using osu.Framework; using osu.Framework.Extensions; +using osu.Framework.Extensions.TypeExtensions; using osu.Framework.IO.File; using osu.Framework.Logging; using osu.Framework.Platform; +using osu.Framework.Threading; using osu.Game.IO; using osu.Game.IO.Archives; using osu.Game.IPC; @@ -109,8 +112,11 @@ namespace osu.Game.Database a.Invoke(); } + private readonly ThreadedTaskScheduler importScheduler; + protected ArchiveModelManager(Storage storage, IDatabaseContextFactory contextFactory, MutableDatabaseBackedStoreWithFileIncludes modelStore, IIpcHost importHost = null) { + importScheduler = new ThreadedTaskScheduler(16, $"{GetType().ReadableName()}.Import"); ContextFactory = contextFactory; ModelStore = modelStore; @@ -130,92 +136,84 @@ namespace osu.Game.Database /// This will post notifications tracking progress. /// /// One or more archive locations on disk. - public void Import(params string[] paths) + public async Task Import(params string[] paths) { var notification = new ProgressNotification { State = ProgressNotificationState.Active }; PostNotification?.Invoke(notification); - Import(notification, paths); + + await Import(notification, paths); } - protected void Import(ProgressNotification notification, params string[] paths) + protected async Task Import(ProgressNotification notification, params string[] paths) { notification.Progress = 0; notification.Text = "Import is initialising..."; var term = $"{typeof(TModel).Name.Replace("Info", "").ToLower()}"; - List imported = new List(); + var tasks = new List(); int current = 0; foreach (string path in paths) { - if (notification.State == ProgressNotificationState.Cancelled) - // user requested abort - return; - - try + tasks.Add(Import(path, notification.CancellationToken).ContinueWith(t => { - var text = "Importing "; - - if (path.Length > 1) - text += $"{++current} of {paths.Length} {term}s.."; - else - text += $"{term}.."; - - // only show the filename if it isn't a temporary one (as those look ugly). - if (!path.Contains(Path.GetTempPath())) - text += $"\n{Path.GetFileName(path)}"; - - notification.Text = text; - - imported.Add(Import(path)); - - notification.Progress = (float)current / paths.Length; - } - catch (Exception e) - { - e = e.InnerException ?? e; - Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); - } - } - - if (imported.Count == 0) - { - notification.Text = "Import failed!"; - notification.State = ProgressNotificationState.Cancelled; - } - else - { - notification.CompletionText = imported.Count == 1 - ? $"Imported {imported.First()}!" - : $"Imported {current} {term}s!"; - - if (imported.Count > 0 && PresentImport != null) - { - notification.CompletionText += " Click to view."; - notification.CompletionClickAction = () => + lock (notification) { - PresentImport?.Invoke(imported); - return true; - }; - } + current++; - notification.State = ProgressNotificationState.Completed; + notification.Text = $"Imported {current} of {paths.Length} {term}s"; + notification.Progress = (float)current / paths.Length; + } + + if (t.Exception != null) + { + var e = t.Exception.InnerException ?? t.Exception; + Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + } + })); } + + await Task.WhenAll(tasks); + + // if (imported.Count == 0) + // { + // notification.Text = "Import failed!"; + // notification.State = ProgressNotificationState.Cancelled; + // } + // else + // { + // notification.CompletionText = imported.Count == 1 + // ? $"Imported {imported.First()}!" + // : $"Imported {current} {term}s!"; + // + // if (imported.Count > 0 && PresentImport != null) + // { + // notification.CompletionText += " Click to view."; + // notification.CompletionClickAction = () => + // { + // PresentImport?.Invoke(imported); + // return true; + // }; + // } + // + // notification.State = ProgressNotificationState.Completed; + // } } /// /// Import one from the filesystem and delete the file on success. /// /// The archive location on disk. + /// An optional cancellation token. /// The imported model, if successful. - public TModel Import(string path) + public async Task Import(string path, CancellationToken cancellationToken = default) { TModel import; using (ArchiveReader reader = getReaderFrom(path)) - import = Import(reader); + import = await Import(reader, cancellationToken); // We may or may not want to delete the file depending on where it is stored. // e.g. reconstructing/repairing database with items from default storage. @@ -243,7 +241,8 @@ namespace osu.Game.Database /// Import an item from an . /// /// The archive to be imported. - public TModel Import(ArchiveReader archive) + /// An optional cancellation token. + public async Task Import(ArchiveReader archive, CancellationToken cancellationToken = default) { try { @@ -253,7 +252,7 @@ namespace osu.Game.Database model.Hash = computeHash(archive); - return Import(model, archive); + return await Import(model, archive, cancellationToken); } catch (Exception e) { @@ -288,7 +287,8 @@ namespace osu.Game.Database /// /// The model to be imported. /// An optional archive to use for model population. - public TModel Import(TModel item, ArchiveReader archive = null) + /// An optional cancellation token. + public async Task Import(TModel item, ArchiveReader archive = null, CancellationToken cancellationToken = default) => await Task.Factory.StartNew(async () => { delayEvents(); @@ -296,17 +296,31 @@ namespace osu.Game.Database { Logger.Log($"Importing {item}...", LoggingTarget.Database); + if (archive != null) + item.Files = createFileInfos(archive, Files); + + var localItem = item; + + try + { + await Populate(item, archive, cancellationToken); + } + catch (TaskCanceledException) + { + return item = null; + } + finally + { + if (!Delete(localItem)) + Files.Dereference(localItem.Files.Select(f => f.FileInfo).ToArray()); + } + using (var write = ContextFactory.GetForWrite()) // used to share a context for full import. keep in mind this will block all writes. { try { if (!write.IsTransactionLeader) throw new InvalidOperationException($"Ensure there is no parent transaction so errors can correctly be handled by {this}"); - if (archive != null) - item.Files = createFileInfos(archive, Files); - - Populate(item, archive); - var existing = CheckForExisting(item); if (existing != null) @@ -332,6 +346,9 @@ namespace osu.Game.Database } catch (Exception e) { + if (!Delete(item)) + Files.Dereference(item.Files.Select(f => f.FileInfo).ToArray()); + write.Errors.Add(e); throw; } @@ -351,7 +368,7 @@ namespace osu.Game.Database } return item; - } + }, CancellationToken.None, TaskCreationOptions.None, importScheduler).Unwrap(); /// /// Perform an update of the specified item. @@ -516,24 +533,24 @@ namespace osu.Game.Database /// /// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future. /// - public Task ImportFromStableAsync() + public async Task ImportFromStableAsync() { var stable = GetStableStorage?.Invoke(); if (stable == null) { Logger.Log("No osu!stable installation available!", LoggingTarget.Information, LogLevel.Error); - return Task.CompletedTask; + return; } if (!stable.ExistsDirectory(ImportFromStablePath)) { // This handles situations like when the user does not have a Skins folder Logger.Log($"No {ImportFromStablePath} folder available in osu!stable installation", LoggingTarget.Information, LogLevel.Error); - return Task.CompletedTask; + return; } - return Task.Factory.StartNew(() => Import(stable.GetDirectories(ImportFromStablePath).Select(f => stable.GetFullPath(f)).ToArray()), TaskCreationOptions.LongRunning); + await Task.Run(async () => await Import(stable.GetDirectories(ImportFromStablePath).Select(f => stable.GetFullPath(f)).ToArray())); } #endregion @@ -552,9 +569,8 @@ namespace osu.Game.Database /// /// The model to populate. /// The archive to use as a reference for population. May be null. - protected virtual void Populate(TModel model, [CanBeNull] ArchiveReader archive) - { - } + /// An optional cancellation token. + protected virtual async Task Populate(TModel model, [CanBeNull] ArchiveReader archive, CancellationToken cancellationToken = default) => await Task.CompletedTask; /// /// Perform any final actions before the import to database executes. diff --git a/osu.Game/Database/ICanAcceptFiles.cs b/osu.Game/Database/ICanAcceptFiles.cs index f55d0c389e..b9f882468d 100644 --- a/osu.Game/Database/ICanAcceptFiles.cs +++ b/osu.Game/Database/ICanAcceptFiles.cs @@ -1,6 +1,8 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System.Threading.Tasks; + namespace osu.Game.Database { /// @@ -12,7 +14,7 @@ namespace osu.Game.Database /// Import the specified paths. /// /// The files which should be imported. - void Import(params string[] paths); + Task Import(params string[] paths); /// /// An array of accepted file extensions (in the standard format of ".abc"). diff --git a/osu.Game/IPC/ArchiveImportIPCChannel.cs b/osu.Game/IPC/ArchiveImportIPCChannel.cs index fc747cd446..484db932f8 100644 --- a/osu.Game/IPC/ArchiveImportIPCChannel.cs +++ b/osu.Game/IPC/ArchiveImportIPCChannel.cs @@ -38,7 +38,7 @@ namespace osu.Game.IPC } if (importer.HandledExtensions.Contains(Path.GetExtension(path)?.ToLowerInvariant())) - importer.Import(path); + await importer.Import(path); } } diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index f9128687d6..d6b8caaf5b 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; +using System.Threading.Tasks; using osu.Framework.Allocation; using osu.Framework.Audio; using osu.Framework.Bindables; @@ -268,13 +269,13 @@ namespace osu.Game private readonly List fileImporters = new List(); - public void Import(params string[] paths) + public async Task Import(params string[] paths) { var extension = Path.GetExtension(paths.First())?.ToLowerInvariant(); foreach (var importer in fileImporters) if (importer.HandledExtensions.Contains(extension)) - importer.Import(paths); + await importer.Import(paths); } public string[] HandledExtensions => fileImporters.SelectMany(i => i.HandledExtensions).ToArray(); diff --git a/osu.Game/Overlays/Notifications/ProgressNotification.cs b/osu.Game/Overlays/Notifications/ProgressNotification.cs index 857a0bda9e..c8e081d29f 100644 --- a/osu.Game/Overlays/Notifications/ProgressNotification.cs +++ b/osu.Game/Overlays/Notifications/ProgressNotification.cs @@ -2,6 +2,7 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.Threading; using osu.Framework.Allocation; using osu.Framework.Graphics; using osu.Framework.Graphics.Containers; @@ -36,6 +37,10 @@ namespace osu.Game.Overlays.Notifications State = state; } + private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + + public CancellationToken CancellationToken => cancellationTokenSource.Token; + public virtual ProgressNotificationState State { get => state; @@ -62,6 +67,8 @@ namespace osu.Game.Overlays.Notifications break; case ProgressNotificationState.Cancelled: + cancellationTokenSource.Cancel(); + Light.Colour = colourCancelled; Light.Pulsate = false; progressBar.Active = false; diff --git a/osu.Game/Screens/Menu/Intro.cs b/osu.Game/Screens/Menu/Intro.cs index 98a2fe8f13..cf5d247482 100644 --- a/osu.Game/Screens/Menu/Intro.cs +++ b/osu.Game/Screens/Menu/Intro.cs @@ -66,7 +66,7 @@ namespace osu.Game.Screens.Menu if (setInfo == null) { // we need to import the default menu background beatmap - setInfo = beatmaps.Import(new ZipArchiveReader(game.Resources.GetStream(@"Tracks/circles.osz"), "circles.osz")); + setInfo = beatmaps.Import(new ZipArchiveReader(game.Resources.GetStream(@"Tracks/circles.osz"), "circles.osz")).Result; setInfo.Protected = true; beatmaps.Update(setInfo); diff --git a/osu.Game/Screens/Play/Player.cs b/osu.Game/Screens/Play/Player.cs index d8389fa6d9..949f9e5ff2 100644 --- a/osu.Game/Screens/Play/Player.cs +++ b/osu.Game/Screens/Play/Player.cs @@ -279,7 +279,7 @@ namespace osu.Game.Screens.Play var score = CreateScore(); if (DrawableRuleset.ReplayScore == null) - scoreManager.Import(score); + scoreManager.Import(score).Wait(); this.Push(CreateResults(score)); diff --git a/osu.Game/Skinning/SkinManager.cs b/osu.Game/Skinning/SkinManager.cs index 3a4d44f608..73cc47ea47 100644 --- a/osu.Game/Skinning/SkinManager.cs +++ b/osu.Game/Skinning/SkinManager.cs @@ -5,6 +5,8 @@ using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using osu.Framework.Audio; using osu.Framework.Audio.Sample; @@ -71,9 +73,9 @@ namespace osu.Game.Skinning protected override SkinInfo CreateModel(ArchiveReader archive) => new SkinInfo { Name = archive.Name }; - protected override void Populate(SkinInfo model, ArchiveReader archive) + protected override async Task Populate(SkinInfo model, ArchiveReader archive, CancellationToken cancellationToken = default) { - base.Populate(model, archive); + await base.Populate(model, archive, cancellationToken); Skin reference = getSkin(model); From 600503ec8ebdb6233beab64b0faa53f02b91af4e Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 12:46:21 +0900 Subject: [PATCH 02/27] Use Task.Run/Wait to avoid warnings --- osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs | 4 ++-- osu.Game/Screens/Select/SongSelect.cs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index ebee358730..c2e8078bd6 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -210,7 +210,7 @@ namespace osu.Game.Tests.Visual.SongSelect AddAssert("start not requested", () => !startRequested); } - private void importForRuleset(int id) => AddStep($"import test map for ruleset {id}", () => manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == id).ToArray()))); + private void importForRuleset(int id) => AddStep($"import test map for ruleset {id}", () => manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == id).ToArray())).Wait()); private static int importId; private int getImportId() => ++importId; @@ -232,7 +232,7 @@ namespace osu.Game.Tests.Visual.SongSelect var usableRulesets = rulesets.AvailableRulesets.Where(r => r.ID != 2).ToArray(); for (int i = 0; i < 100; i += 10) - manager.Import(createTestBeatmapSet(i, usableRulesets)); + manager.Import(createTestBeatmapSet(i, usableRulesets)).Wait(); }); } diff --git a/osu.Game/Screens/Select/SongSelect.cs b/osu.Game/Screens/Select/SongSelect.cs index 6d5be607f4..196d655851 100644 --- a/osu.Game/Screens/Select/SongSelect.cs +++ b/osu.Game/Screens/Select/SongSelect.cs @@ -32,6 +32,7 @@ using osuTK.Input; using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using osu.Framework.Graphics.Sprites; namespace osu.Game.Screens.Select @@ -256,8 +257,8 @@ namespace osu.Game.Screens.Select if (!beatmaps.GetAllUsableBeatmapSets().Any() && beatmaps.StableInstallationAvailable) dialogOverlay.Push(new ImportFromStablePopup(() => { - beatmaps.ImportFromStableAsync(); - skins.ImportFromStableAsync(); + Task.Run(beatmaps.ImportFromStableAsync); + Task.Run(skins.ImportFromStableAsync); })); }); } From b4d2d0bd0b245cba425d9e5d981e3831d9269931 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:19:58 +0900 Subject: [PATCH 03/27] Simplify and combine concurrency of ArchiveModelManager --- osu.Game/Beatmaps/BeatmapManager.cs | 77 ++----------------- osu.Game/Database/ArchiveModelManager.cs | 96 ++++++++++++------------ 2 files changed, 57 insertions(+), 116 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index f5ef52a93b..9e7b6d7971 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -2,7 +2,6 @@ // See the LICENCE file in the repository root for full licence text. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -111,7 +110,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - await Task.WhenAll(beatmapSet.Beatmaps.Select(b => updateQueue.Enqueue(new UpdateItem(b, cancellationToken)).Task).ToArray()); + await Task.WhenAll(beatmapSet.Beatmaps.Select(b => updateQueue.Perform(b, cancellationToken)).ToArray()); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -424,81 +423,24 @@ namespace osu.Game.Beatmaps private class BeatmapUpdateQueue { private readonly IAPIProvider api; - private readonly Queue queue = new Queue(); - private int activeThreads; + private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(4); public BeatmapUpdateQueue(IAPIProvider api) { this.api = api; } - public UpdateItem Enqueue(UpdateItem item) + public Task Perform(BeatmapInfo beatmap, CancellationToken cancellationToken) + => Task.Factory.StartNew(() => perform(beatmap, cancellationToken), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); + + private void perform(BeatmapInfo beatmap, CancellationToken cancellation) { - lock (queue) - { - queue.Enqueue(item); - - if (activeThreads >= 16) - return item; - - new Thread(runWork) { IsBackground = true }.Start(); - activeThreads++; - } - - return item; - } - - private void runWork() - { - while (true) - { - UpdateItem toProcess; - - lock (queue) - { - if (queue.Count == 0) - break; - - toProcess = queue.Dequeue(); - } - - toProcess.PerformUpdate(api); - } - - lock (queue) - activeThreads--; - } - } - - private class UpdateItem - { - public Task Task => tcs.Task; - - private readonly BeatmapInfo beatmap; - private readonly CancellationToken cancellationToken; - - private readonly TaskCompletionSource tcs = new TaskCompletionSource(); - - public UpdateItem(BeatmapInfo beatmap, CancellationToken cancellationToken) - { - this.beatmap = beatmap; - this.cancellationToken = cancellationToken; - } - - public void PerformUpdate(IAPIProvider api) - { - if (cancellationToken.IsCancellationRequested) - { - tcs.SetCanceled(); + if (cancellation.IsCancellationRequested) return; - } if (api?.State != APIState.Online) - { - tcs.SetResult(false); return; - } Logger.Log("Attempting online lookup for the missing values...", LoggingTarget.Database); @@ -512,17 +454,14 @@ namespace osu.Game.Beatmaps beatmap.BeatmapSet.Status = res.BeatmapSet.Status; beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID; beatmap.OnlineBeatmapID = res.OnlineBeatmapID; - - tcs.SetResult(true); }; req.Failure += e => { Logger.Log($"Failed ({e})", LoggingTarget.Database); - - tcs.SetResult(false); }; + // intentionally blocking to limit web request concurrency req.Perform(api); } } diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index afc614772e..e30e1cd3ee 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -1,4 +1,4 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. using System; @@ -11,7 +11,6 @@ using JetBrains.Annotations; using Microsoft.EntityFrameworkCore; using osu.Framework; using osu.Framework.Extensions; -using osu.Framework.Extensions.TypeExtensions; using osu.Framework.IO.File; using osu.Framework.Logging; using osu.Framework.Platform; @@ -32,7 +31,7 @@ namespace osu.Game.Database /// /// The model type. /// The associated file join type. - public abstract class ArchiveModelManager : ICanAcceptFiles + public abstract class ArchiveModelManager : ArchiveModelManager, ICanAcceptFiles where TModel : class, IHasFiles, IHasPrimaryKey, ISoftDelete where TFileModel : INamedFileInfo, new() { @@ -112,11 +111,8 @@ namespace osu.Game.Database a.Invoke(); } - private readonly ThreadedTaskScheduler importScheduler; - protected ArchiveModelManager(Storage storage, IDatabaseContextFactory contextFactory, MutableDatabaseBackedStoreWithFileIncludes modelStore, IIpcHost importHost = null) { - importScheduler = new ThreadedTaskScheduler(16, $"{GetType().ReadableName()}.Import"); ContextFactory = contextFactory; ModelStore = modelStore; @@ -152,55 +148,55 @@ namespace osu.Game.Database var term = $"{typeof(TModel).Name.Replace("Info", "").ToLower()}"; - var tasks = new List(); - int current = 0; - foreach (string path in paths) + var imported = new List(); + + await Task.WhenAll(paths.Select(path => Import(path, notification.CancellationToken).ContinueWith(t => { - tasks.Add(Import(path, notification.CancellationToken).ContinueWith(t => + lock (notification) { - lock (notification) - { - current++; + current++; - notification.Text = $"Imported {current} of {paths.Length} {term}s"; - notification.Progress = (float)current / paths.Length; - } + notification.Text = $"Imported {current} of {paths.Length} {term}s"; + notification.Progress = (float)current / paths.Length; + } - if (t.Exception != null) - { - var e = t.Exception.InnerException ?? t.Exception; - Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); - } - })); + if (t.Exception == null) + { + lock (imported) + imported.Add(t.Result); + } + else + { + var e = t.Exception.InnerException ?? t.Exception; + Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + } + }))); + + if (imported.Count == 0) + { + notification.Text = "Import failed!"; + notification.State = ProgressNotificationState.Cancelled; } + else + { + notification.CompletionText = imported.Count == 1 + ? $"Imported {imported.First()}!" + : $"Imported {current} {term}s!"; - await Task.WhenAll(tasks); + if (imported.Count > 0 && PresentImport != null) + { + notification.CompletionText += " Click to view."; + notification.CompletionClickAction = () => + { + PresentImport?.Invoke(imported); + return true; + }; + } - // if (imported.Count == 0) - // { - // notification.Text = "Import failed!"; - // notification.State = ProgressNotificationState.Cancelled; - // } - // else - // { - // notification.CompletionText = imported.Count == 1 - // ? $"Imported {imported.First()}!" - // : $"Imported {current} {term}s!"; - // - // if (imported.Count > 0 && PresentImport != null) - // { - // notification.CompletionText += " Click to view."; - // notification.CompletionClickAction = () => - // { - // PresentImport?.Invoke(imported); - // return true; - // }; - // } - // - // notification.State = ProgressNotificationState.Completed; - // } + notification.State = ProgressNotificationState.Completed; + } } /// @@ -368,7 +364,7 @@ namespace osu.Game.Database } return item; - }, CancellationToken.None, TaskCreationOptions.None, importScheduler).Unwrap(); + }, CancellationToken.None, TaskCreationOptions.HideScheduler, IMPORT_SCHEDULER).Unwrap(); /// /// Perform an update of the specified item. @@ -615,4 +611,10 @@ namespace osu.Game.Database throw new InvalidFormatException($"{path} is not a valid archive"); } } + + public abstract class ArchiveModelManager + { + // allow sharing static across all generic types + protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(1); + } } From 2d1a54e63489b36cfb4fa5261abd0ed95c092051 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:37:20 +0900 Subject: [PATCH 04/27] Properly implement cancellation --- osu.Game/Database/ArchiveModelManager.cs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index e30e1cd3ee..1d576ff82f 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -154,6 +154,9 @@ namespace osu.Game.Database await Task.WhenAll(paths.Select(path => Import(path, notification.CancellationToken).ContinueWith(t => { + if (notification.CancellationToken.IsCancellationRequested) + return; + lock (notification) { current++; @@ -172,7 +175,7 @@ namespace osu.Game.Database var e = t.Exception.InnerException ?? t.Exception; Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); } - }))); + }, TaskContinuationOptions.NotOnCanceled))); if (imported.Count == 0) { @@ -207,6 +210,8 @@ namespace osu.Game.Database /// The imported model, if successful. public async Task Import(string path, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + TModel import; using (ArchiveReader reader = getReaderFrom(path)) import = await Import(reader, cancellationToken); @@ -240,6 +245,8 @@ namespace osu.Game.Database /// An optional cancellation token. public async Task Import(ArchiveReader archive, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + try { var model = CreateModel(archive); @@ -250,6 +257,10 @@ namespace osu.Game.Database return await Import(model, archive, cancellationToken); } + catch (TaskCanceledException) + { + throw; + } catch (Exception e) { Logger.Error(e, $"Model creation of {archive.Name} failed.", LoggingTarget.Database); @@ -286,6 +297,8 @@ namespace osu.Game.Database /// An optional cancellation token. public async Task Import(TModel item, ArchiveReader archive = null, CancellationToken cancellationToken = default) => await Task.Factory.StartNew(async () => { + cancellationToken.ThrowIfCancellationRequested(); + delayEvents(); try @@ -301,10 +314,6 @@ namespace osu.Game.Database { await Populate(item, archive, cancellationToken); } - catch (TaskCanceledException) - { - return item = null; - } finally { if (!Delete(localItem)) @@ -364,7 +373,7 @@ namespace osu.Game.Database } return item; - }, CancellationToken.None, TaskCreationOptions.HideScheduler, IMPORT_SCHEDULER).Unwrap(); + }, cancellationToken, TaskCreationOptions.HideScheduler, IMPORT_SCHEDULER).Unwrap(); /// /// Perform an update of the specified item. From e12b03e275c40de2b3629b92da24da45545be619 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:46:31 +0900 Subject: [PATCH 05/27] Remove unnecessary reference --- osu.Game.Tests/osu.Game.Tests.csproj | 5 ----- 1 file changed, 5 deletions(-) diff --git a/osu.Game.Tests/osu.Game.Tests.csproj b/osu.Game.Tests/osu.Game.Tests.csproj index 403ae16885..11d70ee7be 100644 --- a/osu.Game.Tests/osu.Game.Tests.csproj +++ b/osu.Game.Tests/osu.Game.Tests.csproj @@ -18,9 +18,4 @@ - - - ..\..\..\..\..\usr\local\share\dotnet\sdk\NuGetFallbackFolder\microsoft.win32.registry\4.5.0\ref\netstandard2.0\Microsoft.Win32.Registry.dll - - \ No newline at end of file From b79fdfc12f2d65da2f0fd80b68fa58aa2aa11882 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:48:58 +0900 Subject: [PATCH 06/27] Fix one more instance of improperly handled cancellation --- osu.Game/Beatmaps/BeatmapManager.cs | 3 +-- osu.Game/Database/ArchiveModelManager.cs | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 9e7b6d7971..cfc6a0be28 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -436,8 +436,7 @@ namespace osu.Game.Beatmaps private void perform(BeatmapInfo beatmap, CancellationToken cancellation) { - if (cancellation.IsCancellationRequested) - return; + cancellation.ThrowIfCancellationRequested(); if (api?.State != APIState.Online) return; diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 1d576ff82f..df96d9984a 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -361,6 +361,10 @@ namespace osu.Game.Database Logger.Log($"Import of {item} successfully completed!", LoggingTarget.Database); } + catch (TaskCanceledException) + { + throw; + } catch (Exception e) { Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); From e4bad93b6668868079dad7227b868b43736e6336 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:52:09 +0900 Subject: [PATCH 07/27] Use variable for web request concurrency for clarity --- osu.Game/Beatmaps/BeatmapManager.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index cfc6a0be28..435edcf722 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -424,7 +424,9 @@ namespace osu.Game.Beatmaps { private readonly IAPIProvider api; - private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(4); + private const int update_queue_request_concurrency = 4; + + private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency); public BeatmapUpdateQueue(IAPIProvider api) { @@ -455,10 +457,7 @@ namespace osu.Game.Beatmaps beatmap.OnlineBeatmapID = res.OnlineBeatmapID; }; - req.Failure += e => - { - Logger.Log($"Failed ({e})", LoggingTarget.Database); - }; + req.Failure += e => { Logger.Log($"Failed ({e})", LoggingTarget.Database); }; // intentionally blocking to limit web request concurrency req.Perform(api); From e19f4935c3cbdd6d037d093d6b41e2f7cb1bf719 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 14:13:36 +0900 Subject: [PATCH 08/27] Fix incorrect undo logic on exception --- osu.Game/Database/ArchiveModelManager.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index df96d9984a..83c51e2abf 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -305,8 +305,7 @@ namespace osu.Game.Database { Logger.Log($"Importing {item}...", LoggingTarget.Database); - if (archive != null) - item.Files = createFileInfos(archive, Files); + item.Files = archive != null ? createFileInfos(archive, Files) : new List(); var localItem = item; @@ -314,7 +313,7 @@ namespace osu.Game.Database { await Populate(item, archive, cancellationToken); } - finally + catch (Exception) { if (!Delete(localItem)) Files.Dereference(localItem.Files.Select(f => f.FileInfo).ToArray()); From f31b19e0d7b2837bfaa1c26fa381d6ab1df43e41 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:02:49 +0900 Subject: [PATCH 09/27] Don't unwrap exception manually --- osu.Game/Database/ArchiveModelManager.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 83c51e2abf..4c4878edee 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -172,8 +172,7 @@ namespace osu.Game.Database } else { - var e = t.Exception.InnerException ?? t.Exception; - Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + Logger.Error(t.Exception, $@"Could not import ({Path.GetFileName(path)})"); } }, TaskContinuationOptions.NotOnCanceled))); From 9bdc8b47bb1e1c8ab8e57b48d0d8604d94ae5d3e Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:13:51 +0900 Subject: [PATCH 10/27] Remove unnecessary async-await pair --- osu.Game/Database/ArchiveModelManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 4c4878edee..fb7f0ffe5d 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -577,7 +577,7 @@ namespace osu.Game.Database /// The model to populate. /// The archive to use as a reference for population. May be null. /// An optional cancellation token. - protected virtual async Task Populate(TModel model, [CanBeNull] ArchiveReader archive, CancellationToken cancellationToken = default) => await Task.CompletedTask; + protected virtual Task Populate(TModel model, [CanBeNull] ArchiveReader archive, CancellationToken cancellationToken = default) => Task.CompletedTask; /// /// Perform any final actions before the import to database executes. From fae32b390171738feed00336df54db7046c9ecd8 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:14:11 +0900 Subject: [PATCH 11/27] Return shorter class name in error messages --- osu.Game/Database/ArchiveModelManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index fb7f0ffe5d..e2a07a860f 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -331,7 +331,7 @@ namespace osu.Game.Database if (CanUndelete(existing, item)) { Undelete(existing); - Logger.Log($"Found existing {typeof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); + Logger.Log($"Found existing {nameof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); handleEvent(() => ItemAdded?.Invoke(existing, true)); return existing; } From 02b376d962d364f7b2525e8478dd4c4e3dd7cf04 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:14:42 +0900 Subject: [PATCH 12/27] Fix rollback logic not necessrily cleaning up file store --- osu.Game/Database/ArchiveModelManager.cs | 63 +++++++++++------------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index e2a07a860f..45460dd11c 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -154,25 +154,22 @@ namespace osu.Game.Database await Task.WhenAll(paths.Select(path => Import(path, notification.CancellationToken).ContinueWith(t => { - if (notification.CancellationToken.IsCancellationRequested) - return; + notification.CancellationToken.ThrowIfCancellationRequested(); - lock (notification) + lock (imported) { - current++; + Interlocked.Increment(ref current); - notification.Text = $"Imported {current} of {paths.Length} {term}s"; - notification.Progress = (float)current / paths.Length; - } - - if (t.Exception == null) - { - lock (imported) + if (t.Exception == null) + { imported.Add(t.Result); - } - else - { - Logger.Error(t.Exception, $@"Could not import ({Path.GetFileName(path)})"); + notification.Text = $"Imported {current} of {paths.Length} {term}s"; + notification.Progress = (float)current / paths.Length; + } + else + { + Logger.Error(t.Exception, $@"Could not import ({Path.GetFileName(path)})"); + } } }, TaskContinuationOptions.NotOnCanceled))); @@ -300,23 +297,23 @@ namespace osu.Game.Database delayEvents(); + void rollback() + { + if (!Delete(item)) + { + // We may have not yet added the model to the underlying table, but should still clean up files. + Logger.Log($"Dereferencing files for incomplete import of {item}.", LoggingTarget.Database); + Files.Dereference(item.Files.Select(f => f.FileInfo).ToArray()); + } + } + try { Logger.Log($"Importing {item}...", LoggingTarget.Database); item.Files = archive != null ? createFileInfos(archive, Files) : new List(); - var localItem = item; - - try - { - await Populate(item, archive, cancellationToken); - } - catch (Exception) - { - if (!Delete(localItem)) - Files.Dereference(localItem.Files.Select(f => f.FileInfo).ToArray()); - } + await Populate(item, archive, cancellationToken); using (var write = ContextFactory.GetForWrite()) // used to share a context for full import. keep in mind this will block all writes. { @@ -333,6 +330,8 @@ namespace osu.Game.Database Undelete(existing); Logger.Log($"Found existing {nameof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); handleEvent(() => ItemAdded?.Invoke(existing, true)); + + rollback(); return existing; } else @@ -349,9 +348,6 @@ namespace osu.Game.Database } catch (Exception e) { - if (!Delete(item)) - Files.Dereference(item.Files.Select(f => f.FileInfo).ToArray()); - write.Errors.Add(e); throw; } @@ -359,13 +355,12 @@ namespace osu.Game.Database Logger.Log($"Import of {item} successfully completed!", LoggingTarget.Database); } - catch (TaskCanceledException) - { - throw; - } catch (Exception e) { - Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); + if (!(e is TaskCanceledException)) + Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); + + rollback(); item = null; } finally From 5b75060b94999e3a50a665a4e40cee3994162fa2 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:45:45 +0900 Subject: [PATCH 13/27] Add test for rollback logic correctly dereferencing files --- osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs | 13 +++++++++++++ osu.Game/IO/FileStore.cs | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 4c9260f640..3f4f40781c 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -12,6 +12,7 @@ using osu.Framework.Platform; using osu.Game.IPC; using osu.Framework.Allocation; using osu.Game.Beatmaps; +using osu.Game.IO; using osu.Game.Tests.Resources; using SharpCompress.Archives.Zip; @@ -97,6 +98,7 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); var manager = osu.Dependencies.Get(); + var files = osu.Dependencies.Get(); int fireCount = 0; @@ -113,6 +115,12 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(0, fireCount -= 2); + Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); + Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); + + Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); + var breakTemp = TestResources.GetTestBeatmapForImport(); MemoryStream brokenOsu = new MemoryStream(new byte[] { 1, 3, 3, 7 }); @@ -131,6 +139,8 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); + Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); + // this will trigger purging of the existing beatmap (online set id match) but should rollback due to broken osu. await manager.Import(breakTemp); @@ -140,6 +150,9 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); + + Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); + Assert.AreEqual(18, files.QueryFiles(f => f.ReferenceCount == 1).Count()); } finally { diff --git a/osu.Game/IO/FileStore.cs b/osu.Game/IO/FileStore.cs index 458f8964f9..370d6786f5 100644 --- a/osu.Game/IO/FileStore.cs +++ b/osu.Game/IO/FileStore.cs @@ -2,8 +2,11 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Linq.Expressions; +using Microsoft.EntityFrameworkCore; using osu.Framework.Extensions; using osu.Framework.IO.Stores; using osu.Framework.Logging; @@ -27,6 +30,13 @@ namespace osu.Game.IO Store = new StorageBackedResourceStore(Storage); } + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// Results from the provided query. + public IEnumerable QueryFiles(Expression> query) => ContextFactory.Get().Set().AsNoTracking().Where(f => f.ReferenceCount > 0).Where(query); + public FileInfo Add(Stream data, bool reference = true) { using (var usage = ContextFactory.GetForWrite()) From 559413f766fef4fef2f8c16e59e61a39f100d785 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 17:12:25 +0900 Subject: [PATCH 14/27] Avoid using ContinueWith in already async context --- osu.Game/Database/ArchiveModelManager.cs | 29 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 45460dd11c..d09aa37d7e 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -152,26 +152,35 @@ namespace osu.Game.Database var imported = new List(); - await Task.WhenAll(paths.Select(path => Import(path, notification.CancellationToken).ContinueWith(t => + await Task.WhenAll(paths.Select(async path => { notification.CancellationToken.ThrowIfCancellationRequested(); - lock (imported) + try { - Interlocked.Increment(ref current); + var model = await Import(path, notification.CancellationToken); - if (t.Exception == null) + lock (imported) { - imported.Add(t.Result); + imported.Add(model); + notification.Text = $"Imported {current} of {paths.Length} {term}s"; notification.Progress = (float)current / paths.Length; } - else - { - Logger.Error(t.Exception, $@"Could not import ({Path.GetFileName(path)})"); - } } - }, TaskContinuationOptions.NotOnCanceled))); + catch (TaskCanceledException) + { + throw; + } + catch (Exception e) + { + Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + } + finally + { + Interlocked.Increment(ref current); + } + })); if (imported.Count == 0) { From c8bd92659bab551a36e0c40dc1b604deca0e4fdb Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 17:12:37 +0900 Subject: [PATCH 15/27] Clean up exception and null handling in Import process --- osu.Game/Database/ArchiveModelManager.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index d09aa37d7e..fa8301bb2e 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -252,15 +252,15 @@ namespace osu.Game.Database { cancellationToken.ThrowIfCancellationRequested(); + TModel model; + try { - var model = CreateModel(archive); + model = CreateModel(archive); if (model == null) return null; model.Hash = computeHash(archive); - - return await Import(model, archive, cancellationToken); } catch (TaskCanceledException) { @@ -271,6 +271,8 @@ namespace osu.Game.Database Logger.Error(e, $"Model creation of {archive.Name} failed.", LoggingTarget.Database); return null; } + + return await Import(model, archive, cancellationToken); } /// @@ -340,7 +342,9 @@ namespace osu.Game.Database Logger.Log($"Found existing {nameof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); handleEvent(() => ItemAdded?.Invoke(existing, true)); + // existing item will be used; rollback new import and exit early. rollback(); + flushEvents(true); return existing; } else @@ -370,14 +374,11 @@ namespace osu.Game.Database Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); rollback(); - item = null; - } - finally - { - // we only want to flush events after we've confirmed the write context didn't have any errors. - flushEvents(item != null); + flushEvents(false); + throw; } + flushEvents(true); return item; }, cancellationToken, TaskCreationOptions.HideScheduler, IMPORT_SCHEDULER).Unwrap(); From dcdb806120c6b581af4be341b60bd84cba5ea9c3 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 17:26:56 +0900 Subject: [PATCH 16/27] Catch newly thrown exception in test --- osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 3f4f40781c..7f08674a95 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -135,14 +135,14 @@ namespace osu.Game.Tests.Beatmaps.IO zip.SaveTo(outStream, SharpCompress.Common.CompressionType.Deflate); } - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); - Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); - - Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); - // this will trigger purging of the existing beatmap (online set id match) but should rollback due to broken osu. - await manager.Import(breakTemp); + try + { + await manager.Import(breakTemp); + } + catch + { + } // no events should be fired in the case of a rollback. Assert.AreEqual(0, fireCount); From 28b2a516e34354eb0f2d958c2f55da91dd8d4f78 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 18:13:33 +0900 Subject: [PATCH 17/27] Ensure exception is only thrown once on rollback --- .../Beatmaps/IO/ImportBeatmapTest.cs | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 7f08674a95..f3680e3c1e 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -11,6 +11,7 @@ using NUnit.Framework; using osu.Framework.Platform; using osu.Game.IPC; using osu.Framework.Allocation; +using osu.Framework.Logging; using osu.Game.Beatmaps; using osu.Game.IO; using osu.Game.Tests.Resources; @@ -96,24 +97,31 @@ namespace osu.Game.Tests.Beatmaps.IO { try { + int itemAddRemoveFireCount = 0; + int loggedExceptionCount = 0; + + Logger.NewEntry += l => + { + if (l.Target == LoggingTarget.Database && l.Exception != null) + Interlocked.Increment(ref loggedExceptionCount); + }; + var osu = loadOsu(host); var manager = osu.Dependencies.Get(); var files = osu.Dependencies.Get(); - int fireCount = 0; - // ReSharper disable once AccessToModifiedClosure - manager.ItemAdded += (_, __) => fireCount++; - manager.ItemRemoved += _ => fireCount++; + manager.ItemAdded += (_, __) => Interlocked.Increment(ref itemAddRemoveFireCount); + manager.ItemRemoved += _ => Interlocked.Increment(ref itemAddRemoveFireCount); var imported = await LoadOszIntoOsu(osu); - Assert.AreEqual(0, fireCount -= 1); + Assert.AreEqual(0, itemAddRemoveFireCount -= 1); imported.Hash += "-changed"; manager.Update(imported); - Assert.AreEqual(0, fireCount -= 2); + Assert.AreEqual(0, itemAddRemoveFireCount -= 2); Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); @@ -145,7 +153,7 @@ namespace osu.Game.Tests.Beatmaps.IO } // no events should be fired in the case of a rollback. - Assert.AreEqual(0, fireCount); + Assert.AreEqual(0, itemAddRemoveFireCount); Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); @@ -153,6 +161,8 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); Assert.AreEqual(18, files.QueryFiles(f => f.ReferenceCount == 1).Count()); + + Assert.AreEqual(1, loggedExceptionCount); } finally { From 12aa264657061f456405a65c0eaaf781f3461304 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 18:35:23 +0900 Subject: [PATCH 18/27] Consolidate tests and check for file reference counts --- .../Beatmaps/IO/ImportBeatmapTest.cs | 51 ++++++++++++------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index f3680e3c1e..5fc05a4b2f 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -77,10 +77,8 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.IsTrue(imported.ID == importedSecondTime.ID); Assert.IsTrue(imported.Beatmaps.First().ID == importedSecondTime.Beatmaps.First().ID); - var manager = osu.Dependencies.Get(); - - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + checkBeatmapSetCount(osu, 1); + checkSingleReferencedFileCount(osu, 18); } finally { @@ -108,7 +106,6 @@ namespace osu.Game.Tests.Beatmaps.IO var osu = loadOsu(host); var manager = osu.Dependencies.Get(); - var files = osu.Dependencies.Get(); // ReSharper disable once AccessToModifiedClosure manager.ItemAdded += (_, __) => Interlocked.Increment(ref itemAddRemoveFireCount); @@ -123,11 +120,9 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(0, itemAddRemoveFireCount -= 2); - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); - Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); - - Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); + checkBeatmapSetCount(osu, 1); + checkBeatmapCount(osu, 12); + checkSingleReferencedFileCount(osu, 18); var breakTemp = TestResources.GetTestBeatmapForImport(); @@ -155,12 +150,10 @@ namespace osu.Game.Tests.Beatmaps.IO // no events should be fired in the case of a rollback. Assert.AreEqual(0, itemAddRemoveFireCount); - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); - Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); + checkBeatmapSetCount(osu, 1); + checkBeatmapCount(osu, 12); - Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); - Assert.AreEqual(18, files.QueryFiles(f => f.ReferenceCount == 1).Count()); + checkSingleReferencedFileCount(osu, 18); Assert.AreEqual(1, loggedExceptionCount); } @@ -193,8 +186,7 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.IsTrue(imported.Beatmaps.First().ID < importedSecondTime.Beatmaps.First().ID); // only one beatmap will exist as the online set ID matched, causing purging of the first import. - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + checkBeatmapSetCount(osu, 1); } finally { @@ -396,11 +388,32 @@ namespace osu.Game.Tests.Beatmaps.IO var manager = osu.Dependencies.Get(); manager.Delete(imported); - Assert.IsTrue(manager.GetAllUsableBeatmapSets().Count == 0); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + checkBeatmapSetCount(osu, 0); + checkBeatmapSetCount(osu, 1, true); + checkSingleReferencedFileCount(osu, 0); + Assert.IsTrue(manager.QueryBeatmapSets(_ => true).First().DeletePending); } + private void checkBeatmapSetCount(OsuGameBase osu, int expected, bool includeDeletePending = false) + { + var manager = osu.Dependencies.Get(); + + Assert.AreEqual(expected, includeDeletePending + ? manager.QueryBeatmapSets(_ => true).ToList().Count + : manager.GetAllUsableBeatmapSets().Count); + } + + private void checkBeatmapCount(OsuGameBase osu, int expected) + { + Assert.AreEqual(expected, osu.Dependencies.Get().QueryBeatmaps(_ => true).ToList().Count); + } + + private void checkSingleReferencedFileCount(OsuGameBase osu, int expected) + { + Assert.AreEqual(expected, osu.Dependencies.Get().QueryFiles(f => f.ReferenceCount == 1).Count()); + } + private OsuGameBase loadOsu(GameHost host) { var osu = new OsuGameBase(); From f7a699e4a2e0031224661370ba840d7d9327df3f Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 18:38:03 +0900 Subject: [PATCH 19/27] Better documentation for import scheduler singleton --- osu.Game/Database/ArchiveModelManager.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index fa8301bb2e..afc2690106 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -630,7 +630,15 @@ namespace osu.Game.Database public abstract class ArchiveModelManager { - // allow sharing static across all generic types - protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(1); + private const int import_queue_request_concurrency = 1; + + /// + /// A singleton scheduler shared by all . + /// + /// + /// This scheduler generally performs IO and CPU intensive work so concurrency is limited harshly. + /// It is mainly being used as a queue mechanism for large imports. + /// + protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(import_queue_request_concurrency); } } From 6cda2cdb8225000f1a7de5e8c842a8d3f0a02aaf Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 18:41:56 +0900 Subject: [PATCH 20/27] Fix exception output to use humanised model name --- osu.Game/Database/ArchiveModelManager.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index afc2690106..93e924fab5 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -146,8 +146,6 @@ namespace osu.Game.Database notification.Progress = 0; notification.Text = "Import is initialising..."; - var term = $"{typeof(TModel).Name.Replace("Info", "").ToLower()}"; - int current = 0; var imported = new List(); @@ -164,7 +162,7 @@ namespace osu.Game.Database { imported.Add(model); - notification.Text = $"Imported {current} of {paths.Length} {term}s"; + notification.Text = $"Imported {current} of {paths.Length} {humanisedModelName}s"; notification.Progress = (float)current / paths.Length; } } @@ -191,7 +189,7 @@ namespace osu.Game.Database { notification.CompletionText = imported.Count == 1 ? $"Imported {imported.First()}!" - : $"Imported {current} {term}s!"; + : $"Imported {current} {humanisedModelName}s!"; if (imported.Count > 0 && PresentImport != null) { @@ -339,7 +337,7 @@ namespace osu.Game.Database if (CanUndelete(existing, item)) { Undelete(existing); - Logger.Log($"Found existing {nameof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); + Logger.Log($"Found existing {humanisedModelName} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); handleEvent(() => ItemAdded?.Invoke(existing, true)); // existing item will be used; rollback new import and exit early. @@ -610,6 +608,8 @@ namespace osu.Game.Database private DbSet queryModel() => ContextFactory.Get().Set(); + private string humanisedModelName => $"{typeof(TModel).Name.Replace("Info", "").ToLower()}"; + /// /// Creates an from a valid storage path. /// From 54497fb1e78dd5447111cd256067f3959c1a61dd Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 19:33:23 +0900 Subject: [PATCH 21/27] Fix prefixing spaces in BeatmapInfo's ToString when metadata is not populated yet --- osu.Game/Beatmaps/BeatmapInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Beatmaps/BeatmapInfo.cs b/osu.Game/Beatmaps/BeatmapInfo.cs index 52238c26fe..3c082bb71e 100644 --- a/osu.Game/Beatmaps/BeatmapInfo.cs +++ b/osu.Game/Beatmaps/BeatmapInfo.cs @@ -119,7 +119,7 @@ namespace osu.Game.Beatmaps /// public List Scores { get; set; } - public override string ToString() => $"{Metadata} [{Version}]"; + public override string ToString() => $"{Metadata} [{Version}]".Trim(); public bool Equals(BeatmapInfo other) { From 29945f27c5f22bcbb36ed693d813fd34707053ee Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 19:33:55 +0900 Subject: [PATCH 22/27] Fix imported count incrementing on failures --- osu.Game/Database/ArchiveModelManager.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 93e924fab5..4e9478dc10 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -162,6 +162,7 @@ namespace osu.Game.Database { imported.Add(model); + Interlocked.Increment(ref current); notification.Text = $"Imported {current} of {paths.Length} {humanisedModelName}s"; notification.Progress = (float)current / paths.Length; } @@ -174,10 +175,6 @@ namespace osu.Game.Database { Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); } - finally - { - Interlocked.Increment(ref current); - } })); if (imported.Count == 0) From 6ca2fcebfce6467398d12fc69911ffb3f1c03eff Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 19:34:32 +0900 Subject: [PATCH 23/27] Centalise and prefix all ArchiveModelManager database logging --- osu.Game/Beatmaps/BeatmapManager.cs | 28 ++++++++++-------- osu.Game/Database/ArchiveModelManager.cs | 36 +++++++++++++++--------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 435edcf722..47411c69ec 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -110,7 +110,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - await Task.WhenAll(beatmapSet.Beatmaps.Select(b => updateQueue.Perform(b, cancellationToken)).ToArray()); + await updateQueue.Perform(beatmapSet, cancellationToken); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -127,7 +127,7 @@ namespace osu.Game.Beatmaps { Delete(existingOnlineId); beatmaps.PurgeDeletable(s => s.ID == existingOnlineId.ID); - Logger.Log($"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been purged.", LoggingTarget.Database); + LogForModel(beatmapSet, $"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been purged."); } } } @@ -433,23 +433,29 @@ namespace osu.Game.Beatmaps this.api = api; } - public Task Perform(BeatmapInfo beatmap, CancellationToken cancellationToken) - => Task.Factory.StartNew(() => perform(beatmap, cancellationToken), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); - - private void perform(BeatmapInfo beatmap, CancellationToken cancellation) + public async Task Perform(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) { - cancellation.ThrowIfCancellationRequested(); - if (api?.State != APIState.Online) return; - Logger.Log("Attempting online lookup for the missing values...", LoggingTarget.Database); + LogForModel(beatmapSet, "Performing online lookups..."); + await Task.WhenAll(beatmapSet.Beatmaps.Select(b => Perform(beatmapSet, b, cancellationToken)).ToArray()); + } + + // todo: expose this when we need to do individual difficulty lookups. + protected Task Perform(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken) + => Task.Factory.StartNew(() => perform(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); + + private void perform(BeatmapSetInfo set, BeatmapInfo beatmap) + { + if (api?.State != APIState.Online) + return; var req = new GetBeatmapRequest(beatmap); req.Success += res => { - Logger.Log($"Successfully mapped to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}.", LoggingTarget.Database); + LogForModel(set, $"Online retrieval mapped {beatmap} to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}."); beatmap.Status = res.Status; beatmap.BeatmapSet.Status = res.BeatmapSet.Status; @@ -457,7 +463,7 @@ namespace osu.Game.Beatmaps beatmap.OnlineBeatmapID = res.OnlineBeatmapID; }; - req.Failure += e => { Logger.Log($"Failed ({e})", LoggingTarget.Database); }; + req.Failure += e => { LogForModel(set, $"Online retrieval failed for {beatmap}", e); }; // intentionally blocking to limit web request concurrency req.Perform(api); diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 4e9478dc10..844ae622a5 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -173,7 +173,7 @@ namespace osu.Game.Database } catch (Exception e) { - Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})", LoggingTarget.Database); } })); @@ -227,7 +227,7 @@ namespace osu.Game.Database } catch (Exception e) { - Logger.Error(e, $@"Could not delete original file after import ({Path.GetFileName(path)})"); + LogForModel(import, $@"Could not delete original file after import ({Path.GetFileName(path)})", e); } return import; @@ -247,7 +247,7 @@ namespace osu.Game.Database { cancellationToken.ThrowIfCancellationRequested(); - TModel model; + TModel model = null; try { @@ -263,7 +263,7 @@ namespace osu.Game.Database } catch (Exception e) { - Logger.Error(e, $"Model creation of {archive.Name} failed.", LoggingTarget.Database); + LogForModel(model, $"Model creation of {archive.Name} failed.", e); return null; } @@ -277,6 +277,16 @@ namespace osu.Game.Database /// protected abstract string[] HashableFileTypes { get; } + protected static void LogForModel(TModel model, string message, Exception e = null) + { + string prefix = $"[{(model?.Hash ?? "?????").Substring(0, 5)}]"; + + if (e != null) + Logger.Error(e, $"{prefix} {message}", LoggingTarget.Database); + else + Logger.Log($"{prefix} {message}", LoggingTarget.Database); + } + /// /// Create a SHA-2 hash from the provided archive based on file content of all files matching . /// @@ -308,14 +318,14 @@ namespace osu.Game.Database if (!Delete(item)) { // We may have not yet added the model to the underlying table, but should still clean up files. - Logger.Log($"Dereferencing files for incomplete import of {item}.", LoggingTarget.Database); + LogForModel(item, "Dereferencing files for incomplete import."); Files.Dereference(item.Files.Select(f => f.FileInfo).ToArray()); } } try { - Logger.Log($"Importing {item}...", LoggingTarget.Database); + LogForModel(item, "Beginning import..."); item.Files = archive != null ? createFileInfos(archive, Files) : new List(); @@ -334,7 +344,7 @@ namespace osu.Game.Database if (CanUndelete(existing, item)) { Undelete(existing); - Logger.Log($"Found existing {humanisedModelName} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); + LogForModel(item, $"Found existing {humanisedModelName} for {item} (ID {existing.ID}) – skipping import."); handleEvent(() => ItemAdded?.Invoke(existing, true)); // existing item will be used; rollback new import and exit early. @@ -342,11 +352,9 @@ namespace osu.Game.Database flushEvents(true); return existing; } - else - { - Delete(existing); - ModelStore.PurgeDeletable(s => s.ID == existing.ID); - } + + Delete(existing); + ModelStore.PurgeDeletable(s => s.ID == existing.ID); } PreImport(item); @@ -361,12 +369,12 @@ namespace osu.Game.Database } } - Logger.Log($"Import of {item} successfully completed!", LoggingTarget.Database); + LogForModel(item, "Import successfully completed!"); } catch (Exception e) { if (!(e is TaskCanceledException)) - Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); + LogForModel(item, "Database import or population failed and has been rolled back.", e); rollback(); flushEvents(false); From 27054a744ed316fd623b33f4a8cfeeec9c787dc8 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 12 Jun 2019 00:35:13 +0900 Subject: [PATCH 24/27] Fill in thread pool names --- osu.Game/Beatmaps/BeatmapManager.cs | 2 +- osu.Game/Database/ArchiveModelManager.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 47411c69ec..67d3382e01 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -426,7 +426,7 @@ namespace osu.Game.Beatmaps private const int update_queue_request_concurrency = 4; - private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency); + private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency, nameof(BeatmapUpdateQueue)); public BeatmapUpdateQueue(IAPIProvider api) { diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 844ae622a5..d764601b32 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -644,6 +644,6 @@ namespace osu.Game.Database /// This scheduler generally performs IO and CPU intensive work so concurrency is limited harshly. /// It is mainly being used as a queue mechanism for large imports. /// - protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(import_queue_request_concurrency); + protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(import_queue_request_concurrency, nameof(ArchiveModelManager)); } } From c4f54d94bc5e8aa9d52b7e17c0dd3508693cc6bc Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Wed, 12 Jun 2019 17:00:27 +0900 Subject: [PATCH 25/27] Rename methods --- osu.Game/Beatmaps/BeatmapManager.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 67d3382e01..5204bda353 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -110,7 +110,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - await updateQueue.Perform(beatmapSet, cancellationToken); + await updateQueue.UpdateAsync(beatmapSet, cancellationToken); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -433,20 +433,20 @@ namespace osu.Game.Beatmaps this.api = api; } - public async Task Perform(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) + public async Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) { if (api?.State != APIState.Online) return; LogForModel(beatmapSet, "Performing online lookups..."); - await Task.WhenAll(beatmapSet.Beatmaps.Select(b => Perform(beatmapSet, b, cancellationToken)).ToArray()); + await Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray()); } // todo: expose this when we need to do individual difficulty lookups. - protected Task Perform(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken) - => Task.Factory.StartNew(() => perform(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); + protected Task UpdateAsync(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken) + => Task.Factory.StartNew(() => update(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); - private void perform(BeatmapSetInfo set, BeatmapInfo beatmap) + private void update(BeatmapSetInfo set, BeatmapInfo beatmap) { if (api?.State != APIState.Online) return; From fd7dc9504e6c91b9502c5e389d28677be49b81d5 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Wed, 12 Jun 2019 17:08:50 +0900 Subject: [PATCH 26/27] Remove async when not required --- osu.Game/Beatmaps/BeatmapManager.cs | 10 +++++----- osu.Game/Database/ArchiveModelManager.cs | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 5204bda353..d90657bff5 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -94,7 +94,7 @@ namespace osu.Game.Beatmaps updateQueue = new BeatmapUpdateQueue(api); } - protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) + protected override Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) { if (archive != null) beatmapSet.Beatmaps = createBeatmapDifficulties(archive); @@ -110,7 +110,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - await updateQueue.UpdateAsync(beatmapSet, cancellationToken); + return updateQueue.UpdateAsync(beatmapSet, cancellationToken); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -433,13 +433,13 @@ namespace osu.Game.Beatmaps this.api = api; } - public async Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) + public Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) { if (api?.State != APIState.Online) - return; + return Task.CompletedTask; LogForModel(beatmapSet, "Performing online lookups..."); - await Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray()); + return Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray()); } // todo: expose this when we need to do individual difficulty lookups. diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index d764601b32..0cf7816250 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -1,4 +1,4 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. using System; @@ -132,13 +132,13 @@ namespace osu.Game.Database /// This will post notifications tracking progress. /// /// One or more archive locations on disk. - public async Task Import(params string[] paths) + public Task Import(params string[] paths) { var notification = new ProgressNotification { State = ProgressNotificationState.Active }; PostNotification?.Invoke(notification); - await Import(notification, paths); + return Import(notification, paths); } protected async Task Import(ProgressNotification notification, params string[] paths) @@ -243,7 +243,7 @@ namespace osu.Game.Database /// /// The archive to be imported. /// An optional cancellation token. - public async Task Import(ArchiveReader archive, CancellationToken cancellationToken = default) + public Task Import(ArchiveReader archive, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -267,7 +267,7 @@ namespace osu.Game.Database return null; } - return await Import(model, archive, cancellationToken); + return Import(model, archive, cancellationToken); } /// @@ -548,24 +548,24 @@ namespace osu.Game.Database /// /// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future. /// - public async Task ImportFromStableAsync() + public Task ImportFromStableAsync() { var stable = GetStableStorage?.Invoke(); if (stable == null) { Logger.Log("No osu!stable installation available!", LoggingTarget.Information, LogLevel.Error); - return; + return Task.CompletedTask; } if (!stable.ExistsDirectory(ImportFromStablePath)) { // This handles situations like when the user does not have a Skins folder Logger.Log($"No {ImportFromStablePath} folder available in osu!stable installation", LoggingTarget.Information, LogLevel.Error); - return; + return Task.CompletedTask; } - await Task.Run(async () => await Import(stable.GetDirectories(ImportFromStablePath).Select(f => stable.GetFullPath(f)).ToArray())); + return Task.Run(async () => await Import(stable.GetDirectories(ImportFromStablePath).Select(f => stable.GetFullPath(f)).ToArray())); } #endregion From 2a67944889c302fe69a87838f70d835c63b50740 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Wed, 12 Jun 2019 17:10:55 +0900 Subject: [PATCH 27/27] Remove interlocked within a lock --- osu.Game/Database/ArchiveModelManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 0cf7816250..1c17adf7b7 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -161,8 +161,8 @@ namespace osu.Game.Database lock (imported) { imported.Add(model); + current++; - Interlocked.Increment(ref current); notification.Text = $"Imported {current} of {paths.Length} {humanisedModelName}s"; notification.Progress = (float)current / paths.Length; }