From e59a00ac6eb1f30cb368392aedb299177fd8a168 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 28 May 2019 14:04:33 +0900 Subject: [PATCH 01/47] Remove excessive selection updating --- .../SongSelect/TestScenePlaySongSelect.cs | 28 ++++++++++++++++++- osu.Game/Screens/Select/SongSelect.cs | 8 +++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index 7e962dbc06..85811e3a0a 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -209,7 +209,33 @@ namespace osu.Game.Tests.Visual.SongSelect AddAssert("start not requested", () => !startRequested); } - private void importForRuleset(int id) => AddStep($"import test map for ruleset {id}", () => manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == id).ToArray()))); + [Test] + public void TestAddNewBeatmap() + { + const int test_count = 10; + int beatmapChangedCount = 0; + createSongSelect(); + AddStep("Setup counter", () => + { + beatmapChangedCount = 0; + songSelect.Carousel.BeatmapSetsChanged += () => beatmapChangedCount++; + }); + AddRepeatStep($"Create beatmaps {test_count} times", () => + { + manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == 0).ToArray())); + + Scheduler.AddDelayed(() => + { + // Wait for debounce + songSelect.Carousel.SelectNextRandom(); + }, 400); + }, test_count); + + AddAssert($"Beatmap changed {test_count} times", () => beatmapChangedCount == test_count); + } + + private void importForRuleset(int id) => AddStep($"import test map for ruleset {id}", + () => manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == id).ToArray()))); private static int importId; private int getImportId() => ++importId; diff --git a/osu.Game/Screens/Select/SongSelect.cs b/osu.Game/Screens/Select/SongSelect.cs index fed1f7a944..f30618ce3f 100644 --- a/osu.Game/Screens/Select/SongSelect.cs +++ b/osu.Game/Screens/Select/SongSelect.cs @@ -597,11 +597,17 @@ namespace osu.Game.Screens.Select { bindBindables(); + // As a selection was already obtained, do not attempt to update the selected beatmap. + if (Carousel.SelectedBeatmapSet != null) + return; + + // Attempt to select the current beatmap on the carousel, if it is valid to be selected. if (!Beatmap.IsDefault && Beatmap.Value.BeatmapSetInfo?.DeletePending == false && Beatmap.Value.BeatmapSetInfo?.Protected == false && Carousel.SelectBeatmap(Beatmap.Value.BeatmapInfo, false)) return; - if (Carousel.SelectedBeatmapSet == null && !Carousel.SelectNextRandom()) + // If the current active beatmap could not be selected, select a new random beatmap. + if (!Carousel.SelectNextRandom()) { // in the case random selection failed, we want to trigger selectionChanged // to show the dummy beatmap (we have nothing else to display). From 436760de967e59400822e25356b502294bd487e3 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 28 May 2019 14:34:52 +0900 Subject: [PATCH 02/47] Change test name --- osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index 85811e3a0a..8e3fe57d92 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -210,7 +210,7 @@ namespace osu.Game.Tests.Visual.SongSelect } [Test] - public void TestAddNewBeatmap() + public void TestAddNewBeatmapWhileSelectingRandom() { const int test_count = 10; int beatmapChangedCount = 0; From 1a871af5520113372c38d70740337dc794fe2b30 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 28 May 2019 19:15:29 +0900 Subject: [PATCH 03/47] Fix hide selection, add test --- .../Visual/SongSelect/TestScenePlaySongSelect.cs | 12 ++++++++++++ osu.Game/Screens/Select/BeatmapCarousel.cs | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index 8e3fe57d92..c33528c3bf 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -234,6 +234,18 @@ namespace osu.Game.Tests.Visual.SongSelect AddAssert($"Beatmap changed {test_count} times", () => beatmapChangedCount == test_count); } + [Test] + public void TestHideSetSelectsCorrectBeatmap() + { + int? previousID = null; + createSongSelect(); + importForRuleset(0); + AddStep("Move to last difficulty", () => songSelect.Carousel.SelectBeatmap(songSelect.Carousel.BeatmapSets.First().Beatmaps.Last())); + AddStep("Store current ID", () => previousID = songSelect.Carousel.SelectedBeatmap.ID); + AddStep("Hide first beatmap", () => manager.Hide(songSelect.Carousel.SelectedBeatmapSet.Beatmaps.First())); + AddAssert("Selected beatmap has not changed", () => songSelect.Carousel.SelectedBeatmap.ID == previousID); + } + private void importForRuleset(int id) => AddStep($"import test map for ruleset {id}", () => manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == id).ToArray()))); diff --git a/osu.Game/Screens/Select/BeatmapCarousel.cs b/osu.Game/Screens/Select/BeatmapCarousel.cs index 63ad3b6ab2..0b3d0c448b 100644 --- a/osu.Game/Screens/Select/BeatmapCarousel.cs +++ b/osu.Game/Screens/Select/BeatmapCarousel.cs @@ -158,6 +158,9 @@ namespace osu.Game.Screens.Select var newSet = createCarouselSet(beatmapSet); + // Since we're about to remove the selected beatmap, store its ID so we can go back if needed. + var previouslySelectedID = selectedBeatmap?.Beatmap.ID; + if (existingSet != null) root.RemoveChild(existingSet); @@ -173,7 +176,7 @@ namespace osu.Game.Screens.Select //check if we can/need to maintain our current selection. if (hadSelection) - select((CarouselItem)newSet.Beatmaps.FirstOrDefault(b => b.Beatmap.ID == selectedBeatmap?.Beatmap.ID) ?? newSet); + select((CarouselItem)newSet.Beatmaps.FirstOrDefault(b => b.Beatmap.ID == previouslySelectedID) ?? newSet); itemsCache.Invalidate(); Schedule(() => BeatmapSetsChanged?.Invoke()); From 4f091417189baf51bf504cf401a914fe8d568234 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 29 May 2019 12:22:34 +0900 Subject: [PATCH 04/47] remove extra bool --- osu.Game/Screens/Select/BeatmapCarousel.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/osu.Game/Screens/Select/BeatmapCarousel.cs b/osu.Game/Screens/Select/BeatmapCarousel.cs index 0b3d0c448b..6e3bec106f 100644 --- a/osu.Game/Screens/Select/BeatmapCarousel.cs +++ b/osu.Game/Screens/Select/BeatmapCarousel.cs @@ -152,15 +152,15 @@ namespace osu.Game.Screens.Select { Schedule(() => { + int? previouslySelectedID = null; CarouselBeatmapSet existingSet = beatmapSets.FirstOrDefault(b => b.BeatmapSet.ID == beatmapSet.ID); - bool hadSelection = existingSet?.State?.Value == CarouselItemState.Selected; + // Since we're about to remove the selected beatmap, store its ID so we can go back if needed. + if (existingSet?.State?.Value == CarouselItemState.Selected) + previouslySelectedID = selectedBeatmap?.Beatmap.ID; var newSet = createCarouselSet(beatmapSet); - // Since we're about to remove the selected beatmap, store its ID so we can go back if needed. - var previouslySelectedID = selectedBeatmap?.Beatmap.ID; - if (existingSet != null) root.RemoveChild(existingSet); @@ -175,7 +175,7 @@ namespace osu.Game.Screens.Select applyActiveCriteria(false, false); //check if we can/need to maintain our current selection. - if (hadSelection) + if (previouslySelectedID != null) select((CarouselItem)newSet.Beatmaps.FirstOrDefault(b => b.Beatmap.ID == previouslySelectedID) ?? newSet); itemsCache.Invalidate(); From 55c0c6a1bbffebe94d7a620689e42d3525d3f84a Mon Sep 17 00:00:00 2001 From: Lucas A Date: Fri, 31 May 2019 17:43:58 +0200 Subject: [PATCH 05/47] Show changelog for current build by clicking on settings footer in settings overlay. --- osu.Game/Overlays/Settings/SettingsFooter.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/osu.Game/Overlays/Settings/SettingsFooter.cs b/osu.Game/Overlays/Settings/SettingsFooter.cs index e8c2c1ffe8..33ad5b101b 100644 --- a/osu.Game/Overlays/Settings/SettingsFooter.cs +++ b/osu.Game/Overlays/Settings/SettingsFooter.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using osu.Framework.Allocation; using osu.Framework.Graphics; using osu.Framework.Graphics.Containers; +using osu.Framework.Input.Events; using osu.Game.Graphics; using osu.Game.Graphics.Containers; using osu.Game.Graphics.Sprites; @@ -17,8 +18,11 @@ namespace osu.Game.Overlays.Settings { public class SettingsFooter : FillFlowContainer { + private OsuGameBase game; + private ChangelogOverlay changelog; + [BackgroundDependencyLoader] - private void load(OsuGameBase game, OsuColour colours, RulesetStore rulesets) + private void load(OsuGameBase game, OsuColour colours, RulesetStore rulesets, ChangelogOverlay changelog) { RelativeSizeAxes = Axes.X; AutoSizeAxes = Axes.Y; @@ -67,6 +71,17 @@ namespace osu.Game.Overlays.Settings Colour = DebugUtils.IsDebug ? colours.Red : Color4.White, }, }; + + this.game = game; + this.changelog = changelog; + } + + protected override bool OnClick(ClickEvent e) + { + if (!game.IsDeployedBuild) return base.OnClick(e); + + changelog?.ShowBuild("lazer", game.Version); + return base.OnClick(e); } } } From 0625f51e65447596f6f1a0a77faf130968958b84 Mon Sep 17 00:00:00 2001 From: Lucas A Date: Fri, 31 May 2019 22:42:09 +0200 Subject: [PATCH 06/47] Allow dependencies to be null in certain cases (Unit tests) --- osu.Game/Overlays/Settings/SettingsFooter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Overlays/Settings/SettingsFooter.cs b/osu.Game/Overlays/Settings/SettingsFooter.cs index 33ad5b101b..317ba2f92d 100644 --- a/osu.Game/Overlays/Settings/SettingsFooter.cs +++ b/osu.Game/Overlays/Settings/SettingsFooter.cs @@ -21,7 +21,7 @@ namespace osu.Game.Overlays.Settings private OsuGameBase game; private ChangelogOverlay changelog; - [BackgroundDependencyLoader] + [BackgroundDependencyLoader(true)] private void load(OsuGameBase game, OsuColour colours, RulesetStore rulesets, ChangelogOverlay changelog) { RelativeSizeAxes = Axes.X; From 58564579e4f679e64d470eeda0a85dfe6e70b670 Mon Sep 17 00:00:00 2001 From: Lucas A Date: Sat, 1 Jun 2019 08:46:38 +0200 Subject: [PATCH 07/47] Invert if statement --- osu.Game/Overlays/Settings/SettingsFooter.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/osu.Game/Overlays/Settings/SettingsFooter.cs b/osu.Game/Overlays/Settings/SettingsFooter.cs index 317ba2f92d..3e0eb6ffde 100644 --- a/osu.Game/Overlays/Settings/SettingsFooter.cs +++ b/osu.Game/Overlays/Settings/SettingsFooter.cs @@ -78,10 +78,11 @@ namespace osu.Game.Overlays.Settings protected override bool OnClick(ClickEvent e) { - if (!game.IsDeployedBuild) return base.OnClick(e); - - changelog?.ShowBuild("lazer", game.Version); - return base.OnClick(e); + if (game.IsDeployedBuild) + { + changelog?.ShowBuild("lazer", game.Version); + } + return true; } } } From 0a867e37aff8782cdc97e1e57f41703bfa429312 Mon Sep 17 00:00:00 2001 From: Lucas A Date: Sun, 2 Jun 2019 12:40:18 +0200 Subject: [PATCH 08/47] Resolve dependencies via Resolved Attribute --- osu.Game/Overlays/Settings/SettingsFooter.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/osu.Game/Overlays/Settings/SettingsFooter.cs b/osu.Game/Overlays/Settings/SettingsFooter.cs index 3e0eb6ffde..306512802b 100644 --- a/osu.Game/Overlays/Settings/SettingsFooter.cs +++ b/osu.Game/Overlays/Settings/SettingsFooter.cs @@ -18,11 +18,14 @@ namespace osu.Game.Overlays.Settings { public class SettingsFooter : FillFlowContainer { - private OsuGameBase game; - private ChangelogOverlay changelog; + [Resolved] + private OsuGameBase game { get; set; } + + [Resolved(CanBeNull = true)] + private ChangelogOverlay changelog { get; set; } [BackgroundDependencyLoader(true)] - private void load(OsuGameBase game, OsuColour colours, RulesetStore rulesets, ChangelogOverlay changelog) + private void load(OsuGameBase game, OsuColour colours, RulesetStore rulesets) { RelativeSizeAxes = Axes.X; AutoSizeAxes = Axes.Y; @@ -71,9 +74,6 @@ namespace osu.Game.Overlays.Settings Colour = DebugUtils.IsDebug ? colours.Red : Color4.White, }, }; - - this.game = game; - this.changelog = changelog; } protected override bool OnClick(ClickEvent e) @@ -82,6 +82,7 @@ namespace osu.Game.Overlays.Settings { changelog?.ShowBuild("lazer", game.Version); } + return true; } } From d8f45f7299dd1531f7b5b485e9feaa6b7a7ba9ba Mon Sep 17 00:00:00 2001 From: Lucas A Date: Sun, 2 Jun 2019 15:17:03 +0200 Subject: [PATCH 09/47] Disallow null references for dependencies loaded via load() --- osu.Game/Overlays/Settings/SettingsFooter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Overlays/Settings/SettingsFooter.cs b/osu.Game/Overlays/Settings/SettingsFooter.cs index 306512802b..8403f70f49 100644 --- a/osu.Game/Overlays/Settings/SettingsFooter.cs +++ b/osu.Game/Overlays/Settings/SettingsFooter.cs @@ -24,7 +24,7 @@ namespace osu.Game.Overlays.Settings [Resolved(CanBeNull = true)] private ChangelogOverlay changelog { get; set; } - [BackgroundDependencyLoader(true)] + [BackgroundDependencyLoader] private void load(OsuGameBase game, OsuColour colours, RulesetStore rulesets) { RelativeSizeAxes = Axes.X; From 115a75e4c6604a214316e7e97752547b3610faff Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 3 Jun 2019 13:16:05 +0900 Subject: [PATCH 10/47] Use a constant for lazer variables --- osu.Desktop/Overlays/VersionManager.cs | 2 +- osu.Game.Tests/Visual/Online/TestSceneChangelogOverlay.cs | 2 +- osu.Game/Online/API/Requests/Responses/APIUpdateStream.cs | 2 +- osu.Game/OsuGameBase.cs | 2 ++ osu.Game/Overlays/Settings/SettingsFooter.cs | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/osu.Desktop/Overlays/VersionManager.cs b/osu.Desktop/Overlays/VersionManager.cs index d2aad99f41..5a8cf32f14 100644 --- a/osu.Desktop/Overlays/VersionManager.cs +++ b/osu.Desktop/Overlays/VersionManager.cs @@ -120,7 +120,7 @@ namespace osu.Desktop.Overlays Activated = delegate { - changelog.ShowBuild("lazer", version); + changelog.ShowBuild(OsuGameBase.CLIENT_STREAM_NAME, version); return true; }; } diff --git a/osu.Game.Tests/Visual/Online/TestSceneChangelogOverlay.cs b/osu.Game.Tests/Visual/Online/TestSceneChangelogOverlay.cs index d1a7730bee..0655611230 100644 --- a/osu.Game.Tests/Visual/Online/TestSceneChangelogOverlay.cs +++ b/osu.Game.Tests/Visual/Online/TestSceneChangelogOverlay.cs @@ -42,7 +42,7 @@ namespace osu.Game.Tests.Visual.Online { Version = "2018.712.0", DisplayVersion = "2018.712.0", - UpdateStream = new APIUpdateStream { Name = "lazer" }, + UpdateStream = new APIUpdateStream { Name = OsuGameBase.CLIENT_STREAM_NAME }, ChangelogEntries = new List { new APIChangelogEntry diff --git a/osu.Game/Online/API/Requests/Responses/APIUpdateStream.cs b/osu.Game/Online/API/Requests/Responses/APIUpdateStream.cs index ef204c7687..d9e48373bb 100644 --- a/osu.Game/Online/API/Requests/Responses/APIUpdateStream.cs +++ b/osu.Game/Online/API/Requests/Responses/APIUpdateStream.cs @@ -45,7 +45,7 @@ namespace osu.Game.Online.API.Requests.Responses case "cuttingedge": return new Color4(238, 170, 0, 255); - case "lazer": + case OsuGameBase.CLIENT_STREAM_NAME: return new Color4(237, 18, 33, 255); case "web": diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 7b9aed8364..00672e82bd 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -44,6 +44,8 @@ namespace osu.Game /// public class OsuGameBase : Framework.Game, ICanAcceptFiles { + public const string CLIENT_STREAM_NAME = "lazer"; + protected OsuConfigManager LocalConfig; protected BeatmapManager BeatmapManager; diff --git a/osu.Game/Overlays/Settings/SettingsFooter.cs b/osu.Game/Overlays/Settings/SettingsFooter.cs index 8403f70f49..298991712b 100644 --- a/osu.Game/Overlays/Settings/SettingsFooter.cs +++ b/osu.Game/Overlays/Settings/SettingsFooter.cs @@ -80,7 +80,7 @@ namespace osu.Game.Overlays.Settings { if (game.IsDeployedBuild) { - changelog?.ShowBuild("lazer", game.Version); + changelog?.ShowBuild(OsuGameBase.CLIENT_STREAM_NAME, game.Version); } return true; From b249fb35446d1176a5bcdec7487fe1d949551e2e Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 3 Jun 2019 13:37:29 +0900 Subject: [PATCH 11/47] Update test scene to support dynamic compilation --- ...stSceneSettings.cs => TestSceneSettingsPanel.cs} | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) rename osu.Game.Tests/Visual/Settings/{TestSceneSettings.cs => TestSceneSettingsPanel.cs} (71%) diff --git a/osu.Game.Tests/Visual/Settings/TestSceneSettings.cs b/osu.Game.Tests/Visual/Settings/TestSceneSettingsPanel.cs similarity index 71% rename from osu.Game.Tests/Visual/Settings/TestSceneSettings.cs rename to osu.Game.Tests/Visual/Settings/TestSceneSettingsPanel.cs index 964754f8d0..27e3cc1590 100644 --- a/osu.Game.Tests/Visual/Settings/TestSceneSettings.cs +++ b/osu.Game.Tests/Visual/Settings/TestSceneSettingsPanel.cs @@ -1,20 +1,29 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System; +using System.Collections.Generic; using NUnit.Framework; using osu.Framework.Allocation; using osu.Framework.Graphics.Containers; using osu.Game.Overlays; +using osu.Game.Overlays.Settings; namespace osu.Game.Tests.Visual.Settings { [TestFixture] - public class TestSceneSettings : OsuTestScene + public class TestSceneSettingsPanel : OsuTestScene { private readonly SettingsPanel settings; private readonly DialogOverlay dialogOverlay; - public TestSceneSettings() + public override IReadOnlyList RequiredTypes => new[] + { + typeof(SettingsFooter), + typeof(SettingsOverlay), + }; + + public TestSceneSettingsPanel() { settings = new SettingsOverlay { From 4e5788959ee6e72953e1a17a01ff20adc90e0840 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 3 Jun 2019 13:38:06 +0900 Subject: [PATCH 12/47] Make clickable text actually a button --- osu.Game/Overlays/Settings/SettingsFooter.cs | 47 ++++++++++++++------ 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/osu.Game/Overlays/Settings/SettingsFooter.cs b/osu.Game/Overlays/Settings/SettingsFooter.cs index 298991712b..7a7f7bdf73 100644 --- a/osu.Game/Overlays/Settings/SettingsFooter.cs +++ b/osu.Game/Overlays/Settings/SettingsFooter.cs @@ -5,10 +5,10 @@ using System.Collections.Generic; using osu.Framework.Allocation; using osu.Framework.Graphics; using osu.Framework.Graphics.Containers; -using osu.Framework.Input.Events; using osu.Game.Graphics; using osu.Game.Graphics.Containers; using osu.Game.Graphics.Sprites; +using osu.Game.Graphics.UserInterface; using osu.Game.Rulesets; using osuTK; using osuTK.Graphics; @@ -18,9 +18,6 @@ namespace osu.Game.Overlays.Settings { public class SettingsFooter : FillFlowContainer { - [Resolved] - private OsuGameBase game { get; set; } - [Resolved(CanBeNull = true)] private ChangelogOverlay changelog { get; set; } @@ -65,25 +62,49 @@ namespace osu.Game.Overlays.Settings Text = game.Name, Font = OsuFont.GetFont(size: 18, weight: FontWeight.Bold), }, - new OsuSpriteText + new BuildDisplay(game.Version, DebugUtils.IsDebug) { Anchor = Anchor.TopCentre, Origin = Anchor.TopCentre, - Font = OsuFont.GetFont(size: 14), - Text = game.Version, - Colour = DebugUtils.IsDebug ? colours.Red : Color4.White, - }, + } }; } - protected override bool OnClick(ClickEvent e) + private class BuildDisplay : OsuAnimatedButton { - if (game.IsDeployedBuild) + private readonly string version; + private readonly bool isDebug; + + [Resolved] + private OsuColour colours { get; set; } + + public BuildDisplay(string version, bool isDebug) { - changelog?.ShowBuild(OsuGameBase.CLIENT_STREAM_NAME, game.Version); + this.version = version; + this.isDebug = isDebug; + + Content.RelativeSizeAxes = Axes.Y; + Content.AutoSizeAxes = AutoSizeAxes = Axes.X; + Height = 20; } - return true; + [BackgroundDependencyLoader(true)] + private void load(ChangelogOverlay changelog) + { + if (!isDebug) + Action = () => changelog?.ShowBuild(OsuGameBase.CLIENT_STREAM_NAME, version); + + Add(new OsuSpriteText + { + Font = OsuFont.GetFont(size: 16), + + Text = version, + Anchor = Anchor.Centre, + Origin = Anchor.Centre, + Padding = new MarginPadding(5), + Colour = isDebug ? colours.Red : Color4.White, + }); + } } } } From 65e3b7c2ae35840c53f0bbd06c388997af9a2054 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 3 Jun 2019 13:58:55 +0900 Subject: [PATCH 13/47] Remove unused DI --- osu.Game/Overlays/Settings/SettingsFooter.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/osu.Game/Overlays/Settings/SettingsFooter.cs b/osu.Game/Overlays/Settings/SettingsFooter.cs index 7a7f7bdf73..b5ee4b4f0c 100644 --- a/osu.Game/Overlays/Settings/SettingsFooter.cs +++ b/osu.Game/Overlays/Settings/SettingsFooter.cs @@ -18,9 +18,6 @@ namespace osu.Game.Overlays.Settings { public class SettingsFooter : FillFlowContainer { - [Resolved(CanBeNull = true)] - private ChangelogOverlay changelog { get; set; } - [BackgroundDependencyLoader] private void load(OsuGameBase game, OsuColour colours, RulesetStore rulesets) { From f090e292c974c2147f7f34b3cc79a5461e7df817 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Tue, 28 May 2019 18:59:21 +0900 Subject: [PATCH 14/47] Move ArchiveModelManager import process to async flow --- .../Beatmaps/IO/ImportBeatmapTest.cs | 50 +++--- osu.Game.Tests/Scores/IO/ImportScoreTest.cs | 30 ++-- .../TestSceneBackgroundScreenBeatmap.cs | 2 +- ...tSceneUpdateableBeatmapBackgroundSprite.cs | 2 +- osu.Game.Tests/osu.Game.Tests.csproj | 5 + osu.Game/Beatmaps/BeatmapManager.cs | 163 +++++++++++++----- osu.Game/Database/ArchiveModelManager.cs | 158 +++++++++-------- osu.Game/Database/ICanAcceptFiles.cs | 4 +- osu.Game/IPC/ArchiveImportIPCChannel.cs | 2 +- osu.Game/OsuGameBase.cs | 5 +- .../Notifications/ProgressNotification.cs | 7 + osu.Game/Screens/Menu/Intro.cs | 2 +- osu.Game/Screens/Play/Player.cs | 2 +- osu.Game/Skinning/SkinManager.cs | 6 +- 14 files changed, 273 insertions(+), 165 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index f020c2a805..4c9260f640 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -21,14 +21,14 @@ namespace osu.Game.Tests.Beatmaps.IO public class ImportBeatmapTest { [Test] - public void TestImportWhenClosed() + public async Task TestImportWhenClosed() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportWhenClosed")) { try { - LoadOszIntoOsu(loadOsu(host)); + await LoadOszIntoOsu(loadOsu(host)); } finally { @@ -38,7 +38,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportThenDelete() + public async Task TestImportThenDelete() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenDelete")) @@ -47,7 +47,7 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); deleteBeatmapSet(imported, osu); } @@ -59,7 +59,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportThenImport() + public async Task TestImportThenImport() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenImport")) @@ -68,8 +68,8 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); - var imported = LoadOszIntoOsu(osu); - var importedSecondTime = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); + var importedSecondTime = await LoadOszIntoOsu(osu); // check the newly "imported" beatmap is actually just the restored previous import. since it matches hash. Assert.IsTrue(imported.ID == importedSecondTime.ID); @@ -88,7 +88,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestRollbackOnFailure() + public async Task TestRollbackOnFailure() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestRollbackOnFailure")) @@ -104,7 +104,7 @@ namespace osu.Game.Tests.Beatmaps.IO manager.ItemAdded += (_, __) => fireCount++; manager.ItemRemoved += _ => fireCount++; - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); Assert.AreEqual(0, fireCount -= 1); @@ -132,7 +132,7 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); // this will trigger purging of the existing beatmap (online set id match) but should rollback due to broken osu. - manager.Import(breakTemp); + await manager.Import(breakTemp); // no events should be fired in the case of a rollback. Assert.AreEqual(0, fireCount); @@ -149,7 +149,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportThenImportDifferentHash() + public async Task TestImportThenImportDifferentHash() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenImportDifferentHash")) @@ -159,12 +159,12 @@ namespace osu.Game.Tests.Beatmaps.IO var osu = loadOsu(host); var manager = osu.Dependencies.Get(); - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); imported.Hash += "-changed"; manager.Update(imported); - var importedSecondTime = LoadOszIntoOsu(osu); + var importedSecondTime = await LoadOszIntoOsu(osu); Assert.IsTrue(imported.ID != importedSecondTime.ID); Assert.IsTrue(imported.Beatmaps.First().ID < importedSecondTime.Beatmaps.First().ID); @@ -181,7 +181,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportThenDeleteThenImport() + public async Task TestImportThenDeleteThenImport() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportThenDeleteThenImport")) @@ -190,11 +190,11 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); deleteBeatmapSet(imported, osu); - var importedSecondTime = LoadOszIntoOsu(osu); + var importedSecondTime = await LoadOszIntoOsu(osu); // check the newly "imported" beatmap is actually just the restored previous import. since it matches hash. Assert.IsTrue(imported.ID == importedSecondTime.ID); @@ -209,7 +209,7 @@ namespace osu.Game.Tests.Beatmaps.IO [TestCase(true)] [TestCase(false)] - public void TestImportThenDeleteThenImportWithOnlineIDMismatch(bool set) + public async Task TestImportThenDeleteThenImportWithOnlineIDMismatch(bool set) { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost($"TestImportThenDeleteThenImport-{set}")) @@ -218,7 +218,7 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); - var imported = LoadOszIntoOsu(osu); + var imported = await LoadOszIntoOsu(osu); if (set) imported.OnlineBeatmapSetID = 1234; @@ -229,7 +229,7 @@ namespace osu.Game.Tests.Beatmaps.IO deleteBeatmapSet(imported, osu); - var importedSecondTime = LoadOszIntoOsu(osu); + var importedSecondTime = await LoadOszIntoOsu(osu); // check the newly "imported" beatmap has been reimported due to mismatch (even though hashes matched) Assert.IsTrue(imported.ID != importedSecondTime.ID); @@ -243,7 +243,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportWithDuplicateBeatmapIDs() + public async Task TestImportWithDuplicateBeatmapIDs() { //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportWithDuplicateBeatmapID")) @@ -284,7 +284,7 @@ namespace osu.Game.Tests.Beatmaps.IO var manager = osu.Dependencies.Get(); - var imported = manager.Import(toImport); + var imported = await manager.Import(toImport); Assert.NotNull(imported); Assert.AreEqual(null, imported.Beatmaps[0].OnlineBeatmapID); @@ -330,7 +330,7 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public void TestImportWhenFileOpen() + public async Task TestImportWhenFileOpen() { using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportWhenFileOpen")) { @@ -339,7 +339,7 @@ namespace osu.Game.Tests.Beatmaps.IO var osu = loadOsu(host); var temp = TestResources.GetTestBeatmapForImport(); using (File.OpenRead(temp)) - osu.Dependencies.Get().Import(temp); + await osu.Dependencies.Get().Import(temp); ensureLoaded(osu); File.Delete(temp); Assert.IsFalse(File.Exists(temp), "We likely held a read lock on the file when we shouldn't"); @@ -351,13 +351,13 @@ namespace osu.Game.Tests.Beatmaps.IO } } - public static BeatmapSetInfo LoadOszIntoOsu(OsuGameBase osu, string path = null) + public static async Task LoadOszIntoOsu(OsuGameBase osu, string path = null) { var temp = path ?? TestResources.GetTestBeatmapForImport(); var manager = osu.Dependencies.Get(); - manager.Import(temp); + await manager.Import(temp); var imported = manager.GetAllUsableBeatmapSets(); diff --git a/osu.Game.Tests/Scores/IO/ImportScoreTest.cs b/osu.Game.Tests/Scores/IO/ImportScoreTest.cs index e39f18c3cd..4babb07213 100644 --- a/osu.Game.Tests/Scores/IO/ImportScoreTest.cs +++ b/osu.Game.Tests/Scores/IO/ImportScoreTest.cs @@ -23,13 +23,13 @@ namespace osu.Game.Tests.Scores.IO public class ImportScoreTest { [Test] - public void TestBasicImport() + public async Task TestBasicImport() { using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestBasicImport")) { try { - var osu = loadOsu(host); + var osu = await loadOsu(host); var toImport = new ScoreInfo { @@ -43,7 +43,7 @@ namespace osu.Game.Tests.Scores.IO OnlineScoreID = 12345, }; - var imported = loadIntoOsu(osu, toImport); + var imported = await loadIntoOsu(osu, toImport); Assert.AreEqual(toImport.Rank, imported.Rank); Assert.AreEqual(toImport.TotalScore, imported.TotalScore); @@ -62,20 +62,20 @@ namespace osu.Game.Tests.Scores.IO } [Test] - public void TestImportMods() + public async Task TestImportMods() { using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportMods")) { try { - var osu = loadOsu(host); + var osu = await loadOsu(host); var toImport = new ScoreInfo { Mods = new Mod[] { new OsuModHardRock(), new OsuModDoubleTime() }, }; - var imported = loadIntoOsu(osu, toImport); + var imported = await loadIntoOsu(osu, toImport); Assert.IsTrue(imported.Mods.Any(m => m is OsuModHardRock)); Assert.IsTrue(imported.Mods.Any(m => m is OsuModDoubleTime)); @@ -88,13 +88,13 @@ namespace osu.Game.Tests.Scores.IO } [Test] - public void TestImportStatistics() + public async Task TestImportStatistics() { using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestImportStatistics")) { try { - var osu = loadOsu(host); + var osu = await loadOsu(host); var toImport = new ScoreInfo { @@ -105,7 +105,7 @@ namespace osu.Game.Tests.Scores.IO } }; - var imported = loadIntoOsu(osu, toImport); + var imported = await loadIntoOsu(osu, toImport); Assert.AreEqual(toImport.Statistics[HitResult.Perfect], imported.Statistics[HitResult.Perfect]); Assert.AreEqual(toImport.Statistics[HitResult.Miss], imported.Statistics[HitResult.Miss]); @@ -117,7 +117,7 @@ namespace osu.Game.Tests.Scores.IO } } - private ScoreInfo loadIntoOsu(OsuGameBase osu, ScoreInfo score) + private async Task loadIntoOsu(OsuGameBase osu, ScoreInfo score) { var beatmapManager = osu.Dependencies.Get(); @@ -125,20 +125,24 @@ namespace osu.Game.Tests.Scores.IO score.Ruleset = new OsuRuleset().RulesetInfo; var scoreManager = osu.Dependencies.Get(); - scoreManager.Import(score); + await scoreManager.Import(score); return scoreManager.GetAllUsableScores().First(); } - private OsuGameBase loadOsu(GameHost host) + private async Task loadOsu(GameHost host) { var osu = new OsuGameBase(); + +#pragma warning disable 4014 Task.Run(() => host.Run(osu)); +#pragma warning restore 4014 + waitForOrAssert(() => osu.IsLoaded, @"osu! failed to start in a reasonable amount of time"); var beatmapFile = TestResources.GetTestBeatmapForImport(); var beatmapManager = osu.Dependencies.Get(); - beatmapManager.Import(beatmapFile); + await beatmapManager.Import(beatmapFile); return osu; } diff --git a/osu.Game.Tests/Visual/Background/TestSceneBackgroundScreenBeatmap.cs b/osu.Game.Tests/Visual/Background/TestSceneBackgroundScreenBeatmap.cs index 7104a420a3..8b941e4633 100644 --- a/osu.Game.Tests/Visual/Background/TestSceneBackgroundScreenBeatmap.cs +++ b/osu.Game.Tests/Visual/Background/TestSceneBackgroundScreenBeatmap.cs @@ -72,7 +72,7 @@ namespace osu.Game.Tests.Visual.Background Dependencies.Cache(manager = new BeatmapManager(LocalStorage, factory, rulesets, null, audio, host, Beatmap.Default)); Dependencies.Cache(new OsuConfigManager(LocalStorage)); - manager.Import(TestResources.GetTestBeatmapForImport()); + manager.Import(TestResources.GetTestBeatmapForImport()).Wait(); Beatmap.SetDefault(); } diff --git a/osu.Game.Tests/Visual/UserInterface/TestSceneUpdateableBeatmapBackgroundSprite.cs b/osu.Game.Tests/Visual/UserInterface/TestSceneUpdateableBeatmapBackgroundSprite.cs index f59458ef8d..c361598354 100644 --- a/osu.Game.Tests/Visual/UserInterface/TestSceneUpdateableBeatmapBackgroundSprite.cs +++ b/osu.Game.Tests/Visual/UserInterface/TestSceneUpdateableBeatmapBackgroundSprite.cs @@ -32,7 +32,7 @@ namespace osu.Game.Tests.Visual.UserInterface this.api = api; this.rulesets = rulesets; - testBeatmap = ImportBeatmapTest.LoadOszIntoOsu(osu); + testBeatmap = ImportBeatmapTest.LoadOszIntoOsu(osu).Result; } [Test] diff --git a/osu.Game.Tests/osu.Game.Tests.csproj b/osu.Game.Tests/osu.Game.Tests.csproj index 11d70ee7be..403ae16885 100644 --- a/osu.Game.Tests/osu.Game.Tests.csproj +++ b/osu.Game.Tests/osu.Game.Tests.csproj @@ -18,4 +18,9 @@ + + + ..\..\..\..\..\usr\local\share\dotnet\sdk\NuGetFallbackFolder\microsoft.win32.registry\4.5.0\ref\netstandard2.0\Microsoft.Win32.Registry.dll + + \ No newline at end of file diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index b6fe7f88fa..f5ef52a93b 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -2,10 +2,12 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Linq.Expressions; +using System.Threading; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using osu.Framework.Audio; @@ -14,6 +16,7 @@ using osu.Framework.Extensions; using osu.Framework.Graphics.Textures; using osu.Framework.Logging; using osu.Framework.Platform; +using osu.Framework.Threading; using osu.Game.Beatmaps.Formats; using osu.Game.Database; using osu.Game.IO.Archives; @@ -72,6 +75,8 @@ namespace osu.Game.Beatmaps private readonly List currentDownloads = new List(); + private readonly BeatmapUpdateQueue updateQueue; + public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, AudioManager audioManager, GameHost host = null, WorkingBeatmap defaultBeatmap = null) : base(storage, contextFactory, new BeatmapStore(contextFactory), host) @@ -86,9 +91,11 @@ namespace osu.Game.Beatmaps beatmaps = (BeatmapStore)ModelStore; beatmaps.BeatmapHidden += b => BeatmapHidden?.Invoke(b); beatmaps.BeatmapRestored += b => BeatmapRestored?.Invoke(b); + + updateQueue = new BeatmapUpdateQueue(api); } - protected override void Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive) + protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) { if (archive != null) beatmapSet.Beatmaps = createBeatmapDifficulties(archive); @@ -104,8 +111,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - foreach (BeatmapInfo b in beatmapSet.Beatmaps) - fetchAndPopulateOnlineValues(b); + await Task.WhenAll(beatmapSet.Beatmaps.Select(b => updateQueue.Enqueue(new UpdateItem(b, cancellationToken)).Task).ToArray()); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -181,10 +187,10 @@ namespace osu.Game.Beatmaps request.Success += filename => { - Task.Factory.StartNew(() => + Task.Factory.StartNew(async () => { // This gets scheduled back to the update thread, but we want the import to run in the background. - Import(downloadNotification, filename); + await Import(downloadNotification, filename); currentDownloads.Remove(request); }, TaskCreationOptions.LongRunning); }; @@ -381,47 +387,6 @@ namespace osu.Game.Beatmaps return beatmapInfos; } - /// - /// Query the API to populate missing values like OnlineBeatmapID / OnlineBeatmapSetID or (Rank-)Status. - /// - /// The beatmap to populate. - /// Whether to re-query if the provided beatmap already has populated values. - /// True if population was successful. - private bool fetchAndPopulateOnlineValues(BeatmapInfo beatmap, bool force = false) - { - if (api?.State != APIState.Online) - return false; - - if (!force && beatmap.OnlineBeatmapID != null && beatmap.BeatmapSet.OnlineBeatmapSetID != null - && beatmap.Status != BeatmapSetOnlineStatus.None && beatmap.BeatmapSet.Status != BeatmapSetOnlineStatus.None) - return true; - - Logger.Log("Attempting online lookup for the missing values...", LoggingTarget.Database); - - try - { - var req = new GetBeatmapRequest(beatmap); - - req.Perform(api); - - var res = req.Result; - - Logger.Log($"Successfully mapped to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}.", LoggingTarget.Database); - - beatmap.Status = res.Status; - beatmap.BeatmapSet.Status = res.BeatmapSet.Status; - beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID; - beatmap.OnlineBeatmapID = res.OnlineBeatmapID; - - return true; - } - catch (Exception e) - { - Logger.Log($"Failed ({e})", LoggingTarget.Database); - return false; - } - } - /// /// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation. /// @@ -455,5 +420,111 @@ namespace osu.Game.Beatmaps public override bool IsImportant => false; } } + + private class BeatmapUpdateQueue + { + private readonly IAPIProvider api; + private readonly Queue queue = new Queue(); + + private int activeThreads; + + public BeatmapUpdateQueue(IAPIProvider api) + { + this.api = api; + } + + public UpdateItem Enqueue(UpdateItem item) + { + lock (queue) + { + queue.Enqueue(item); + + if (activeThreads >= 16) + return item; + + new Thread(runWork) { IsBackground = true }.Start(); + activeThreads++; + } + + return item; + } + + private void runWork() + { + while (true) + { + UpdateItem toProcess; + + lock (queue) + { + if (queue.Count == 0) + break; + + toProcess = queue.Dequeue(); + } + + toProcess.PerformUpdate(api); + } + + lock (queue) + activeThreads--; + } + } + + private class UpdateItem + { + public Task Task => tcs.Task; + + private readonly BeatmapInfo beatmap; + private readonly CancellationToken cancellationToken; + + private readonly TaskCompletionSource tcs = new TaskCompletionSource(); + + public UpdateItem(BeatmapInfo beatmap, CancellationToken cancellationToken) + { + this.beatmap = beatmap; + this.cancellationToken = cancellationToken; + } + + public void PerformUpdate(IAPIProvider api) + { + if (cancellationToken.IsCancellationRequested) + { + tcs.SetCanceled(); + return; + } + + if (api?.State != APIState.Online) + { + tcs.SetResult(false); + return; + } + + Logger.Log("Attempting online lookup for the missing values...", LoggingTarget.Database); + + var req = new GetBeatmapRequest(beatmap); + + req.Success += res => + { + Logger.Log($"Successfully mapped to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}.", LoggingTarget.Database); + + beatmap.Status = res.Status; + beatmap.BeatmapSet.Status = res.BeatmapSet.Status; + beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID; + beatmap.OnlineBeatmapID = res.OnlineBeatmapID; + + tcs.SetResult(true); + }; + + req.Failure += e => + { + Logger.Log($"Failed ({e})", LoggingTarget.Database); + + tcs.SetResult(false); + }; + + req.Perform(api); + } + } } } diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 54dbae9ddc..afc614772e 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -5,14 +5,17 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; using Microsoft.EntityFrameworkCore; using osu.Framework; using osu.Framework.Extensions; +using osu.Framework.Extensions.TypeExtensions; using osu.Framework.IO.File; using osu.Framework.Logging; using osu.Framework.Platform; +using osu.Framework.Threading; using osu.Game.IO; using osu.Game.IO.Archives; using osu.Game.IPC; @@ -109,8 +112,11 @@ namespace osu.Game.Database a.Invoke(); } + private readonly ThreadedTaskScheduler importScheduler; + protected ArchiveModelManager(Storage storage, IDatabaseContextFactory contextFactory, MutableDatabaseBackedStoreWithFileIncludes modelStore, IIpcHost importHost = null) { + importScheduler = new ThreadedTaskScheduler(16, $"{GetType().ReadableName()}.Import"); ContextFactory = contextFactory; ModelStore = modelStore; @@ -130,92 +136,84 @@ namespace osu.Game.Database /// This will post notifications tracking progress. /// /// One or more archive locations on disk. - public void Import(params string[] paths) + public async Task Import(params string[] paths) { var notification = new ProgressNotification { State = ProgressNotificationState.Active }; PostNotification?.Invoke(notification); - Import(notification, paths); + + await Import(notification, paths); } - protected void Import(ProgressNotification notification, params string[] paths) + protected async Task Import(ProgressNotification notification, params string[] paths) { notification.Progress = 0; notification.Text = "Import is initialising..."; var term = $"{typeof(TModel).Name.Replace("Info", "").ToLower()}"; - List imported = new List(); + var tasks = new List(); int current = 0; foreach (string path in paths) { - if (notification.State == ProgressNotificationState.Cancelled) - // user requested abort - return; - - try + tasks.Add(Import(path, notification.CancellationToken).ContinueWith(t => { - var text = "Importing "; - - if (path.Length > 1) - text += $"{++current} of {paths.Length} {term}s.."; - else - text += $"{term}.."; - - // only show the filename if it isn't a temporary one (as those look ugly). - if (!path.Contains(Path.GetTempPath())) - text += $"\n{Path.GetFileName(path)}"; - - notification.Text = text; - - imported.Add(Import(path)); - - notification.Progress = (float)current / paths.Length; - } - catch (Exception e) - { - e = e.InnerException ?? e; - Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); - } - } - - if (imported.Count == 0) - { - notification.Text = "Import failed!"; - notification.State = ProgressNotificationState.Cancelled; - } - else - { - notification.CompletionText = imported.Count == 1 - ? $"Imported {imported.First()}!" - : $"Imported {current} {term}s!"; - - if (imported.Count > 0 && PresentImport != null) - { - notification.CompletionText += " Click to view."; - notification.CompletionClickAction = () => + lock (notification) { - PresentImport?.Invoke(imported); - return true; - }; - } + current++; - notification.State = ProgressNotificationState.Completed; + notification.Text = $"Imported {current} of {paths.Length} {term}s"; + notification.Progress = (float)current / paths.Length; + } + + if (t.Exception != null) + { + var e = t.Exception.InnerException ?? t.Exception; + Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + } + })); } + + await Task.WhenAll(tasks); + + // if (imported.Count == 0) + // { + // notification.Text = "Import failed!"; + // notification.State = ProgressNotificationState.Cancelled; + // } + // else + // { + // notification.CompletionText = imported.Count == 1 + // ? $"Imported {imported.First()}!" + // : $"Imported {current} {term}s!"; + // + // if (imported.Count > 0 && PresentImport != null) + // { + // notification.CompletionText += " Click to view."; + // notification.CompletionClickAction = () => + // { + // PresentImport?.Invoke(imported); + // return true; + // }; + // } + // + // notification.State = ProgressNotificationState.Completed; + // } } /// /// Import one from the filesystem and delete the file on success. /// /// The archive location on disk. + /// An optional cancellation token. /// The imported model, if successful. - public TModel Import(string path) + public async Task Import(string path, CancellationToken cancellationToken = default) { TModel import; using (ArchiveReader reader = getReaderFrom(path)) - import = Import(reader); + import = await Import(reader, cancellationToken); // We may or may not want to delete the file depending on where it is stored. // e.g. reconstructing/repairing database with items from default storage. @@ -243,7 +241,8 @@ namespace osu.Game.Database /// Import an item from an . /// /// The archive to be imported. - public TModel Import(ArchiveReader archive) + /// An optional cancellation token. + public async Task Import(ArchiveReader archive, CancellationToken cancellationToken = default) { try { @@ -253,7 +252,7 @@ namespace osu.Game.Database model.Hash = computeHash(archive); - return Import(model, archive); + return await Import(model, archive, cancellationToken); } catch (Exception e) { @@ -288,7 +287,8 @@ namespace osu.Game.Database /// /// The model to be imported. /// An optional archive to use for model population. - public TModel Import(TModel item, ArchiveReader archive = null) + /// An optional cancellation token. + public async Task Import(TModel item, ArchiveReader archive = null, CancellationToken cancellationToken = default) => await Task.Factory.StartNew(async () => { delayEvents(); @@ -296,17 +296,31 @@ namespace osu.Game.Database { Logger.Log($"Importing {item}...", LoggingTarget.Database); + if (archive != null) + item.Files = createFileInfos(archive, Files); + + var localItem = item; + + try + { + await Populate(item, archive, cancellationToken); + } + catch (TaskCanceledException) + { + return item = null; + } + finally + { + if (!Delete(localItem)) + Files.Dereference(localItem.Files.Select(f => f.FileInfo).ToArray()); + } + using (var write = ContextFactory.GetForWrite()) // used to share a context for full import. keep in mind this will block all writes. { try { if (!write.IsTransactionLeader) throw new InvalidOperationException($"Ensure there is no parent transaction so errors can correctly be handled by {this}"); - if (archive != null) - item.Files = createFileInfos(archive, Files); - - Populate(item, archive); - var existing = CheckForExisting(item); if (existing != null) @@ -332,6 +346,9 @@ namespace osu.Game.Database } catch (Exception e) { + if (!Delete(item)) + Files.Dereference(item.Files.Select(f => f.FileInfo).ToArray()); + write.Errors.Add(e); throw; } @@ -351,7 +368,7 @@ namespace osu.Game.Database } return item; - } + }, CancellationToken.None, TaskCreationOptions.None, importScheduler).Unwrap(); /// /// Perform an update of the specified item. @@ -516,24 +533,24 @@ namespace osu.Game.Database /// /// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future. /// - public Task ImportFromStableAsync() + public async Task ImportFromStableAsync() { var stable = GetStableStorage?.Invoke(); if (stable == null) { Logger.Log("No osu!stable installation available!", LoggingTarget.Information, LogLevel.Error); - return Task.CompletedTask; + return; } if (!stable.ExistsDirectory(ImportFromStablePath)) { // This handles situations like when the user does not have a Skins folder Logger.Log($"No {ImportFromStablePath} folder available in osu!stable installation", LoggingTarget.Information, LogLevel.Error); - return Task.CompletedTask; + return; } - return Task.Factory.StartNew(() => Import(stable.GetDirectories(ImportFromStablePath).Select(f => stable.GetFullPath(f)).ToArray()), TaskCreationOptions.LongRunning); + await Task.Run(async () => await Import(stable.GetDirectories(ImportFromStablePath).Select(f => stable.GetFullPath(f)).ToArray())); } #endregion @@ -552,9 +569,8 @@ namespace osu.Game.Database /// /// The model to populate. /// The archive to use as a reference for population. May be null. - protected virtual void Populate(TModel model, [CanBeNull] ArchiveReader archive) - { - } + /// An optional cancellation token. + protected virtual async Task Populate(TModel model, [CanBeNull] ArchiveReader archive, CancellationToken cancellationToken = default) => await Task.CompletedTask; /// /// Perform any final actions before the import to database executes. diff --git a/osu.Game/Database/ICanAcceptFiles.cs b/osu.Game/Database/ICanAcceptFiles.cs index f55d0c389e..b9f882468d 100644 --- a/osu.Game/Database/ICanAcceptFiles.cs +++ b/osu.Game/Database/ICanAcceptFiles.cs @@ -1,6 +1,8 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System.Threading.Tasks; + namespace osu.Game.Database { /// @@ -12,7 +14,7 @@ namespace osu.Game.Database /// Import the specified paths. /// /// The files which should be imported. - void Import(params string[] paths); + Task Import(params string[] paths); /// /// An array of accepted file extensions (in the standard format of ".abc"). diff --git a/osu.Game/IPC/ArchiveImportIPCChannel.cs b/osu.Game/IPC/ArchiveImportIPCChannel.cs index fc747cd446..484db932f8 100644 --- a/osu.Game/IPC/ArchiveImportIPCChannel.cs +++ b/osu.Game/IPC/ArchiveImportIPCChannel.cs @@ -38,7 +38,7 @@ namespace osu.Game.IPC } if (importer.HandledExtensions.Contains(Path.GetExtension(path)?.ToLowerInvariant())) - importer.Import(path); + await importer.Import(path); } } diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index f9128687d6..d6b8caaf5b 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; +using System.Threading.Tasks; using osu.Framework.Allocation; using osu.Framework.Audio; using osu.Framework.Bindables; @@ -268,13 +269,13 @@ namespace osu.Game private readonly List fileImporters = new List(); - public void Import(params string[] paths) + public async Task Import(params string[] paths) { var extension = Path.GetExtension(paths.First())?.ToLowerInvariant(); foreach (var importer in fileImporters) if (importer.HandledExtensions.Contains(extension)) - importer.Import(paths); + await importer.Import(paths); } public string[] HandledExtensions => fileImporters.SelectMany(i => i.HandledExtensions).ToArray(); diff --git a/osu.Game/Overlays/Notifications/ProgressNotification.cs b/osu.Game/Overlays/Notifications/ProgressNotification.cs index 857a0bda9e..c8e081d29f 100644 --- a/osu.Game/Overlays/Notifications/ProgressNotification.cs +++ b/osu.Game/Overlays/Notifications/ProgressNotification.cs @@ -2,6 +2,7 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.Threading; using osu.Framework.Allocation; using osu.Framework.Graphics; using osu.Framework.Graphics.Containers; @@ -36,6 +37,10 @@ namespace osu.Game.Overlays.Notifications State = state; } + private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + + public CancellationToken CancellationToken => cancellationTokenSource.Token; + public virtual ProgressNotificationState State { get => state; @@ -62,6 +67,8 @@ namespace osu.Game.Overlays.Notifications break; case ProgressNotificationState.Cancelled: + cancellationTokenSource.Cancel(); + Light.Colour = colourCancelled; Light.Pulsate = false; progressBar.Active = false; diff --git a/osu.Game/Screens/Menu/Intro.cs b/osu.Game/Screens/Menu/Intro.cs index 98a2fe8f13..cf5d247482 100644 --- a/osu.Game/Screens/Menu/Intro.cs +++ b/osu.Game/Screens/Menu/Intro.cs @@ -66,7 +66,7 @@ namespace osu.Game.Screens.Menu if (setInfo == null) { // we need to import the default menu background beatmap - setInfo = beatmaps.Import(new ZipArchiveReader(game.Resources.GetStream(@"Tracks/circles.osz"), "circles.osz")); + setInfo = beatmaps.Import(new ZipArchiveReader(game.Resources.GetStream(@"Tracks/circles.osz"), "circles.osz")).Result; setInfo.Protected = true; beatmaps.Update(setInfo); diff --git a/osu.Game/Screens/Play/Player.cs b/osu.Game/Screens/Play/Player.cs index d8389fa6d9..949f9e5ff2 100644 --- a/osu.Game/Screens/Play/Player.cs +++ b/osu.Game/Screens/Play/Player.cs @@ -279,7 +279,7 @@ namespace osu.Game.Screens.Play var score = CreateScore(); if (DrawableRuleset.ReplayScore == null) - scoreManager.Import(score); + scoreManager.Import(score).Wait(); this.Push(CreateResults(score)); diff --git a/osu.Game/Skinning/SkinManager.cs b/osu.Game/Skinning/SkinManager.cs index 3a4d44f608..73cc47ea47 100644 --- a/osu.Game/Skinning/SkinManager.cs +++ b/osu.Game/Skinning/SkinManager.cs @@ -5,6 +5,8 @@ using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using osu.Framework.Audio; using osu.Framework.Audio.Sample; @@ -71,9 +73,9 @@ namespace osu.Game.Skinning protected override SkinInfo CreateModel(ArchiveReader archive) => new SkinInfo { Name = archive.Name }; - protected override void Populate(SkinInfo model, ArchiveReader archive) + protected override async Task Populate(SkinInfo model, ArchiveReader archive, CancellationToken cancellationToken = default) { - base.Populate(model, archive); + await base.Populate(model, archive, cancellationToken); Skin reference = getSkin(model); From 600503ec8ebdb6233beab64b0faa53f02b91af4e Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 12:46:21 +0900 Subject: [PATCH 15/47] Use Task.Run/Wait to avoid warnings --- osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs | 4 ++-- osu.Game/Screens/Select/SongSelect.cs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index ebee358730..c2e8078bd6 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -210,7 +210,7 @@ namespace osu.Game.Tests.Visual.SongSelect AddAssert("start not requested", () => !startRequested); } - private void importForRuleset(int id) => AddStep($"import test map for ruleset {id}", () => manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == id).ToArray()))); + private void importForRuleset(int id) => AddStep($"import test map for ruleset {id}", () => manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == id).ToArray())).Wait()); private static int importId; private int getImportId() => ++importId; @@ -232,7 +232,7 @@ namespace osu.Game.Tests.Visual.SongSelect var usableRulesets = rulesets.AvailableRulesets.Where(r => r.ID != 2).ToArray(); for (int i = 0; i < 100; i += 10) - manager.Import(createTestBeatmapSet(i, usableRulesets)); + manager.Import(createTestBeatmapSet(i, usableRulesets)).Wait(); }); } diff --git a/osu.Game/Screens/Select/SongSelect.cs b/osu.Game/Screens/Select/SongSelect.cs index 6d5be607f4..196d655851 100644 --- a/osu.Game/Screens/Select/SongSelect.cs +++ b/osu.Game/Screens/Select/SongSelect.cs @@ -32,6 +32,7 @@ using osuTK.Input; using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using osu.Framework.Graphics.Sprites; namespace osu.Game.Screens.Select @@ -256,8 +257,8 @@ namespace osu.Game.Screens.Select if (!beatmaps.GetAllUsableBeatmapSets().Any() && beatmaps.StableInstallationAvailable) dialogOverlay.Push(new ImportFromStablePopup(() => { - beatmaps.ImportFromStableAsync(); - skins.ImportFromStableAsync(); + Task.Run(beatmaps.ImportFromStableAsync); + Task.Run(skins.ImportFromStableAsync); })); }); } From b4d2d0bd0b245cba425d9e5d981e3831d9269931 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:19:58 +0900 Subject: [PATCH 16/47] Simplify and combine concurrency of ArchiveModelManager --- osu.Game/Beatmaps/BeatmapManager.cs | 77 ++----------------- osu.Game/Database/ArchiveModelManager.cs | 96 ++++++++++++------------ 2 files changed, 57 insertions(+), 116 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index f5ef52a93b..9e7b6d7971 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -2,7 +2,6 @@ // See the LICENCE file in the repository root for full licence text. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -111,7 +110,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - await Task.WhenAll(beatmapSet.Beatmaps.Select(b => updateQueue.Enqueue(new UpdateItem(b, cancellationToken)).Task).ToArray()); + await Task.WhenAll(beatmapSet.Beatmaps.Select(b => updateQueue.Perform(b, cancellationToken)).ToArray()); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -424,81 +423,24 @@ namespace osu.Game.Beatmaps private class BeatmapUpdateQueue { private readonly IAPIProvider api; - private readonly Queue queue = new Queue(); - private int activeThreads; + private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(4); public BeatmapUpdateQueue(IAPIProvider api) { this.api = api; } - public UpdateItem Enqueue(UpdateItem item) + public Task Perform(BeatmapInfo beatmap, CancellationToken cancellationToken) + => Task.Factory.StartNew(() => perform(beatmap, cancellationToken), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); + + private void perform(BeatmapInfo beatmap, CancellationToken cancellation) { - lock (queue) - { - queue.Enqueue(item); - - if (activeThreads >= 16) - return item; - - new Thread(runWork) { IsBackground = true }.Start(); - activeThreads++; - } - - return item; - } - - private void runWork() - { - while (true) - { - UpdateItem toProcess; - - lock (queue) - { - if (queue.Count == 0) - break; - - toProcess = queue.Dequeue(); - } - - toProcess.PerformUpdate(api); - } - - lock (queue) - activeThreads--; - } - } - - private class UpdateItem - { - public Task Task => tcs.Task; - - private readonly BeatmapInfo beatmap; - private readonly CancellationToken cancellationToken; - - private readonly TaskCompletionSource tcs = new TaskCompletionSource(); - - public UpdateItem(BeatmapInfo beatmap, CancellationToken cancellationToken) - { - this.beatmap = beatmap; - this.cancellationToken = cancellationToken; - } - - public void PerformUpdate(IAPIProvider api) - { - if (cancellationToken.IsCancellationRequested) - { - tcs.SetCanceled(); + if (cancellation.IsCancellationRequested) return; - } if (api?.State != APIState.Online) - { - tcs.SetResult(false); return; - } Logger.Log("Attempting online lookup for the missing values...", LoggingTarget.Database); @@ -512,17 +454,14 @@ namespace osu.Game.Beatmaps beatmap.BeatmapSet.Status = res.BeatmapSet.Status; beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID; beatmap.OnlineBeatmapID = res.OnlineBeatmapID; - - tcs.SetResult(true); }; req.Failure += e => { Logger.Log($"Failed ({e})", LoggingTarget.Database); - - tcs.SetResult(false); }; + // intentionally blocking to limit web request concurrency req.Perform(api); } } diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index afc614772e..e30e1cd3ee 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -1,4 +1,4 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. using System; @@ -11,7 +11,6 @@ using JetBrains.Annotations; using Microsoft.EntityFrameworkCore; using osu.Framework; using osu.Framework.Extensions; -using osu.Framework.Extensions.TypeExtensions; using osu.Framework.IO.File; using osu.Framework.Logging; using osu.Framework.Platform; @@ -32,7 +31,7 @@ namespace osu.Game.Database /// /// The model type. /// The associated file join type. - public abstract class ArchiveModelManager : ICanAcceptFiles + public abstract class ArchiveModelManager : ArchiveModelManager, ICanAcceptFiles where TModel : class, IHasFiles, IHasPrimaryKey, ISoftDelete where TFileModel : INamedFileInfo, new() { @@ -112,11 +111,8 @@ namespace osu.Game.Database a.Invoke(); } - private readonly ThreadedTaskScheduler importScheduler; - protected ArchiveModelManager(Storage storage, IDatabaseContextFactory contextFactory, MutableDatabaseBackedStoreWithFileIncludes modelStore, IIpcHost importHost = null) { - importScheduler = new ThreadedTaskScheduler(16, $"{GetType().ReadableName()}.Import"); ContextFactory = contextFactory; ModelStore = modelStore; @@ -152,55 +148,55 @@ namespace osu.Game.Database var term = $"{typeof(TModel).Name.Replace("Info", "").ToLower()}"; - var tasks = new List(); - int current = 0; - foreach (string path in paths) + var imported = new List(); + + await Task.WhenAll(paths.Select(path => Import(path, notification.CancellationToken).ContinueWith(t => { - tasks.Add(Import(path, notification.CancellationToken).ContinueWith(t => + lock (notification) { - lock (notification) - { - current++; + current++; - notification.Text = $"Imported {current} of {paths.Length} {term}s"; - notification.Progress = (float)current / paths.Length; - } + notification.Text = $"Imported {current} of {paths.Length} {term}s"; + notification.Progress = (float)current / paths.Length; + } - if (t.Exception != null) - { - var e = t.Exception.InnerException ?? t.Exception; - Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); - } - })); + if (t.Exception == null) + { + lock (imported) + imported.Add(t.Result); + } + else + { + var e = t.Exception.InnerException ?? t.Exception; + Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + } + }))); + + if (imported.Count == 0) + { + notification.Text = "Import failed!"; + notification.State = ProgressNotificationState.Cancelled; } + else + { + notification.CompletionText = imported.Count == 1 + ? $"Imported {imported.First()}!" + : $"Imported {current} {term}s!"; - await Task.WhenAll(tasks); + if (imported.Count > 0 && PresentImport != null) + { + notification.CompletionText += " Click to view."; + notification.CompletionClickAction = () => + { + PresentImport?.Invoke(imported); + return true; + }; + } - // if (imported.Count == 0) - // { - // notification.Text = "Import failed!"; - // notification.State = ProgressNotificationState.Cancelled; - // } - // else - // { - // notification.CompletionText = imported.Count == 1 - // ? $"Imported {imported.First()}!" - // : $"Imported {current} {term}s!"; - // - // if (imported.Count > 0 && PresentImport != null) - // { - // notification.CompletionText += " Click to view."; - // notification.CompletionClickAction = () => - // { - // PresentImport?.Invoke(imported); - // return true; - // }; - // } - // - // notification.State = ProgressNotificationState.Completed; - // } + notification.State = ProgressNotificationState.Completed; + } } /// @@ -368,7 +364,7 @@ namespace osu.Game.Database } return item; - }, CancellationToken.None, TaskCreationOptions.None, importScheduler).Unwrap(); + }, CancellationToken.None, TaskCreationOptions.HideScheduler, IMPORT_SCHEDULER).Unwrap(); /// /// Perform an update of the specified item. @@ -615,4 +611,10 @@ namespace osu.Game.Database throw new InvalidFormatException($"{path} is not a valid archive"); } } + + public abstract class ArchiveModelManager + { + // allow sharing static across all generic types + protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(1); + } } From 2d1a54e63489b36cfb4fa5261abd0ed95c092051 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:37:20 +0900 Subject: [PATCH 17/47] Properly implement cancellation --- osu.Game/Database/ArchiveModelManager.cs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index e30e1cd3ee..1d576ff82f 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -154,6 +154,9 @@ namespace osu.Game.Database await Task.WhenAll(paths.Select(path => Import(path, notification.CancellationToken).ContinueWith(t => { + if (notification.CancellationToken.IsCancellationRequested) + return; + lock (notification) { current++; @@ -172,7 +175,7 @@ namespace osu.Game.Database var e = t.Exception.InnerException ?? t.Exception; Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); } - }))); + }, TaskContinuationOptions.NotOnCanceled))); if (imported.Count == 0) { @@ -207,6 +210,8 @@ namespace osu.Game.Database /// The imported model, if successful. public async Task Import(string path, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + TModel import; using (ArchiveReader reader = getReaderFrom(path)) import = await Import(reader, cancellationToken); @@ -240,6 +245,8 @@ namespace osu.Game.Database /// An optional cancellation token. public async Task Import(ArchiveReader archive, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + try { var model = CreateModel(archive); @@ -250,6 +257,10 @@ namespace osu.Game.Database return await Import(model, archive, cancellationToken); } + catch (TaskCanceledException) + { + throw; + } catch (Exception e) { Logger.Error(e, $"Model creation of {archive.Name} failed.", LoggingTarget.Database); @@ -286,6 +297,8 @@ namespace osu.Game.Database /// An optional cancellation token. public async Task Import(TModel item, ArchiveReader archive = null, CancellationToken cancellationToken = default) => await Task.Factory.StartNew(async () => { + cancellationToken.ThrowIfCancellationRequested(); + delayEvents(); try @@ -301,10 +314,6 @@ namespace osu.Game.Database { await Populate(item, archive, cancellationToken); } - catch (TaskCanceledException) - { - return item = null; - } finally { if (!Delete(localItem)) @@ -364,7 +373,7 @@ namespace osu.Game.Database } return item; - }, CancellationToken.None, TaskCreationOptions.HideScheduler, IMPORT_SCHEDULER).Unwrap(); + }, cancellationToken, TaskCreationOptions.HideScheduler, IMPORT_SCHEDULER).Unwrap(); /// /// Perform an update of the specified item. From e12b03e275c40de2b3629b92da24da45545be619 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:46:31 +0900 Subject: [PATCH 18/47] Remove unnecessary reference --- osu.Game.Tests/osu.Game.Tests.csproj | 5 ----- 1 file changed, 5 deletions(-) diff --git a/osu.Game.Tests/osu.Game.Tests.csproj b/osu.Game.Tests/osu.Game.Tests.csproj index 403ae16885..11d70ee7be 100644 --- a/osu.Game.Tests/osu.Game.Tests.csproj +++ b/osu.Game.Tests/osu.Game.Tests.csproj @@ -18,9 +18,4 @@ - - - ..\..\..\..\..\usr\local\share\dotnet\sdk\NuGetFallbackFolder\microsoft.win32.registry\4.5.0\ref\netstandard2.0\Microsoft.Win32.Registry.dll - - \ No newline at end of file From b79fdfc12f2d65da2f0fd80b68fa58aa2aa11882 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:48:58 +0900 Subject: [PATCH 19/47] Fix one more instance of improperly handled cancellation --- osu.Game/Beatmaps/BeatmapManager.cs | 3 +-- osu.Game/Database/ArchiveModelManager.cs | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 9e7b6d7971..cfc6a0be28 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -436,8 +436,7 @@ namespace osu.Game.Beatmaps private void perform(BeatmapInfo beatmap, CancellationToken cancellation) { - if (cancellation.IsCancellationRequested) - return; + cancellation.ThrowIfCancellationRequested(); if (api?.State != APIState.Online) return; diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 1d576ff82f..df96d9984a 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -361,6 +361,10 @@ namespace osu.Game.Database Logger.Log($"Import of {item} successfully completed!", LoggingTarget.Database); } + catch (TaskCanceledException) + { + throw; + } catch (Exception e) { Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); From e4bad93b6668868079dad7227b868b43736e6336 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 13:52:09 +0900 Subject: [PATCH 20/47] Use variable for web request concurrency for clarity --- osu.Game/Beatmaps/BeatmapManager.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index cfc6a0be28..435edcf722 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -424,7 +424,9 @@ namespace osu.Game.Beatmaps { private readonly IAPIProvider api; - private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(4); + private const int update_queue_request_concurrency = 4; + + private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency); public BeatmapUpdateQueue(IAPIProvider api) { @@ -455,10 +457,7 @@ namespace osu.Game.Beatmaps beatmap.OnlineBeatmapID = res.OnlineBeatmapID; }; - req.Failure += e => - { - Logger.Log($"Failed ({e})", LoggingTarget.Database); - }; + req.Failure += e => { Logger.Log($"Failed ({e})", LoggingTarget.Database); }; // intentionally blocking to limit web request concurrency req.Perform(api); From e19f4935c3cbdd6d037d093d6b41e2f7cb1bf719 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 14:13:36 +0900 Subject: [PATCH 21/47] Fix incorrect undo logic on exception --- osu.Game/Database/ArchiveModelManager.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index df96d9984a..83c51e2abf 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -305,8 +305,7 @@ namespace osu.Game.Database { Logger.Log($"Importing {item}...", LoggingTarget.Database); - if (archive != null) - item.Files = createFileInfos(archive, Files); + item.Files = archive != null ? createFileInfos(archive, Files) : new List(); var localItem = item; @@ -314,7 +313,7 @@ namespace osu.Game.Database { await Populate(item, archive, cancellationToken); } - finally + catch (Exception) { if (!Delete(localItem)) Files.Dereference(localItem.Files.Select(f => f.FileInfo).ToArray()); From f31b19e0d7b2837bfaa1c26fa381d6ab1df43e41 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:02:49 +0900 Subject: [PATCH 22/47] Don't unwrap exception manually --- osu.Game/Database/ArchiveModelManager.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 83c51e2abf..4c4878edee 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -172,8 +172,7 @@ namespace osu.Game.Database } else { - var e = t.Exception.InnerException ?? t.Exception; - Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + Logger.Error(t.Exception, $@"Could not import ({Path.GetFileName(path)})"); } }, TaskContinuationOptions.NotOnCanceled))); From 9bdc8b47bb1e1c8ab8e57b48d0d8604d94ae5d3e Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:13:51 +0900 Subject: [PATCH 23/47] Remove unnecessary async-await pair --- osu.Game/Database/ArchiveModelManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 4c4878edee..fb7f0ffe5d 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -577,7 +577,7 @@ namespace osu.Game.Database /// The model to populate. /// The archive to use as a reference for population. May be null. /// An optional cancellation token. - protected virtual async Task Populate(TModel model, [CanBeNull] ArchiveReader archive, CancellationToken cancellationToken = default) => await Task.CompletedTask; + protected virtual Task Populate(TModel model, [CanBeNull] ArchiveReader archive, CancellationToken cancellationToken = default) => Task.CompletedTask; /// /// Perform any final actions before the import to database executes. From fae32b390171738feed00336df54db7046c9ecd8 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:14:11 +0900 Subject: [PATCH 24/47] Return shorter class name in error messages --- osu.Game/Database/ArchiveModelManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index fb7f0ffe5d..e2a07a860f 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -331,7 +331,7 @@ namespace osu.Game.Database if (CanUndelete(existing, item)) { Undelete(existing); - Logger.Log($"Found existing {typeof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); + Logger.Log($"Found existing {nameof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); handleEvent(() => ItemAdded?.Invoke(existing, true)); return existing; } From 02b376d962d364f7b2525e8478dd4c4e3dd7cf04 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:14:42 +0900 Subject: [PATCH 25/47] Fix rollback logic not necessrily cleaning up file store --- osu.Game/Database/ArchiveModelManager.cs | 63 +++++++++++------------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index e2a07a860f..45460dd11c 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -154,25 +154,22 @@ namespace osu.Game.Database await Task.WhenAll(paths.Select(path => Import(path, notification.CancellationToken).ContinueWith(t => { - if (notification.CancellationToken.IsCancellationRequested) - return; + notification.CancellationToken.ThrowIfCancellationRequested(); - lock (notification) + lock (imported) { - current++; + Interlocked.Increment(ref current); - notification.Text = $"Imported {current} of {paths.Length} {term}s"; - notification.Progress = (float)current / paths.Length; - } - - if (t.Exception == null) - { - lock (imported) + if (t.Exception == null) + { imported.Add(t.Result); - } - else - { - Logger.Error(t.Exception, $@"Could not import ({Path.GetFileName(path)})"); + notification.Text = $"Imported {current} of {paths.Length} {term}s"; + notification.Progress = (float)current / paths.Length; + } + else + { + Logger.Error(t.Exception, $@"Could not import ({Path.GetFileName(path)})"); + } } }, TaskContinuationOptions.NotOnCanceled))); @@ -300,23 +297,23 @@ namespace osu.Game.Database delayEvents(); + void rollback() + { + if (!Delete(item)) + { + // We may have not yet added the model to the underlying table, but should still clean up files. + Logger.Log($"Dereferencing files for incomplete import of {item}.", LoggingTarget.Database); + Files.Dereference(item.Files.Select(f => f.FileInfo).ToArray()); + } + } + try { Logger.Log($"Importing {item}...", LoggingTarget.Database); item.Files = archive != null ? createFileInfos(archive, Files) : new List(); - var localItem = item; - - try - { - await Populate(item, archive, cancellationToken); - } - catch (Exception) - { - if (!Delete(localItem)) - Files.Dereference(localItem.Files.Select(f => f.FileInfo).ToArray()); - } + await Populate(item, archive, cancellationToken); using (var write = ContextFactory.GetForWrite()) // used to share a context for full import. keep in mind this will block all writes. { @@ -333,6 +330,8 @@ namespace osu.Game.Database Undelete(existing); Logger.Log($"Found existing {nameof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); handleEvent(() => ItemAdded?.Invoke(existing, true)); + + rollback(); return existing; } else @@ -349,9 +348,6 @@ namespace osu.Game.Database } catch (Exception e) { - if (!Delete(item)) - Files.Dereference(item.Files.Select(f => f.FileInfo).ToArray()); - write.Errors.Add(e); throw; } @@ -359,13 +355,12 @@ namespace osu.Game.Database Logger.Log($"Import of {item} successfully completed!", LoggingTarget.Database); } - catch (TaskCanceledException) - { - throw; - } catch (Exception e) { - Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); + if (!(e is TaskCanceledException)) + Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); + + rollback(); item = null; } finally From 5b75060b94999e3a50a665a4e40cee3994162fa2 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 16:45:45 +0900 Subject: [PATCH 26/47] Add test for rollback logic correctly dereferencing files --- osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs | 13 +++++++++++++ osu.Game/IO/FileStore.cs | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 4c9260f640..3f4f40781c 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -12,6 +12,7 @@ using osu.Framework.Platform; using osu.Game.IPC; using osu.Framework.Allocation; using osu.Game.Beatmaps; +using osu.Game.IO; using osu.Game.Tests.Resources; using SharpCompress.Archives.Zip; @@ -97,6 +98,7 @@ namespace osu.Game.Tests.Beatmaps.IO { var osu = loadOsu(host); var manager = osu.Dependencies.Get(); + var files = osu.Dependencies.Get(); int fireCount = 0; @@ -113,6 +115,12 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(0, fireCount -= 2); + Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); + Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); + + Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); + var breakTemp = TestResources.GetTestBeatmapForImport(); MemoryStream brokenOsu = new MemoryStream(new byte[] { 1, 3, 3, 7 }); @@ -131,6 +139,8 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); + Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); + // this will trigger purging of the existing beatmap (online set id match) but should rollback due to broken osu. await manager.Import(breakTemp); @@ -140,6 +150,9 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); + + Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); + Assert.AreEqual(18, files.QueryFiles(f => f.ReferenceCount == 1).Count()); } finally { diff --git a/osu.Game/IO/FileStore.cs b/osu.Game/IO/FileStore.cs index 458f8964f9..370d6786f5 100644 --- a/osu.Game/IO/FileStore.cs +++ b/osu.Game/IO/FileStore.cs @@ -2,8 +2,11 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Linq.Expressions; +using Microsoft.EntityFrameworkCore; using osu.Framework.Extensions; using osu.Framework.IO.Stores; using osu.Framework.Logging; @@ -27,6 +30,13 @@ namespace osu.Game.IO Store = new StorageBackedResourceStore(Storage); } + /// + /// Perform a lookup query on available s. + /// + /// The query. + /// Results from the provided query. + public IEnumerable QueryFiles(Expression> query) => ContextFactory.Get().Set().AsNoTracking().Where(f => f.ReferenceCount > 0).Where(query); + public FileInfo Add(Stream data, bool reference = true) { using (var usage = ContextFactory.GetForWrite()) From 559413f766fef4fef2f8c16e59e61a39f100d785 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 17:12:25 +0900 Subject: [PATCH 27/47] Avoid using ContinueWith in already async context --- osu.Game/Database/ArchiveModelManager.cs | 29 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 45460dd11c..d09aa37d7e 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -152,26 +152,35 @@ namespace osu.Game.Database var imported = new List(); - await Task.WhenAll(paths.Select(path => Import(path, notification.CancellationToken).ContinueWith(t => + await Task.WhenAll(paths.Select(async path => { notification.CancellationToken.ThrowIfCancellationRequested(); - lock (imported) + try { - Interlocked.Increment(ref current); + var model = await Import(path, notification.CancellationToken); - if (t.Exception == null) + lock (imported) { - imported.Add(t.Result); + imported.Add(model); + notification.Text = $"Imported {current} of {paths.Length} {term}s"; notification.Progress = (float)current / paths.Length; } - else - { - Logger.Error(t.Exception, $@"Could not import ({Path.GetFileName(path)})"); - } } - }, TaskContinuationOptions.NotOnCanceled))); + catch (TaskCanceledException) + { + throw; + } + catch (Exception e) + { + Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + } + finally + { + Interlocked.Increment(ref current); + } + })); if (imported.Count == 0) { From c8bd92659bab551a36e0c40dc1b604deca0e4fdb Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 17:12:37 +0900 Subject: [PATCH 28/47] Clean up exception and null handling in Import process --- osu.Game/Database/ArchiveModelManager.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index d09aa37d7e..fa8301bb2e 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -252,15 +252,15 @@ namespace osu.Game.Database { cancellationToken.ThrowIfCancellationRequested(); + TModel model; + try { - var model = CreateModel(archive); + model = CreateModel(archive); if (model == null) return null; model.Hash = computeHash(archive); - - return await Import(model, archive, cancellationToken); } catch (TaskCanceledException) { @@ -271,6 +271,8 @@ namespace osu.Game.Database Logger.Error(e, $"Model creation of {archive.Name} failed.", LoggingTarget.Database); return null; } + + return await Import(model, archive, cancellationToken); } /// @@ -340,7 +342,9 @@ namespace osu.Game.Database Logger.Log($"Found existing {nameof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); handleEvent(() => ItemAdded?.Invoke(existing, true)); + // existing item will be used; rollback new import and exit early. rollback(); + flushEvents(true); return existing; } else @@ -370,14 +374,11 @@ namespace osu.Game.Database Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); rollback(); - item = null; - } - finally - { - // we only want to flush events after we've confirmed the write context didn't have any errors. - flushEvents(item != null); + flushEvents(false); + throw; } + flushEvents(true); return item; }, cancellationToken, TaskCreationOptions.HideScheduler, IMPORT_SCHEDULER).Unwrap(); From dcdb806120c6b581af4be341b60bd84cba5ea9c3 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 17:26:56 +0900 Subject: [PATCH 29/47] Catch newly thrown exception in test --- osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 3f4f40781c..7f08674a95 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -135,14 +135,14 @@ namespace osu.Game.Tests.Beatmaps.IO zip.SaveTo(outStream, SharpCompress.Common.CompressionType.Deflate); } - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); - Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); - - Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); - // this will trigger purging of the existing beatmap (online set id match) but should rollback due to broken osu. - await manager.Import(breakTemp); + try + { + await manager.Import(breakTemp); + } + catch + { + } // no events should be fired in the case of a rollback. Assert.AreEqual(0, fireCount); From 28b2a516e34354eb0f2d958c2f55da91dd8d4f78 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 18:13:33 +0900 Subject: [PATCH 30/47] Ensure exception is only thrown once on rollback --- .../Beatmaps/IO/ImportBeatmapTest.cs | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 7f08674a95..f3680e3c1e 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -11,6 +11,7 @@ using NUnit.Framework; using osu.Framework.Platform; using osu.Game.IPC; using osu.Framework.Allocation; +using osu.Framework.Logging; using osu.Game.Beatmaps; using osu.Game.IO; using osu.Game.Tests.Resources; @@ -96,24 +97,31 @@ namespace osu.Game.Tests.Beatmaps.IO { try { + int itemAddRemoveFireCount = 0; + int loggedExceptionCount = 0; + + Logger.NewEntry += l => + { + if (l.Target == LoggingTarget.Database && l.Exception != null) + Interlocked.Increment(ref loggedExceptionCount); + }; + var osu = loadOsu(host); var manager = osu.Dependencies.Get(); var files = osu.Dependencies.Get(); - int fireCount = 0; - // ReSharper disable once AccessToModifiedClosure - manager.ItemAdded += (_, __) => fireCount++; - manager.ItemRemoved += _ => fireCount++; + manager.ItemAdded += (_, __) => Interlocked.Increment(ref itemAddRemoveFireCount); + manager.ItemRemoved += _ => Interlocked.Increment(ref itemAddRemoveFireCount); var imported = await LoadOszIntoOsu(osu); - Assert.AreEqual(0, fireCount -= 1); + Assert.AreEqual(0, itemAddRemoveFireCount -= 1); imported.Hash += "-changed"; manager.Update(imported); - Assert.AreEqual(0, fireCount -= 2); + Assert.AreEqual(0, itemAddRemoveFireCount -= 2); Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); @@ -145,7 +153,7 @@ namespace osu.Game.Tests.Beatmaps.IO } // no events should be fired in the case of a rollback. - Assert.AreEqual(0, fireCount); + Assert.AreEqual(0, itemAddRemoveFireCount); Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); @@ -153,6 +161,8 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); Assert.AreEqual(18, files.QueryFiles(f => f.ReferenceCount == 1).Count()); + + Assert.AreEqual(1, loggedExceptionCount); } finally { From 1aa865c3fb154d0ce88321f38cbef309c7799c22 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Mon, 10 Jun 2019 18:34:24 +0900 Subject: [PATCH 31/47] split out method for reuse --- .../Visual/SongSelect/TestScenePlaySongSelect.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index 8636890081..d6b5e0175f 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -138,7 +138,7 @@ namespace osu.Game.Tests.Visual.SongSelect { createSongSelect(); changeRuleset(2); - importForRuleset(0); + addRulesetImportStep(0); AddUntilStep("no selection", () => songSelect.Carousel.SelectedBeatmap == null); } @@ -147,8 +147,8 @@ namespace osu.Game.Tests.Visual.SongSelect { createSongSelect(); changeRuleset(2); - importForRuleset(2); - importForRuleset(1); + addRulesetImportStep(2); + addRulesetImportStep(1); AddUntilStep("has selection", () => songSelect.Carousel.SelectedBeatmap.RulesetID == 2); changeRuleset(1); @@ -223,7 +223,7 @@ namespace osu.Game.Tests.Visual.SongSelect }); AddRepeatStep($"Create beatmaps {test_count} times", () => { - manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == 0).ToArray())); + importForRuleset(0); Scheduler.AddDelayed(() => { @@ -240,15 +240,16 @@ namespace osu.Game.Tests.Visual.SongSelect { int? previousID = null; createSongSelect(); - importForRuleset(0); + addRulesetImportStep(0); AddStep("Move to last difficulty", () => songSelect.Carousel.SelectBeatmap(songSelect.Carousel.BeatmapSets.First().Beatmaps.Last())); AddStep("Store current ID", () => previousID = songSelect.Carousel.SelectedBeatmap.ID); AddStep("Hide first beatmap", () => manager.Hide(songSelect.Carousel.SelectedBeatmapSet.Beatmaps.First())); AddAssert("Selected beatmap has not changed", () => songSelect.Carousel.SelectedBeatmap.ID == previousID); } - private void importForRuleset(int id) => AddStep($"import test map for ruleset {id}", - () => manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == id).ToArray()))); + private void addRulesetImportStep(int id) => AddStep($"import test map for ruleset {id}", () => importForRuleset(id)); + + private void importForRuleset(int id) => manager.Import(createTestBeatmapSet(getImportId(), rulesets.AvailableRulesets.Where(r => r.ID == id).ToArray())); private static int importId; private int getImportId() => ++importId; From 12aa264657061f456405a65c0eaaf781f3461304 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 18:35:23 +0900 Subject: [PATCH 32/47] Consolidate tests and check for file reference counts --- .../Beatmaps/IO/ImportBeatmapTest.cs | 51 ++++++++++++------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index f3680e3c1e..5fc05a4b2f 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -77,10 +77,8 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.IsTrue(imported.ID == importedSecondTime.ID); Assert.IsTrue(imported.Beatmaps.First().ID == importedSecondTime.Beatmaps.First().ID); - var manager = osu.Dependencies.Get(); - - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + checkBeatmapSetCount(osu, 1); + checkSingleReferencedFileCount(osu, 18); } finally { @@ -108,7 +106,6 @@ namespace osu.Game.Tests.Beatmaps.IO var osu = loadOsu(host); var manager = osu.Dependencies.Get(); - var files = osu.Dependencies.Get(); // ReSharper disable once AccessToModifiedClosure manager.ItemAdded += (_, __) => Interlocked.Increment(ref itemAddRemoveFireCount); @@ -123,11 +120,9 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.AreEqual(0, itemAddRemoveFireCount -= 2); - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); - Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); - - Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); + checkBeatmapSetCount(osu, 1); + checkBeatmapCount(osu, 12); + checkSingleReferencedFileCount(osu, 18); var breakTemp = TestResources.GetTestBeatmapForImport(); @@ -155,12 +150,10 @@ namespace osu.Game.Tests.Beatmaps.IO // no events should be fired in the case of a rollback. Assert.AreEqual(0, itemAddRemoveFireCount); - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); - Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); + checkBeatmapSetCount(osu, 1); + checkBeatmapCount(osu, 12); - Assert.AreEqual(18, files.QueryFiles(_ => true).Count()); - Assert.AreEqual(18, files.QueryFiles(f => f.ReferenceCount == 1).Count()); + checkSingleReferencedFileCount(osu, 18); Assert.AreEqual(1, loggedExceptionCount); } @@ -193,8 +186,7 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.IsTrue(imported.Beatmaps.First().ID < importedSecondTime.Beatmaps.First().ID); // only one beatmap will exist as the online set ID matched, causing purging of the first import. - Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + checkBeatmapSetCount(osu, 1); } finally { @@ -396,11 +388,32 @@ namespace osu.Game.Tests.Beatmaps.IO var manager = osu.Dependencies.Get(); manager.Delete(imported); - Assert.IsTrue(manager.GetAllUsableBeatmapSets().Count == 0); - Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + checkBeatmapSetCount(osu, 0); + checkBeatmapSetCount(osu, 1, true); + checkSingleReferencedFileCount(osu, 0); + Assert.IsTrue(manager.QueryBeatmapSets(_ => true).First().DeletePending); } + private void checkBeatmapSetCount(OsuGameBase osu, int expected, bool includeDeletePending = false) + { + var manager = osu.Dependencies.Get(); + + Assert.AreEqual(expected, includeDeletePending + ? manager.QueryBeatmapSets(_ => true).ToList().Count + : manager.GetAllUsableBeatmapSets().Count); + } + + private void checkBeatmapCount(OsuGameBase osu, int expected) + { + Assert.AreEqual(expected, osu.Dependencies.Get().QueryBeatmaps(_ => true).ToList().Count); + } + + private void checkSingleReferencedFileCount(OsuGameBase osu, int expected) + { + Assert.AreEqual(expected, osu.Dependencies.Get().QueryFiles(f => f.ReferenceCount == 1).Count()); + } + private OsuGameBase loadOsu(GameHost host) { var osu = new OsuGameBase(); From f7a699e4a2e0031224661370ba840d7d9327df3f Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 18:38:03 +0900 Subject: [PATCH 33/47] Better documentation for import scheduler singleton --- osu.Game/Database/ArchiveModelManager.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index fa8301bb2e..afc2690106 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -630,7 +630,15 @@ namespace osu.Game.Database public abstract class ArchiveModelManager { - // allow sharing static across all generic types - protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(1); + private const int import_queue_request_concurrency = 1; + + /// + /// A singleton scheduler shared by all . + /// + /// + /// This scheduler generally performs IO and CPU intensive work so concurrency is limited harshly. + /// It is mainly being used as a queue mechanism for large imports. + /// + protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(import_queue_request_concurrency); } } From 41e3e2222bc7d9c75e44a1340144c2d52e1f7acb Mon Sep 17 00:00:00 2001 From: David Zhao Date: Mon, 10 Jun 2019 18:40:49 +0900 Subject: [PATCH 34/47] fix test --- osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index d6b5e0175f..bc15bc6b6d 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -219,7 +219,7 @@ namespace osu.Game.Tests.Visual.SongSelect AddStep("Setup counter", () => { beatmapChangedCount = 0; - songSelect.Carousel.BeatmapSetsChanged += () => beatmapChangedCount++; + songSelect.Carousel.SelectionChanged += _ => beatmapChangedCount++; }); AddRepeatStep($"Create beatmaps {test_count} times", () => { From 6cda2cdb8225000f1a7de5e8c842a8d3f0a02aaf Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 18:41:56 +0900 Subject: [PATCH 35/47] Fix exception output to use humanised model name --- osu.Game/Database/ArchiveModelManager.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index afc2690106..93e924fab5 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -146,8 +146,6 @@ namespace osu.Game.Database notification.Progress = 0; notification.Text = "Import is initialising..."; - var term = $"{typeof(TModel).Name.Replace("Info", "").ToLower()}"; - int current = 0; var imported = new List(); @@ -164,7 +162,7 @@ namespace osu.Game.Database { imported.Add(model); - notification.Text = $"Imported {current} of {paths.Length} {term}s"; + notification.Text = $"Imported {current} of {paths.Length} {humanisedModelName}s"; notification.Progress = (float)current / paths.Length; } } @@ -191,7 +189,7 @@ namespace osu.Game.Database { notification.CompletionText = imported.Count == 1 ? $"Imported {imported.First()}!" - : $"Imported {current} {term}s!"; + : $"Imported {current} {humanisedModelName}s!"; if (imported.Count > 0 && PresentImport != null) { @@ -339,7 +337,7 @@ namespace osu.Game.Database if (CanUndelete(existing, item)) { Undelete(existing); - Logger.Log($"Found existing {nameof(TModel)} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); + Logger.Log($"Found existing {humanisedModelName} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); handleEvent(() => ItemAdded?.Invoke(existing, true)); // existing item will be used; rollback new import and exit early. @@ -610,6 +608,8 @@ namespace osu.Game.Database private DbSet queryModel() => ContextFactory.Get().Set(); + private string humanisedModelName => $"{typeof(TModel).Name.Replace("Info", "").ToLower()}"; + /// /// Creates an from a valid storage path. /// From 54497fb1e78dd5447111cd256067f3959c1a61dd Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 19:33:23 +0900 Subject: [PATCH 36/47] Fix prefixing spaces in BeatmapInfo's ToString when metadata is not populated yet --- osu.Game/Beatmaps/BeatmapInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Beatmaps/BeatmapInfo.cs b/osu.Game/Beatmaps/BeatmapInfo.cs index 52238c26fe..3c082bb71e 100644 --- a/osu.Game/Beatmaps/BeatmapInfo.cs +++ b/osu.Game/Beatmaps/BeatmapInfo.cs @@ -119,7 +119,7 @@ namespace osu.Game.Beatmaps /// public List Scores { get; set; } - public override string ToString() => $"{Metadata} [{Version}]"; + public override string ToString() => $"{Metadata} [{Version}]".Trim(); public bool Equals(BeatmapInfo other) { From 29945f27c5f22bcbb36ed693d813fd34707053ee Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 19:33:55 +0900 Subject: [PATCH 37/47] Fix imported count incrementing on failures --- osu.Game/Database/ArchiveModelManager.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 93e924fab5..4e9478dc10 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -162,6 +162,7 @@ namespace osu.Game.Database { imported.Add(model); + Interlocked.Increment(ref current); notification.Text = $"Imported {current} of {paths.Length} {humanisedModelName}s"; notification.Progress = (float)current / paths.Length; } @@ -174,10 +175,6 @@ namespace osu.Game.Database { Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); } - finally - { - Interlocked.Increment(ref current); - } })); if (imported.Count == 0) From 6ca2fcebfce6467398d12fc69911ffb3f1c03eff Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 10 Jun 2019 19:34:32 +0900 Subject: [PATCH 38/47] Centalise and prefix all ArchiveModelManager database logging --- osu.Game/Beatmaps/BeatmapManager.cs | 28 ++++++++++-------- osu.Game/Database/ArchiveModelManager.cs | 36 +++++++++++++++--------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 435edcf722..47411c69ec 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -110,7 +110,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - await Task.WhenAll(beatmapSet.Beatmaps.Select(b => updateQueue.Perform(b, cancellationToken)).ToArray()); + await updateQueue.Perform(beatmapSet, cancellationToken); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -127,7 +127,7 @@ namespace osu.Game.Beatmaps { Delete(existingOnlineId); beatmaps.PurgeDeletable(s => s.ID == existingOnlineId.ID); - Logger.Log($"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been purged.", LoggingTarget.Database); + LogForModel(beatmapSet, $"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been purged."); } } } @@ -433,23 +433,29 @@ namespace osu.Game.Beatmaps this.api = api; } - public Task Perform(BeatmapInfo beatmap, CancellationToken cancellationToken) - => Task.Factory.StartNew(() => perform(beatmap, cancellationToken), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); - - private void perform(BeatmapInfo beatmap, CancellationToken cancellation) + public async Task Perform(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) { - cancellation.ThrowIfCancellationRequested(); - if (api?.State != APIState.Online) return; - Logger.Log("Attempting online lookup for the missing values...", LoggingTarget.Database); + LogForModel(beatmapSet, "Performing online lookups..."); + await Task.WhenAll(beatmapSet.Beatmaps.Select(b => Perform(beatmapSet, b, cancellationToken)).ToArray()); + } + + // todo: expose this when we need to do individual difficulty lookups. + protected Task Perform(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken) + => Task.Factory.StartNew(() => perform(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); + + private void perform(BeatmapSetInfo set, BeatmapInfo beatmap) + { + if (api?.State != APIState.Online) + return; var req = new GetBeatmapRequest(beatmap); req.Success += res => { - Logger.Log($"Successfully mapped to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}.", LoggingTarget.Database); + LogForModel(set, $"Online retrieval mapped {beatmap} to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}."); beatmap.Status = res.Status; beatmap.BeatmapSet.Status = res.BeatmapSet.Status; @@ -457,7 +463,7 @@ namespace osu.Game.Beatmaps beatmap.OnlineBeatmapID = res.OnlineBeatmapID; }; - req.Failure += e => { Logger.Log($"Failed ({e})", LoggingTarget.Database); }; + req.Failure += e => { LogForModel(set, $"Online retrieval failed for {beatmap}", e); }; // intentionally blocking to limit web request concurrency req.Perform(api); diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 4e9478dc10..844ae622a5 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -173,7 +173,7 @@ namespace osu.Game.Database } catch (Exception e) { - Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})"); + Logger.Error(e, $@"Could not import ({Path.GetFileName(path)})", LoggingTarget.Database); } })); @@ -227,7 +227,7 @@ namespace osu.Game.Database } catch (Exception e) { - Logger.Error(e, $@"Could not delete original file after import ({Path.GetFileName(path)})"); + LogForModel(import, $@"Could not delete original file after import ({Path.GetFileName(path)})", e); } return import; @@ -247,7 +247,7 @@ namespace osu.Game.Database { cancellationToken.ThrowIfCancellationRequested(); - TModel model; + TModel model = null; try { @@ -263,7 +263,7 @@ namespace osu.Game.Database } catch (Exception e) { - Logger.Error(e, $"Model creation of {archive.Name} failed.", LoggingTarget.Database); + LogForModel(model, $"Model creation of {archive.Name} failed.", e); return null; } @@ -277,6 +277,16 @@ namespace osu.Game.Database /// protected abstract string[] HashableFileTypes { get; } + protected static void LogForModel(TModel model, string message, Exception e = null) + { + string prefix = $"[{(model?.Hash ?? "?????").Substring(0, 5)}]"; + + if (e != null) + Logger.Error(e, $"{prefix} {message}", LoggingTarget.Database); + else + Logger.Log($"{prefix} {message}", LoggingTarget.Database); + } + /// /// Create a SHA-2 hash from the provided archive based on file content of all files matching . /// @@ -308,14 +318,14 @@ namespace osu.Game.Database if (!Delete(item)) { // We may have not yet added the model to the underlying table, but should still clean up files. - Logger.Log($"Dereferencing files for incomplete import of {item}.", LoggingTarget.Database); + LogForModel(item, "Dereferencing files for incomplete import."); Files.Dereference(item.Files.Select(f => f.FileInfo).ToArray()); } } try { - Logger.Log($"Importing {item}...", LoggingTarget.Database); + LogForModel(item, "Beginning import..."); item.Files = archive != null ? createFileInfos(archive, Files) : new List(); @@ -334,7 +344,7 @@ namespace osu.Game.Database if (CanUndelete(existing, item)) { Undelete(existing); - Logger.Log($"Found existing {humanisedModelName} for {item} (ID {existing.ID}). Skipping import.", LoggingTarget.Database); + LogForModel(item, $"Found existing {humanisedModelName} for {item} (ID {existing.ID}) – skipping import."); handleEvent(() => ItemAdded?.Invoke(existing, true)); // existing item will be used; rollback new import and exit early. @@ -342,11 +352,9 @@ namespace osu.Game.Database flushEvents(true); return existing; } - else - { - Delete(existing); - ModelStore.PurgeDeletable(s => s.ID == existing.ID); - } + + Delete(existing); + ModelStore.PurgeDeletable(s => s.ID == existing.ID); } PreImport(item); @@ -361,12 +369,12 @@ namespace osu.Game.Database } } - Logger.Log($"Import of {item} successfully completed!", LoggingTarget.Database); + LogForModel(item, "Import successfully completed!"); } catch (Exception e) { if (!(e is TaskCanceledException)) - Logger.Error(e, $"Import of {item} failed and has been rolled back.", LoggingTarget.Database); + LogForModel(item, "Database import or population failed and has been rolled back.", e); rollback(); flushEvents(false); From d4ba67747b37f4db9a0a482a5fef54470a6bcbfc Mon Sep 17 00:00:00 2001 From: David Zhao Date: Mon, 10 Jun 2019 18:58:03 +0900 Subject: [PATCH 39/47] fix test count --- .idea/.idea.osu/.idea/.name | 1 + .../Visual/SongSelect/TestScenePlaySongSelect.cs | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 .idea/.idea.osu/.idea/.name diff --git a/.idea/.idea.osu/.idea/.name b/.idea/.idea.osu/.idea/.name new file mode 100644 index 0000000000..21cb4db60e --- /dev/null +++ b/.idea/.idea.osu/.idea/.name @@ -0,0 +1 @@ +osu \ No newline at end of file diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index bc15bc6b6d..fa73a14252 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -11,6 +11,7 @@ using osu.Framework.Allocation; using osu.Framework.Audio; using osu.Framework.Bindables; using osu.Framework.Extensions; +using osu.Framework.Logging; using osu.Framework.MathUtils; using osu.Framework.Platform; using osu.Framework.Screens; @@ -215,11 +216,13 @@ namespace osu.Game.Tests.Visual.SongSelect { const int test_count = 10; int beatmapChangedCount = 0; + int debounceCount = 0; createSongSelect(); - AddStep("Setup counter", () => + AddStep("Setup counters", () => { beatmapChangedCount = 0; - songSelect.Carousel.SelectionChanged += _ => beatmapChangedCount++; + debounceCount = 0; + songSelect.Carousel.SelectionChanged += _ => beatmapChangedCount += 1; }); AddRepeatStep($"Create beatmaps {test_count} times", () => { @@ -229,10 +232,14 @@ namespace osu.Game.Tests.Visual.SongSelect { // Wait for debounce songSelect.Carousel.SelectNextRandom(); + ++debounceCount; }, 400); }, test_count); - AddAssert($"Beatmap changed {test_count} times", () => beatmapChangedCount == test_count); + AddUntilStep("Debounce limit reached", () => debounceCount == test_count); + + // The selected beatmap should have changed an additional 2 times since both initially loading songselect and the first import also triggers selectionChanged + AddAssert($"Beatmap changed {test_count + 2} times", () => beatmapChangedCount == test_count + 2); } [Test] From 975bb3db8a2aba107858543eb12b419f363cad44 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 11 Jun 2019 15:51:14 +0900 Subject: [PATCH 40/47] cleanup --- .idea/.idea.osu/.idea/.name | 1 - osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 .idea/.idea.osu/.idea/.name diff --git a/.idea/.idea.osu/.idea/.name b/.idea/.idea.osu/.idea/.name deleted file mode 100644 index 21cb4db60e..0000000000 --- a/.idea/.idea.osu/.idea/.name +++ /dev/null @@ -1 +0,0 @@ -osu \ No newline at end of file diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index fa73a14252..198da2c5b1 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -222,7 +222,7 @@ namespace osu.Game.Tests.Visual.SongSelect { beatmapChangedCount = 0; debounceCount = 0; - songSelect.Carousel.SelectionChanged += _ => beatmapChangedCount += 1; + songSelect.Carousel.SelectionChanged += _ => beatmapChangedCount++; }); AddRepeatStep($"Create beatmaps {test_count} times", () => { From a53ade07a5ea39046b5b90d240435664010021a3 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 11 Jun 2019 15:51:57 +0900 Subject: [PATCH 41/47] remove unused using --- osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs index 198da2c5b1..2664c7a42c 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestScenePlaySongSelect.cs @@ -11,7 +11,6 @@ using osu.Framework.Allocation; using osu.Framework.Audio; using osu.Framework.Bindables; using osu.Framework.Extensions; -using osu.Framework.Logging; using osu.Framework.MathUtils; using osu.Framework.Platform; using osu.Framework.Screens; From 27054a744ed316fd623b33f4a8cfeeec9c787dc8 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 12 Jun 2019 00:35:13 +0900 Subject: [PATCH 42/47] Fill in thread pool names --- osu.Game/Beatmaps/BeatmapManager.cs | 2 +- osu.Game/Database/ArchiveModelManager.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 47411c69ec..67d3382e01 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -426,7 +426,7 @@ namespace osu.Game.Beatmaps private const int update_queue_request_concurrency = 4; - private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency); + private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency, nameof(BeatmapUpdateQueue)); public BeatmapUpdateQueue(IAPIProvider api) { diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 844ae622a5..d764601b32 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -644,6 +644,6 @@ namespace osu.Game.Database /// This scheduler generally performs IO and CPU intensive work so concurrency is limited harshly. /// It is mainly being used as a queue mechanism for large imports. /// - protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(import_queue_request_concurrency); + protected static readonly ThreadedTaskScheduler IMPORT_SCHEDULER = new ThreadedTaskScheduler(import_queue_request_concurrency, nameof(ArchiveModelManager)); } } From 94e65a324456172971e1b387f039ec1fd7343c0d Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 12 Jun 2019 15:16:59 +0900 Subject: [PATCH 43/47] Fix settings checkboxes not being searchable --- osu.Game/Overlays/Settings/SettingsCheckbox.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/osu.Game/Overlays/Settings/SettingsCheckbox.cs b/osu.Game/Overlays/Settings/SettingsCheckbox.cs index a1501d8015..a554159fd7 100644 --- a/osu.Game/Overlays/Settings/SettingsCheckbox.cs +++ b/osu.Game/Overlays/Settings/SettingsCheckbox.cs @@ -10,11 +10,14 @@ namespace osu.Game.Overlays.Settings { private OsuCheckbox checkbox; + private string labelText; + protected override Drawable CreateControl() => checkbox = new OsuCheckbox(); public override string LabelText { - set => checkbox.LabelText = value; + get => labelText; + set => checkbox.LabelText = labelText = value; } } } From 13234fb4a4b5e9a26a3480c3ac3c960fd7f56c60 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Wed, 12 Jun 2019 16:07:35 +0900 Subject: [PATCH 44/47] Adjust comments a bit --- osu.Game/Screens/Select/BeatmapCarousel.cs | 2 +- osu.Game/Screens/Select/SongSelect.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game/Screens/Select/BeatmapCarousel.cs b/osu.Game/Screens/Select/BeatmapCarousel.cs index 6e3bec106f..cf21c78c7f 100644 --- a/osu.Game/Screens/Select/BeatmapCarousel.cs +++ b/osu.Game/Screens/Select/BeatmapCarousel.cs @@ -155,7 +155,7 @@ namespace osu.Game.Screens.Select int? previouslySelectedID = null; CarouselBeatmapSet existingSet = beatmapSets.FirstOrDefault(b => b.BeatmapSet.ID == beatmapSet.ID); - // Since we're about to remove the selected beatmap, store its ID so we can go back if needed. + // If the selected beatmap is about to be removed, store its ID so it can be re-selected if required if (existingSet?.State?.Value == CarouselItemState.Selected) previouslySelectedID = selectedBeatmap?.Beatmap.ID; diff --git a/osu.Game/Screens/Select/SongSelect.cs b/osu.Game/Screens/Select/SongSelect.cs index dcfe1de8f2..d0645dbab6 100644 --- a/osu.Game/Screens/Select/SongSelect.cs +++ b/osu.Game/Screens/Select/SongSelect.cs @@ -594,7 +594,7 @@ namespace osu.Game.Screens.Select { bindBindables(); - // As a selection was already obtained, do not attempt to update the selected beatmap. + // If a selection was already obtained, do not attempt to update the selected beatmap. if (Carousel.SelectedBeatmapSet != null) return; From c4f54d94bc5e8aa9d52b7e17c0dd3508693cc6bc Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Wed, 12 Jun 2019 17:00:27 +0900 Subject: [PATCH 45/47] Rename methods --- osu.Game/Beatmaps/BeatmapManager.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 67d3382e01..5204bda353 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -110,7 +110,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - await updateQueue.Perform(beatmapSet, cancellationToken); + await updateQueue.UpdateAsync(beatmapSet, cancellationToken); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -433,20 +433,20 @@ namespace osu.Game.Beatmaps this.api = api; } - public async Task Perform(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) + public async Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) { if (api?.State != APIState.Online) return; LogForModel(beatmapSet, "Performing online lookups..."); - await Task.WhenAll(beatmapSet.Beatmaps.Select(b => Perform(beatmapSet, b, cancellationToken)).ToArray()); + await Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray()); } // todo: expose this when we need to do individual difficulty lookups. - protected Task Perform(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken) - => Task.Factory.StartNew(() => perform(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); + protected Task UpdateAsync(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken) + => Task.Factory.StartNew(() => update(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler, updateScheduler); - private void perform(BeatmapSetInfo set, BeatmapInfo beatmap) + private void update(BeatmapSetInfo set, BeatmapInfo beatmap) { if (api?.State != APIState.Online) return; From fd7dc9504e6c91b9502c5e389d28677be49b81d5 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Wed, 12 Jun 2019 17:08:50 +0900 Subject: [PATCH 46/47] Remove async when not required --- osu.Game/Beatmaps/BeatmapManager.cs | 10 +++++----- osu.Game/Database/ArchiveModelManager.cs | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 5204bda353..d90657bff5 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -94,7 +94,7 @@ namespace osu.Game.Beatmaps updateQueue = new BeatmapUpdateQueue(api); } - protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) + protected override Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default) { if (archive != null) beatmapSet.Beatmaps = createBeatmapDifficulties(archive); @@ -110,7 +110,7 @@ namespace osu.Game.Beatmaps validateOnlineIds(beatmapSet); - await updateQueue.UpdateAsync(beatmapSet, cancellationToken); + return updateQueue.UpdateAsync(beatmapSet, cancellationToken); } protected override void PreImport(BeatmapSetInfo beatmapSet) @@ -433,13 +433,13 @@ namespace osu.Game.Beatmaps this.api = api; } - public async Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) + public Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken) { if (api?.State != APIState.Online) - return; + return Task.CompletedTask; LogForModel(beatmapSet, "Performing online lookups..."); - await Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray()); + return Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray()); } // todo: expose this when we need to do individual difficulty lookups. diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index d764601b32..0cf7816250 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -1,4 +1,4 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. using System; @@ -132,13 +132,13 @@ namespace osu.Game.Database /// This will post notifications tracking progress. /// /// One or more archive locations on disk. - public async Task Import(params string[] paths) + public Task Import(params string[] paths) { var notification = new ProgressNotification { State = ProgressNotificationState.Active }; PostNotification?.Invoke(notification); - await Import(notification, paths); + return Import(notification, paths); } protected async Task Import(ProgressNotification notification, params string[] paths) @@ -243,7 +243,7 @@ namespace osu.Game.Database /// /// The archive to be imported. /// An optional cancellation token. - public async Task Import(ArchiveReader archive, CancellationToken cancellationToken = default) + public Task Import(ArchiveReader archive, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -267,7 +267,7 @@ namespace osu.Game.Database return null; } - return await Import(model, archive, cancellationToken); + return Import(model, archive, cancellationToken); } /// @@ -548,24 +548,24 @@ namespace osu.Game.Database /// /// This is a temporary method and will likely be replaced by a full-fledged (and more correctly placed) migration process in the future. /// - public async Task ImportFromStableAsync() + public Task ImportFromStableAsync() { var stable = GetStableStorage?.Invoke(); if (stable == null) { Logger.Log("No osu!stable installation available!", LoggingTarget.Information, LogLevel.Error); - return; + return Task.CompletedTask; } if (!stable.ExistsDirectory(ImportFromStablePath)) { // This handles situations like when the user does not have a Skins folder Logger.Log($"No {ImportFromStablePath} folder available in osu!stable installation", LoggingTarget.Information, LogLevel.Error); - return; + return Task.CompletedTask; } - await Task.Run(async () => await Import(stable.GetDirectories(ImportFromStablePath).Select(f => stable.GetFullPath(f)).ToArray())); + return Task.Run(async () => await Import(stable.GetDirectories(ImportFromStablePath).Select(f => stable.GetFullPath(f)).ToArray())); } #endregion From 2a67944889c302fe69a87838f70d835c63b50740 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Wed, 12 Jun 2019 17:10:55 +0900 Subject: [PATCH 47/47] Remove interlocked within a lock --- osu.Game/Database/ArchiveModelManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 0cf7816250..1c17adf7b7 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -161,8 +161,8 @@ namespace osu.Game.Database lock (imported) { imported.Add(model); + current++; - Interlocked.Increment(ref current); notification.Text = $"Imported {current} of {paths.Length} {humanisedModelName}s"; notification.Progress = (float)current / paths.Length; }