From 2a87b851fae5d3fb783c936782155968006f1900 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 28 May 2018 18:38:42 +0900 Subject: [PATCH 01/16] Add proper transaction rollback logic on exception --- osu.Game/Database/DatabaseWriteUsage.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/DatabaseWriteUsage.cs b/osu.Game/Database/DatabaseWriteUsage.cs index 7858c1a0d1..07bebbf0c3 100644 --- a/osu.Game/Database/DatabaseWriteUsage.cs +++ b/osu.Game/Database/DatabaseWriteUsage.cs @@ -28,8 +28,19 @@ namespace osu.Game.Database if (isDisposed) return; isDisposed = true; - PerformedWrite |= Context.SaveChanges(transaction) > 0; - usageCompleted?.Invoke(this); + try + { + PerformedWrite |= Context.SaveChanges(transaction) > 0; + } + catch (Exception e) + { + transaction?.Rollback(); + throw; + } + finally + { + usageCompleted?.Invoke(this); + } } public void Dispose() From d4e7f08c20ce4311e9dba4ff1a6feade891d50aa Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 28 May 2018 18:43:53 +0900 Subject: [PATCH 02/16] Bring entity framework up-to-date and re-enable transactions --- osu.Desktop/osu.Desktop.csproj | 6 +++--- osu.Game/Database/OsuDbContext.cs | 3 +-- osu.Game/osu.Game.csproj | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/osu.Desktop/osu.Desktop.csproj b/osu.Desktop/osu.Desktop.csproj index 3d64cab84e..3a35568f8f 100644 --- a/osu.Desktop/osu.Desktop.csproj +++ b/osu.Desktop/osu.Desktop.csproj @@ -1,4 +1,4 @@ - + net471;netcoreapp2.0 @@ -30,10 +30,10 @@ - + - + \ No newline at end of file diff --git a/osu.Game/Database/OsuDbContext.cs b/osu.Game/Database/OsuDbContext.cs index 1979ce3648..06503ffc5f 100644 --- a/osu.Game/Database/OsuDbContext.cs +++ b/osu.Game/Database/OsuDbContext.cs @@ -106,8 +106,7 @@ namespace osu.Game.Database public IDbContextTransaction BeginTransaction() { - // return Database.BeginTransaction(); - return null; + return Database.BeginTransaction(); } public int SaveChanges(IDbContextTransaction transaction = null) diff --git a/osu.Game/osu.Game.csproj b/osu.Game/osu.Game.csproj index 1a75f1979a..afb656a260 100644 --- a/osu.Game/osu.Game.csproj +++ b/osu.Game/osu.Game.csproj @@ -17,7 +17,7 @@ - + From bcb04f616895f166cdfff223e781d11dda77aba0 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 28 May 2018 19:56:27 +0900 Subject: [PATCH 03/16] Improve transaction handling flexibility --- osu.Game/Database/ArchiveModelManager.cs | 66 +++++++++++++++++---- osu.Game/Database/DatabaseContextFactory.cs | 23 ++++++- osu.Game/Database/DatabaseWriteUsage.cs | 16 +++-- osu.Game/Database/OsuDbContext.cs | 13 ---- 4 files changed, 84 insertions(+), 34 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index e04559d547..28c25c873f 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -56,13 +56,42 @@ namespace osu.Game.Database // ReSharper disable once NotAccessedField.Local (we should keep a reference to this so it is not finalised) private ArchiveImportIPCChannel ipc; + private readonly List cachedEvents = new List(); + + private bool delayingEvents; + + private void cacheEvents() + { + delayingEvents = true; + } + + private void flushEvents(bool perform) + { + if (perform) + { + foreach (var a in cachedEvents) + a.Invoke(); + } + + cachedEvents.Clear(); + delayingEvents = false; + } + + private void handleEvent(Action a) + { + if (delayingEvents) + cachedEvents.Add(a); + else + a.Invoke(); + } + protected ArchiveModelManager(Storage storage, IDatabaseContextFactory contextFactory, MutableDatabaseBackedStore modelStore, IIpcHost importHost = null) { ContextFactory = contextFactory; ModelStore = modelStore; - ModelStore.ItemAdded += s => ItemAdded?.Invoke(s); - ModelStore.ItemRemoved += s => ItemRemoved?.Invoke(s); + ModelStore.ItemAdded += s => handleEvent(() => ItemAdded?.Invoke(s)); + ModelStore.ItemRemoved += s => handleEvent(() => ItemRemoved?.Invoke(s)); Files = new FileStore(contextFactory, storage); @@ -138,24 +167,37 @@ namespace osu.Game.Database /// The archive to be imported. public TModel Import(ArchiveReader archive) { - using (ContextFactory.GetForWrite()) // used to share a context for full import. keep in mind this will block all writes. + TModel item = null; + cacheEvents(); + + try { - // create a new model (don't yet add to database) - var item = CreateModel(archive); + using (var write = ContextFactory.GetForWrite()) // used to share a context for full import. keep in mind this will block all writes. + { + if (!write.IsTransactionLeader) throw new InvalidOperationException($"Ensure there is no parent transaction so errors can correctly be handled by {this}"); - var existing = CheckForExisting(item); + // create a new model (don't yet add to database) + item = CreateModel(archive); - if (existing != null) return existing; + var existing = CheckForExisting(item); - item.Files = createFileInfos(archive, Files); + if (existing != null) return existing; - Populate(item, archive); + item.Files = createFileInfos(archive, Files); - // import to store - ModelStore.Add(item); + Populate(item, archive); - return item; + // import to store + ModelStore.Add(item); + } } + catch + { + item = null; + } + + flushEvents(item != null); + return item; } /// diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index 71960303b5..65a19e23c0 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -1,7 +1,9 @@ // Copyright (c) 2007-2018 ppy Pty Ltd . // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE +using System.Linq; using System.Threading; +using Microsoft.EntityFrameworkCore.Storage; using osu.Framework.Platform; namespace osu.Game.Database @@ -17,8 +19,12 @@ namespace osu.Game.Database private readonly object writeLock = new object(); private bool currentWriteDidWrite; + private bool currentWriteDidError; + private int currentWriteUsages; + private IDbContextTransaction currentWriteTransaction; + public DatabaseContextFactory(GameHost host) { this.host = host; @@ -40,9 +46,12 @@ namespace osu.Game.Database { Monitor.Enter(writeLock); + if (currentWriteTransaction == null) + currentWriteTransaction = threadContexts.Value.Database.BeginTransaction(); + Interlocked.Increment(ref currentWriteUsages); - return new DatabaseWriteUsage(threadContexts.Value, usageCompleted); + return new DatabaseWriteUsage(threadContexts.Value, usageCompleted) { IsTransactionLeader = currentWriteUsages == 1 }; } private void usageCompleted(DatabaseWriteUsage usage) @@ -52,16 +61,24 @@ namespace osu.Game.Database try { currentWriteDidWrite |= usage.PerformedWrite; + currentWriteDidError |= usage.Errors.Any(); if (usages > 0) return; + if (currentWriteDidError) + currentWriteTransaction.Rollback(); + else + currentWriteTransaction.Commit(); + + currentWriteTransaction = null; + currentWriteDidWrite = false; + currentWriteDidError = false; + if (currentWriteDidWrite) { // explicitly dispose to ensure any outstanding flushes happen as soon as possible (and underlying resources are purged). usage.Context.Dispose(); - currentWriteDidWrite = false; - // once all writes are complete, we want to refresh thread-specific contexts to make sure they don't have stale local caches. recycleThreadContexts(); } diff --git a/osu.Game/Database/DatabaseWriteUsage.cs b/osu.Game/Database/DatabaseWriteUsage.cs index 07bebbf0c3..8216c04b22 100644 --- a/osu.Game/Database/DatabaseWriteUsage.cs +++ b/osu.Game/Database/DatabaseWriteUsage.cs @@ -2,26 +2,31 @@ // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE using System; -using Microsoft.EntityFrameworkCore.Storage; +using System.Collections.Generic; namespace osu.Game.Database { public class DatabaseWriteUsage : IDisposable { public readonly OsuDbContext Context; - private readonly IDbContextTransaction transaction; private readonly Action usageCompleted; public DatabaseWriteUsage(OsuDbContext context, Action onCompleted) { Context = context; - transaction = Context.BeginTransaction(); usageCompleted = onCompleted; } public bool PerformedWrite { get; private set; } private bool isDisposed; + public List Errors = new List(); + + /// + /// Whether this write usage will commit a transaction on completion. + /// If false, there is a parent usage responsible for transaction commit. + /// + public bool IsTransactionLeader = false; protected void Dispose(bool disposing) { @@ -30,12 +35,11 @@ namespace osu.Game.Database try { - PerformedWrite |= Context.SaveChanges(transaction) > 0; + PerformedWrite |= Context.SaveChanges() > 0; } catch (Exception e) { - transaction?.Rollback(); - throw; + Errors.Add(e); } finally { diff --git a/osu.Game/Database/OsuDbContext.cs b/osu.Game/Database/OsuDbContext.cs index 06503ffc5f..0ae197d62d 100644 --- a/osu.Game/Database/OsuDbContext.cs +++ b/osu.Game/Database/OsuDbContext.cs @@ -3,7 +3,6 @@ using System; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Storage; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.Logging; using osu.Framework.Logging; @@ -104,18 +103,6 @@ namespace osu.Game.Database modelBuilder.Entity().HasOne(b => b.BaseDifficulty); } - public IDbContextTransaction BeginTransaction() - { - return Database.BeginTransaction(); - } - - public int SaveChanges(IDbContextTransaction transaction = null) - { - var ret = base.SaveChanges(); - if (ret > 0) transaction?.Commit(); - return ret; - } - private class OsuDbLoggerFactory : ILoggerFactory { #region Disposal From a3287b8cf2e3734e606b20490c9f2f8d5ab4a962 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 28 May 2018 21:45:05 +0900 Subject: [PATCH 04/16] Correctly rollback failed imports --- osu.Game/Database/ArchiveModelManager.cs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 28c25c873f..86af2fd0ed 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -58,13 +58,20 @@ namespace osu.Game.Database private readonly List cachedEvents = new List(); + /// + /// Allows delaying of outwards events until an operation is confirmed (at a database level). + /// private bool delayingEvents; - private void cacheEvents() - { - delayingEvents = true; - } + /// + /// Begin delaying outwards events. + /// + private void delayEvents() => delayingEvents = true; + /// + /// Flush delayed events and disable delaying. + /// + /// Whether the flushed events should be performed. private void flushEvents(bool perform) { if (perform) @@ -167,8 +174,8 @@ namespace osu.Game.Database /// The archive to be imported. public TModel Import(ArchiveReader archive) { - TModel item = null; - cacheEvents(); + TModel item; + delayEvents(); try { @@ -196,6 +203,7 @@ namespace osu.Game.Database item = null; } + // we only want to flush events after we've confirmed the write context didn't have any errors. flushEvents(item != null); return item; } From 80806be0471d1547e4b6af8b7b6f2ec520e780f4 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 29 May 2018 10:59:39 +0900 Subject: [PATCH 05/16] Don't start transactions for migration It looks like transactions are used internally during migration. --- osu.Game/Database/DatabaseContextFactory.cs | 11 ++++++----- osu.Game/Database/IDatabaseContextFactory.cs | 3 ++- osu.Game/Database/SingletonContextFactory.cs | 2 +- osu.Game/OsuGameBase.cs | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index 65a19e23c0..ec408456e3 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -41,17 +41,18 @@ namespace osu.Game.Database /// Request a context for write usage. Can be consumed in a nested fashion (and will return the same underlying context). /// This method may block if a write is already active on a different thread. /// + /// Whether to start a transaction for this write. /// A usage containing a usable context. - public DatabaseWriteUsage GetForWrite() + public DatabaseWriteUsage GetForWrite(bool withTransaction = true) { Monitor.Enter(writeLock); - if (currentWriteTransaction == null) + if (currentWriteTransaction == null && withTransaction) currentWriteTransaction = threadContexts.Value.Database.BeginTransaction(); Interlocked.Increment(ref currentWriteUsages); - return new DatabaseWriteUsage(threadContexts.Value, usageCompleted) { IsTransactionLeader = currentWriteUsages == 1 }; + return new DatabaseWriteUsage(threadContexts.Value, usageCompleted) { IsTransactionLeader = currentWriteTransaction != null && currentWriteUsages == 1 }; } private void usageCompleted(DatabaseWriteUsage usage) @@ -66,9 +67,9 @@ namespace osu.Game.Database if (usages > 0) return; if (currentWriteDidError) - currentWriteTransaction.Rollback(); + currentWriteTransaction?.Rollback(); else - currentWriteTransaction.Commit(); + currentWriteTransaction?.Commit(); currentWriteTransaction = null; currentWriteDidWrite = false; diff --git a/osu.Game/Database/IDatabaseContextFactory.cs b/osu.Game/Database/IDatabaseContextFactory.cs index 372e1770e4..d38d15b252 100644 --- a/osu.Game/Database/IDatabaseContextFactory.cs +++ b/osu.Game/Database/IDatabaseContextFactory.cs @@ -14,7 +14,8 @@ namespace osu.Game.Database /// Request a context for write usage. Can be consumed in a nested fashion (and will return the same underlying context). /// This method may block if a write is already active on a different thread. /// + /// Whether to start a transaction for this write. /// A usage containing a usable context. - DatabaseWriteUsage GetForWrite(); + DatabaseWriteUsage GetForWrite(bool withTransaction = true); } } diff --git a/osu.Game/Database/SingletonContextFactory.cs b/osu.Game/Database/SingletonContextFactory.cs index 74951e8433..ce3fbf6881 100644 --- a/osu.Game/Database/SingletonContextFactory.cs +++ b/osu.Game/Database/SingletonContextFactory.cs @@ -14,6 +14,6 @@ namespace osu.Game.Database public OsuDbContext Get() => context; - public DatabaseWriteUsage GetForWrite() => new DatabaseWriteUsage(context, null); + public DatabaseWriteUsage GetForWrite(bool withTransaction = true) => new DatabaseWriteUsage(context, null); } } diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index a3a081d6d1..b9d32a6322 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -208,7 +208,7 @@ namespace osu.Game { try { - using (var db = contextFactory.GetForWrite()) + using (var db = contextFactory.GetForWrite(false)) db.Context.Migrate(); } catch (MigrationFailedException e) @@ -220,7 +220,7 @@ namespace osu.Game contextFactory.ResetDatabase(); Logger.Log("Database purged successfully.", LoggingTarget.Database, LogLevel.Important); - using (var db = contextFactory.GetForWrite()) + using (var db = contextFactory.GetForWrite(false)) db.Context.Migrate(); } } From 72da640059e8aa05f193d0d3800bc0d6fee50cf6 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 29 May 2018 11:37:47 +0900 Subject: [PATCH 06/16] Change order of event firing in Update calls A remove event should not be fired before the update is successful. --- osu.Game/Database/MutableDatabaseBackedStore.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/osu.Game/Database/MutableDatabaseBackedStore.cs b/osu.Game/Database/MutableDatabaseBackedStore.cs index 8569d81f01..69a1f57cc4 100644 --- a/osu.Game/Database/MutableDatabaseBackedStore.cs +++ b/osu.Game/Database/MutableDatabaseBackedStore.cs @@ -50,11 +50,10 @@ namespace osu.Game.Database /// The item to update. public void Update(T item) { - ItemRemoved?.Invoke(item); - using (var usage = ContextFactory.GetForWrite()) usage.Context.Update(item); + ItemRemoved?.Invoke(item); ItemAdded?.Invoke(item); } From 015fd9d0e7d256410dea6fe583b3e5e7c0043a02 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 29 May 2018 11:44:42 +0900 Subject: [PATCH 07/16] Fix loading test method returning oldest import rather than newest --- osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index f60caf2397..740692395a 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -219,7 +219,7 @@ namespace osu.Game.Tests.Beatmaps.IO waitForOrAssert(() => !File.Exists(temp), "Temporary file still exists after standard import", 5000); - return imported.FirstOrDefault(); + return imported.LastOrDefault(); } private void deleteBeatmapSet(BeatmapSetInfo imported, OsuGameBase osu) From cc081cad5a0d78367420b31bfdb8b7a2baff0686 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 29 May 2018 11:57:11 +0900 Subject: [PATCH 08/16] Simplify test osz instantiation --- .../Beatmaps/IO/ImportBeatmapTest.cs | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 740692395a..38fd6e649b 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -162,8 +162,7 @@ namespace osu.Game.Tests.Beatmaps.IO var osu = loadOsu(host); - var temp = prepareTempCopy(osz_path); - Assert.IsTrue(File.Exists(temp)); + var temp = createTemporaryBeatmap(); var importer = new ArchiveImportIPCChannel(client); if (!importer.ImportAsync(temp).Wait(10000)) @@ -188,8 +187,7 @@ namespace osu.Game.Tests.Beatmaps.IO try { var osu = loadOsu(host); - var temp = prepareTempCopy(osz_path); - Assert.IsTrue(File.Exists(temp), "Temporary file copy never substantiated"); + var temp = createTemporaryBeatmap(); using (File.OpenRead(temp)) osu.Dependencies.Get().Import(temp); ensureLoaded(osu); @@ -203,11 +201,16 @@ namespace osu.Game.Tests.Beatmaps.IO } } - private BeatmapSetInfo loadOszIntoOsu(OsuGameBase osu) + private string createTemporaryBeatmap() { - var temp = prepareTempCopy(osz_path); - + var temp = new FileInfo(osz_path).CopyTo(Path.GetTempFileName(), true).FullName; Assert.IsTrue(File.Exists(temp)); + return temp; + } + + private BeatmapSetInfo loadOszIntoOsu(OsuGameBase osu, string path = null) + { + var temp = path ?? createTemporaryBeatmap(); var manager = osu.Dependencies.Get(); @@ -232,12 +235,6 @@ namespace osu.Game.Tests.Beatmaps.IO Assert.IsTrue(manager.QueryBeatmapSets(_ => true).First().DeletePending); } - private string prepareTempCopy(string path) - { - var temp = Path.GetTempFileName(); - return new FileInfo(path).CopyTo(temp, true).FullName; - } - private OsuGameBase loadOsu(GameHost host) { var osu = new OsuGameBase(); From 3d3026a80cdce1dc15f04b88f242120ec09cfffc Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 29 May 2018 13:48:14 +0900 Subject: [PATCH 09/16] Report any error during import to the write context to allow for rollback --- osu.Game/Database/ArchiveModelManager.cs | 26 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 86af2fd0ed..99f1e9f581 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -181,21 +181,29 @@ namespace osu.Game.Database { using (var write = ContextFactory.GetForWrite()) // used to share a context for full import. keep in mind this will block all writes. { - if (!write.IsTransactionLeader) throw new InvalidOperationException($"Ensure there is no parent transaction so errors can correctly be handled by {this}"); + try + { + if (!write.IsTransactionLeader) throw new InvalidOperationException($"Ensure there is no parent transaction so errors can correctly be handled by {this}"); - // create a new model (don't yet add to database) - item = CreateModel(archive); + // create a new model (don't yet add to database) + item = CreateModel(archive); - var existing = CheckForExisting(item); + var existing = CheckForExisting(item); - if (existing != null) return existing; + if (existing != null) return existing; - item.Files = createFileInfos(archive, Files); + item.Files = createFileInfos(archive, Files); - Populate(item, archive); + Populate(item, archive); - // import to store - ModelStore.Add(item); + // import to store + ModelStore.Add(item); + } + catch (Exception e) + { + write.Errors.Add(e); + throw; + } } } catch From 203691b1c776728fb48564c51bef60a20bc0d499 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 29 May 2018 13:48:27 +0900 Subject: [PATCH 10/16] Add import rollback test --- .../Beatmaps/IO/ImportBeatmapTest.cs | 75 +++++++++++++++++-- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 38fd6e649b..586217a05f 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -12,6 +12,7 @@ using osu.Framework.Platform; using osu.Game.IPC; using osu.Framework.Allocation; using osu.Game.Beatmaps; +using SharpCompress.Archives.Zip; namespace osu.Game.Tests.Beatmaps.IO { @@ -77,8 +78,69 @@ namespace osu.Game.Tests.Beatmaps.IO var manager = osu.Dependencies.Get(); - Assert.IsTrue(manager.GetAllUsableBeatmapSets().Count == 1); - Assert.IsTrue(manager.QueryBeatmapSets(_ => true).ToList().Count == 1); + Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); + Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + } + finally + { + host.Exit(); + } + } + } + + [Test] + public void TestRollbackOnFailure() + { + //unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. + using (HeadlessGameHost host = new CleanRunHeadlessGameHost("TestRollbackOnFailure")) + { + try + { + var osu = loadOsu(host); + var manager = osu.Dependencies.Get(); + + int fireCount = 0; + + // ReSharper disable once AccessToModifiedClosure + manager.ItemAdded += _ => fireCount++; + manager.ItemRemoved += _ => fireCount++; + + var imported = loadOszIntoOsu(osu); + + Assert.AreEqual(0, fireCount -= 1); + + imported.Hash += "-changed"; + manager.Update(imported); + + Assert.AreEqual(0, fireCount -= 2); + + var breakTemp = createTemporaryBeatmap(); + + MemoryStream brokenOsu = new MemoryStream(new byte[] { 1, 3, 3, 7 }); + MemoryStream brokenOsz = new MemoryStream(File.ReadAllBytes(breakTemp)); + + File.Delete(breakTemp); + + using (var outStream = File.Open(breakTemp, FileMode.CreateNew)) + using (var zip = ZipArchive.Open(brokenOsz)) + { + zip.AddEntry("broken.osu", brokenOsu, false); + zip.SaveTo(outStream, SharpCompress.Common.CompressionType.Deflate); + } + + Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); + Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); + + // this will trigger purging of the existing beatmap (online set id match) but should rollback due to broken osu. + manager.Import(breakTemp); + + // no events should be fired in the case of a rollback. + Assert.AreEqual(0, fireCount); + + Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); + Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); + Assert.AreEqual(12, manager.QueryBeatmaps(_ => true).ToList().Count); } finally { @@ -100,18 +162,17 @@ namespace osu.Game.Tests.Beatmaps.IO var imported = loadOszIntoOsu(osu); - //var change = manager.QueryBeatmapSets(_ => true).First(); imported.Hash += "-changed"; manager.Update(imported); var importedSecondTime = loadOszIntoOsu(osu); - // check the newly "imported" beatmap is actually just the restored previous import. since it matches hash. Assert.IsTrue(imported.ID != importedSecondTime.ID); Assert.IsTrue(imported.Beatmaps.First().ID < importedSecondTime.Beatmaps.First().ID); - Assert.IsTrue(manager.GetAllUsableBeatmapSets().Count == 1); - Assert.IsTrue(manager.QueryBeatmapSets(_ => true).ToList().Count == 1); + // only one beatmap will exist as the online set ID matched, causing purging of the first import. + Assert.AreEqual(1, manager.GetAllUsableBeatmapSets().Count); + Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); } finally { @@ -231,7 +292,7 @@ namespace osu.Game.Tests.Beatmaps.IO manager.Delete(imported); Assert.IsTrue(manager.GetAllUsableBeatmapSets().Count == 0); - Assert.IsTrue(manager.QueryBeatmapSets(_ => true).ToList().Count == 1); + Assert.AreEqual(1, manager.QueryBeatmapSets(_ => true).ToList().Count); Assert.IsTrue(manager.QueryBeatmapSets(_ => true).First().DeletePending); } From c1f416b1ccb6cea656d90c128c120048d9619a81 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 29 May 2018 14:25:27 +0900 Subject: [PATCH 11/16] Add back missing rethrow --- osu.Game/Database/DatabaseWriteUsage.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/osu.Game/Database/DatabaseWriteUsage.cs b/osu.Game/Database/DatabaseWriteUsage.cs index 8216c04b22..64ab24e824 100644 --- a/osu.Game/Database/DatabaseWriteUsage.cs +++ b/osu.Game/Database/DatabaseWriteUsage.cs @@ -40,6 +40,7 @@ namespace osu.Game.Database catch (Exception e) { Errors.Add(e); + throw; } finally { From 4a18951cce0279731bf7afae07f330dda4d2d37f Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 29 May 2018 18:37:45 +0900 Subject: [PATCH 12/16] Report full error to log file --- osu.Game/Database/ArchiveModelManager.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 99f1e9f581..6a31182370 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -206,8 +206,9 @@ namespace osu.Game.Database } } } - catch + catch (Exception e) { + Logger.Error(e, $"Import of {archive.Name} failed and has been rolled back.", LoggingTarget.Database); item = null; } From 31ab6f240841b5be96077c72e7761109e690c5e5 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 29 May 2018 19:43:52 +0900 Subject: [PATCH 13/16] Fix event flushing sticking on early return --- osu.Game/Database/ArchiveModelManager.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 6a31182370..62d8c16946 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -174,7 +174,7 @@ namespace osu.Game.Database /// The archive to be imported. public TModel Import(ArchiveReader archive) { - TModel item; + TModel item = null; delayEvents(); try @@ -211,9 +211,12 @@ namespace osu.Game.Database Logger.Error(e, $"Import of {archive.Name} failed and has been rolled back.", LoggingTarget.Database); item = null; } + finally + { + // we only want to flush events after we've confirmed the write context didn't have any errors. + flushEvents(item != null); + } - // we only want to flush events after we've confirmed the write context didn't have any errors. - flushEvents(item != null); return item; } From e23e2bd3485045eb079b4c69f71b5cdf50b98260 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 30 May 2018 13:37:52 +0900 Subject: [PATCH 14/16] Fix recycling never being performed due to incorrect ordering --- osu.Game/Database/DatabaseContextFactory.cs | 33 +++++++++++---------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index ec408456e3..f57246453e 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -64,24 +64,25 @@ namespace osu.Game.Database currentWriteDidWrite |= usage.PerformedWrite; currentWriteDidError |= usage.Errors.Any(); - if (usages > 0) return; - - if (currentWriteDidError) - currentWriteTransaction?.Rollback(); - else - currentWriteTransaction?.Commit(); - - currentWriteTransaction = null; - currentWriteDidWrite = false; - currentWriteDidError = false; - - if (currentWriteDidWrite) + if (usages == 0) { - // explicitly dispose to ensure any outstanding flushes happen as soon as possible (and underlying resources are purged). - usage.Context.Dispose(); + if (currentWriteDidError) + currentWriteTransaction?.Rollback(); + else + currentWriteTransaction?.Commit(); - // once all writes are complete, we want to refresh thread-specific contexts to make sure they don't have stale local caches. - recycleThreadContexts(); + if (currentWriteDidWrite || currentWriteDidError) + { + // explicitly dispose to ensure any outstanding flushes happen as soon as possible (and underlying resources are purged). + usage.Context.Dispose(); + + // once all writes are complete, we want to refresh thread-specific contexts to make sure they don't have stale local caches. + recycleThreadContexts(); + } + + currentWriteTransaction = null; + currentWriteDidWrite = false; + currentWriteDidError = false; } } finally From 4a7de043e080aeb087de9d67eb7b3627a3471bf7 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 30 May 2018 13:43:25 +0900 Subject: [PATCH 15/16] Recycle all contexts on beginning a write operation for the time being --- osu.Game/Database/DatabaseContextFactory.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/osu.Game/Database/DatabaseContextFactory.cs b/osu.Game/Database/DatabaseContextFactory.cs index f57246453e..a1d371f431 100644 --- a/osu.Game/Database/DatabaseContextFactory.cs +++ b/osu.Game/Database/DatabaseContextFactory.cs @@ -48,7 +48,14 @@ namespace osu.Game.Database Monitor.Enter(writeLock); if (currentWriteTransaction == null && withTransaction) + { + // this mitigates the fact that changes on tracked entities will not be rolled back with the transaction by ensuring write operations are always executed in isolated contexts. + // if this results in sub-optimal efficiency, we may need to look into removing Database-level transactions in favour of running SaveChanges where we currently commit the transaction. + if (threadContexts.IsValueCreated) + recycleThreadContexts(); + currentWriteTransaction = threadContexts.Value.Database.BeginTransaction(); + } Interlocked.Increment(ref currentWriteUsages); From eb893174947ae5cf8d050d932b27658cfab990cb Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 30 May 2018 13:43:43 +0900 Subject: [PATCH 16/16] Remove performance optimisation tracking disables to keep things simple for now --- osu.Game/Database/ArchiveModelManager.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 62d8c16946..1505ac0549 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -240,12 +240,8 @@ namespace osu.Game.Database /// The item to delete. public void Delete(TModel item) { - using (var usage = ContextFactory.GetForWrite()) + using (ContextFactory.GetForWrite()) { - var context = usage.Context; - - context.ChangeTracker.AutoDetectChangesEnabled = false; - // re-fetch the model on the import context. var foundModel = queryModel().Include(s => s.Files).ThenInclude(f => f.FileInfo).First(s => s.ID == item.ID); @@ -253,8 +249,6 @@ namespace osu.Game.Database if (ModelStore.Delete(foundModel)) Files.Dereference(foundModel.Files.Select(f => f.FileInfo).ToArray()); - - context.ChangeTracker.AutoDetectChangesEnabled = true; } }