sim1222/osukey
1
1
Fork 0
mirror of https://github.com/osukey/osukey.git synced 2025-08-04 15:16:38 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into storyboard-fixes

Browse Source
This commit is contained in:
Dean Herbert
2021-10-02 00:35:51 +09:00
committed by GitHub
parent 4c28749d73 a620b55d55
commit d6dbdc911f
47 changed files with 2024 additions and 1231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

669
osu.Game/Beatmaps/BeatmapManager.cs
View File

@ -6,111 +6,67 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using JetBrains.Annotations;
using Microsoft.EntityFrameworkCore;
using osu.Framework.Audio;
using osu.Framework.Audio.Track;
using osu.Framework.Bindables;
using osu.Framework.Extensions;
using osu.Framework.Graphics.Textures;
using osu.Framework.IO.Stores;
using osu.Framework.Lists;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Framework.Statistics;
using osu.Framework.Testing;
using osu.Game.Beatmaps.Formats;
using osu.Game.Database;
using osu.Game.IO;
using osu.Game.IO.Archives;
using osu.Game.Online.API;
using osu.Game.Online.API.Requests;
using osu.Game.Overlays.Notifications;
using osu.Game.Rulesets;
using osu.Game.Rulesets.Objects;
using osu.Game.Skinning;
using osu.Game.Users;
using Decoder = osu.Game.Beatmaps.Formats.Decoder;
namespace osu.Game.Beatmaps
{
/// <summary>
/// Handles the storage and retrieval of Beatmaps/WorkingBeatmaps.
/// Handles general operations related to global beatmap management.
/// </summary>
[ExcludeFromDynamicCompile]
public partial class BeatmapManager : DownloadableArchiveModelManager<BeatmapSetInfo, BeatmapSetFileInfo>, IDisposable, IBeatmapResourceProvider
public class BeatmapManager : IModelDownloader<BeatmapSetInfo>, IModelManager<BeatmapSetInfo>, IModelFileManager<BeatmapSetInfo, BeatmapSetFileInfo>, ICanAcceptFiles, IWorkingBeatmapCache, IDisposable
{
/// <summary>
/// Fired when a single difficulty has been hidden.
/// </summary>
public IBindable<WeakReference<BeatmapInfo>> BeatmapHidden => beatmapHidden;
private readonly BeatmapModelManager beatmapModelManager;
private readonly BeatmapModelDownloader beatmapModelDownloader;
private readonly Bindable<WeakReference<BeatmapInfo>> beatmapHidden = new Bindable<WeakReference<BeatmapInfo>>();
/// <summary>
/// Fired when a single difficulty has been restored.
/// </summary>
public IBindable<WeakReference<BeatmapInfo>> BeatmapRestored => beatmapRestored;
private readonly Bindable<WeakReference<BeatmapInfo>> beatmapRestored = new Bindable<WeakReference<BeatmapInfo>>();
/// <summary>
/// A default representation of a WorkingBeatmap to use when no beatmap is available.
/// </summary>
public readonly WorkingBeatmap DefaultBeatmap;
public override IEnumerable<string> HandledExtensions => new[] { ".osz" };
protected override string[] HashableFileTypes => new[] { ".osu" };
protected override string ImportFromStablePath => ".";
protected override Storage PrepareStableStorage(StableStorage stableStorage) => stableStorage.GetSongStorage();
private readonly RulesetStore rulesets;
private readonly BeatmapStore beatmaps;
private readonly AudioManager audioManager;
private readonly IResourceStore<byte[]> resources;
private readonly LargeTextureStore largeTextureStore;
private readonly ITrackStore trackStore;
[CanBeNull]
private readonly GameHost host;
[CanBeNull]
private readonly BeatmapOnlineLookupQueue onlineLookupQueue;
private readonly WorkingBeatmapCache workingBeatmapCache;
private readonly BeatmapOnlineLookupQueue onlineBetamapLookupQueue;
public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, [NotNull] AudioManager audioManager, IResourceStore<byte[]> resources, GameHost host = null,
WorkingBeatmap defaultBeatmap = null, bool performOnlineLookups = false)
: base(storage, contextFactory, api, new BeatmapStore(contextFactory), host)
{
this.rulesets = rulesets;
this.audioManager = audioManager;
this.resources = resources;
this.host = host;
beatmapModelManager = CreateBeatmapModelManager(storage, contextFactory, rulesets, api, host);
beatmapModelDownloader = CreateBeatmapModelDownloader(beatmapModelManager, api, host);
workingBeatmapCache = CreateWorkingBeatmapCache(audioManager, resources, new FileStore(contextFactory, storage).Store, defaultBeatmap, host);
DefaultBeatmap = defaultBeatmap;
beatmaps = (BeatmapStore)ModelStore;
beatmaps.BeatmapHidden += b => beatmapHidden.Value = new WeakReference<BeatmapInfo>(b);
beatmaps.BeatmapRestored += b => beatmapRestored.Value = new WeakReference<BeatmapInfo>(b);
beatmaps.ItemRemoved += removeWorkingCache;
beatmaps.ItemUpdated += removeWorkingCache;
workingBeatmapCache.BeatmapManager = beatmapModelManager;
if (performOnlineLookups)
onlineLookupQueue = new BeatmapOnlineLookupQueue(api, storage);
largeTextureStore = new LargeTextureStore(host?.CreateTextureLoaderStore(Files.Store));
trackStore = audioManager.GetTrackStore(Files.Store);
{
onlineBetamapLookupQueue = new BeatmapOnlineLookupQueue(api, storage);
beatmapModelManager.OnlineLookupQueue = onlineBetamapLookupQueue;
}
}
protected override ArchiveDownloadRequest<BeatmapSetInfo> CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) =>
new DownloadBeatmapSetRequest(set, minimiseDownloadSize);
protected virtual BeatmapModelDownloader CreateBeatmapModelDownloader(BeatmapModelManager modelManager, IAPIProvider api, GameHost host)
{
return new BeatmapModelDownloader(modelManager, api, host);
}
protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path)?.ToLowerInvariant() == ".osz";
protected virtual WorkingBeatmapCache CreateWorkingBeatmapCache(AudioManager audioManager, IResourceStore<byte[]> resources, IResourceStore<byte[]> storage, WorkingBeatmap defaultBeatmap, GameHost host) =>
new WorkingBeatmapCache(audioManager, resources, storage, defaultBeatmap, host);
protected virtual BeatmapModelManager CreateBeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, IAPIProvider api, GameHost host) =>
new BeatmapModelManager(storage, contextFactory, rulesets, host);
/// <summary>
/// Create a new <see cref="WorkingBeatmap"/>.
/// </summary>
public WorkingBeatmap CreateNew(RulesetInfo ruleset, User user)
{
var metadata = new BeatmapMetadata
@ -134,112 +90,21 @@ namespace osu.Game.Beatmaps
}
};
var working = Import(set).Result;
var working = beatmapModelManager.Import(set).Result;
return GetWorkingBeatmap(working.Beatmaps.First());
}
protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default)
{
if (archive != null)
beatmapSet.Beatmaps = createBeatmapDifficulties(beatmapSet.Files);
foreach (BeatmapInfo b in beatmapSet.Beatmaps)
{
// remove metadata from difficulties where it matches the set
if (beatmapSet.Metadata.Equals(b.Metadata))
b.Metadata = null;
b.BeatmapSet = beatmapSet;
}
validateOnlineIds(beatmapSet);
bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0);
if (onlineLookupQueue != null)
await onlineLookupQueue.UpdateAsync(beatmapSet, cancellationToken).ConfigureAwait(false);
// ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID.
if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0))
{
if (beatmapSet.OnlineBeatmapSetID != null)
{
beatmapSet.OnlineBeatmapSetID = null;
LogForModel(beatmapSet, "Disassociating beatmap set ID due to loss of all beatmap IDs");
}
}
}
protected override void PreImport(BeatmapSetInfo beatmapSet)
{
if (beatmapSet.Beatmaps.Any(b => b.BaseDifficulty == null))
throw new InvalidOperationException($"Cannot import {nameof(BeatmapInfo)} with null {nameof(BeatmapInfo.BaseDifficulty)}.");
// check if a set already exists with the same online id, delete if it does.
if (beatmapSet.OnlineBeatmapSetID != null)
{
var existingOnlineId = beatmaps.ConsumableItems.FirstOrDefault(b => b.OnlineBeatmapSetID == beatmapSet.OnlineBeatmapSetID);
if (existingOnlineId != null)
{
Delete(existingOnlineId);
// in order to avoid a unique key constraint, immediately remove the online ID from the previous set.
existingOnlineId.OnlineBeatmapSetID = null;
foreach (var b in existingOnlineId.Beatmaps)
b.OnlineBeatmapID = null;
LogForModel(beatmapSet, $"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been deleted.");
}
}
}
private void validateOnlineIds(BeatmapSetInfo beatmapSet)
{
var beatmapIds = beatmapSet.Beatmaps.Where(b => b.OnlineBeatmapID.HasValue).Select(b => b.OnlineBeatmapID).ToList();
// ensure all IDs are unique
if (beatmapIds.GroupBy(b => b).Any(g => g.Count() > 1))
{
LogForModel(beatmapSet, "Found non-unique IDs, resetting...");
resetIds();
return;
}
// find any existing beatmaps in the database that have matching online ids
var existingBeatmaps = QueryBeatmaps(b => beatmapIds.Contains(b.OnlineBeatmapID)).ToList();
if (existingBeatmaps.Count > 0)
{
// reset the import ids (to force a re-fetch) *unless* they match the candidate CheckForExisting set.
// we can ignore the case where the new ids are contained by the CheckForExisting set as it will either be used (import skipped) or deleted.
var existing = CheckForExisting(beatmapSet);
if (existing == null || existingBeatmaps.Any(b => !existing.Beatmaps.Contains(b)))
{
LogForModel(beatmapSet, "Found existing import with IDs already, resetting...");
resetIds();
}
}
void resetIds() => beatmapSet.Beatmaps.ForEach(b => b.OnlineBeatmapID = null);
}
protected override bool CheckLocalAvailability(BeatmapSetInfo model, IQueryable<BeatmapSetInfo> items)
=> base.CheckLocalAvailability(model, items)
|| (model.OnlineBeatmapSetID != null && items.Any(b => b.OnlineBeatmapSetID == model.OnlineBeatmapSetID));
#region Delegation to BeatmapModelManager (methods which previously existed locally).
/// <summary>
/// Delete a beatmap difficulty.
/// Fired when a single difficulty has been hidden.
/// </summary>
/// <param name="beatmap">The beatmap difficulty to hide.</param>
public void Hide(BeatmapInfo beatmap) => beatmaps.Hide(beatmap);
public IBindable<WeakReference<BeatmapInfo>> BeatmapHidden => beatmapModelManager.BeatmapHidden;
/// <summary>
/// Restore a beatmap difficulty.
/// Fired when a single difficulty has been restored.
/// </summary>
/// <param name="beatmap">The beatmap difficulty to restore.</param>
public void Restore(BeatmapInfo beatmap) => beatmaps.Restore(beatmap);
public IBindable<WeakReference<BeatmapInfo>> BeatmapRestored => beatmapModelManager.BeatmapRestored;
/// <summary>
/// Saves an <see cref="IBeatmap"/> file against a given <see cref="BeatmapInfo"/>.
@ -247,109 +112,13 @@ namespace osu.Game.Beatmaps
/// <param name="info">The <see cref="BeatmapInfo"/> to save the content against. The file referenced by <see cref="BeatmapInfo.Path"/> will be replaced.</param>
/// <param name="beatmapContent">The <see cref="IBeatmap"/> content to write.</param>
/// <param name="beatmapSkin">The beatmap <see cref="ISkin"/> content to write, null if to be omitted.</param>
public virtual void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null)
{
var setInfo = info.BeatmapSet;
using (var stream = new MemoryStream())
{
using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true))
new LegacyBeatmapEncoder(beatmapContent, beatmapSkin).Encode(sw);
stream.Seek(0, SeekOrigin.Begin);
using (ContextFactory.GetForWrite())
{
var beatmapInfo = setInfo.Beatmaps.Single(b => b.ID == info.ID);
var metadata = beatmapInfo.Metadata ?? setInfo.Metadata;
// grab the original file (or create a new one if not found).
var fileInfo = setInfo.Files.SingleOrDefault(f => string.Equals(f.Filename, beatmapInfo.Path, StringComparison.OrdinalIgnoreCase)) ?? new BeatmapSetFileInfo();
// metadata may have changed; update the path with the standard format.
beatmapInfo.Path = $"{metadata.Artist} - {metadata.Title} ({metadata.Author}) [{beatmapInfo.Version}].osu";
beatmapInfo.MD5Hash = stream.ComputeMD5Hash();
// update existing or populate new file's filename.
fileInfo.Filename = beatmapInfo.Path;
stream.Seek(0, SeekOrigin.Begin);
ReplaceFile(setInfo, fileInfo, stream);
}
}
removeWorkingCache(info);
}
private readonly WeakList<BeatmapManagerWorkingBeatmap> workingCache = new WeakList<BeatmapManagerWorkingBeatmap>();
/// <summary>
/// Retrieve a <see cref="WorkingBeatmap"/> instance for the provided <see cref="BeatmapInfo"/>
/// </summary>
/// <param name="beatmapInfo">The beatmap to lookup.</param>
/// <returns>A <see cref="WorkingBeatmap"/> instance correlating to the provided <see cref="BeatmapInfo"/>.</returns>
public virtual WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo)
{
// if there are no files, presume the full beatmap info has not yet been fetched from the database.
if (beatmapInfo?.BeatmapSet?.Files.Count == 0)
{
int lookupId = beatmapInfo.ID;
beatmapInfo = QueryBeatmap(b => b.ID == lookupId);
}
if (beatmapInfo?.BeatmapSet == null)
return DefaultBeatmap;
lock (workingCache)
{
var working = workingCache.FirstOrDefault(w => w.BeatmapInfo?.ID == beatmapInfo.ID);
if (working != null)
return working;
beatmapInfo.Metadata ??= beatmapInfo.BeatmapSet.Metadata;
workingCache.Add(working = new BeatmapManagerWorkingBeatmap(beatmapInfo, this));
// best effort; may be higher than expected.
GlobalStatistics.Get<int>(nameof(Beatmaps), $"Cached {nameof(WorkingBeatmap)}s").Value = workingCache.Count();
return working;
}
}
/// <summary>
/// Perform a lookup query on available <see cref="BeatmapSetInfo"/>s.
/// </summary>
/// <param name="query">The query.</param>
/// <returns>The first result for the provided query, or null if no results were found.</returns>
public BeatmapSetInfo QueryBeatmapSet(Expression<Func<BeatmapSetInfo, bool>> query) => beatmaps.ConsumableItems.AsNoTracking().FirstOrDefault(query);
protected override bool CanSkipImport(BeatmapSetInfo existing, BeatmapSetInfo import)
{
if (!base.CanSkipImport(existing, import))
return false;
return existing.Beatmaps.Any(b => b.OnlineBeatmapID != null);
}
protected override bool CanReuseExisting(BeatmapSetInfo existing, BeatmapSetInfo import)
{
if (!base.CanReuseExisting(existing, import))
return false;
var existingIds = existing.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i);
var importIds = import.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i);
// force re-import if we are not in a sane state.
return existing.OnlineBeatmapSetID == import.OnlineBeatmapSetID && existingIds.SequenceEqual(importIds);
}
public virtual void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null) => beatmapModelManager.Save(info, beatmapContent, beatmapSkin);
/// <summary>
/// Returns a list of all usable <see cref="BeatmapSetInfo"/>s.
/// </summary>
/// <returns>A list of available <see cref="BeatmapSetInfo"/>.</returns>
public List<BeatmapSetInfo> GetAllUsableBeatmapSets(IncludedDetails includes = IncludedDetails.All, bool includeProtected = false) =>
GetAllUsableBeatmapSetsEnumerable(includes, includeProtected).ToList();
public List<BeatmapSetInfo> GetAllUsableBeatmapSets(IncludedDetails includes = IncludedDetails.All, bool includeProtected = false) => beatmapModelManager.GetAllUsableBeatmapSets(includes, includeProtected);
/// <summary>
/// Returns a list of all usable <see cref="BeatmapSetInfo"/>s. Note that files are not populated.
@ -357,34 +126,7 @@ namespace osu.Game.Beatmaps
/// <param name="includes">The level of detail to include in the returned objects.</param>
/// <param name="includeProtected">Whether to include protected (system) beatmaps. These should not be included for gameplay playable use cases.</param>
/// <returns>A list of available <see cref="BeatmapSetInfo"/>.</returns>
public IEnumerable<BeatmapSetInfo> GetAllUsableBeatmapSetsEnumerable(IncludedDetails includes, bool includeProtected = false)
{
IQueryable<BeatmapSetInfo> queryable;
switch (includes)
{
case IncludedDetails.Minimal:
queryable = beatmaps.BeatmapSetsOverview;
break;
case IncludedDetails.AllButRuleset:
queryable = beatmaps.BeatmapSetsWithoutRuleset;
break;
case IncludedDetails.AllButFiles:
queryable = beatmaps.BeatmapSetsWithoutFiles;
break;
default:
queryable = beatmaps.ConsumableItems;
break;
}
// AsEnumerable used here to avoid applying the WHERE in sql. When done so, ef core 2.x uses an incorrect ORDER BY
// clause which causes queries to take 5-10x longer.
// TODO: remove if upgrading to EF core 3.x.
return queryable.AsEnumerable().Where(s => !s.DeletePending && (includeProtected || !s.Protected));
}
public IEnumerable<BeatmapSetInfo> GetAllUsableBeatmapSetsEnumerable(IncludedDetails includes, bool includeProtected = false) => beatmapModelManager.GetAllUsableBeatmapSetsEnumerable(includes, includeProtected);
/// <summary>
/// Perform a lookup query on available <see cref="BeatmapSetInfo"/>s.
@ -392,207 +134,204 @@ namespace osu.Game.Beatmaps
/// <param name="query">The query.</param>
/// <param name="includes">The level of detail to include in the returned objects.</param>
/// <returns>Results from the provided query.</returns>
public IEnumerable<BeatmapSetInfo> QueryBeatmapSets(Expression<Func<BeatmapSetInfo, bool>> query, IncludedDetails includes = IncludedDetails.All)
{
IQueryable<BeatmapSetInfo> queryable;
switch (includes)
{
case IncludedDetails.Minimal:
queryable = beatmaps.BeatmapSetsOverview;
break;
case IncludedDetails.AllButRuleset:
queryable = beatmaps.BeatmapSetsWithoutRuleset;
break;
case IncludedDetails.AllButFiles:
queryable = beatmaps.BeatmapSetsWithoutFiles;
break;
default:
queryable = beatmaps.ConsumableItems;
break;
}
return queryable.AsNoTracking().Where(query);
}
public IEnumerable<BeatmapSetInfo> QueryBeatmapSets(Expression<Func<BeatmapSetInfo, bool>> query, IncludedDetails includes = IncludedDetails.All) => beatmapModelManager.QueryBeatmapSets(query, includes);
/// <summary>
/// Perform a lookup query on available <see cref="BeatmapInfo"/>s.
/// Perform a lookup query on available <see cref="BeatmapSetInfo"/>s.
/// </summary>
/// <param name="query">The query.</param>
/// <returns>The first result for the provided query, or null if no results were found.</returns>
public BeatmapInfo QueryBeatmap(Expression<Func<BeatmapInfo, bool>> query) => beatmaps.Beatmaps.AsNoTracking().FirstOrDefault(query);
public BeatmapSetInfo QueryBeatmapSet(Expression<Func<BeatmapSetInfo, bool>> query) => beatmapModelManager.QueryBeatmapSet(query);
/// <summary>
/// Perform a lookup query on available <see cref="BeatmapInfo"/>s.
/// </summary>
/// <param name="query">The query.</param>
/// <returns>Results from the provided query.</returns>
public IQueryable<BeatmapInfo> QueryBeatmaps(Expression<Func<BeatmapInfo, bool>> query) => beatmaps.Beatmaps.AsNoTracking().Where(query);
public IQueryable<BeatmapInfo> QueryBeatmaps(Expression<Func<BeatmapInfo, bool>> query) => beatmapModelManager.QueryBeatmaps(query);
protected override string HumanisedModelName => "beatmap";
/// <summary>
/// Perform a lookup query on available <see cref="BeatmapInfo"/>s.
/// </summary>
/// <param name="query">The query.</param>
/// <returns>The first result for the provided query, or null if no results were found.</returns>
public BeatmapInfo QueryBeatmap(Expression<Func<BeatmapInfo, bool>> query) => beatmapModelManager.QueryBeatmap(query);
protected override BeatmapSetInfo CreateModel(ArchiveReader reader)
/// <summary>
/// A default representation of a WorkingBeatmap to use when no beatmap is available.
/// </summary>
public WorkingBeatmap DefaultBeatmap => workingBeatmapCache.DefaultBeatmap;
/// <summary>
/// Fired when a notification should be presented to the user.
/// </summary>
public Action<Notification> PostNotification
{
// let's make sure there are actually .osu files to import.
string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrEmpty(mapName))
set
{
Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database);
return null;
beatmapModelManager.PostNotification = value;
beatmapModelDownloader.PostNotification = value;
}
Beatmap beatmap;
using (var stream = new LineBufferedReader(reader.GetStream(mapName)))
beatmap = Decoder.GetDecoder<Beatmap>(stream).Decode(stream);
return new BeatmapSetInfo
{
OnlineBeatmapSetID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID,
Beatmaps = new List<BeatmapInfo>(),
Metadata = beatmap.Metadata,
DateAdded = DateTimeOffset.UtcNow
};
}
/// <summary>
/// Create all required <see cref="BeatmapInfo"/>s for the provided archive.
/// Fired when the user requests to view the resulting import.
/// </summary>
private List<BeatmapInfo> createBeatmapDifficulties(List<BeatmapSetFileInfo> files)
{
var beatmapInfos = new List<BeatmapInfo>();
public Action<IEnumerable<BeatmapSetInfo>> PresentImport { set => beatmapModelManager.PresentImport = value; }
foreach (var file in files.Where(f => f.Filename.EndsWith(".osu", StringComparison.OrdinalIgnoreCase)))
{
using (var raw = Files.Store.GetStream(file.FileInfo.StoragePath))
using (var ms = new MemoryStream()) // we need a memory stream so we can seek
using (var sr = new LineBufferedReader(ms))
{
raw.CopyTo(ms);
ms.Position = 0;
/// <summary>
/// Delete a beatmap difficulty.
/// </summary>
/// <param name="beatmap">The beatmap difficulty to hide.</param>
public void Hide(BeatmapInfo beatmap) => beatmapModelManager.Hide(beatmap);
var decoder = Decoder.GetDecoder<Beatmap>(sr);
IBeatmap beatmap = decoder.Decode(sr);
string hash = ms.ComputeSHA2Hash();
if (beatmapInfos.Any(b => b.Hash == hash))
continue;
beatmap.BeatmapInfo.Path = file.Filename;
beatmap.BeatmapInfo.Hash = hash;
beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash();
var ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID);
beatmap.BeatmapInfo.Ruleset = ruleset;
// TODO: this should be done in a better place once we actually need to dynamically update it.
beatmap.BeatmapInfo.StarDifficulty = ruleset?.CreateInstance().CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating ?? 0;
beatmap.BeatmapInfo.Length = calculateLength(beatmap);
beatmap.BeatmapInfo.BPM = 60000 / beatmap.GetMostCommonBeatLength();
beatmapInfos.Add(beatmap.BeatmapInfo);
}
}
return beatmapInfos;
}
private double calculateLength(IBeatmap b)
{
if (!b.HitObjects.Any())
return 0;
var lastObject = b.HitObjects.Last();
//TODO: this isn't always correct (consider mania where a non-last object may last for longer than the last in the list).
double endTime = lastObject.GetEndTime();
double startTime = b.HitObjects.First().StartTime;
return endTime - startTime;
}
private void removeWorkingCache(BeatmapSetInfo info)
{
if (info.Beatmaps == null) return;
foreach (var b in info.Beatmaps)
removeWorkingCache(b);
}
private void removeWorkingCache(BeatmapInfo info)
{
lock (workingCache)
{
var working = workingCache.FirstOrDefault(w => w.BeatmapInfo?.ID == info.ID);
if (working != null)
workingCache.Remove(working);
}
}
public void Dispose()
{
onlineLookupQueue?.Dispose();
}
#region IResourceStorageProvider
TextureStore IBeatmapResourceProvider.LargeTextureStore => largeTextureStore;
ITrackStore IBeatmapResourceProvider.Tracks => trackStore;
AudioManager IStorageResourceProvider.AudioManager => audioManager;
IResourceStore<byte[]> IStorageResourceProvider.Files => Files.Store;
IResourceStore<byte[]> IStorageResourceProvider.Resources => resources;
IResourceStore<TextureUpload> IStorageResourceProvider.CreateTextureLoaderStore(IResourceStore<byte[]> underlyingStore) => host?.CreateTextureLoaderStore(underlyingStore);
/// <summary>
/// Restore a beatmap difficulty.
/// </summary>
/// <param name="beatmap">The beatmap difficulty to restore.</param>
public void Restore(BeatmapInfo beatmap) => beatmapModelManager.Restore(beatmap);
#endregion
/// <summary>
/// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation.
/// </summary>
private class DummyConversionBeatmap : WorkingBeatmap
#region Implementation of IModelManager<BeatmapSetInfo>
public bool IsAvailableLocally(BeatmapSetInfo model)
{
private readonly IBeatmap beatmap;
public DummyConversionBeatmap(IBeatmap beatmap)
: base(beatmap.BeatmapInfo, null)
{
this.beatmap = beatmap;
}
protected override IBeatmap GetBeatmap() => beatmap;
protected override Texture GetBackground() => null;
protected override Track GetBeatmapTrack() => null;
protected internal override ISkin GetSkin() => null;
public override Stream GetStream(string storagePath) => null;
return beatmapModelManager.IsAvailableLocally(model);
}
}
/// <summary>
/// The level of detail to include in database results.
/// </summary>
public enum IncludedDetails
{
/// <summary>
/// Only include beatmap difficulties and set level metadata.
/// </summary>
Minimal,
public IBindable<WeakReference<BeatmapSetInfo>> ItemUpdated => beatmapModelManager.ItemUpdated;
/// <summary>
/// Include all difficulties, rulesets, difficulty metadata but no files.
/// </summary>
AllButFiles,
public IBindable<WeakReference<BeatmapSetInfo>> ItemRemoved => beatmapModelManager.ItemRemoved;
/// <summary>
/// Include everything except ruleset. Used for cases where we aren't sure the ruleset is present but still want to consume the beatmap.
/// </summary>
AllButRuleset,
public Task ImportFromStableAsync(StableStorage stableStorage)
{
return beatmapModelManager.ImportFromStableAsync(stableStorage);
}
/// <summary>
/// Include everything.
/// </summary>
All
public void Export(BeatmapSetInfo item)
{
beatmapModelManager.Export(item);
}
public void ExportModelTo(BeatmapSetInfo model, Stream outputStream)
{
beatmapModelManager.ExportModelTo(model, outputStream);
}
public void Update(BeatmapSetInfo item)
{
beatmapModelManager.Update(item);
}
public bool Delete(BeatmapSetInfo item)
{
return beatmapModelManager.Delete(item);
}
public void Delete(List<BeatmapSetInfo> items, bool silent = false)
{
beatmapModelManager.Delete(items, silent);
}
public void Undelete(List<BeatmapSetInfo> items, bool silent = false)
{
beatmapModelManager.Undelete(items, silent);
}
public void Undelete(BeatmapSetInfo item)
{
beatmapModelManager.Undelete(item);
}
#endregion
#region Implementation of IModelDownloader<BeatmapSetInfo>
public IBindable<WeakReference<ArchiveDownloadRequest<BeatmapSetInfo>>> DownloadBegan => beatmapModelDownloader.DownloadBegan;
public IBindable<WeakReference<ArchiveDownloadRequest<BeatmapSetInfo>>> DownloadFailed => beatmapModelDownloader.DownloadFailed;
public bool Download(BeatmapSetInfo model, bool minimiseDownloadSize = false)
{
return beatmapModelDownloader.Download(model, minimiseDownloadSize);
}
public ArchiveDownloadRequest<BeatmapSetInfo> GetExistingDownload(BeatmapSetInfo model)
{
return beatmapModelDownloader.GetExistingDownload(model);
}
#endregion
#region Implementation of ICanAcceptFiles
public Task Import(params string[] paths)
{
return beatmapModelManager.Import(paths);
}
public Task Import(params ImportTask[] tasks)
{
return beatmapModelManager.Import(tasks);
}
public Task<IEnumerable<BeatmapSetInfo>> Import(ProgressNotification notification, params ImportTask[] tasks)
{
return beatmapModelManager.Import(notification, tasks);
}
public Task<BeatmapSetInfo> Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default)
{
return beatmapModelManager.Import(task, lowPriority, cancellationToken);
}
public Task<BeatmapSetInfo> Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default)
{
return beatmapModelManager.Import(archive, lowPriority, cancellationToken);
}
public Task<BeatmapSetInfo> Import(BeatmapSetInfo item, ArchiveReader archive = null, bool lowPriority = false, CancellationToken cancellationToken = default)
{
return beatmapModelManager.Import(item, archive, lowPriority, cancellationToken);
}
public IEnumerable<string> HandledExtensions => beatmapModelManager.HandledExtensions;
#endregion
#region Implementation of IWorkingBeatmapCache
public WorkingBeatmap GetWorkingBeatmap(BeatmapInfo importedBeatmap) => workingBeatmapCache.GetWorkingBeatmap(importedBeatmap);
#endregion
#region Implementation of IModelFileManager<in BeatmapSetInfo,in BeatmapSetFileInfo>
public void ReplaceFile(BeatmapSetInfo model, BeatmapSetFileInfo file, Stream contents, string filename = null)
{
beatmapModelManager.ReplaceFile(model, file, contents, filename);
}
public void DeleteFile(BeatmapSetInfo model, BeatmapSetFileInfo file)
{
beatmapModelManager.DeleteFile(model, file);
}
public void AddFile(BeatmapSetInfo model, Stream contents, string filename)
{
beatmapModelManager.AddFile(model, contents, filename);
}
#endregion
#region Implementation of IDisposable
public void Dispose()
{
onlineBetamapLookupQueue?.Dispose();
}
#endregion
}
}

215
osu.Game/Beatmaps/BeatmapManager_BeatmapOnlineLookupQueue.cs
View File

@ -1,215 +0,0 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using osu.Framework.Development;
using osu.Framework.IO.Network;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Framework.Testing;
using osu.Framework.Threading;
using osu.Game.Database;
using osu.Game.Online.API;
using osu.Game.Online.API.Requests;
using SharpCompress.Compressors;
using SharpCompress.Compressors.BZip2;
namespace osu.Game.Beatmaps
{
public partial class BeatmapManager
{
[ExcludeFromDynamicCompile]
private class BeatmapOnlineLookupQueue : IDisposable
{
private readonly IAPIProvider api;
private readonly Storage storage;
private const int update_queue_request_concurrency = 4;
private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency, nameof(BeatmapOnlineLookupQueue));
private FileWebRequest cacheDownloadRequest;
private const string cache_database_name = "online.db";
public BeatmapOnlineLookupQueue(IAPIProvider api, Storage storage)
{
this.api = api;
this.storage = storage;
// avoid downloading / using cache for unit tests.
if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name))
prepareLocalCache();
}
public Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken)
{
return Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray());
}
// todo: expose this when we need to do individual difficulty lookups.
protected Task UpdateAsync(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken)
=> Task.Factory.StartNew(() => lookup(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler | TaskCreationOptions.RunContinuationsAsynchronously, updateScheduler);
private void lookup(BeatmapSetInfo set, BeatmapInfo beatmap)
{
if (checkLocalCache(set, beatmap))
return;
if (api?.State.Value != APIState.Online)
return;
var req = new GetBeatmapRequest(beatmap);
req.Failure += fail;
try
{
// intentionally blocking to limit web request concurrency
api.Perform(req);
var res = req.Result;
if (res != null)
{
beatmap.Status = res.Status;
beatmap.BeatmapSet.Status = res.BeatmapSet.Status;
beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID;
beatmap.OnlineBeatmapID = res.OnlineBeatmapID;
if (beatmap.Metadata != null)
beatmap.Metadata.AuthorID = res.AuthorID;
if (beatmap.BeatmapSet.Metadata != null)
beatmap.BeatmapSet.Metadata.AuthorID = res.AuthorID;
LogForModel(set, $"Online retrieval mapped {beatmap} to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}.");
}
}
catch (Exception e)
{
fail(e);
}
void fail(Exception e)
{
beatmap.OnlineBeatmapID = null;
LogForModel(set, $"Online retrieval failed for {beatmap} ({e.Message})");
}
}
private void prepareLocalCache()
{
string cacheFilePath = storage.GetFullPath(cache_database_name);
string compressedCacheFilePath = $"{cacheFilePath}.bz2";
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}");
cacheDownloadRequest.Failed += ex =>
{
File.Delete(compressedCacheFilePath);
File.Delete(cacheFilePath);
Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache download failed: {ex}", LoggingTarget.Database);
};
cacheDownloadRequest.Finished += () =>
{
try
{
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
using (var outStream = File.OpenWrite(cacheFilePath))
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
bz2.CopyTo(outStream);
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null;
}
catch (Exception ex)
{
Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
File.Delete(cacheFilePath);
}
finally
{
File.Delete(compressedCacheFilePath);
}
};
cacheDownloadRequest.PerformAsync();
}
private bool checkLocalCache(BeatmapSetInfo set, BeatmapInfo beatmap)
{
// download is in progress (or was, and failed).
if (cacheDownloadRequest != null)
return false;
// database is unavailable.
if (!storage.Exists(cache_database_name))
return false;
if (string.IsNullOrEmpty(beatmap.MD5Hash)
&& string.IsNullOrEmpty(beatmap.Path)
&& beatmap.OnlineBeatmapID == null)
return false;
try
{
using (var db = new SqliteConnection(DatabaseContextFactory.CreateDatabaseConnectionString("online.db", storage)))
{
db.Open();
using (var cmd = db.CreateCommand())
{
cmd.CommandText = "SELECT beatmapset_id, beatmap_id, approved, user_id FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineBeatmapID OR filename = @Path";
cmd.Parameters.Add(new SqliteParameter("@MD5Hash", beatmap.MD5Hash));
cmd.Parameters.Add(new SqliteParameter("@OnlineBeatmapID", beatmap.OnlineBeatmapID ?? (object)DBNull.Value));
cmd.Parameters.Add(new SqliteParameter("@Path", beatmap.Path));
using (var reader = cmd.ExecuteReader())
{
if (reader.Read())
{
var status = (BeatmapSetOnlineStatus)reader.GetByte(2);
beatmap.Status = status;
beatmap.BeatmapSet.Status = status;
beatmap.BeatmapSet.OnlineBeatmapSetID = reader.GetInt32(0);
beatmap.OnlineBeatmapID = reader.GetInt32(1);
if (beatmap.Metadata != null)
beatmap.Metadata.AuthorID = reader.GetInt32(3);
if (beatmap.BeatmapSet.Metadata != null)
beatmap.BeatmapSet.Metadata.AuthorID = reader.GetInt32(3);
LogForModel(set, $"Cached local retrieval for {beatmap}.");
return true;
}
}
}
}
}
catch (Exception ex)
{
LogForModel(set, $"Cached local retrieval for {beatmap} failed with {ex}.");
}
return false;
}
public void Dispose()
{
cacheDownloadRequest?.Dispose();
updateScheduler?.Dispose();
}
}
}
}

21
osu.Game/Beatmaps/BeatmapModelDownloader.cs Normal file
View File

@ -0,0 +1,21 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using osu.Framework.Platform;
using osu.Game.Database;
using osu.Game.Online.API;
using osu.Game.Online.API.Requests;
namespace osu.Game.Beatmaps
{
public class BeatmapModelDownloader : ModelDownloader<BeatmapSetInfo>
{
protected override ArchiveDownloadRequest<BeatmapSetInfo> CreateDownloadRequest(BeatmapSetInfo set, bool minimiseDownloadSize) =>
new DownloadBeatmapSetRequest(set, minimiseDownloadSize);
public BeatmapModelDownloader(BeatmapModelManager beatmapModelManager, IAPIProvider api, GameHost host = null)
: base(beatmapModelManager, api, host)
{
}
}
}

473
osu.Game/Beatmaps/BeatmapModelManager.cs Normal file
View File

@ -0,0 +1,473 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using osu.Framework.Audio.Track;
using osu.Framework.Bindables;
using osu.Framework.Extensions;
using osu.Framework.Graphics.Textures;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Framework.Testing;
using osu.Game.Beatmaps.Formats;
using osu.Game.Database;
using osu.Game.IO;
using osu.Game.IO.Archives;
using osu.Game.Rulesets;
using osu.Game.Rulesets.Objects;
using osu.Game.Skinning;
using Decoder = osu.Game.Beatmaps.Formats.Decoder;
namespace osu.Game.Beatmaps
{
/// <summary>
/// Handles ef-core storage of beatmaps.
/// </summary>
[ExcludeFromDynamicCompile]
public class BeatmapModelManager : ArchiveModelManager<BeatmapSetInfo, BeatmapSetFileInfo>
{
/// <summary>
/// Fired when a single difficulty has been hidden.
/// </summary>
public IBindable<WeakReference<BeatmapInfo>> BeatmapHidden => beatmapHidden;
private readonly Bindable<WeakReference<BeatmapInfo>> beatmapHidden = new Bindable<WeakReference<BeatmapInfo>>();
/// <summary>
/// Fired when a single difficulty has been restored.
/// </summary>
public IBindable<WeakReference<BeatmapInfo>> BeatmapRestored => beatmapRestored;
/// <summary>
/// An online lookup queue component which handles populating online beatmap metadata.
/// </summary>
public BeatmapOnlineLookupQueue OnlineLookupQueue { private get; set; }
/// <summary>
/// The game working beatmap cache, used to invalidate entries on changes.
/// </summary>
public WorkingBeatmapCache WorkingBeatmapCache { private get; set; }
private readonly Bindable<WeakReference<BeatmapInfo>> beatmapRestored = new Bindable<WeakReference<BeatmapInfo>>();
public override IEnumerable<string> HandledExtensions => new[] { ".osz" };
protected override string[] HashableFileTypes => new[] { ".osu" };
protected override string ImportFromStablePath => ".";
protected override Storage PrepareStableStorage(StableStorage stableStorage) => stableStorage.GetSongStorage();
private readonly BeatmapStore beatmaps;
private readonly RulesetStore rulesets;
public BeatmapModelManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, GameHost host = null)
: base(storage, contextFactory, new BeatmapStore(contextFactory), host)
{
this.rulesets = rulesets;
beatmaps = (BeatmapStore)ModelStore;
beatmaps.BeatmapHidden += b => beatmapHidden.Value = new WeakReference<BeatmapInfo>(b);
beatmaps.BeatmapRestored += b => beatmapRestored.Value = new WeakReference<BeatmapInfo>(b);
beatmaps.ItemRemoved += b => WorkingBeatmapCache?.Invalidate(b);
beatmaps.ItemUpdated += obj => WorkingBeatmapCache?.Invalidate(obj);
}
protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path)?.ToLowerInvariant() == ".osz";
protected override async Task Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive, CancellationToken cancellationToken = default)
{
if (archive != null)
beatmapSet.Beatmaps = createBeatmapDifficulties(beatmapSet.Files);
foreach (BeatmapInfo b in beatmapSet.Beatmaps)
{
// remove metadata from difficulties where it matches the set
if (beatmapSet.Metadata.Equals(b.Metadata))
b.Metadata = null;
b.BeatmapSet = beatmapSet;
}
validateOnlineIds(beatmapSet);
bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0);
if (OnlineLookupQueue != null)
await OnlineLookupQueue.UpdateAsync(beatmapSet, cancellationToken).ConfigureAwait(false);
// ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID.
if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineBeatmapID > 0))
{
if (beatmapSet.OnlineBeatmapSetID != null)
{
beatmapSet.OnlineBeatmapSetID = null;
LogForModel(beatmapSet, "Disassociating beatmap set ID due to loss of all beatmap IDs");
}
}
}
protected override void PreImport(BeatmapSetInfo beatmapSet)
{
if (beatmapSet.Beatmaps.Any(b => b.BaseDifficulty == null))
throw new InvalidOperationException($"Cannot import {nameof(BeatmapInfo)} with null {nameof(BeatmapInfo.BaseDifficulty)}.");
// check if a set already exists with the same online id, delete if it does.
if (beatmapSet.OnlineBeatmapSetID != null)
{
var existingOnlineId = beatmaps.ConsumableItems.FirstOrDefault(b => b.OnlineBeatmapSetID == beatmapSet.OnlineBeatmapSetID);
if (existingOnlineId != null)
{
Delete(existingOnlineId);
// in order to avoid a unique key constraint, immediately remove the online ID from the previous set.
existingOnlineId.OnlineBeatmapSetID = null;
foreach (var b in existingOnlineId.Beatmaps)
b.OnlineBeatmapID = null;
LogForModel(beatmapSet, $"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been deleted.");
}
}
}
private void validateOnlineIds(BeatmapSetInfo beatmapSet)
{
var beatmapIds = beatmapSet.Beatmaps.Where(b => b.OnlineBeatmapID.HasValue).Select(b => b.OnlineBeatmapID).ToList();
// ensure all IDs are unique
if (beatmapIds.GroupBy(b => b).Any(g => g.Count() > 1))
{
LogForModel(beatmapSet, "Found non-unique IDs, resetting...");
resetIds();
return;
}
// find any existing beatmaps in the database that have matching online ids
var existingBeatmaps = QueryBeatmaps(b => beatmapIds.Contains(b.OnlineBeatmapID)).ToList();
if (existingBeatmaps.Count > 0)
{
// reset the import ids (to force a re-fetch) *unless* they match the candidate CheckForExisting set.
// we can ignore the case where the new ids are contained by the CheckForExisting set as it will either be used (import skipped) or deleted.
var existing = CheckForExisting(beatmapSet);
if (existing == null || existingBeatmaps.Any(b => !existing.Beatmaps.Contains(b)))
{
LogForModel(beatmapSet, "Found existing import with IDs already, resetting...");
resetIds();
}
}
void resetIds() => beatmapSet.Beatmaps.ForEach(b => b.OnlineBeatmapID = null);
}
/// <summary>
/// Delete a beatmap difficulty.
/// </summary>
/// <param name="beatmap">The beatmap difficulty to hide.</param>
public void Hide(BeatmapInfo beatmap) => beatmaps.Hide(beatmap);
/// <summary>
/// Restore a beatmap difficulty.
/// </summary>
/// <param name="beatmap">The beatmap difficulty to restore.</param>
public void Restore(BeatmapInfo beatmap) => beatmaps.Restore(beatmap);
/// <summary>
/// Saves an <see cref="IBeatmap"/> file against a given <see cref="BeatmapInfo"/>.
/// </summary>
/// <param name="info">The <see cref="BeatmapInfo"/> to save the content against. The file referenced by <see cref="BeatmapInfo.Path"/> will be replaced.</param>
/// <param name="beatmapContent">The <see cref="IBeatmap"/> content to write.</param>
/// <param name="beatmapSkin">The beatmap <see cref="ISkin"/> content to write, null if to be omitted.</param>
public virtual void Save(BeatmapInfo info, IBeatmap beatmapContent, ISkin beatmapSkin = null)
{
var setInfo = info.BeatmapSet;
using (var stream = new MemoryStream())
{
using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true))
new LegacyBeatmapEncoder(beatmapContent, beatmapSkin).Encode(sw);
stream.Seek(0, SeekOrigin.Begin);
using (ContextFactory.GetForWrite())
{
var beatmapInfo = setInfo.Beatmaps.Single(b => b.ID == info.ID);
var metadata = beatmapInfo.Metadata ?? setInfo.Metadata;
// grab the original file (or create a new one if not found).
var fileInfo = setInfo.Files.SingleOrDefault(f => string.Equals(f.Filename, beatmapInfo.Path, StringComparison.OrdinalIgnoreCase)) ?? new BeatmapSetFileInfo();
// metadata may have changed; update the path with the standard format.
beatmapInfo.Path = $"{metadata.Artist} - {metadata.Title} ({metadata.Author}) [{beatmapInfo.Version}].osu";
beatmapInfo.MD5Hash = stream.ComputeMD5Hash();
// update existing or populate new file's filename.
fileInfo.Filename = beatmapInfo.Path;
stream.Seek(0, SeekOrigin.Begin);
ReplaceFile(setInfo, fileInfo, stream);
}
}
WorkingBeatmapCache?.Invalidate(info);
}
/// <summary>
/// Perform a lookup query on available <see cref="BeatmapSetInfo"/>s.
/// </summary>
/// <param name="query">The query.</param>
/// <returns>The first result for the provided query, or null if no results were found.</returns>
public BeatmapSetInfo QueryBeatmapSet(Expression<Func<BeatmapSetInfo, bool>> query) => beatmaps.ConsumableItems.AsNoTracking().FirstOrDefault(query);
protected override bool CanSkipImport(BeatmapSetInfo existing, BeatmapSetInfo import)
{
if (!base.CanSkipImport(existing, import))
return false;
return existing.Beatmaps.Any(b => b.OnlineBeatmapID != null);
}
protected override bool CanReuseExisting(BeatmapSetInfo existing, BeatmapSetInfo import)
{
if (!base.CanReuseExisting(existing, import))
return false;
var existingIds = existing.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i);
var importIds = import.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i);
// force re-import if we are not in a sane state.
return existing.OnlineBeatmapSetID == import.OnlineBeatmapSetID && existingIds.SequenceEqual(importIds);
}
/// <summary>
/// Returns a list of all usable <see cref="BeatmapSetInfo"/>s.
/// </summary>
/// <returns>A list of available <see cref="BeatmapSetInfo"/>.</returns>
public List<BeatmapSetInfo> GetAllUsableBeatmapSets(IncludedDetails includes = IncludedDetails.All, bool includeProtected = false) =>
GetAllUsableBeatmapSetsEnumerable(includes, includeProtected).ToList();
/// <summary>
/// Returns a list of all usable <see cref="BeatmapSetInfo"/>s. Note that files are not populated.
/// </summary>
/// <param name="includes">The level of detail to include in the returned objects.</param>
/// <param name="includeProtected">Whether to include protected (system) beatmaps. These should not be included for gameplay playable use cases.</param>
/// <returns>A list of available <see cref="BeatmapSetInfo"/>.</returns>
public IEnumerable<BeatmapSetInfo> GetAllUsableBeatmapSetsEnumerable(IncludedDetails includes, bool includeProtected = false)
{
IQueryable<BeatmapSetInfo> queryable;
switch (includes)
{
case IncludedDetails.Minimal:
queryable = beatmaps.BeatmapSetsOverview;
break;
case IncludedDetails.AllButRuleset:
queryable = beatmaps.BeatmapSetsWithoutRuleset;
break;
case IncludedDetails.AllButFiles:
queryable = beatmaps.BeatmapSetsWithoutFiles;
break;
default:
queryable = beatmaps.ConsumableItems;
break;
}
// AsEnumerable used here to avoid applying the WHERE in sql. When done so, ef core 2.x uses an incorrect ORDER BY
// clause which causes queries to take 5-10x longer.
// TODO: remove if upgrading to EF core 3.x.
return queryable.AsEnumerable().Where(s => !s.DeletePending && (includeProtected || !s.Protected));
}
/// <summary>
/// Perform a lookup query on available <see cref="BeatmapSetInfo"/>s.
/// </summary>
/// <param name="query">The query.</param>
/// <param name="includes">The level of detail to include in the returned objects.</param>
/// <returns>Results from the provided query.</returns>
public IEnumerable<BeatmapSetInfo> QueryBeatmapSets(Expression<Func<BeatmapSetInfo, bool>> query, IncludedDetails includes = IncludedDetails.All)
{
IQueryable<BeatmapSetInfo> queryable;
switch (includes)
{
case IncludedDetails.Minimal:
queryable = beatmaps.BeatmapSetsOverview;
break;
case IncludedDetails.AllButRuleset:
queryable = beatmaps.BeatmapSetsWithoutRuleset;
break;
case IncludedDetails.AllButFiles:
queryable = beatmaps.BeatmapSetsWithoutFiles;
break;
default:
queryable = beatmaps.ConsumableItems;
break;
}
return queryable.AsNoTracking().Where(query);
}
/// <summary>
/// Perform a lookup query on available <see cref="BeatmapInfo"/>s.
/// </summary>
/// <param name="query">The query.</param>
/// <returns>The first result for the provided query, or null if no results were found.</returns>
public BeatmapInfo QueryBeatmap(Expression<Func<BeatmapInfo, bool>> query) => beatmaps.Beatmaps.AsNoTracking().FirstOrDefault(query);
/// <summary>
/// Perform a lookup query on available <see cref="BeatmapInfo"/>s.
/// </summary>
/// <param name="query">The query.</param>
/// <returns>Results from the provided query.</returns>
public IQueryable<BeatmapInfo> QueryBeatmaps(Expression<Func<BeatmapInfo, bool>> query) => beatmaps.Beatmaps.AsNoTracking().Where(query);
public override string HumanisedModelName => "beatmap";
protected override bool CheckLocalAvailability(BeatmapSetInfo model, IQueryable<BeatmapSetInfo> items)
=> base.CheckLocalAvailability(model, items)
|| (model.OnlineBeatmapSetID != null && items.Any(b => b.OnlineBeatmapSetID == model.OnlineBeatmapSetID));
protected override BeatmapSetInfo CreateModel(ArchiveReader reader)
{
// let's make sure there are actually .osu files to import.
string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrEmpty(mapName))
{
Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database);
return null;
}
Beatmap beatmap;
using (var stream = new LineBufferedReader(reader.GetStream(mapName)))
beatmap = Decoder.GetDecoder<Beatmap>(stream).Decode(stream);
return new BeatmapSetInfo
{
OnlineBeatmapSetID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID,
Beatmaps = new List<BeatmapInfo>(),
Metadata = beatmap.Metadata,
DateAdded = DateTimeOffset.UtcNow
};
}
/// <summary>
/// Create all required <see cref="BeatmapInfo"/>s for the provided archive.
/// </summary>
private List<BeatmapInfo> createBeatmapDifficulties(List<BeatmapSetFileInfo> files)
{
var beatmapInfos = new List<BeatmapInfo>();
foreach (var file in files.Where(f => f.Filename.EndsWith(".osu", StringComparison.OrdinalIgnoreCase)))
{
using (var raw = Files.Store.GetStream(file.FileInfo.StoragePath))
using (var ms = new MemoryStream()) // we need a memory stream so we can seek
using (var sr = new LineBufferedReader(ms))
{
raw.CopyTo(ms);
ms.Position = 0;
var decoder = Decoder.GetDecoder<Beatmap>(sr);
IBeatmap beatmap = decoder.Decode(sr);
string hash = ms.ComputeSHA2Hash();
if (beatmapInfos.Any(b => b.Hash == hash))
continue;
beatmap.BeatmapInfo.Path = file.Filename;
beatmap.BeatmapInfo.Hash = hash;
beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash();
var ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID);
beatmap.BeatmapInfo.Ruleset = ruleset;
// TODO: this should be done in a better place once we actually need to dynamically update it.
beatmap.BeatmapInfo.StarDifficulty = ruleset?.CreateInstance().CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating ?? 0;
beatmap.BeatmapInfo.Length = calculateLength(beatmap);
beatmap.BeatmapInfo.BPM = 60000 / beatmap.GetMostCommonBeatLength();
beatmapInfos.Add(beatmap.BeatmapInfo);
}
}
return beatmapInfos;
}
private double calculateLength(IBeatmap b)
{
if (!b.HitObjects.Any())
return 0;
var lastObject = b.HitObjects.Last();
//TODO: this isn't always correct (consider mania where a non-last object may last for longer than the last in the list).
double endTime = lastObject.GetEndTime();
double startTime = b.HitObjects.First().StartTime;
return endTime - startTime;
}
/// <summary>
/// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation.
/// </summary>
private class DummyConversionBeatmap : WorkingBeatmap
{
private readonly IBeatmap beatmap;
public DummyConversionBeatmap(IBeatmap beatmap)
: base(beatmap.BeatmapInfo, null)
{
this.beatmap = beatmap;
}
protected override IBeatmap GetBeatmap() => beatmap;
protected override Texture GetBackground() => null;
protected override Track GetBeatmapTrack() => null;
protected internal override ISkin GetSkin() => null;
public override Stream GetStream(string storagePath) => null;
}
}
/// <summary>
/// The level of detail to include in database results.
/// </summary>
public enum IncludedDetails
{
/// <summary>
/// Only include beatmap difficulties and set level metadata.
/// </summary>
Minimal,
/// <summary>
/// Include all difficulties, rulesets, difficulty metadata but no files.
/// </summary>
AllButFiles,
/// <summary>
/// Include everything except ruleset. Used for cases where we aren't sure the ruleset is present but still want to consume the beatmap.
/// </summary>
AllButRuleset,
/// <summary>
/// Include everything.
/// </summary>
All
}
}

222
osu.Game/Beatmaps/BeatmapOnlineLookupQueue.cs Normal file
View File

@ -0,0 +1,222 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
using osu.Framework.Development;
using osu.Framework.IO.Network;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Framework.Testing;
using osu.Framework.Threading;
using osu.Game.Database;
using osu.Game.Online.API;
using osu.Game.Online.API.Requests;
using SharpCompress.Compressors;
using SharpCompress.Compressors.BZip2;
namespace osu.Game.Beatmaps
{
/// <summary>
/// A component which handles population of online IDs for beatmaps using a two part lookup procedure.
/// </summary>
/// <remarks>
/// On creating the component, a copy of a database containing metadata for a large subset of beatmaps (stored to <see cref="cache_database_name"/>) will be downloaded if not already present locally.
/// This will always be checked before doing a second online query to get required metadata.
/// </remarks>
[ExcludeFromDynamicCompile]
public class BeatmapOnlineLookupQueue : IDisposable
{
private readonly IAPIProvider api;
private readonly Storage storage;
private const int update_queue_request_concurrency = 4;
private readonly ThreadedTaskScheduler updateScheduler = new ThreadedTaskScheduler(update_queue_request_concurrency, nameof(BeatmapOnlineLookupQueue));
private FileWebRequest cacheDownloadRequest;
private const string cache_database_name = "online.db";
public BeatmapOnlineLookupQueue(IAPIProvider api, Storage storage)
{
this.api = api;
this.storage = storage;
// avoid downloading / using cache for unit tests.
if (!DebugUtils.IsNUnitRunning && !storage.Exists(cache_database_name))
prepareLocalCache();
}
public Task UpdateAsync(BeatmapSetInfo beatmapSet, CancellationToken cancellationToken)
{
return Task.WhenAll(beatmapSet.Beatmaps.Select(b => UpdateAsync(beatmapSet, b, cancellationToken)).ToArray());
}
// todo: expose this when we need to do individual difficulty lookups.
protected Task UpdateAsync(BeatmapSetInfo beatmapSet, BeatmapInfo beatmap, CancellationToken cancellationToken)
=> Task.Factory.StartNew(() => lookup(beatmapSet, beatmap), cancellationToken, TaskCreationOptions.HideScheduler | TaskCreationOptions.RunContinuationsAsynchronously, updateScheduler);
private void lookup(BeatmapSetInfo set, BeatmapInfo beatmap)
{
if (checkLocalCache(set, beatmap))
return;
if (api?.State.Value != APIState.Online)
return;
var req = new GetBeatmapRequest(beatmap);
req.Failure += fail;
try
{
// intentionally blocking to limit web request concurrency
api.Perform(req);
var res = req.Result;
if (res != null)
{
beatmap.Status = res.Status;
beatmap.BeatmapSet.Status = res.BeatmapSet.Status;
beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID;
beatmap.OnlineBeatmapID = res.OnlineBeatmapID;
if (beatmap.Metadata != null)
beatmap.Metadata.AuthorID = res.AuthorID;
if (beatmap.BeatmapSet.Metadata != null)
beatmap.BeatmapSet.Metadata.AuthorID = res.AuthorID;
logForModel(set, $"Online retrieval mapped {beatmap} to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}.");
}
}
catch (Exception e)
{
fail(e);
}
void fail(Exception e)
{
beatmap.OnlineBeatmapID = null;
logForModel(set, $"Online retrieval failed for {beatmap} ({e.Message})");
}
}
private void prepareLocalCache()
{
string cacheFilePath = storage.GetFullPath(cache_database_name);
string compressedCacheFilePath = $"{cacheFilePath}.bz2";
cacheDownloadRequest = new FileWebRequest(compressedCacheFilePath, $"https://assets.ppy.sh/client-resources/{cache_database_name}.bz2?{DateTimeOffset.UtcNow:yyyyMMdd}");
cacheDownloadRequest.Failed += ex =>
{
File.Delete(compressedCacheFilePath);
File.Delete(cacheFilePath);
Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache download failed: {ex}", LoggingTarget.Database);
};
cacheDownloadRequest.Finished += () =>
{
try
{
using (var stream = File.OpenRead(cacheDownloadRequest.Filename))
using (var outStream = File.OpenWrite(cacheFilePath))
using (var bz2 = new BZip2Stream(stream, CompressionMode.Decompress, false))
bz2.CopyTo(outStream);
// set to null on completion to allow lookups to begin using the new source
cacheDownloadRequest = null;
}
catch (Exception ex)
{
Logger.Log($"{nameof(BeatmapOnlineLookupQueue)}'s online cache extraction failed: {ex}", LoggingTarget.Database);
File.Delete(cacheFilePath);
}
finally
{
File.Delete(compressedCacheFilePath);
}
};
cacheDownloadRequest.PerformAsync();
}
private bool checkLocalCache(BeatmapSetInfo set, BeatmapInfo beatmap)
{
// download is in progress (or was, and failed).
if (cacheDownloadRequest != null)
return false;
// database is unavailable.
if (!storage.Exists(cache_database_name))
return false;
if (string.IsNullOrEmpty(beatmap.MD5Hash)
&& string.IsNullOrEmpty(beatmap.Path)
&& beatmap.OnlineBeatmapID == null)
return false;
try
{
using (var db = new SqliteConnection(DatabaseContextFactory.CreateDatabaseConnectionString("online.db", storage)))
{
db.Open();
using (var cmd = db.CreateCommand())
{
cmd.CommandText = "SELECT beatmapset_id, beatmap_id, approved, user_id FROM osu_beatmaps WHERE checksum = @MD5Hash OR beatmap_id = @OnlineBeatmapID OR filename = @Path";
cmd.Parameters.Add(new SqliteParameter("@MD5Hash", beatmap.MD5Hash));
cmd.Parameters.Add(new SqliteParameter("@OnlineBeatmapID", beatmap.OnlineBeatmapID ?? (object)DBNull.Value));
cmd.Parameters.Add(new SqliteParameter("@Path", beatmap.Path));
using (var reader = cmd.ExecuteReader())
{
if (reader.Read())
{
var status = (BeatmapSetOnlineStatus)reader.GetByte(2);
beatmap.Status = status;
beatmap.BeatmapSet.Status = status;
beatmap.BeatmapSet.OnlineBeatmapSetID = reader.GetInt32(0);
beatmap.OnlineBeatmapID = reader.GetInt32(1);
if (beatmap.Metadata != null)
beatmap.Metadata.AuthorID = reader.GetInt32(3);
if (beatmap.BeatmapSet.Metadata != null)
beatmap.BeatmapSet.Metadata.AuthorID = reader.GetInt32(3);
logForModel(set, $"Cached local retrieval for {beatmap}.");
return true;
}
}
}
}
}
catch (Exception ex)
{
logForModel(set, $"Cached local retrieval for {beatmap} failed with {ex}.");
}
return false;
}
private void logForModel(BeatmapSetInfo set, string message) =>
ArchiveModelManager<BeatmapSetInfo, BeatmapSetFileInfo>.LogForModel(set, $"[{nameof(BeatmapOnlineLookupQueue)}] {message}");
public void Dispose()
{
cacheDownloadRequest?.Dispose();
updateScheduler?.Dispose();
}
}
}

15
osu.Game/Beatmaps/IWorkingBeatmapCache.cs Normal file
View File

@ -0,0 +1,15 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
namespace osu.Game.Beatmaps
{
public interface IWorkingBeatmapCache
{
/// <summary>
/// Retrieve a <see cref="WorkingBeatmap"/> instance for the provided <see cref="BeatmapInfo"/>
/// </summary>
/// <param name="beatmapInfo">The beatmap to lookup.</param>
/// <returns>A <see cref="WorkingBeatmap"/> instance correlating to the provided <see cref="BeatmapInfo"/>.</returns>
WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo);
}
}

100
osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs → osu.Game/Beatmaps/WorkingBeatmapCache.cs
View File

@ -1,12 +1,18 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using JetBrains.Annotations;
using osu.Framework.Audio;
using osu.Framework.Audio.Track;
using osu.Framework.Graphics.Textures;
using osu.Framework.IO.Stores;
using osu.Framework.Lists;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Framework.Statistics;
using osu.Framework.Testing;
using osu.Game.Beatmaps.Formats;
using osu.Game.IO;
@ -15,8 +21,96 @@ using osu.Game.Storyboards;
namespace osu.Game.Beatmaps
{
public partial class BeatmapManager
public class WorkingBeatmapCache : IBeatmapResourceProvider, IWorkingBeatmapCache
{
private readonly WeakList<BeatmapManagerWorkingBeatmap> workingCache = new WeakList<BeatmapManagerWorkingBeatmap>();
/// <summary>
/// A default representation of a WorkingBeatmap to use when no beatmap is available.
/// </summary>
public readonly WorkingBeatmap DefaultBeatmap;
public BeatmapModelManager BeatmapManager { private get; set; }
private readonly AudioManager audioManager;
private readonly IResourceStore<byte[]> resources;
private readonly LargeTextureStore largeTextureStore;
private readonly ITrackStore trackStore;
private readonly IResourceStore<byte[]> files;
[CanBeNull]
private readonly GameHost host;
public WorkingBeatmapCache([NotNull] AudioManager audioManager, IResourceStore<byte[]> resources, IResourceStore<byte[]> files, WorkingBeatmap defaultBeatmap = null, GameHost host = null)
{
DefaultBeatmap = defaultBeatmap;
this.audioManager = audioManager;
this.resources = resources;
this.host = host;
this.files = files;
largeTextureStore = new LargeTextureStore(host?.CreateTextureLoaderStore(files));
trackStore = audioManager.GetTrackStore(files);
}
public void Invalidate(BeatmapSetInfo info)
{
if (info.Beatmaps == null) return;
foreach (var b in info.Beatmaps)
Invalidate(b);
}
public void Invalidate(BeatmapInfo info)
{
lock (workingCache)
{
var working = workingCache.FirstOrDefault(w => w.BeatmapInfo?.ID == info.ID);
if (working != null)
workingCache.Remove(working);
}
}
public virtual WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo)
{
// if there are no files, presume the full beatmap info has not yet been fetched from the database.
if (beatmapInfo?.BeatmapSet?.Files.Count == 0)
{
int lookupId = beatmapInfo.ID;
beatmapInfo = BeatmapManager.QueryBeatmap(b => b.ID == lookupId);
}
if (beatmapInfo?.BeatmapSet == null)
return DefaultBeatmap;
lock (workingCache)
{
var working = workingCache.FirstOrDefault(w => w.BeatmapInfo?.ID == beatmapInfo.ID);
if (working != null)
return working;
beatmapInfo.Metadata ??= beatmapInfo.BeatmapSet.Metadata;
workingCache.Add(working = new BeatmapManagerWorkingBeatmap(beatmapInfo, this));
// best effort; may be higher than expected.
GlobalStatistics.Get<int>(nameof(Beatmaps), $"Cached {nameof(WorkingBeatmap)}s").Value = workingCache.Count();
return working;
}
}
#region IResourceStorageProvider
TextureStore IBeatmapResourceProvider.LargeTextureStore => largeTextureStore;
ITrackStore IBeatmapResourceProvider.Tracks => trackStore;
AudioManager IStorageResourceProvider.AudioManager => audioManager;
IResourceStore<byte[]> IStorageResourceProvider.Files => files;
IResourceStore<byte[]> IStorageResourceProvider.Resources => resources;
IResourceStore<TextureUpload> IStorageResourceProvider.CreateTextureLoaderStore(IResourceStore<byte[]> underlyingStore) => host?.CreateTextureLoaderStore(underlyingStore);
#endregion
[ExcludeFromDynamicCompile]
private class BeatmapManagerWorkingBeatmap : WorkingBeatmap
{