sim1222/osukey
1
1
Fork 0
mirror of https://github.com/osukey/osukey.git synced 2025-08-05 07:33:55 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into guard-url-protocols

Browse Source
This commit is contained in:
Dean Herbert
2022-12-16 18:23:16 +09:00
parent 27c497145f 965be5123d
commit ba54551313
45 changed files with 439 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
osu.Game/Database/ICanAcceptFiles.cs
View File

@ -31,7 +31,8 @@ namespace osu.Game.Database
/// This will post notifications tracking progress.
/// </remarks>
/// <param name="tasks">The import tasks from which the files should be imported.</param>
Task Import(params ImportTask[] tasks);
/// <param name="parameters">Parameters to further configure the import process.</param>
Task Import(ImportTask[] tasks, ImportParameters parameters = default);
/// <summary>
/// An array of accepted file extensions (in the standard format of ".abc").

3
osu.Game/Database/IModelImporter.cs
View File

@ -20,8 +20,9 @@ namespace osu.Game.Database
/// </summary>
/// <param name="notification">The notification to update.</param>
/// <param name="tasks">The import tasks.</param>
/// <param name="parameters">Parameters to further configure the import process.</param>
/// <returns>The imported models.</returns>
Task<IEnumerable<Live<TModel>>> Import(ProgressNotification notification, params ImportTask[] tasks);
Task<IEnumerable<Live<TModel>>> Import(ProgressNotification notification, ImportTask[] tasks, ImportParameters parameters = default);
/// <summary>
/// Process a single import as an update for an existing model.

25
osu.Game/Database/ImportParameters.cs Normal file
View File

@ -0,0 +1,25 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.
namespace osu.Game.Database
{
public struct ImportParameters
{
/// <summary>
/// Whether this import is part of a larger batch.
/// </summary>
/// <remarks>
/// May skip intensive pre-import checks in favour of faster processing.
///
/// More specifically, imports will be skipped before they begin, given an existing model matches on hash and filenames. Should generally only be used for large batch imports, as it may defy user expectations when updating an existing model.
///
/// Will also change scheduling behaviour to run at a lower priority.
/// </remarks>
public bool Batch { get; set; }
/// <summary>
/// Whether this import should use hard links rather than file copy operations if available.
/// </summary>
public bool PreferHardLinks { get; set; }
}
}

13
osu.Game/Database/LegacyImportManager.cs
View File

@ -54,6 +54,19 @@ namespace osu.Game.Database
public void UpdateStorage(string stablePath) => cachedStorage = new StableStorage(stablePath, gameHost as DesktopGameHost);
public bool CheckHardLinkAvailability()
{
var stableStorage = GetCurrentStableStorage();
if (stableStorage == null || gameHost is not DesktopGameHost desktopGameHost)
return false;
string testExistingPath = stableStorage.GetFullPath(string.Empty);
string testDestinationPath = desktopGameHost.Storage.GetFullPath(string.Empty);
return HardLinkHelper.CheckAvailability(testDestinationPath, testExistingPath);
}
public virtual async Task<int> GetImportCount(StableContent content, CancellationToken cancellationToken)
{
var stableStorage = GetCurrentStableStorage();

7
osu.Game/Database/LegacyModelImporter.cs
View File

@ -57,7 +57,12 @@ namespace osu.Game.Database
return Task.CompletedTask;
}
return Task.Run(async () => await Importer.Import(GetStableImportPaths(storage).ToArray()).ConfigureAwait(false));
return Task.Run(async () =>
{
var tasks = GetStableImportPaths(storage).Select(p => new ImportTask(p)).ToArray();
await Importer.Import(tasks, new ImportParameters { Batch = true, PreferHardLinks = true }).ConfigureAwait(false);
});
}
/// <summary>

2
osu.Game/Database/ModelDownloader.cs
View File

@ -73,7 +73,7 @@ namespace osu.Game.Database
if (originalModel != null)
importSuccessful = (await importer.ImportAsUpdate(notification, new ImportTask(filename), originalModel).ConfigureAwait(false)) != null;
else
importSuccessful = (await importer.Import(notification, new ImportTask(filename)).ConfigureAwait(false)).Any();
importSuccessful = (await importer.Import(notification, new[] { new ImportTask(filename) }).ConfigureAwait(false)).Any();
// for now a failed import will be marked as a failed download for simplicity.
if (!importSuccessful)

38
osu.Game/Database/RealmArchiveModelImporter.cs
View File

@ -81,16 +81,16 @@ namespace osu.Game.Database
public Task Import(params string[] paths) => Import(paths.Select(p => new ImportTask(p)).ToArray());
public Task Import(params ImportTask[] tasks)
public Task Import(ImportTask[] tasks, ImportParameters parameters = default)
{
var notification = new ProgressNotification { State = ProgressNotificationState.Active };
PostNotification?.Invoke(notification);
return Import(notification, tasks);
return Import(notification, tasks, parameters);
}
public async Task<IEnumerable<Live<TModel>>> Import(ProgressNotification notification, params ImportTask[] tasks)
public async Task<IEnumerable<Live<TModel>>> Import(ProgressNotification notification, ImportTask[] tasks, ImportParameters parameters = default)
{
if (tasks.Length == 0)
{
@ -106,7 +106,7 @@ namespace osu.Game.Database
var imported = new List<Live<TModel>>();
bool isBatchImport = tasks.Length >= minimum_items_considered_batch_import;
parameters.Batch |= tasks.Length >= minimum_items_considered_batch_import;
await Task.WhenAll(tasks.Select(async task =>
{
@ -115,7 +115,7 @@ namespace osu.Game.Database
try
{
var model = await Import(task, isBatchImport, notification.CancellationToken).ConfigureAwait(false);
var model = await Import(task, parameters, notification.CancellationToken).ConfigureAwait(false);
lock (imported)
{
@ -176,16 +176,16 @@ namespace osu.Game.Database
/// Note that this bypasses the UI flow and should only be used for special cases or testing.
/// </summary>
/// <param name="task">The <see cref="ImportTask"/> containing data about the <typeparamref name="TModel"/> to import.</param>
/// <param name="batchImport">Whether this import is part of a larger batch.</param>
/// <param name="parameters">Parameters to further configure the import process.</param>
/// <param name="cancellationToken">An optional cancellation token.</param>
/// <returns>The imported model, if successful.</returns>
public async Task<Live<TModel>?> Import(ImportTask task, bool batchImport = false, CancellationToken cancellationToken = default)
public async Task<Live<TModel>?> Import(ImportTask task, ImportParameters parameters = default, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
Live<TModel>? import;
using (ArchiveReader reader = task.GetReader())
import = await importFromArchive(reader, batchImport, cancellationToken).ConfigureAwait(false);
import = await importFromArchive(reader, parameters, cancellationToken).ConfigureAwait(false);
// We may or may not want to delete the file depending on where it is stored.
// e.g. reconstructing/repairing database with items from default storage.
@ -211,9 +211,9 @@ namespace osu.Game.Database
/// This method also handled queueing the import task on a relevant import thread pool.
/// </remarks>
/// <param name="archive">The archive to be imported.</param>
/// <param name="batchImport">Whether this import is part of a larger batch.</param>
/// <param name="parameters">Parameters to further configure the import process.</param>
/// <param name="cancellationToken">An optional cancellation token.</param>
private async Task<Live<TModel>?> importFromArchive(ArchiveReader archive, bool batchImport = false, CancellationToken cancellationToken = default)
private async Task<Live<TModel>?> importFromArchive(ArchiveReader archive, ImportParameters parameters = default, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
@ -236,10 +236,10 @@ namespace osu.Game.Database
return null;
}
var scheduledImport = Task.Factory.StartNew(() => ImportModel(model, archive, batchImport, cancellationToken),
var scheduledImport = Task.Factory.StartNew(() => ImportModel(model, archive, parameters, cancellationToken),
cancellationToken,
TaskCreationOptions.HideScheduler,
batchImport ? import_scheduler_batch : import_scheduler);
parameters.Batch ? import_scheduler_batch : import_scheduler);
return await scheduledImport.ConfigureAwait(false);
}
@ -249,15 +249,15 @@ namespace osu.Game.Database
/// </summary>
/// <param name="item">The model to be imported.</param>
/// <param name="archive">An optional archive to use for model population.</param>
/// <param name="batchImport">If <c>true</c>, imports will be skipped before they begin, given an existing model matches on hash and filenames. Should generally only be used for large batch imports, as it may defy user expectations when updating an existing model.</param>
/// <param name="parameters">Parameters to further configure the import process.</param>
/// <param name="cancellationToken">An optional cancellation token.</param>
public virtual Live<TModel>? ImportModel(TModel item, ArchiveReader? archive = null, bool batchImport = false, CancellationToken cancellationToken = default) => Realm.Run(realm =>
public virtual Live<TModel>? ImportModel(TModel item, ArchiveReader? archive = null, ImportParameters parameters = default, CancellationToken cancellationToken = default) => Realm.Run(realm =>
{
cancellationToken.ThrowIfCancellationRequested();
TModel? existing;
if (batchImport && archive != null)
if (parameters.Batch && archive != null)
{
// this is a fast bail condition to improve large import performance.
item.Hash = computeHashFast(archive);
@ -303,7 +303,7 @@ namespace osu.Game.Database
foreach (var filenames in getShortenedFilenames(archive))
{
using (Stream s = archive.GetStream(filenames.original))
files.Add(new RealmNamedFileUsage(Files.Add(s, realm, false), filenames.shortened));
files.Add(new RealmNamedFileUsage(Files.Add(s, realm, false, parameters.PreferHardLinks), filenames.shortened));
}
}
@ -358,7 +358,7 @@ namespace osu.Game.Database
// import to store
realm.Add(item);
PostImport(item, realm, batchImport);
PostImport(item, realm, parameters);
transaction.Commit();
}
@ -493,8 +493,8 @@ namespace osu.Game.Database
/// </summary>
/// <param name="model">The model prepared for import.</param>
/// <param name="realm">The current realm context.</param>
/// <param name="batchImport">Whether the import was part of a batch.</param>
protected virtual void PostImport(TModel model, Realm realm, bool batchImport)
/// <param name="parameters">Parameters to further configure the import process.</param>
protected virtual void PostImport(TModel model, Realm realm, ImportParameters parameters)
{
}

16
osu.Game/Database/RealmFileStore.cs
View File

@ -4,12 +4,14 @@
using System;
using System.IO;
using System.Linq;
using osu.Framework;
using osu.Framework.Extensions;
using osu.Framework.IO.Stores;
using osu.Framework.Logging;
using osu.Framework.Platform;
using osu.Framework.Testing;
using osu.Game.Extensions;
using osu.Game.IO;
using osu.Game.Models;
using Realms;
@ -41,7 +43,8 @@ namespace osu.Game.Database
/// <param name="data">The file data stream.</param>
/// <param name="realm">The realm instance to add to. Should already be in a transaction.</param>
/// <param name="addToRealm">Whether the <see cref="RealmFile"/> should immediately be added to the underlying realm. If <c>false</c> is provided here, the instance must be manually added.</param>
public RealmFile Add(Stream data, Realm realm, bool addToRealm = true)
/// <param name="preferHardLinks">Whether this import should use hard links rather than file copy operations if available.</param>
public RealmFile Add(Stream data, Realm realm, bool addToRealm = true, bool preferHardLinks = false)
{
string hash = data.ComputeSHA2Hash();
@ -50,7 +53,7 @@ namespace osu.Game.Database
var file = existing ?? new RealmFile { Hash = hash };
if (!checkFileExistsAndMatchesHash(file))
copyToStore(file, data);
copyToStore(file, data, preferHardLinks);
if (addToRealm && !file.IsManaged)
realm.Add(file);
@ -58,8 +61,15 @@ namespace osu.Game.Database
return file;
}
private void copyToStore(RealmFile file, Stream data)
private void copyToStore(RealmFile file, Stream data, bool preferHardLinks)
{
if (RuntimeInfo.OS == RuntimeInfo.Platform.Windows && data is FileStream fs && preferHardLinks)
{
// attempt to do a fast hard link rather than copy.
if (HardLinkHelper.CreateHardLink(Storage.GetFullPath(file.GetStoragePath(), true), fs.Name, IntPtr.Zero))
return;
}
data.Seek(0, SeekOrigin.Begin);
using (var output = Storage.CreateFileSafely(file.GetStoragePath()))