From d95a62fa563ae10ef8453bff223f88dd5418616a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 30 Sep 2021 23:45:09 +0900 Subject: [PATCH] Add models and stores for beatmap manager requirements --- osu.Game/Database/RealmContextFactory.cs | 23 + osu.Game/Stores/ArchiveModelImporter.cs | 557 +++++++++++++++++++++++ osu.Game/Stores/BeatmapImporter.cs | 336 ++++++++++++++ 3 files changed, 916 insertions(+) create mode 100644 osu.Game/Stores/ArchiveModelImporter.cs create mode 100644 osu.Game/Stores/BeatmapImporter.cs diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 82d51e365e..933454f860 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -2,12 +2,14 @@ // See the LICENCE file in the repository root for full licence text. using System; +using System.Linq; using System.Threading; using osu.Framework.Allocation; using osu.Framework.Development; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Statistics; +using osu.Game.Models; using Realms; #nullable enable @@ -70,6 +72,27 @@ namespace osu.Game.Database if (!Filename.EndsWith(realm_extension, StringComparison.Ordinal)) Filename += realm_extension; + + cleanupPendingDeletions(); + } + + private void cleanupPendingDeletions() + { + using (var realm = CreateContext()) + using (var transaction = realm.BeginWrite()) + { + var pendingDeleteSets = realm.All().Where(s => s.DeletePending); + + foreach (var s in pendingDeleteSets) + { + foreach (var b in s.Beatmaps) + realm.Remove(b); + + realm.Remove(s); + } + + transaction.Commit(); + } } /// diff --git a/osu.Game/Stores/ArchiveModelImporter.cs b/osu.Game/Stores/ArchiveModelImporter.cs new file mode 100644 index 0000000000..c165dbecd8 --- /dev/null +++ b/osu.Game/Stores/ArchiveModelImporter.cs @@ -0,0 +1,557 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Humanizer; +using NuGet.Packaging; +using osu.Framework.Extensions; +using osu.Framework.Extensions.IEnumerableExtensions; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Framework.Threading; +using osu.Game.Database; +using osu.Game.IO.Archives; +using osu.Game.Models; +using osu.Game.Overlays.Notifications; +using Realms; + +#nullable enable + +namespace osu.Game.Stores +{ + /// + /// Encapsulates a model store class to give it import functionality. + /// Adds cross-functionality with to give access to the central file store for the provided model. + /// + /// The model type. + public abstract class ArchiveModelImporter : IModelImporter + where TModel : RealmObject, IHasRealmFiles, IHasGuidPrimaryKey, ISoftDelete + { + private const int import_queue_request_concurrency = 1; + + /// + /// The size of a batch import operation before considering it a lower priority operation. + /// + private const int low_priority_import_batch_size = 1; + + /// + /// A singleton scheduler shared by all . + /// + /// + /// This scheduler generally performs IO and CPU intensive work so concurrency is limited harshly. + /// It is mainly being used as a queue mechanism for large imports. + /// + private static readonly ThreadedTaskScheduler import_scheduler = new ThreadedTaskScheduler(import_queue_request_concurrency, nameof(ArchiveModelImporter)); + + /// + /// A second scheduler for lower priority imports. + /// For simplicity, these will just run in parallel with normal priority imports, but a future refactor would see this implemented via a custom scheduler/queue. + /// See https://gist.github.com/peppy/f0e118a14751fc832ca30dd48ba3876b for an incomplete version of this. + /// + private static readonly ThreadedTaskScheduler import_scheduler_low_priority = new ThreadedTaskScheduler(import_queue_request_concurrency, nameof(ArchiveModelImporter)); + + public virtual IEnumerable HandledExtensions => new[] { @".zip" }; + + protected readonly RealmFileStore Files; + + protected readonly RealmContextFactory ContextFactory; + + /// + /// Fired when the user requests to view the resulting import. + /// + public Action>>? PresentImport; + + /// + /// Set an endpoint for notifications to be posted to. + /// + public Action? PostNotification { protected get; set; } + + protected ArchiveModelImporter(Storage storage, RealmContextFactory contextFactory) + { + ContextFactory = contextFactory; + + Files = new RealmFileStore(contextFactory, storage); + } + + /// + /// Import one or more items from filesystem . + /// + /// + /// This will be treated as a low priority import if more than one path is specified; use to always import at standard priority. + /// This will post notifications tracking progress. + /// + /// One or more archive locations on disk. + public Task Import(params string[] paths) + { + var notification = new ProgressNotification { State = ProgressNotificationState.Active }; + + PostNotification?.Invoke(notification); + + return Import(notification, paths.Select(p => new ImportTask(p)).ToArray()); + } + + public Task Import(params ImportTask[] tasks) + { + var notification = new ProgressNotification { State = ProgressNotificationState.Active }; + + PostNotification?.Invoke(notification); + + return Import(notification, tasks); + } + + public async Task>> Import(ProgressNotification notification, params ImportTask[] tasks) + { + if (tasks.Length == 0) + { + notification.CompletionText = $"No {HumanisedModelName}s were found to import!"; + notification.State = ProgressNotificationState.Completed; + return Enumerable.Empty>(); + } + + notification.Progress = 0; + notification.Text = $"{HumanisedModelName.Humanize(LetterCasing.Title)} import is initialising..."; + + int current = 0; + + var imported = new List>(); + + bool isLowPriorityImport = tasks.Length > low_priority_import_batch_size; + + try + { + await Task.WhenAll(tasks.Select(async task => + { + notification.CancellationToken.ThrowIfCancellationRequested(); + + try + { + var model = await Import(task, isLowPriorityImport, notification.CancellationToken).ConfigureAwait(false); + + lock (imported) + { + if (model != null) + imported.Add(model); + current++; + + notification.Text = $"Imported {current} of {tasks.Length} {HumanisedModelName}s"; + notification.Progress = (float)current / tasks.Length; + } + } + catch (TaskCanceledException) + { + throw; + } + catch (Exception e) + { + Logger.Error(e, $@"Could not import ({task})", LoggingTarget.Database); + } + })).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + if (imported.Count == 0) + { + notification.State = ProgressNotificationState.Cancelled; + return imported; + } + } + + if (imported.Count == 0) + { + notification.Text = $"{HumanisedModelName.Humanize(LetterCasing.Title)} import failed!"; + notification.State = ProgressNotificationState.Cancelled; + } + else + { + notification.CompletionText = imported.Count == 1 + ? $"Imported {imported.First()}!" + : $"Imported {imported.Count} {HumanisedModelName}s!"; + + if (imported.Count > 0 && PresentImport != null) + { + notification.CompletionText += " Click to view."; + notification.CompletionClickAction = () => + { + PresentImport?.Invoke(imported); + return true; + }; + } + + notification.State = ProgressNotificationState.Completed; + } + + return imported; + } + + /// + /// Import one from the filesystem and delete the file on success. + /// Note that this bypasses the UI flow and should only be used for special cases or testing. + /// + /// The containing data about the to import. + /// Whether this is a low priority import. + /// An optional cancellation token. + /// The imported model, if successful. + public async Task?> Import(ImportTask task, bool lowPriority = false, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + ILive? import; + using (ArchiveReader reader = task.GetReader()) + import = await Import(reader, lowPriority, cancellationToken).ConfigureAwait(false); + + // We may or may not want to delete the file depending on where it is stored. + // e.g. reconstructing/repairing database with items from default storage. + // Also, not always a single file, i.e. for LegacyFilesystemReader + // TODO: Add a check to prevent files from storage to be deleted. + try + { + if (import != null && File.Exists(task.Path) && ShouldDeleteArchive(task.Path)) + File.Delete(task.Path); + } + catch (Exception e) + { + Logger.Error(e, $@"Could not delete original file after import ({task})"); + } + + return import; + } + + /// + /// Silently import an item from an . + /// + /// The archive to be imported. + /// Whether this is a low priority import. + /// An optional cancellation token. + public async Task?> Import(ArchiveReader archive, bool lowPriority = false, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + TModel? model = null; + + try + { + model = CreateModel(archive); + + if (model == null) + return null; + } + catch (TaskCanceledException) + { + throw; + } + catch (Exception e) + { + LogForModel(model, @$"Model creation of {archive.Name} failed.", e); + return null; + } + + var scheduledImport = Task.Factory.StartNew(async () => await Import(model, archive, lowPriority, cancellationToken).ConfigureAwait(false), + cancellationToken, TaskCreationOptions.HideScheduler, lowPriority ? import_scheduler_low_priority : import_scheduler).Unwrap(); + + return await scheduledImport.ConfigureAwait(true); + } + + /// + /// Any file extensions which should be included in hash creation. + /// Generally should include all file types which determine the file's uniqueness. + /// Large files should be avoided if possible. + /// + /// + /// This is only used by the default hash implementation. If is overridden, it will not be used. + /// + protected abstract string[] HashableFileTypes { get; } + + internal static void LogForModel(TModel? model, string message, Exception? e = null) + { + string trimmedHash; + if (model == null || !model.IsValid || string.IsNullOrEmpty(model.Hash)) + trimmedHash = "?????"; + else + trimmedHash = model.Hash.Substring(0, 5); + + string prefix = $"[{trimmedHash}]"; + + if (e != null) + Logger.Error(e, $"{prefix} {message}", LoggingTarget.Database); + else + Logger.Log($"{prefix} {message}", LoggingTarget.Database); + } + + /// + /// Whether the implementation overrides with a custom implementation. + /// Custom hash implementations must bypass the early exit in the import flow (see usage). + /// + protected virtual bool HasCustomHashFunction => false; + + /// + /// Create a SHA-2 hash from the provided archive based on file content of all files matching . + /// + /// + /// In the case of no matching files, a hash will be generated from the passed archive's . + /// + protected virtual string ComputeHash(TModel item, ArchiveReader? reader = null) + { + if (reader != null) + // fast hashing for cases where the item's files may not be populated. + return computeHashFast(reader); + + // for now, concatenate all hashable files in the set to create a unique hash. + MemoryStream hashable = new MemoryStream(); + + foreach (RealmNamedFileUsage file in item.Files.Where(f => HashableFileTypes.Any(ext => f.Filename.EndsWith(ext, StringComparison.OrdinalIgnoreCase))).OrderBy(f => f.Filename)) + { + using (Stream s = Files.Store.GetStream(file.File.StoragePath)) + s.CopyTo(hashable); + } + + if (hashable.Length > 0) + return hashable.ComputeSHA2Hash(); + + return item.Hash; + } + + /// + /// Silently import an item from a . + /// + /// The model to be imported. + /// An optional archive to use for model population. + /// Whether this is a low priority import. + /// An optional cancellation token. + public virtual async Task?> Import(TModel item, ArchiveReader? archive = null, bool lowPriority = false, CancellationToken cancellationToken = default) + { + using (var realm = ContextFactory.CreateContext()) + { + cancellationToken.ThrowIfCancellationRequested(); + + bool checkedExisting = false; + TModel? existing = null; + + if (archive != null && !HasCustomHashFunction) + { + // this is a fast bail condition to improve large import performance. + item.Hash = computeHashFast(archive); + + checkedExisting = true; + existing = CheckForExisting(item, realm); + + if (existing != null) + { + // bare minimum comparisons + // + // note that this should really be checking filesizes on disk (of existing files) for some degree of sanity. + // or alternatively doing a faster hash check. either of these require database changes and reprocessing of existing files. + if (CanSkipImport(existing, item) && + getFilenames(existing.Files).SequenceEqual(getShortenedFilenames(archive).Select(p => p.shortened).OrderBy(f => f))) + { + LogForModel(item, @$"Found existing (optimised) {HumanisedModelName} for {item} (ID {existing.ID}) – skipping import."); + + using (var transaction = realm.BeginWrite()) + { + existing.DeletePending = false; + transaction.Commit(); + } + + return existing.ToLive(); + } + + LogForModel(item, @"Found existing (optimised) but failed pre-check."); + } + } + + try + { + LogForModel(item, @"Beginning import..."); + + // TODO: do we want to make the transaction this local? not 100% sure, will need further investigation. + using (var transaction = realm.BeginWrite()) + { + if (archive != null) + // TODO: look into rollback of file additions (or delayed commit). + item.Files.AddRange(createFileInfos(archive, Files, realm)); + + item.Hash = ComputeHash(item, archive); + + // TODO: we may want to run this outside of the transaction. + await Populate(item, archive, realm, cancellationToken).ConfigureAwait(false); + + if (!checkedExisting) + existing = CheckForExisting(item, realm); + + if (existing != null) + { + if (CanReuseExisting(existing, item)) + { + LogForModel(item, @$"Found existing {HumanisedModelName} for {item} (ID {existing.ID}) – skipping import."); + existing.DeletePending = false; + + return existing.ToLive(); + } + + LogForModel(item, @"Found existing but failed re-use check."); + + existing.DeletePending = true; + + // todo: actually delete? i don't think this is required... + // ModelStore.PurgeDeletable(s => s.ID == existing.ID); + } + + PreImport(item, realm); + + // import to store + realm.Add(item); + + transaction.Commit(); + } + + LogForModel(item, @"Import successfully completed!"); + } + catch (Exception e) + { + if (!(e is TaskCanceledException)) + LogForModel(item, @"Database import or population failed and has been rolled back.", e); + + throw; + } + + return item.ToLive(); + } + } + + private string computeHashFast(ArchiveReader reader) + { + MemoryStream hashable = new MemoryStream(); + + foreach (var file in reader.Filenames.Where(f => HashableFileTypes.Any(ext => f.EndsWith(ext, StringComparison.OrdinalIgnoreCase))).OrderBy(f => f)) + { + using (Stream s = reader.GetStream(file)) + s.CopyTo(hashable); + } + + if (hashable.Length > 0) + return hashable.ComputeSHA2Hash(); + + return reader.Name.ComputeSHA2Hash(); + } + + /// + /// Create all required s for the provided archive, adding them to the global file store. + /// + private List createFileInfos(ArchiveReader reader, RealmFileStore files, Realm realm) + { + var fileInfos = new List(); + + // import files to manager + foreach (var filenames in getShortenedFilenames(reader)) + { + using (Stream s = reader.GetStream(filenames.original)) + { + var item = new RealmNamedFileUsage(files.Add(s, realm), filenames.shortened); + fileInfos.Add(item); + } + } + + return fileInfos; + } + + private IEnumerable<(string original, string shortened)> getShortenedFilenames(ArchiveReader reader) + { + string prefix = reader.Filenames.GetCommonPrefix(); + if (!(prefix.EndsWith('/') || prefix.EndsWith('\\'))) + prefix = string.Empty; + + // import files to manager + foreach (string file in reader.Filenames) + yield return (file, file.Substring(prefix.Length).ToStandardisedPath()); + } + + /// + /// Create a barebones model from the provided archive. + /// Actual expensive population should be done in ; this should just prepare for duplicate checking. + /// + /// The archive to create the model for. + /// A model populated with minimal information. Returning a null will abort importing silently. + protected abstract TModel? CreateModel(ArchiveReader archive); + + /// + /// Populate the provided model completely from the given archive. + /// After this method, the model should be in a state ready to commit to a store. + /// + /// The model to populate. + /// The archive to use as a reference for population. May be null. + /// The current realm context. + /// An optional cancellation token. + protected abstract Task Populate(TModel model, ArchiveReader? archive, Realm realm, CancellationToken cancellationToken = default); + + /// + /// Perform any final actions before the import to database executes. + /// + /// The model prepared for import. + /// The current realm context. + protected virtual void PreImport(TModel model, Realm realm) + { + } + + /// + /// Check whether an existing model already exists for a new import item. + /// + /// The new model proposed for import. + /// The current realm context. + /// An existing model which matches the criteria to skip importing, else null. + protected TModel? CheckForExisting(TModel model, Realm realm) => string.IsNullOrEmpty(model.Hash) ? null : realm.All().FirstOrDefault(b => b.Hash == model.Hash); + + /// + /// Whether import can be skipped after finding an existing import early in the process. + /// Only valid when is not overridden. + /// + /// The existing model. + /// The newly imported model. + /// Whether to skip this import completely. + protected virtual bool CanSkipImport(TModel existing, TModel import) => true; + + /// + /// After an existing is found during an import process, the default behaviour is to use/restore the existing + /// item and skip the import. This method allows changing that behaviour. + /// + /// The existing model. + /// The newly imported model. + /// Whether the existing model should be restored and used. Returning false will delete the existing and force a re-import. + protected virtual bool CanReuseExisting(TModel existing, TModel import) => + // for the best or worst, we copy and import files of a new import before checking whether + // it is a duplicate. so to check if anything has changed, we can just compare all File IDs. + getIDs(existing.Files).SequenceEqual(getIDs(import.Files)) && + getFilenames(existing.Files).SequenceEqual(getFilenames(import.Files)); + + /// + /// Whether this specified path should be removed after successful import. + /// + /// The path for consideration. May be a file or a directory. + /// Whether to perform deletion. + protected virtual bool ShouldDeleteArchive(string path) => false; + + private IEnumerable getIDs(IEnumerable files) + { + foreach (var f in files.OrderBy(f => f.Filename)) + yield return f.File.Hash; + } + + private IEnumerable getFilenames(IEnumerable files) + { + foreach (var f in files.OrderBy(f => f.Filename)) + yield return f.Filename; + } + + public virtual string HumanisedModelName => $"{typeof(TModel).Name.Replace(@"Info", "").ToLower()}"; + + private string getValidFilename(string filename) + { + foreach (char c in Path.GetInvalidFileNameChars()) + filename = filename.Replace(c, '_'); + return filename; + } + } +} diff --git a/osu.Game/Stores/BeatmapImporter.cs b/osu.Game/Stores/BeatmapImporter.cs new file mode 100644 index 0000000000..95d2c4fe7b --- /dev/null +++ b/osu.Game/Stores/BeatmapImporter.cs @@ -0,0 +1,336 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Packaging; +using osu.Framework.Audio.Track; +using osu.Framework.Extensions; +using osu.Framework.Extensions.IEnumerableExtensions; +using osu.Framework.Graphics.Textures; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Framework.Testing; +using osu.Game.Beatmaps; +using osu.Game.Beatmaps.Formats; +using osu.Game.Database; +using osu.Game.IO; +using osu.Game.IO.Archives; +using osu.Game.Models; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Objects; +using osu.Game.Skinning; +using Realms; + +#nullable enable + +namespace osu.Game.Stores +{ + /// + /// Handles the storage and retrieval of Beatmaps/WorkingBeatmaps. + /// + [ExcludeFromDynamicCompile] + public class BeatmapImporter : ArchiveModelImporter, IDisposable + { + public override IEnumerable HandledExtensions => new[] { ".osz" }; + + protected override string[] HashableFileTypes => new[] { ".osu" }; + + // protected override string ImportFromStablePath => "."; + // + // protected override Storage PrepareStableStorage(StableStorage stableStorage) => stableStorage.GetSongStorage(); + // + // protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path)?.ToLowerInvariant() == ".osz"; + // + // protected override bool CheckLocalAvailability(RealmBeatmapSet model, System.Linq.IQueryable items) + // => base.CheckLocalAvailability(model, items) || (model.OnlineID != null && items.Any(b => b.OnlineID == model.OnlineID)); + + private readonly dynamic? onlineLookupQueue = null; // todo: BeatmapOnlineLookupQueue is private + + public BeatmapImporter(RealmContextFactory contextFactory, Storage storage, bool performOnlineLookups = false) + : base(storage, contextFactory) + { + if (performOnlineLookups) + { + // onlineLookupQueue = new BeatmapOnlineLookupQueue(api, storage); + } + } + + protected override bool ShouldDeleteArchive(string path) => Path.GetExtension(path).ToLowerInvariant() == ".osz"; + + protected override async Task Populate(RealmBeatmapSet beatmapSet, ArchiveReader? archive, Realm realm, CancellationToken cancellationToken = default) + { + if (archive != null) + beatmapSet.Beatmaps.AddRange(createBeatmapDifficulties(beatmapSet.Files, realm)); + + foreach (RealmBeatmap b in beatmapSet.Beatmaps) + b.BeatmapSet = beatmapSet; + + validateOnlineIds(beatmapSet, realm); + + bool hadOnlineBeatmapIDs = beatmapSet.Beatmaps.Any(b => b.OnlineID > 0); + + if (onlineLookupQueue != null) + await onlineLookupQueue.UpdateAsync(beatmapSet, cancellationToken).ConfigureAwait(false); + + // ensure at least one beatmap was able to retrieve or keep an online ID, else drop the set ID. + if (hadOnlineBeatmapIDs && !beatmapSet.Beatmaps.Any(b => b.OnlineID > 0)) + { + if (beatmapSet.OnlineID != null) + { + beatmapSet.OnlineID = null; + LogForModel(beatmapSet, "Disassociating beatmap set ID due to loss of all beatmap IDs"); + } + } + } + + protected override void PreImport(RealmBeatmapSet beatmapSet, Realm realm) + { + // if (beatmapSet.Beatmaps.Any(b => b.Difficulty == null)) + // throw new InvalidOperationException($"Cannot import {nameof(IBeatmapInfo)} with null {nameof(IBeatmapInfo.Difficulty)}."); + + // check if a set already exists with the same online id, delete if it does. + if (beatmapSet.OnlineID != null) + { + var existingOnlineId = realm.All().FirstOrDefault(b => b.OnlineID == beatmapSet.OnlineID); + + if (existingOnlineId != null) + { + existingOnlineId.DeletePending = true; + + // in order to avoid a unique key constraint, immediately remove the online ID from the previous set. + existingOnlineId.OnlineID = null; + foreach (var b in existingOnlineId.Beatmaps) + b.OnlineID = null; + + LogForModel(beatmapSet, $"Found existing beatmap set with same OnlineID ({beatmapSet.OnlineID}). It has been deleted."); + } + } + } + + private void validateOnlineIds(RealmBeatmapSet beatmapSet, Realm realm) + { + var beatmapIds = beatmapSet.Beatmaps.Where(b => b.OnlineID.HasValue).Select(b => b.OnlineID).ToList(); + + // ensure all IDs are unique + if (beatmapIds.GroupBy(b => b).Any(g => g.Count() > 1)) + { + LogForModel(beatmapSet, "Found non-unique IDs, resetting..."); + resetIds(); + return; + } + + // find any existing beatmaps in the database that have matching online ids + List existingBeatmaps = new List(); + + foreach (var id in beatmapIds) + existingBeatmaps.AddRange(realm.All().Where(b => b.OnlineID == id)); + + if (existingBeatmaps.Any()) + { + // reset the import ids (to force a re-fetch) *unless* they match the candidate CheckForExisting set. + // we can ignore the case where the new ids are contained by the CheckForExisting set as it will either be used (import skipped) or deleted. + + var existing = CheckForExisting(beatmapSet, realm); + + if (existing == null || existingBeatmaps.Any(b => !existing.Beatmaps.Contains(b))) + { + LogForModel(beatmapSet, "Found existing import with online IDs already, resetting..."); + resetIds(); + } + } + + void resetIds() => beatmapSet.Beatmaps.ForEach(b => b.OnlineID = null); + } + + protected override bool CanSkipImport(RealmBeatmapSet existing, RealmBeatmapSet import) + { + if (!base.CanSkipImport(existing, import)) + return false; + + return existing.Beatmaps.Any(b => b.OnlineID != null); + } + + protected override bool CanReuseExisting(RealmBeatmapSet existing, RealmBeatmapSet import) + { + if (!base.CanReuseExisting(existing, import)) + return false; + + var existingIds = existing.Beatmaps.Select(b => b.OnlineID).OrderBy(i => i); + var importIds = import.Beatmaps.Select(b => b.OnlineID).OrderBy(i => i); + + // force re-import if we are not in a sane state. + return existing.OnlineID == import.OnlineID && existingIds.SequenceEqual(importIds); + } + + public override string HumanisedModelName => "beatmap"; + + protected override RealmBeatmapSet? CreateModel(ArchiveReader reader) + { + // let's make sure there are actually .osu files to import. + string? mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu", StringComparison.OrdinalIgnoreCase)); + + if (string.IsNullOrEmpty(mapName)) + { + Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database); + return null; + } + + Beatmap beatmap; + using (var stream = new LineBufferedReader(reader.GetStream(mapName))) + beatmap = Decoder.GetDecoder(stream).Decode(stream); + + return new RealmBeatmapSet + { + OnlineID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID, + // Metadata = beatmap.Metadata, + DateAdded = DateTimeOffset.UtcNow + }; + } + + /// + /// Create all required s for the provided archive. + /// + private List createBeatmapDifficulties(IList files, Realm realm) + { + var beatmaps = new List(); + + foreach (var file in files.Where(f => f.Filename.EndsWith(".osu", StringComparison.OrdinalIgnoreCase))) + { + using (var raw = Files.Store.GetStream(file.File.StoragePath)) + using (var ms = new MemoryStream()) // we need a memory stream so we can seek + using (var sr = new LineBufferedReader(ms)) + { + raw.CopyTo(ms); + ms.Position = 0; + + var decoder = Decoder.GetDecoder(sr); + IBeatmap beatmap = decoder.Decode(sr); + + string hash = ms.ComputeSHA2Hash(); + + if (beatmaps.Any(b => b.Hash == hash)) + continue; + + var beatmapInfo = beatmap.BeatmapInfo; + + var ruleset = realm.All().FirstOrDefault(r => r.OnlineID == beatmapInfo.RulesetID); + var rulesetInstance = (ruleset as IRulesetInfo)?.CreateInstance(); + + if (ruleset == null || rulesetInstance == null) + { + Logger.Log($"Skipping import due to missing local ruleset {beatmapInfo.RulesetID}.", LoggingTarget.Database); + continue; + } + + beatmapInfo.Path = file.Filename; + beatmapInfo.Hash = hash; + beatmapInfo.MD5Hash = ms.ComputeMD5Hash(); + + // TODO: this should be done in a better place once we actually need to dynamically update it. + beatmap.BeatmapInfo.Ruleset = rulesetInstance.RulesetInfo; + beatmap.BeatmapInfo.StarDifficulty = rulesetInstance.CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating; + beatmap.BeatmapInfo.Length = calculateLength(beatmap); + beatmap.BeatmapInfo.BPM = 60000 / beatmap.GetMostCommonBeatLength(); + + var difficulty = new RealmBeatmapDifficulty + { + DrainRate = beatmapInfo.BaseDifficulty.DrainRate, + CircleSize = beatmapInfo.BaseDifficulty.CircleSize, + OverallDifficulty = beatmapInfo.BaseDifficulty.OverallDifficulty, + ApproachRate = beatmapInfo.BaseDifficulty.ApproachRate, + SliderMultiplier = beatmapInfo.BaseDifficulty.SliderMultiplier, + SliderTickRate = beatmapInfo.BaseDifficulty.SliderTickRate, + }; + + var metadata = new RealmBeatmapMetadata + { + Title = beatmap.Metadata.Title, + TitleUnicode = beatmap.Metadata.TitleUnicode, + Artist = beatmap.Metadata.Artist, + ArtistUnicode = beatmap.Metadata.ArtistUnicode, + Author = beatmap.Metadata.AuthorString, + Source = beatmap.Metadata.Source, + Tags = beatmap.Metadata.Tags, + PreviewTime = beatmap.Metadata.PreviewTime, + AudioFile = beatmap.Metadata.AudioFile, + BackgroundFile = beatmap.Metadata.BackgroundFile, + }; + + var realmBeatmap = new RealmBeatmap(ruleset, difficulty, metadata) + { + DifficultyName = beatmapInfo.Version, + OnlineID = beatmapInfo.OnlineBeatmapID, + Length = beatmapInfo.Length, + BPM = beatmapInfo.BPM, + Hash = beatmapInfo.Hash, + StarRating = beatmapInfo.StarDifficulty, + MD5Hash = beatmapInfo.MD5Hash, + Hidden = beatmapInfo.Hidden, + AudioLeadIn = beatmapInfo.AudioLeadIn, + StackLeniency = beatmapInfo.StackLeniency, + SpecialStyle = beatmapInfo.SpecialStyle, + LetterboxInBreaks = beatmapInfo.LetterboxInBreaks, + WidescreenStoryboard = beatmapInfo.WidescreenStoryboard, + EpilepsyWarning = beatmapInfo.EpilepsyWarning, + SamplesMatchPlaybackRate = beatmapInfo.SamplesMatchPlaybackRate, + DistanceSpacing = beatmapInfo.DistanceSpacing, + BeatDivisor = beatmapInfo.BeatDivisor, + GridSize = beatmapInfo.GridSize, + TimelineZoom = beatmapInfo.TimelineZoom, + }; + + // TODO: IBeatmap.BeatmapInfo needs to be updated to the new interface. + // beatmaps.Add(beatmap.BeatmapInfo); + + beatmaps.Add(realmBeatmap); + } + } + + return beatmaps; + } + + public void Dispose() + { + onlineLookupQueue?.Dispose(); + } + + private double calculateLength(IBeatmap b) + { + if (!b.HitObjects.Any()) + return 0; + + var lastObject = b.HitObjects.Last(); + + //TODO: this isn't always correct (consider mania where a non-last object may last for longer than the last in the list). + double endTime = lastObject.GetEndTime(); + double startTime = b.HitObjects.First().StartTime; + + return endTime - startTime; + } + + /// + /// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation. + /// + private class DummyConversionBeatmap : WorkingBeatmap + { + private readonly IBeatmap beatmap; + + public DummyConversionBeatmap(IBeatmap beatmap) + : base(beatmap.BeatmapInfo, null) + { + this.beatmap = beatmap; + } + + protected override IBeatmap GetBeatmap() => beatmap; + protected override Texture? GetBackground() => null; + protected override Track? GetBeatmapTrack() => null; + protected internal override ISkin? GetSkin() => null; + public override Stream? GetStream(string storagePath) => null; + } + } +}