// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Linq.Expressions; using System.Text; using System.Threading; using System.Threading.Tasks; using osu.Framework.Audio; using osu.Framework.Audio.Track; using osu.Framework.Extensions; using osu.Framework.IO.Stores; using osu.Framework.Platform; using osu.Game.Beatmaps.Formats; using osu.Game.Database; using osu.Game.Extensions; using osu.Game.IO.Archives; using osu.Game.Models; using osu.Game.Online.API; using osu.Game.Online.API.Requests.Responses; using osu.Game.Overlays.Notifications; using osu.Game.Rulesets; using osu.Game.Skinning; using osu.Game.Utils; using Realms; namespace osu.Game.Beatmaps { /// /// Handles general operations related to global beatmap management. /// public class BeatmapManager : ModelManager, IModelImporter, IWorkingBeatmapCache { public ITrackStore BeatmapTrackStore { get; } private readonly BeatmapImporter beatmapImporter; private readonly WorkingBeatmapCache workingBeatmapCache; private readonly BeatmapExporter beatmapExporter; private readonly LegacyBeatmapExporter legacyBeatmapExporter; public ProcessBeatmapDelegate? ProcessBeatmap { private get; set; } public override bool PauseImports { get => base.PauseImports; set { base.PauseImports = value; beatmapImporter.PauseImports = value; } } public BeatmapManager(Storage storage, RealmAccess realm, IAPIProvider? api, AudioManager audioManager, IResourceStore gameResources, GameHost? host = null, WorkingBeatmap? defaultBeatmap = null, BeatmapDifficultyCache? difficultyCache = null, bool performOnlineLookups = false) : base(storage, realm) { if (performOnlineLookups) { if (api == null) throw new ArgumentNullException(nameof(api), "API must be provided if online lookups are required."); if (difficultyCache == null) throw new ArgumentNullException(nameof(difficultyCache), "Difficulty cache must be provided if online lookups are required."); } var userResources = new RealmFileStore(realm, storage).Store; BeatmapTrackStore = audioManager.GetTrackStore(userResources); beatmapImporter = CreateBeatmapImporter(storage, realm); beatmapImporter.ProcessBeatmap = (beatmapSet, scope) => ProcessBeatmap?.Invoke(beatmapSet, scope); beatmapImporter.PostNotification = obj => PostNotification?.Invoke(obj); workingBeatmapCache = CreateWorkingBeatmapCache(audioManager, gameResources, userResources, defaultBeatmap, host); beatmapExporter = new BeatmapExporter(storage) { PostNotification = obj => PostNotification?.Invoke(obj) }; legacyBeatmapExporter = new LegacyBeatmapExporter(storage) { PostNotification = obj => PostNotification?.Invoke(obj) }; } protected virtual WorkingBeatmapCache CreateWorkingBeatmapCache(AudioManager audioManager, IResourceStore resources, IResourceStore storage, WorkingBeatmap? defaultBeatmap, GameHost? host) { return new WorkingBeatmapCache(BeatmapTrackStore, audioManager, resources, storage, defaultBeatmap, host); } protected virtual BeatmapImporter CreateBeatmapImporter(Storage storage, RealmAccess realm) => new BeatmapImporter(storage, realm); /// /// Create a new beatmap set, backed by a model, /// with a single difficulty which is backed by a model /// and represented by the returned usable . /// public WorkingBeatmap CreateNew(RulesetInfo ruleset, APIUser user) { var metadata = new BeatmapMetadata { Author = new RealmUser { OnlineID = user.OnlineID, Username = user.Username, } }; var beatmapSet = new BeatmapSetInfo { DateAdded = DateTimeOffset.UtcNow, Beatmaps = { new BeatmapInfo(ruleset, new BeatmapDifficulty(), metadata) } }; foreach (BeatmapInfo b in beatmapSet.Beatmaps) b.BeatmapSet = beatmapSet; var imported = beatmapImporter.ImportModel(beatmapSet); if (imported == null) throw new InvalidOperationException("Failed to import new beatmap"); return imported.PerformRead(s => GetWorkingBeatmap(s.Beatmaps.First())); } /// /// Add a new difficulty to the provided based on the provided . /// The new difficulty will be backed by a model /// and represented by the returned . /// /// /// Contrary to , this method does not preserve hitobjects and beatmap-level settings from . /// The created beatmap will have zero hitobjects and will have default settings (including difficulty settings), but will preserve metadata and existing timing points. /// /// The to add the new difficulty to. /// The to use as a baseline reference when creating the new difficulty. /// The ruleset with which the new difficulty should be created. public virtual WorkingBeatmap CreateNewDifficulty(BeatmapSetInfo targetBeatmapSet, WorkingBeatmap referenceWorkingBeatmap, RulesetInfo rulesetInfo) { var newBeatmapInfo = new BeatmapInfo(rulesetInfo, new BeatmapDifficulty(), referenceWorkingBeatmap.Metadata.DeepClone()) { DifficultyName = NamingUtils.GetNextBestName(targetBeatmapSet.Beatmaps.Select(b => b.DifficultyName), "New Difficulty") }; var newBeatmap = new Beatmap { BeatmapInfo = newBeatmapInfo }; foreach (var timingPoint in referenceWorkingBeatmap.Beatmap.ControlPointInfo.TimingPoints) newBeatmap.ControlPointInfo.Add(timingPoint.Time, timingPoint.DeepClone()); return addDifficultyToSet(targetBeatmapSet, newBeatmap, referenceWorkingBeatmap.Skin); } /// /// Add a copy of the provided to the provided . /// The new difficulty will be backed by a model /// and represented by the returned . /// /// /// Contrary to , this method creates a nearly-exact copy of /// (with the exception of a few key properties that cannot be copied under any circumstance, like difficulty name, beatmap hash, or online status). /// /// The to add the copy to. /// The to be copied. public virtual WorkingBeatmap CopyExistingDifficulty(BeatmapSetInfo targetBeatmapSet, WorkingBeatmap referenceWorkingBeatmap) { var newBeatmap = referenceWorkingBeatmap.GetPlayableBeatmap(referenceWorkingBeatmap.BeatmapInfo.Ruleset).Clone(); BeatmapInfo newBeatmapInfo; newBeatmap.BeatmapInfo = newBeatmapInfo = referenceWorkingBeatmap.BeatmapInfo.Clone(); // assign a new ID to the clone. newBeatmapInfo.ID = Guid.NewGuid(); // add "(copy)" suffix to difficulty name, and additionally ensure that it doesn't conflict with any other potentially pre-existing copies. newBeatmapInfo.DifficultyName = NamingUtils.GetNextBestName( targetBeatmapSet.Beatmaps.Select(b => b.DifficultyName), $"{newBeatmapInfo.DifficultyName} (copy)"); // clear the hash, as that's what is used to match .osu files with their corresponding realm beatmaps. newBeatmapInfo.Hash = string.Empty; // clear online properties. newBeatmapInfo.ResetOnlineInfo(); return addDifficultyToSet(targetBeatmapSet, newBeatmap, referenceWorkingBeatmap.Skin); } private WorkingBeatmap addDifficultyToSet(BeatmapSetInfo targetBeatmapSet, IBeatmap newBeatmap, ISkin beatmapSkin) { // populate circular beatmap set info <-> beatmap info references manually. // several places like `Save()` or `GetWorkingBeatmap()` // rely on them being freely traversable in both directions for correct operation. targetBeatmapSet.Beatmaps.Add(newBeatmap.BeatmapInfo); newBeatmap.BeatmapInfo.BeatmapSet = targetBeatmapSet; save(newBeatmap.BeatmapInfo, newBeatmap, beatmapSkin, transferCollections: false); workingBeatmapCache.Invalidate(targetBeatmapSet); return GetWorkingBeatmap(newBeatmap.BeatmapInfo); } /// /// Delete a beatmap difficulty. /// /// The beatmap difficulty to hide. public void Hide(BeatmapInfo beatmapInfo) { Realm.Run(r => { using (var transaction = r.BeginWrite()) { if (!beatmapInfo.IsManaged) beatmapInfo = r.Find(beatmapInfo.ID)!; beatmapInfo.Hidden = true; transaction.Commit(); } }); } /// /// Restore a beatmap difficulty. /// /// The beatmap difficulty to restore. public void Restore(BeatmapInfo beatmapInfo) { Realm.Run(r => { using (var transaction = r.BeginWrite()) { if (!beatmapInfo.IsManaged) beatmapInfo = r.Find(beatmapInfo.ID)!; beatmapInfo.Hidden = false; transaction.Commit(); } }); } public void RestoreAll() { Realm.Run(r => { using (var transaction = r.BeginWrite()) { foreach (var beatmap in r.All().Where(b => b.Hidden)) beatmap.Hidden = false; transaction.Commit(); } }); } /// /// Returns a list of all usable s. /// /// A list of available . public List GetAllUsableBeatmapSets() { return Realm.Run(r => { r.Refresh(); return r.All().Where(b => !b.DeletePending).Detach(); }); } /// /// Perform a lookup query on available s. /// /// The query. /// The first result for the provided query, or null if no results were found. public Live? QueryBeatmapSet(Expression> query) { return Realm.Run(r => r.All().FirstOrDefault(query)?.ToLive(Realm)); } /// /// Perform a lookup query on available s. /// /// The query. /// The first result for the provided query, or null if no results were found. public BeatmapInfo? QueryBeatmap(Expression> query) => Realm.Run(r => r.All().Filter($"{nameof(BeatmapInfo.BeatmapSet)}.{nameof(BeatmapSetInfo.DeletePending)} == false").FirstOrDefault(query)?.Detach()); /// /// A default representation of a WorkingBeatmap to use when no beatmap is available. /// public IWorkingBeatmap DefaultBeatmap => workingBeatmapCache.DefaultBeatmap; /// /// Saves an existing file against a given . /// /// /// This method will also update any user beatmap collection hash references to the new post-saved hash. /// /// The to save the content against. The file referenced by will be replaced. /// The content to write. /// The beatmap content to write, null if to be omitted. public virtual void Save(BeatmapInfo beatmapInfo, IBeatmap beatmapContent, ISkin? beatmapSkin = null) => save(beatmapInfo, beatmapContent, beatmapSkin, transferCollections: true); public void DeleteAllVideos() { Realm.Write(r => { var items = r.All().Where(s => !s.DeletePending && !s.Protected); DeleteVideos(items.ToList()); }); } public void Delete(Expression>? filter = null, bool silent = false) { Realm.Run(r => { var items = r.All().Where(s => !s.DeletePending && !s.Protected); if (filter != null) items = items.Where(filter); Delete(items.ToList(), silent); }); } /// /// Delete a beatmap difficulty immediately. /// /// /// There's no undoing this operation, as we don't have a soft-deletion flag on . /// This may be a future consideration if there's a user requirement for undeleting support. /// public void DeleteDifficultyImmediately(BeatmapInfo beatmapInfo) { Realm.Write(r => { if (!beatmapInfo.IsManaged) beatmapInfo = r.Find(beatmapInfo.ID)!; Debug.Assert(beatmapInfo.BeatmapSet != null); Debug.Assert(beatmapInfo.File != null); var setInfo = beatmapInfo.BeatmapSet; DeleteFile(setInfo, beatmapInfo.File); setInfo.Beatmaps.Remove(beatmapInfo); r.Remove(beatmapInfo.Metadata); r.Remove(beatmapInfo); updateHashAndMarkDirty(setInfo); workingBeatmapCache.Invalidate(setInfo); }); } /// /// Delete videos from a list of beatmaps. /// This will post notifications tracking progress. /// public void DeleteVideos(List items, bool silent = false) { const string no_videos_message = "No videos found to delete!"; if (items.Count == 0) { if (!silent) PostNotification?.Invoke(new ProgressCompletionNotification { Text = no_videos_message }); return; } var notification = new ProgressNotification { Progress = 0, Text = $"Preparing to delete all {HumanisedModelName} videos...", CompletionText = no_videos_message, State = ProgressNotificationState.Active, }; if (!silent) PostNotification?.Invoke(notification); int i = 0; int deleted = 0; foreach (var b in items) { if (notification.State == ProgressNotificationState.Cancelled) // user requested abort return; var video = b.Files.FirstOrDefault(f => OsuGameBase.VIDEO_EXTENSIONS.Any(ex => f.Filename.EndsWith(ex, StringComparison.OrdinalIgnoreCase))); if (video != null) { DeleteFile(b, video); deleted++; notification.CompletionText = $"Deleted {deleted} {HumanisedModelName} video(s)!"; } notification.Text = $"Deleting videos from {HumanisedModelName}s ({deleted} deleted)"; notification.Progress = (float)++i / items.Count; } notification.State = ProgressNotificationState.Completed; } public void UndeleteAll() { Realm.Run(r => Undelete(r.All().Where(s => s.DeletePending).ToList())); } public Task?> ImportAsUpdate(ProgressNotification notification, ImportTask importTask, BeatmapSetInfo original) => beatmapImporter.ImportAsUpdate(notification, importTask, original); public Task> BeginExternalEditing(BeatmapSetInfo model) => beatmapImporter.BeginExternalEditing(model); public Task Export(BeatmapSetInfo beatmap) => beatmapExporter.ExportAsync(beatmap.ToLive(Realm)); public Task ExportLegacy(BeatmapSetInfo beatmap) => legacyBeatmapExporter.ExportAsync(beatmap.ToLive(Realm)); private void updateHashAndMarkDirty(BeatmapSetInfo setInfo) { setInfo.Hash = beatmapImporter.ComputeHash(setInfo); setInfo.Status = BeatmapOnlineStatus.LocallyModified; } private void save(BeatmapInfo beatmapInfo, IBeatmap beatmapContent, ISkin? beatmapSkin, bool transferCollections) { var setInfo = beatmapInfo.BeatmapSet; Debug.Assert(setInfo != null); // Difficulty settings must be copied first due to the clone in `Beatmap<>.BeatmapInfo_Set`. // This should hopefully be temporary, assuming said clone is eventually removed. // Warning: The directionality here is important. Changes have to be copied *from* beatmapContent (which comes from editor and is being saved) // *to* the beatmapInfo (which is a database model and needs to receive values without the taiko slider velocity multiplier for correct operation). // CopyTo() will undo such adjustments, while CopyFrom() will not. beatmapContent.Difficulty.CopyTo(beatmapInfo.Difficulty); // All changes to metadata are made in the provided beatmapInfo, so this should be copied to the `IBeatmap` before encoding. beatmapContent.BeatmapInfo = beatmapInfo; // Since now this is a locally-modified beatmap, we also set all relevant flags to indicate this. // Importantly, the `ResetOnlineInfo()` call must happen before encoding, as online ID is encoded into the `.osu` file, // which influences the beatmap checksums. beatmapInfo.LastLocalUpdate = DateTimeOffset.Now; beatmapInfo.Status = BeatmapOnlineStatus.LocallyModified; beatmapInfo.ResetOnlineInfo(); Realm.Write(r => { using var stream = new MemoryStream(); using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true)) new LegacyBeatmapEncoder(beatmapContent, beatmapSkin).Encode(sw); stream.Seek(0, SeekOrigin.Begin); // AddFile generally handles updating/replacing files, but this is a case where the filename may have also changed so let's delete for simplicity. var existingFileInfo = beatmapInfo.Path != null ? setInfo.GetFile(beatmapInfo.Path) : null; string targetFilename = createBeatmapFilenameFromMetadata(beatmapInfo); // ensure that two difficulties from the set don't point at the same beatmap file. if (setInfo.Beatmaps.Any(b => b.ID != beatmapInfo.ID && string.Equals(b.Path, targetFilename, StringComparison.OrdinalIgnoreCase))) throw new InvalidOperationException($"{setInfo.GetDisplayString()} already has a difficulty with the name of '{beatmapInfo.DifficultyName}'."); if (existingFileInfo != null) DeleteFile(setInfo, existingFileInfo); string oldMd5Hash = beatmapInfo.MD5Hash; beatmapInfo.MD5Hash = stream.ComputeMD5Hash(); beatmapInfo.Hash = stream.ComputeSHA2Hash(); AddFile(setInfo, stream, createBeatmapFilenameFromMetadata(beatmapInfo)); updateHashAndMarkDirty(setInfo); var liveBeatmapSet = r.Find(setInfo.ID)!; setInfo.CopyChangesToRealm(liveBeatmapSet); if (transferCollections) beatmapInfo.TransferCollectionReferences(r, oldMd5Hash); liveBeatmapSet.Beatmaps.Single(b => b.ID == beatmapInfo.ID) .UpdateLocalScores(r); // do not look up metadata. // this is a locally-modified set now, so looking up metadata is busy work at best and harmful at worst. ProcessBeatmap?.Invoke(liveBeatmapSet, MetadataLookupScope.None); }); Debug.Assert(beatmapInfo.BeatmapSet != null); static string createBeatmapFilenameFromMetadata(BeatmapInfo beatmapInfo) { var metadata = beatmapInfo.Metadata; return $"{metadata.Artist} - {metadata.Title} ({metadata.Author.Username}) [{beatmapInfo.DifficultyName}].osu".GetValidFilename(); } } #region Implementation of ICanAcceptFiles public Task Import(params string[] paths) => beatmapImporter.Import(paths); public Task Import(ImportTask[] tasks, ImportParameters parameters = default) => beatmapImporter.Import(tasks, parameters); public Task>> Import(ProgressNotification notification, ImportTask[] tasks, ImportParameters parameters = default) => beatmapImporter.Import(notification, tasks, parameters); public Task?> Import(ImportTask task, ImportParameters parameters = default, CancellationToken cancellationToken = default) => beatmapImporter.Import(task, parameters, cancellationToken); public Live? Import(BeatmapSetInfo item, ArchiveReader? archive = null, CancellationToken cancellationToken = default) => beatmapImporter.ImportModel(item, archive, default, cancellationToken); public IEnumerable HandledExtensions => beatmapImporter.HandledExtensions; #endregion #region Implementation of IWorkingBeatmapCache /// /// Retrieve a instance for the provided /// /// The beatmap to lookup. /// Whether to force a refetch from the database to ensure is up-to-date. /// A instance correlating to the provided . public WorkingBeatmap GetWorkingBeatmap(BeatmapInfo? beatmapInfo, bool refetch = false) { if (beatmapInfo != null) { if (refetch) workingBeatmapCache.Invalidate(beatmapInfo); // Detached beatmapsets don't come with files as an optimisation (see `RealmObjectExtensions.beatmap_set_mapper`). // If we seem to be missing files, now is a good time to re-fetch. bool missingFiles = beatmapInfo.BeatmapSet?.Files.Count == 0; if (refetch || beatmapInfo.IsManaged || missingFiles) { Guid id = beatmapInfo.ID; beatmapInfo = Realm.Run(r => r.Find(id)?.Detach()) ?? beatmapInfo; } Debug.Assert(beatmapInfo.IsManaged != true); } return workingBeatmapCache.GetWorkingBeatmap(beatmapInfo); } WorkingBeatmap IWorkingBeatmapCache.GetWorkingBeatmap(BeatmapInfo beatmapInfo) => GetWorkingBeatmap(beatmapInfo); void IWorkingBeatmapCache.Invalidate(BeatmapSetInfo beatmapSetInfo) => workingBeatmapCache.Invalidate(beatmapSetInfo); void IWorkingBeatmapCache.Invalidate(BeatmapInfo beatmapInfo) => workingBeatmapCache.Invalidate(beatmapInfo); public event Action? OnInvalidated { add => workingBeatmapCache.OnInvalidated += value; remove => workingBeatmapCache.OnInvalidated -= value; } public override bool IsAvailableLocally(BeatmapSetInfo model) => Realm.Run(realm => realm.All().Any(s => s.OnlineID == model.OnlineID)); #endregion #region Implementation of IPostImports public Action>>? PresentImport { set => beatmapImporter.PresentImport = value; } #endregion public override string HumanisedModelName => "beatmap"; } /// /// Delegate type for beatmap processing callbacks. /// /// The beatmap set to be processed. /// The scope to use when looking up metadata. public delegate void ProcessBeatmapDelegate(BeatmapSetInfo beatmapSet, MetadataLookupScope lookupScope); }