// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Linq.Expressions; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using osu.Framework.Audio; using osu.Framework.Audio.Track; using osu.Framework.Extensions; using osu.Framework.Graphics.Textures; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Game.Beatmaps.Formats; using osu.Game.Database; using osu.Game.IO.Archives; using osu.Game.Online.API; using osu.Game.Online.API.Requests; using osu.Game.Overlays.Notifications; using osu.Game.Rulesets; namespace osu.Game.Beatmaps { /// /// Handles the storage and retrieval of Beatmaps/WorkingBeatmaps. /// public partial class BeatmapManager : ArchiveModelManager { /// /// Fired when a single difficulty has been hidden. /// public event Action BeatmapHidden; /// /// Fired when a single difficulty has been restored. /// public event Action BeatmapRestored; /// /// Fired when a beatmap download begins. /// public event Action BeatmapDownloadBegan; /// /// Fired when a beatmap download is interrupted, due to user cancellation or other failures. /// public event Action BeatmapDownloadFailed; /// /// A default representation of a WorkingBeatmap to use when no beatmap is available. /// public readonly WorkingBeatmap DefaultBeatmap; public override string[] HandledExtensions => new[] { ".osz" }; protected override string[] HashableFileTypes => new[] { ".osu" }; protected override string ImportFromStablePath => "Songs"; private readonly RulesetStore rulesets; private readonly BeatmapStore beatmaps; private readonly APIAccess api; private readonly AudioManager audioManager; private readonly GameHost host; private readonly List currentDownloads = new List(); public BeatmapManager(Storage storage, IDatabaseContextFactory contextFactory, RulesetStore rulesets, APIAccess api, AudioManager audioManager, GameHost host = null, WorkingBeatmap defaultBeatmap = null) : base(storage, contextFactory, new BeatmapStore(contextFactory), host) { this.rulesets = rulesets; this.api = api; this.audioManager = audioManager; this.host = host; DefaultBeatmap = defaultBeatmap; beatmaps = (BeatmapStore)ModelStore; beatmaps.BeatmapHidden += b => BeatmapHidden?.Invoke(b); beatmaps.BeatmapRestored += b => BeatmapRestored?.Invoke(b); } protected override void Populate(BeatmapSetInfo beatmapSet, ArchiveReader archive) { if (archive != null) beatmapSet.Beatmaps = createBeatmapDifficulties(archive); foreach (BeatmapInfo b in beatmapSet.Beatmaps) { // remove metadata from difficulties where it matches the set if (beatmapSet.Metadata.Equals(b.Metadata)) b.Metadata = null; b.BeatmapSet = beatmapSet; } validateOnlineIds(beatmapSet); foreach (BeatmapInfo b in beatmapSet.Beatmaps) fetchAndPopulateOnlineValues(b, beatmapSet.Beatmaps); } protected override void PreImport(BeatmapSetInfo beatmapSet) { // check if a set already exists with the same online id, delete if it does. if (beatmapSet.OnlineBeatmapSetID != null) { var existingOnlineId = beatmaps.ConsumableItems.FirstOrDefault(b => b.OnlineBeatmapSetID == beatmapSet.OnlineBeatmapSetID); if (existingOnlineId != null) { Delete(existingOnlineId); beatmaps.PurgeDeletable(s => s.ID == existingOnlineId.ID); Logger.Log($"Found existing beatmap set with same OnlineBeatmapSetID ({beatmapSet.OnlineBeatmapSetID}). It has been purged.", LoggingTarget.Database); } } } private void validateOnlineIds(BeatmapSetInfo beatmapSet) { var beatmapIds = beatmapSet.Beatmaps.Where(b => b.OnlineBeatmapID.HasValue).Select(b => b.OnlineBeatmapID).ToList(); // ensure all IDs are unique if (beatmapIds.GroupBy(b => b).Any(g => g.Count() > 1)) { resetIds(); return; } // find any existing beatmaps in the database that have matching online ids var existingBeatmaps = QueryBeatmaps(b => beatmapIds.Contains(b.OnlineBeatmapID)).ToList(); if (existingBeatmaps.Count > 0) { // reset the import ids (to force a re-fetch) *unless* they match the candidate CheckForExisting set. // we can ignore the case where the new ids are contained by the CheckForExisting set as it will either be used (import skipped) or deleted. var existing = CheckForExisting(beatmapSet); if (existing == null || existingBeatmaps.Any(b => !existing.Beatmaps.Contains(b))) resetIds(); } void resetIds() => beatmapSet.Beatmaps.ForEach(b => b.OnlineBeatmapID = null); } /// /// Downloads a beatmap. /// This will post notifications tracking progress. /// /// The to be downloaded. /// Whether the beatmap should be downloaded without video. Defaults to false. /// Downloading can happen public bool Download(BeatmapSetInfo beatmapSetInfo, bool noVideo = false) { var existing = GetExistingDownload(beatmapSetInfo); if (existing != null || api == null) return false; var downloadNotification = new DownloadNotification { Text = $"Downloading {beatmapSetInfo}", }; var request = new DownloadBeatmapSetRequest(beatmapSetInfo, noVideo); request.DownloadProgressed += progress => { downloadNotification.State = ProgressNotificationState.Active; downloadNotification.Progress = progress; }; request.Success += filename => { Task.Factory.StartNew(() => { // This gets scheduled back to the update thread, but we want the import to run in the background. Import(downloadNotification, filename); currentDownloads.Remove(request); }, TaskCreationOptions.LongRunning); }; request.Failure += error => { BeatmapDownloadFailed?.Invoke(request); if (error is OperationCanceledException) return; downloadNotification.State = ProgressNotificationState.Cancelled; Logger.Error(error, "Beatmap download failed!"); currentDownloads.Remove(request); }; downloadNotification.CancelRequested += () => { request.Cancel(); currentDownloads.Remove(request); downloadNotification.State = ProgressNotificationState.Cancelled; return true; }; currentDownloads.Add(request); PostNotification?.Invoke(downloadNotification); // don't run in the main api queue as this is a long-running task. Task.Factory.StartNew(() => { try { request.Perform(api); } catch (Exception e) { // no need to handle here as exceptions will filter down to request.Failure above. } }, TaskCreationOptions.LongRunning); BeatmapDownloadBegan?.Invoke(request); return true; } /// /// Get an existing download request if it exists. /// /// The whose download request is wanted. /// The object if it exists, or null. public DownloadBeatmapSetRequest GetExistingDownload(BeatmapSetInfo beatmap) => currentDownloads.Find(d => d.BeatmapSet.OnlineBeatmapSetID == beatmap.OnlineBeatmapSetID); /// /// Delete a beatmap difficulty. /// /// The beatmap difficulty to hide. public void Hide(BeatmapInfo beatmap) => beatmaps.Hide(beatmap); /// /// Restore a beatmap difficulty. /// /// The beatmap difficulty to restore. public void Restore(BeatmapInfo beatmap) => beatmaps.Restore(beatmap); /// /// Retrieve a instance for the provided /// /// The beatmap to lookup. /// The currently loaded . Allows for optimisation where elements are shared with the new beatmap. May be returned if beatmapInfo requested matches /// A instance correlating to the provided . public WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo, WorkingBeatmap previous = null) { if (beatmapInfo?.ID > 0 && previous != null && previous.BeatmapInfo?.ID == beatmapInfo.ID) return previous; if (beatmapInfo?.BeatmapSet == null || beatmapInfo == DefaultBeatmap?.BeatmapInfo) return DefaultBeatmap; if (beatmapInfo.Metadata == null) beatmapInfo.Metadata = beatmapInfo.BeatmapSet.Metadata; WorkingBeatmap working = new BeatmapManagerWorkingBeatmap(Files.Store, new LargeTextureStore(host?.CreateTextureLoaderStore(Files.Store)), beatmapInfo, audioManager); previous?.TransferTo(working); return working; } /// /// Perform a lookup query on available s. /// /// The query. /// The first result for the provided query, or null if no results were found. public BeatmapSetInfo QueryBeatmapSet(Expression> query) => beatmaps.ConsumableItems.AsNoTracking().FirstOrDefault(query); protected override bool CanUndelete(BeatmapSetInfo existing, BeatmapSetInfo import) { if (!base.CanUndelete(existing, import)) return false; var existingIds = existing.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); var importIds = import.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); // force re-import if we are not in a sane state. return existing.OnlineBeatmapSetID == import.OnlineBeatmapSetID && existingIds.SequenceEqual(importIds); } /// /// Returns a list of all usable s. /// /// A list of available . public List GetAllUsableBeatmapSets() => GetAllUsableBeatmapSetsEnumerable().ToList(); /// /// Returns a list of all usable s. /// /// A list of available . public IQueryable GetAllUsableBeatmapSetsEnumerable() => beatmaps.ConsumableItems.Where(s => !s.DeletePending && !s.Protected); /// /// Perform a lookup query on available s. /// /// The query. /// Results from the provided query. public IEnumerable QueryBeatmapSets(Expression> query) => beatmaps.ConsumableItems.AsNoTracking().Where(query); /// /// Perform a lookup query on available s. /// /// The query. /// The first result for the provided query, or null if no results were found. public BeatmapInfo QueryBeatmap(Expression> query) => beatmaps.Beatmaps.AsNoTracking().FirstOrDefault(query); /// /// Perform a lookup query on available s. /// /// The query. /// Results from the provided query. public IQueryable QueryBeatmaps(Expression> query) => beatmaps.Beatmaps.AsNoTracking().Where(query); protected override BeatmapSetInfo CreateModel(ArchiveReader reader) { // let's make sure there are actually .osu files to import. string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu")); if (string.IsNullOrEmpty(mapName)) { Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database); return null; } Beatmap beatmap; using (var stream = new StreamReader(reader.GetStream(mapName))) beatmap = Decoder.GetDecoder(stream).Decode(stream); return new BeatmapSetInfo { OnlineBeatmapSetID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID, Beatmaps = new List(), Metadata = beatmap.Metadata, }; } /// /// Create all required s for the provided archive. /// private List createBeatmapDifficulties(ArchiveReader reader) { var beatmapInfos = new List(); foreach (var name in reader.Filenames.Where(f => f.EndsWith(".osu"))) { using (var raw = reader.GetStream(name)) using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit using (var sr = new StreamReader(ms)) { raw.CopyTo(ms); ms.Position = 0; var decoder = Decoder.GetDecoder(sr); IBeatmap beatmap = decoder.Decode(sr); beatmap.BeatmapInfo.Path = name; beatmap.BeatmapInfo.Hash = ms.ComputeSHA2Hash(); beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash(); var ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID); beatmap.BeatmapInfo.Ruleset = ruleset; // TODO: this should be done in a better place once we actually need to dynamically update it. beatmap.BeatmapInfo.StarDifficulty = ruleset?.CreateInstance().CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating ?? 0; beatmapInfos.Add(beatmap.BeatmapInfo); } } return beatmapInfos; } /// /// Query the API to populate missing values like OnlineBeatmapID / OnlineBeatmapSetID or (Rank-)Status. /// /// The beatmap to populate. /// The other beatmaps contained within this set. /// Whether to re-query if the provided beatmap already has populated values. /// True if population was successful. private bool fetchAndPopulateOnlineValues(BeatmapInfo beatmap, IEnumerable otherBeatmaps, bool force = false) { if (api?.State != APIState.Online) return false; if (!force && beatmap.OnlineBeatmapID != null && beatmap.BeatmapSet.OnlineBeatmapSetID != null && beatmap.Status != BeatmapSetOnlineStatus.None && beatmap.BeatmapSet.Status != BeatmapSetOnlineStatus.None) return true; Logger.Log("Attempting online lookup for the missing values...", LoggingTarget.Database); try { var req = new GetBeatmapRequest(beatmap); req.Perform(api); var res = req.Result; Logger.Log($"Successfully mapped to {res.OnlineBeatmapSetID} / {res.OnlineBeatmapID}.", LoggingTarget.Database); beatmap.Status = res.Status; beatmap.BeatmapSet.Status = res.BeatmapSet.Status; if (otherBeatmaps.Any(b => b.OnlineBeatmapID == res.OnlineBeatmapID)) { Logger.Log("Another beatmap in the same set already mapped to this ID. We'll skip adding it this time.", LoggingTarget.Database); return false; } beatmap.BeatmapSet.OnlineBeatmapSetID = res.OnlineBeatmapSetID; beatmap.OnlineBeatmapID = res.OnlineBeatmapID; return true; } catch (Exception e) { Logger.Log($"Failed ({e})", LoggingTarget.Database); return false; } } /// /// A dummy WorkingBeatmap for the purpose of retrieving a beatmap for star difficulty calculation. /// private class DummyConversionBeatmap : WorkingBeatmap { private readonly IBeatmap beatmap; public DummyConversionBeatmap(IBeatmap beatmap) : base(beatmap.BeatmapInfo) { this.beatmap = beatmap; } protected override IBeatmap GetBeatmap() => beatmap; protected override Texture GetBackground() => null; protected override Track GetTrack() => null; } private class DownloadNotification : ProgressNotification { public override bool IsImportant => false; protected override Notification CreateCompletionNotification() => new SilencedProgressCompletionNotification { Activated = CompletionClickAction, Text = CompletionText }; private class SilencedProgressCompletionNotification : ProgressCompletionNotification { public override bool IsImportant => false; } } } }