From 898a60109873697ed8b165ab3f5e8f885af41843 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 26 Jul 2017 20:22:02 +0900 Subject: [PATCH] Introduce a reference counting file store --- .../Tests/TestCasePlaySongSelect.cs | 2 +- .../Beatmaps/IO/LegacyFilesystemReader.cs | 43 ------ osu.Desktop/Program.cs | 3 - osu.Desktop/osu.Desktop.csproj | 1 - .../Beatmaps/Formats/OsuLegacyDecoderTest.cs | 6 - .../Beatmaps/IO/OszArchiveReaderTest.cs | 9 +- osu.Game/Beatmaps/BeatmapDatabase.cs | 43 ++++-- osu.Game/Beatmaps/BeatmapInfo.cs | 4 +- osu.Game/Beatmaps/BeatmapSetFileInfo.cs | 17 +++ osu.Game/Beatmaps/BeatmapSetInfo.cs | 8 +- osu.Game/Beatmaps/BeatmapStore.cs | 132 +++++++++++------- .../Beatmaps/BeatmapStoreWorkingBeatmap.cs | 18 +-- osu.Game/Beatmaps/Formats/BeatmapDecoder.cs | 5 + osu.Game/Beatmaps/IO/ArchiveReader.cs | 36 +---- .../Beatmaps/IO/LegacyFilesystemReader.cs | 33 +++++ osu.Game/Beatmaps/IO/OszArchiveReader.cs | 32 ++--- osu.Game/Database/DatabaseStore.cs | 18 +-- osu.Game/IO/FileDatabase.cs | 114 +++++++++++++++ osu.Game/IO/FileInfo.cs | 24 ++++ osu.Game/OsuGameBase.cs | 9 +- osu.Game/Screens/Menu/Intro.cs | 15 +- osu.Game/osu.Game.csproj | 4 + 22 files changed, 361 insertions(+), 215 deletions(-) delete mode 100644 osu.Desktop/Beatmaps/IO/LegacyFilesystemReader.cs create mode 100644 osu.Game/Beatmaps/BeatmapSetFileInfo.cs create mode 100644 osu.Game/Beatmaps/IO/LegacyFilesystemReader.cs create mode 100644 osu.Game/IO/FileDatabase.cs create mode 100644 osu.Game/IO/FileInfo.cs diff --git a/osu.Desktop.VisualTests/Tests/TestCasePlaySongSelect.cs b/osu.Desktop.VisualTests/Tests/TestCasePlaySongSelect.cs index 6d41be72c7..a17b873b0d 100644 --- a/osu.Desktop.VisualTests/Tests/TestCasePlaySongSelect.cs +++ b/osu.Desktop.VisualTests/Tests/TestCasePlaySongSelect.cs @@ -31,7 +31,7 @@ namespace osu.Desktop.VisualTests.Tests var backingDatabase = storage.GetDatabase(@"client"); rulesets = new RulesetDatabase(backingDatabase); - store = new BeatmapStore(storage, backingDatabase, rulesets); + store = new BeatmapStore(storage, null, backingDatabase, rulesets); var sets = new List(); diff --git a/osu.Desktop/Beatmaps/IO/LegacyFilesystemReader.cs b/osu.Desktop/Beatmaps/IO/LegacyFilesystemReader.cs deleted file mode 100644 index 8772fc9f28..0000000000 --- a/osu.Desktop/Beatmaps/IO/LegacyFilesystemReader.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2007-2017 ppy Pty Ltd . -// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE - -using System.IO; -using System.Linq; -using osu.Game.Beatmaps.IO; - -namespace osu.Desktop.Beatmaps.IO -{ - /// - /// Reads an extracted legacy beatmap from disk. - /// - public class LegacyFilesystemReader : ArchiveReader - { - public static void Register() => AddReader((storage, path) => Directory.Exists(path)); - - private readonly string basePath; - - public LegacyFilesystemReader(string path) - { - basePath = path; - - BeatmapFilenames = Directory.GetFiles(basePath, @"*.osu").Select(Path.GetFileName).ToArray(); - - if (BeatmapFilenames.Length == 0) - throw new FileNotFoundException(@"This directory contains no beatmaps"); - - StoryboardFilename = Directory.GetFiles(basePath, @"*.osb").Select(Path.GetFileName).FirstOrDefault(); - } - - public override Stream GetStream(string name) - { - return File.OpenRead(Path.Combine(basePath, name)); - } - - public override void Dispose() - { - // no-op - } - - public override Stream GetUnderlyingStream() => null; - } -} diff --git a/osu.Desktop/Program.cs b/osu.Desktop/Program.cs index 210f780078..3b63239525 100644 --- a/osu.Desktop/Program.cs +++ b/osu.Desktop/Program.cs @@ -3,7 +3,6 @@ using System; using System.IO; -using osu.Desktop.Beatmaps.IO; using osu.Framework.Desktop; using osu.Framework.Desktop.Platform; using osu.Game.IPC; @@ -15,8 +14,6 @@ namespace osu.Desktop [STAThread] public static int Main(string[] args) { - LegacyFilesystemReader.Register(); - // Back up the cwd before DesktopGameHost changes it var cwd = Environment.CurrentDirectory; diff --git a/osu.Desktop/osu.Desktop.csproj b/osu.Desktop/osu.Desktop.csproj index e69603602c..82fbefec7a 100644 --- a/osu.Desktop/osu.Desktop.csproj +++ b/osu.Desktop/osu.Desktop.csproj @@ -228,7 +228,6 @@ - diff --git a/osu.Game.Tests/Beatmaps/Formats/OsuLegacyDecoderTest.cs b/osu.Game.Tests/Beatmaps/Formats/OsuLegacyDecoderTest.cs index 4814af984e..da3b448f74 100644 --- a/osu.Game.Tests/Beatmaps/Formats/OsuLegacyDecoderTest.cs +++ b/osu.Game.Tests/Beatmaps/Formats/OsuLegacyDecoderTest.cs @@ -16,12 +16,6 @@ namespace osu.Game.Tests.Beatmaps.Formats [TestFixture] public class OsuLegacyDecoderTest { - [OneTimeSetUpAttribute] - public void SetUp() - { - OsuLegacyDecoder.Register(); - } - [Test] public void TestDecodeMetadata() { diff --git a/osu.Game.Tests/Beatmaps/IO/OszArchiveReaderTest.cs b/osu.Game.Tests/Beatmaps/IO/OszArchiveReaderTest.cs index fadfa872ce..7a7a8a58bc 100644 --- a/osu.Game.Tests/Beatmaps/IO/OszArchiveReaderTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/OszArchiveReaderTest.cs @@ -2,6 +2,7 @@ // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE using System.IO; +using System.Linq; using NUnit.Framework; using osu.Game.Beatmaps; using osu.Game.Beatmaps.IO; @@ -13,12 +14,6 @@ namespace osu.Game.Tests.Beatmaps.IO [TestFixture] public class OszArchiveReaderTest { - [OneTimeSetUpAttribute] - public void SetUp() - { - OszArchiveReader.Register(); - } - [Test] public void TestReadBeatmaps() { @@ -40,7 +35,7 @@ namespace osu.Game.Tests.Beatmaps.IO "Soleily - Renatus (MMzz) [Muzukashii].osu", "Soleily - Renatus (MMzz) [Oni].osu" }; - var maps = reader.BeatmapFilenames; + var maps = reader.Filenames.ToArray(); foreach (var map in expected) Assert.Contains(map, maps); } diff --git a/osu.Game/Beatmaps/BeatmapDatabase.cs b/osu.Game/Beatmaps/BeatmapDatabase.cs index ca607c87eb..1fd3f5b52f 100644 --- a/osu.Game/Beatmaps/BeatmapDatabase.cs +++ b/osu.Game/Beatmaps/BeatmapDatabase.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; using osu.Framework.Logging; using osu.Game.Database; -using osu.Game.Screens.Menu; using SQLite.Net; using SQLiteNetExtensions.Extensions; @@ -19,11 +18,13 @@ namespace osu.Game.Beatmaps public event Action BeatmapSetAdded; public event Action BeatmapSetRemoved; - public BeatmapDatabase(SQLiteConnection connection) : base(connection) + public BeatmapDatabase(SQLiteConnection connection) + : base(connection) { } - protected override Type[] ValidTypes => new[] { + protected override Type[] ValidTypes => new[] + { typeof(BeatmapSetInfo), typeof(BeatmapInfo), typeof(BeatmapMetadata), @@ -37,17 +38,21 @@ namespace osu.Game.Beatmaps Connection.DropTable(); Connection.DropTable(); Connection.DropTable(); + Connection.DropTable(); Connection.DropTable(); } Connection.CreateTable(); Connection.CreateTable(); Connection.CreateTable(); + Connection.CreateTable(); Connection.CreateTable(); deletePending(); } + public void Update(BeatmapSetInfo setInfo) => Connection.Update(setInfo); + public void Import(IEnumerable beatmapSets) { lock (Connection) @@ -64,24 +69,32 @@ namespace osu.Game.Beatmaps } } - public void Delete(IEnumerable beatmapSets) + public bool Delete(BeatmapSetInfo set) { - foreach (var s in beatmapSets) - { - s.DeletePending = true; - Update(s, false); - BeatmapSetRemoved?.Invoke(s); - } + if (set.DeletePending) return false; + + set.DeletePending = true; + Connection.Update(set); + + BeatmapSetRemoved?.Invoke(set); + return true; + } + + public bool Undelete(BeatmapSetInfo set) + { + if (!set.DeletePending) return false; + + set.DeletePending = false; + Connection.Update(set); + + BeatmapSetAdded?.Invoke(set); + return true; } private void deletePending() { - foreach (var b in GetAllWithChildren(b => b.DeletePending)) + foreach (var b in GetAllWithChildren(b => b.DeletePending && !b.Protected)) { - if (b.Hash == Intro.MENU_MUSIC_BEATMAP_HASH) - // this is a bit hacky, but will do for now. - continue; - try { foreach (var i in b.Beatmaps) diff --git a/osu.Game/Beatmaps/BeatmapInfo.cs b/osu.Game/Beatmaps/BeatmapInfo.cs index a154fc008a..32d50145a6 100644 --- a/osu.Game/Beatmaps/BeatmapInfo.cs +++ b/osu.Game/Beatmaps/BeatmapInfo.cs @@ -95,11 +95,11 @@ namespace osu.Game.Beatmaps } public bool AudioEquals(BeatmapInfo other) => other != null && BeatmapSet != null && other.BeatmapSet != null && - BeatmapSet.Path == other.BeatmapSet.Path && + BeatmapSet.Hash == other.BeatmapSet.Hash && (Metadata ?? BeatmapSet.Metadata).AudioFile == (other.Metadata ?? other.BeatmapSet.Metadata).AudioFile; public bool BackgroundEquals(BeatmapInfo other) => other != null && BeatmapSet != null && other.BeatmapSet != null && - BeatmapSet.Path == other.BeatmapSet.Path && + BeatmapSet.Hash == other.BeatmapSet.Hash && (Metadata ?? BeatmapSet.Metadata).BackgroundFile == (other.Metadata ?? other.BeatmapSet.Metadata).BackgroundFile; } } diff --git a/osu.Game/Beatmaps/BeatmapSetFileInfo.cs b/osu.Game/Beatmaps/BeatmapSetFileInfo.cs new file mode 100644 index 0000000000..d18b1e833b --- /dev/null +++ b/osu.Game/Beatmaps/BeatmapSetFileInfo.cs @@ -0,0 +1,17 @@ +// Copyright (c) 2007-2017 ppy Pty Ltd . +// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE + +using osu.Game.IO; +using SQLiteNetExtensions.Attributes; + +namespace osu.Game.Beatmaps +{ + public class BeatmapSetFileInfo + { + [ForeignKey(typeof(BeatmapSetInfo))] + public int BeatmapSetInfoID { get; set; } + + [ForeignKey(typeof(FileInfo))] + public int FileInfoID { get; set; } + } +} \ No newline at end of file diff --git a/osu.Game/Beatmaps/BeatmapSetInfo.cs b/osu.Game/Beatmaps/BeatmapSetInfo.cs index 2e41ded28a..6a466fb3b2 100644 --- a/osu.Game/Beatmaps/BeatmapSetInfo.cs +++ b/osu.Game/Beatmaps/BeatmapSetInfo.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; +using osu.Game.IO; using SQLite.Net.Attributes; using SQLiteNetExtensions.Attributes; @@ -34,8 +35,11 @@ namespace osu.Game.Beatmaps public string Hash { get; set; } - public string Path { get; set; } + public string StoryboardFile => Files.FirstOrDefault(f => f.Filename.EndsWith(".osb"))?.Filename; - public string StoryboardFile { get; set; } + [ManyToMany(typeof(BeatmapSetFileInfo))] + public List Files { get; set; } + + public bool Protected { get; set; } } } diff --git a/osu.Game/Beatmaps/BeatmapStore.cs b/osu.Game/Beatmaps/BeatmapStore.cs index 4e5eeee0a4..b84d249893 100644 --- a/osu.Game/Beatmaps/BeatmapStore.cs +++ b/osu.Game/Beatmaps/BeatmapStore.cs @@ -5,14 +5,17 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using Ionic.Zip; using osu.Framework.Extensions; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Game.Beatmaps.Formats; using osu.Game.Beatmaps.IO; +using osu.Game.IO; using osu.Game.IPC; using osu.Game.Rulesets; using SQLite.Net; +using FileInfo = osu.Game.IO.FileInfo; namespace osu.Game.Beatmaps { @@ -25,6 +28,7 @@ namespace osu.Game.Beatmaps public readonly BeatmapDatabase Database; private readonly Storage storage; + private readonly FileDatabase files; private readonly RulesetDatabase rulesets; @@ -39,14 +43,16 @@ namespace osu.Game.Beatmaps /// public WorkingBeatmap DefaultBeatmap { private get; set; } - public BeatmapStore(Storage storage, SQLiteConnection connection, RulesetDatabase rulesets, IIpcHost importHost = null) + public BeatmapStore(Storage storage, FileDatabase files, SQLiteConnection connection, RulesetDatabase rulesets, IIpcHost importHost = null) { Database = new BeatmapDatabase(connection); Database.BeatmapSetAdded += s => BeatmapSetAdded?.Invoke(s); Database.BeatmapSetRemoved += s => BeatmapSetRemoved?.Invoke(s); this.storage = storage; + this.files = files; this.rulesets = rulesets; + if (importHost != null) ipc = new BeatmapIPCChannel(importHost, this); } @@ -61,7 +67,8 @@ namespace osu.Game.Beatmaps { try { - Import(ArchiveReader.GetReader(storage, path)); + using (ArchiveReader reader = getReaderFrom(path)) + Import(reader); // We may or may not want to delete the file depending on where it is stored. // e.g. reconstructing/repairing database with beatmaps from default storage. @@ -88,28 +95,44 @@ namespace osu.Game.Beatmaps /// Import a beatmap from an . /// /// The beatmap to be imported. - public void Import(ArchiveReader archiveReader) + public BeatmapSetInfo Import(ArchiveReader archiveReader) { BeatmapSetInfo set = importToStorage(archiveReader); //If we have an ID then we already exist in the database. if (set.ID == 0) Database.Import(new[] { set }); + + return set; } /// /// Delete a beatmap from the store. /// /// The beatmap to delete. - public void Delete(BeatmapSetInfo beatmapSet) => Database.Delete(new[] { beatmapSet }); + public void Delete(BeatmapSetInfo beatmapSet) + { + if (!Database.Delete(beatmapSet)) return; + + if (!beatmapSet.Protected) + files.Dereference(beatmapSet.Files); + } + + public void Undelete(BeatmapSetInfo beatmapSet) + { + if (!Database.Undelete(beatmapSet)) return; + + files.Reference(beatmapSet.Files); + } public WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo, WorkingBeatmap previous = null) { if (beatmapInfo == null || beatmapInfo == DefaultBeatmap?.BeatmapInfo) return DefaultBeatmap; - if (beatmapInfo.BeatmapSet == null || beatmapInfo.Ruleset == null) - beatmapInfo = Database.GetChildren(beatmapInfo, true); + beatmapInfo = Database.GetChildren(beatmapInfo, true); + + Database.GetChildren(beatmapInfo.BeatmapSet, true); if (beatmapInfo.BeatmapSet == null) throw new InvalidOperationException($@"Beatmap set {beatmapInfo.BeatmapSetInfoID} is not in the local database."); @@ -117,7 +140,7 @@ namespace osu.Game.Beatmaps if (beatmapInfo.Metadata == null) beatmapInfo.Metadata = beatmapInfo.BeatmapSet.Metadata; - WorkingBeatmap working = new BeatmapStoreWorkingBeatmap(() => string.IsNullOrEmpty(beatmapInfo.BeatmapSet.Path) ? null : ArchiveReader.GetReader(storage, beatmapInfo.BeatmapSet.Path), beatmapInfo); + WorkingBeatmap working = new BeatmapStoreWorkingBeatmap(files.Store, beatmapInfo); previous?.TransferTo(working); @@ -132,38 +155,43 @@ namespace osu.Game.Beatmaps Database.Reset(); } + private ArchiveReader getReaderFrom(string path) + { + if (ZipFile.IsZipFile(path)) + return new OszArchiveReader(storage.GetStream(path)); + else + return new LegacyFilesystemReader(path); + } + private BeatmapSetInfo importToStorage(ArchiveReader archiveReader) { BeatmapMetadata metadata; - using (var stream = new StreamReader(archiveReader.GetStream(archiveReader.BeatmapFilenames[0]))) + using (var stream = new StreamReader(archiveReader.GetStream(archiveReader.Filenames.First(f => f.EndsWith(".osu"))))) metadata = BeatmapDecoder.GetDecoder(stream).Decode(stream).Metadata; - string hash; - string path; + MemoryStream hashable = new MemoryStream(); - using (var input = archiveReader.GetUnderlyingStream()) + List fileInfos = new List(); + + foreach (string file in archiveReader.Filenames) { - hash = input.GetMd5Hash(); - input.Seek(0, SeekOrigin.Begin); - path = Path.Combine(@"beatmaps", hash.Remove(1), hash.Remove(2), hash); - if (!storage.Exists(path)) - using (var output = storage.GetStream(path, FileAccess.Write)) - input.CopyTo(output); + using (Stream s = archiveReader.GetStream(file)) + { + fileInfos.Add(files.Add(s, file)); + s.CopyTo(hashable); + } } - var existing = Database.Query().FirstOrDefault(b => b.Hash == hash); + var overallHash = hashable.GetMd5Hash(); + + var existing = Database.Query().FirstOrDefault(b => b.Hash == overallHash); if (existing != null) { Database.GetChildren(existing); - if (existing.DeletePending) - { - existing.DeletePending = false; - Database.Update(existing, false); - BeatmapSetAdded?.Invoke(existing); - } + Undelete(existing); return existing; } @@ -172,41 +200,51 @@ namespace osu.Game.Beatmaps { OnlineBeatmapSetID = metadata.OnlineBeatmapSetID, Beatmaps = new List(), - Path = path, - Hash = hash, + Hash = overallHash, + Files = fileInfos, Metadata = metadata }; - using (var archive = ArchiveReader.GetReader(storage, path)) + var mapNames = archiveReader.Filenames.Where(f => f.EndsWith(".osu")); + + foreach (var name in mapNames) { - string[] mapNames = archive.BeatmapFilenames; - foreach (var name in mapNames) - using (var raw = archive.GetStream(name)) - using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit - using (var sr = new StreamReader(ms)) - { - raw.CopyTo(ms); - ms.Position = 0; + using (var raw = archiveReader.GetStream(name)) + using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit + using (var sr = new StreamReader(ms)) + { + raw.CopyTo(ms); + ms.Position = 0; - var decoder = BeatmapDecoder.GetDecoder(sr); - Beatmap beatmap = decoder.Decode(sr); + var decoder = BeatmapDecoder.GetDecoder(sr); + Beatmap beatmap = decoder.Decode(sr); - beatmap.BeatmapInfo.Path = name; - beatmap.BeatmapInfo.Hash = ms.GetMd5Hash(); + beatmap.BeatmapInfo.Path = name; + beatmap.BeatmapInfo.Hash = ms.GetMd5Hash(); - // TODO: Diff beatmap metadata with set metadata and leave it here if necessary - beatmap.BeatmapInfo.Metadata = null; + // TODO: Diff beatmap metadata with set metadata and leave it here if necessary + beatmap.BeatmapInfo.Metadata = null; - // TODO: this should be done in a better place once we actually need to dynamically update it. - beatmap.BeatmapInfo.Ruleset = rulesets.Query().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID); - beatmap.BeatmapInfo.StarDifficulty = rulesets.Query().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID)?.CreateInstance()?.CreateDifficultyCalculator(beatmap).Calculate() ?? 0; + // TODO: this should be done in a better place once we actually need to dynamically update it. + beatmap.BeatmapInfo.Ruleset = rulesets.Query().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID); + beatmap.BeatmapInfo.StarDifficulty = rulesets.Query().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID)?.CreateInstance()?.CreateDifficultyCalculator(beatmap) + .Calculate() ?? 0; - beatmapSet.Beatmaps.Add(beatmap.BeatmapInfo); - } - beatmapSet.StoryboardFile = archive.StoryboardFilename; + beatmapSet.Beatmaps.Add(beatmap.BeatmapInfo); + } } return beatmapSet; } + + public BeatmapSetInfo QueryBeatmapSet(Func func) + { + BeatmapSetInfo set = Database.Query().FirstOrDefault(func); + + if (set != null) + Database.GetChildren(set, true); + + return set; + } } } diff --git a/osu.Game/Beatmaps/BeatmapStoreWorkingBeatmap.cs b/osu.Game/Beatmaps/BeatmapStoreWorkingBeatmap.cs index 478cd7d87d..3ddd7eecd6 100644 --- a/osu.Game/Beatmaps/BeatmapStoreWorkingBeatmap.cs +++ b/osu.Game/Beatmaps/BeatmapStoreWorkingBeatmap.cs @@ -1,8 +1,8 @@ // Copyright (c) 2007-2017 ppy Pty Ltd . // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE -using System; using System.IO; +using System.Linq; using osu.Framework.Audio.Track; using osu.Framework.Graphics.Textures; using osu.Framework.IO.Stores; @@ -12,12 +12,12 @@ namespace osu.Game.Beatmaps { internal class BeatmapStoreWorkingBeatmap : WorkingBeatmap { - private readonly Func> getStore; + private readonly IResourceStore store; - public BeatmapStoreWorkingBeatmap(Func> getStore, BeatmapInfo beatmapInfo) + public BeatmapStoreWorkingBeatmap(IResourceStore store, BeatmapInfo beatmapInfo) : base(beatmapInfo) { - this.getStore = getStore; + this.store = store; } protected override Beatmap GetBeatmap() @@ -27,7 +27,7 @@ namespace osu.Game.Beatmaps Beatmap beatmap; BeatmapDecoder decoder; - using (var stream = new StreamReader(getStore().GetStream(BeatmapInfo.Path))) + using (var stream = new StreamReader(store.GetStream(getPathForFile(BeatmapInfo.Path)))) { decoder = BeatmapDecoder.GetDecoder(stream); beatmap = decoder.Decode(stream); @@ -36,7 +36,7 @@ namespace osu.Game.Beatmaps if (beatmap == null || BeatmapSetInfo.StoryboardFile == null) return beatmap; - using (var stream = new StreamReader(getStore().GetStream(BeatmapSetInfo.StoryboardFile))) + using (var stream = new StreamReader(store.GetStream(getPathForFile(BeatmapSetInfo.StoryboardFile)))) decoder.Decode(stream, beatmap); @@ -45,6 +45,8 @@ namespace osu.Game.Beatmaps catch { return null; } } + private string getPathForFile(string filename) => BeatmapSetInfo.Files.First(f => f.Filename == filename).StoragePath; + protected override Texture GetBackground() { if (Metadata?.BackgroundFile == null) @@ -52,7 +54,7 @@ namespace osu.Game.Beatmaps try { - return new TextureStore(new RawTextureLoaderStore(getStore()), false).Get(Metadata.BackgroundFile); + return new TextureStore(new RawTextureLoaderStore(store), false).Get(getPathForFile(Metadata.BackgroundFile)); } catch { return null; } } @@ -61,7 +63,7 @@ namespace osu.Game.Beatmaps { try { - var trackData = getStore().GetStream(Metadata.AudioFile); + var trackData = store.GetStream(getPathForFile(Metadata.AudioFile)); return trackData == null ? null : new TrackBass(trackData); } catch { return new TrackVirtual(); } diff --git a/osu.Game/Beatmaps/Formats/BeatmapDecoder.cs b/osu.Game/Beatmaps/Formats/BeatmapDecoder.cs index a365974cfa..1c3eadc91e 100644 --- a/osu.Game/Beatmaps/Formats/BeatmapDecoder.cs +++ b/osu.Game/Beatmaps/Formats/BeatmapDecoder.cs @@ -12,6 +12,11 @@ namespace osu.Game.Beatmaps.Formats { private static readonly Dictionary decoders = new Dictionary(); + static BeatmapDecoder() + { + OsuLegacyDecoder.Register(); + } + public static BeatmapDecoder GetDecoder(StreamReader stream) { string line = stream.ReadLine()?.Trim(); diff --git a/osu.Game/Beatmaps/IO/ArchiveReader.cs b/osu.Game/Beatmaps/IO/ArchiveReader.cs index 7ff5668b6f..3af2a7ea2a 100644 --- a/osu.Game/Beatmaps/IO/ArchiveReader.cs +++ b/osu.Game/Beatmaps/IO/ArchiveReader.cs @@ -5,45 +5,11 @@ using System; using System.Collections.Generic; using System.IO; using osu.Framework.IO.Stores; -using osu.Framework.Platform; namespace osu.Game.Beatmaps.IO { public abstract class ArchiveReader : IDisposable, IResourceStore { - private class Reader - { - public Func Test; - public Type Type; - } - - private static readonly List readers = new List(); - - public static ArchiveReader GetReader(Storage storage, string path) - { - foreach (var reader in readers) - { - if (reader.Test(storage, path)) - return (ArchiveReader)Activator.CreateInstance(reader.Type, storage.GetStream(path)); - } - throw new IOException(@"Unknown file format"); - } - - protected static void AddReader(Func test) where T : ArchiveReader - { - readers.Add(new Reader { Test = test, Type = typeof(T) }); - } - - /// - /// Gets a list of beatmap file names. - /// - public string[] BeatmapFilenames { get; protected set; } - - /// - /// The storyboard filename. Null if no storyboard is present. - /// - public string StoryboardFilename { get; protected set; } - /// /// Opens a stream for reading a specific file from this archive. /// @@ -51,6 +17,8 @@ namespace osu.Game.Beatmaps.IO public abstract void Dispose(); + public abstract IEnumerable Filenames { get; } + public virtual byte[] Get(string name) { using (Stream input = GetStream(name)) diff --git a/osu.Game/Beatmaps/IO/LegacyFilesystemReader.cs b/osu.Game/Beatmaps/IO/LegacyFilesystemReader.cs new file mode 100644 index 0000000000..dc38181717 --- /dev/null +++ b/osu.Game/Beatmaps/IO/LegacyFilesystemReader.cs @@ -0,0 +1,33 @@ +// Copyright (c) 2007-2017 ppy Pty Ltd . +// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE + +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace osu.Game.Beatmaps.IO +{ + /// + /// Reads an extracted legacy beatmap from disk. + /// + public class LegacyFilesystemReader : ArchiveReader + { + private readonly string path; + + public LegacyFilesystemReader(string path) + { + this.path = path; + } + + public override Stream GetStream(string name) => File.OpenRead(Path.Combine(path, name)); + + public override void Dispose() + { + // no-op + } + + public override IEnumerable Filenames => Directory.GetFiles(path).Select(Path.GetFileName).ToArray(); + + public override Stream GetUnderlyingStream() => null; + } +} diff --git a/osu.Game/Beatmaps/IO/OszArchiveReader.cs b/osu.Game/Beatmaps/IO/OszArchiveReader.cs index a22a5f4cce..4e0c40d28e 100644 --- a/osu.Game/Beatmaps/IO/OszArchiveReader.cs +++ b/osu.Game/Beatmaps/IO/OszArchiveReader.cs @@ -1,25 +1,15 @@ // Copyright (c) 2007-2017 ppy Pty Ltd . // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE +using System.Collections.Generic; using System.IO; using System.Linq; using Ionic.Zip; -using osu.Game.Beatmaps.Formats; namespace osu.Game.Beatmaps.IO { public sealed class OszArchiveReader : ArchiveReader { - public static void Register() - { - AddReader((storage, path) => - { - using (var stream = storage.GetStream(path)) - return stream != null && ZipFile.IsZipFile(stream, false); - }); - OsuLegacyDecoder.Register(); - } - private readonly Stream archiveStream; private readonly ZipFile archive; @@ -27,13 +17,6 @@ namespace osu.Game.Beatmaps.IO { this.archiveStream = archiveStream; archive = ZipFile.Read(archiveStream); - - BeatmapFilenames = archive.Entries.Where(e => e.FileName.EndsWith(@".osu")).Select(e => e.FileName).ToArray(); - - if (BeatmapFilenames.Length == 0) - throw new FileNotFoundException(@"This directory contains no beatmaps"); - - StoryboardFilename = archive.Entries.Where(e => e.FileName.EndsWith(@".osb")).Select(e => e.FileName).FirstOrDefault(); } public override Stream GetStream(string name) @@ -41,7 +24,16 @@ namespace osu.Game.Beatmaps.IO ZipEntry entry = archive.Entries.SingleOrDefault(e => e.FileName == name); if (entry == null) throw new FileNotFoundException(); - return entry.OpenReader(); + + // allow seeking + MemoryStream copy = new MemoryStream(); + + using (Stream s = entry.OpenReader()) + s.CopyTo(copy); + + copy.Position = 0; + + return copy; } public override void Dispose() @@ -50,6 +42,8 @@ namespace osu.Game.Beatmaps.IO archiveStream.Dispose(); } + public override IEnumerable Filenames => archive.Entries.Select(e => e.FileName).ToArray(); + public override Stream GetUnderlyingStream() => archiveStream; } } \ No newline at end of file diff --git a/osu.Game/Database/DatabaseStore.cs b/osu.Game/Database/DatabaseStore.cs index 0102998604..5187769bb2 100644 --- a/osu.Game/Database/DatabaseStore.cs +++ b/osu.Game/Database/DatabaseStore.cs @@ -3,9 +3,9 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Linq.Expressions; using osu.Framework.Logging; +using osu.Framework.Platform; using SQLite.Net; using SQLiteNetExtensions.Extensions; @@ -13,10 +13,12 @@ namespace osu.Game.Database { public abstract class DatabaseStore { - protected SQLiteConnection Connection { get; } + protected readonly Storage Storage; + protected readonly SQLiteConnection Connection; - protected DatabaseStore(SQLiteConnection connection) + protected DatabaseStore(SQLiteConnection connection, Storage storage = null) { + Storage = storage; Connection = connection; try @@ -63,15 +65,5 @@ namespace osu.Game.Database } protected abstract Type[] ValidTypes { get; } - - public void Update(T record, bool cascade = true) where T : class - { - if (ValidTypes.All(t => t != typeof(T))) - throw new ArgumentException("Must be a type managed by BeatmapDatabase", nameof(T)); - if (cascade) - Connection.UpdateWithChildren(record); - else - Connection.Update(record); - } } } \ No newline at end of file diff --git a/osu.Game/IO/FileDatabase.cs b/osu.Game/IO/FileDatabase.cs new file mode 100644 index 0000000000..b61fbdb0e1 --- /dev/null +++ b/osu.Game/IO/FileDatabase.cs @@ -0,0 +1,114 @@ +// Copyright (c) 2007-2017 ppy Pty Ltd . +// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using osu.Framework.Extensions; +using osu.Framework.IO.Stores; +using osu.Framework.Logging; +using osu.Framework.Platform; +using osu.Game.Database; +using SQLite.Net; + +namespace osu.Game.IO +{ + /// + /// Handles the Store and retrieval of Files/FileSets to the database backing + /// + public class FileDatabase : DatabaseStore + { + private const string prefix = "files"; + + public readonly ResourceStore Store; + + public FileDatabase(SQLiteConnection connection, Storage storage) : base(connection, storage) + { + Store = new NamespacedResourceStore(new StorageBackedResourceStore(storage), prefix); + } + + protected override Type[] ValidTypes => new[] { + typeof(FileInfo), + }; + + protected override void Prepare(bool reset = false) + { + if (reset) + Connection.DropTable(); + + Connection.CreateTable(); + + deletePending(); + } + + public FileInfo Add(Stream data, string filename = null) + { + string hash = data.GetMd5Hash(); + + var info = new FileInfo + { + Filename = filename, + Hash = hash, + }; + + var existing = Connection.Table().FirstOrDefault(f => f.Hash == info.Hash); + + if (existing != null) + { + info = existing; + } + else + { + string path = Path.Combine(prefix, info.StoragePath); + + data.Seek(0, SeekOrigin.Begin); + + if (!Storage.Exists(path)) + using (var output = Storage.GetStream(path, FileAccess.Write)) + data.CopyTo(output); + + data.Seek(0, SeekOrigin.Begin); + + Connection.Insert(info); + } + + Reference(new[] { info }); + return info; + } + + public void Reference(IEnumerable files) + { + foreach (var f in files) + { + f.ReferenceCount++; + Connection.Update(f); + } + } + + public void Dereference(IEnumerable files) + { + foreach (var f in files) + { + f.ReferenceCount--; + Connection.Update(f); + } + } + + private void deletePending() + { + foreach (var f in GetAllWithChildren(f => f.ReferenceCount < 1)) + { + try + { + Connection.Delete(f); + Storage.Delete(Path.Combine(prefix, f.Hash)); + } + catch (Exception e) + { + Logger.Error(e, $@"Could not delete beatmap {f}"); + } + } + } + } +} \ No newline at end of file diff --git a/osu.Game/IO/FileInfo.cs b/osu.Game/IO/FileInfo.cs new file mode 100644 index 0000000000..6f4c4b26e8 --- /dev/null +++ b/osu.Game/IO/FileInfo.cs @@ -0,0 +1,24 @@ +// Copyright (c) 2007-2017 ppy Pty Ltd . +// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE + +using System.IO; +using SQLite.Net.Attributes; + +namespace osu.Game.IO +{ + public class FileInfo + { + [PrimaryKey, AutoIncrement] + public int ID { get; set; } + + public string Filename { get; set; } + + [Indexed(Unique = true)] + public string Hash { get; set; } + + public string StoragePath => Path.Combine(Hash.Remove(1), Hash.Remove(2), Hash); + + [Indexed] + public int ReferenceCount { get; set; } + } +} diff --git a/osu.Game/OsuGameBase.cs b/osu.Game/OsuGameBase.cs index 82e4b5d4ca..8ad07cd7bc 100644 --- a/osu.Game/OsuGameBase.cs +++ b/osu.Game/OsuGameBase.cs @@ -11,7 +11,6 @@ using osu.Framework.Graphics.Containers; using osu.Framework.IO.Stores; using osu.Framework.Platform; using osu.Game.Beatmaps; -using osu.Game.Beatmaps.IO; using osu.Game.Configuration; using osu.Game.Graphics; using osu.Game.Graphics.Cursor; @@ -19,6 +18,7 @@ using osu.Game.Graphics.Processing; using osu.Game.Online.API; using SQLite.Net; using osu.Framework.Graphics.Performance; +using osu.Game.IO; using osu.Game.Rulesets; using osu.Game.Rulesets.Scoring; @@ -32,6 +32,8 @@ namespace osu.Game protected RulesetDatabase RulesetDatabase; + protected FileDatabase FileDatabase; + protected ScoreDatabase ScoreDatabase; protected override string MainResourceFile => @"osu.Game.Resources.dll"; @@ -96,7 +98,8 @@ namespace osu.Game SQLiteConnection connection = Host.Storage.GetDatabase(@"client"); dependencies.Cache(RulesetDatabase = new RulesetDatabase(connection)); - dependencies.Cache(BeatmapStore = new BeatmapStore(Host.Storage, connection, RulesetDatabase, Host)); + dependencies.Cache(FileDatabase = new FileDatabase(connection, Host.Storage)); + dependencies.Cache(BeatmapStore = new BeatmapStore(Host.Storage, FileDatabase, connection, RulesetDatabase, Host)); dependencies.Cache(ScoreDatabase = new ScoreDatabase(Host.Storage, connection, Host, BeatmapStore)); dependencies.Cache(new OsuColour()); @@ -131,8 +134,6 @@ namespace osu.Game Beatmap = new NonNullableBindable(defaultBeatmap); BeatmapStore.DefaultBeatmap = defaultBeatmap; - OszArchiveReader.Register(); - dependencies.Cache(API = new APIAccess { Username = LocalConfig.Get(OsuSetting.Username), diff --git a/osu.Game/Screens/Menu/Intro.cs b/osu.Game/Screens/Menu/Intro.cs index a8f2c3c78f..55eedaba9a 100644 --- a/osu.Game/Screens/Menu/Intro.cs +++ b/osu.Game/Screens/Menu/Intro.cs @@ -22,7 +22,7 @@ namespace osu.Game.Screens.Menu { private readonly OsuLogo logo; - public const string MENU_MUSIC_BEATMAP_HASH = "21c1271b91234385978b5418881fdd88"; + private const string menu_music_beatmap_hash = "715a09144f885d746644c1983e285044"; /// /// Whether we have loaded the menu previously. @@ -84,23 +84,18 @@ namespace osu.Game.Screens.Menu if (setInfo == null) { - var query = beatmaps.Database.Query().Where(b => b.Hash == MENU_MUSIC_BEATMAP_HASH); - - setInfo = query.FirstOrDefault(); + setInfo = beatmaps.QueryBeatmapSet(b => b.Hash == menu_music_beatmap_hash); if (setInfo == null) { // we need to import the default menu background beatmap - beatmaps.Import(new OszArchiveReader(game.Resources.GetStream(@"Tracks/circles.osz"))); + setInfo = beatmaps.Import(new OszArchiveReader(game.Resources.GetStream(@"Tracks/circles.osz"))); - setInfo = query.First(); - - setInfo.DeletePending = true; - beatmaps.Database.Update(setInfo, false); + setInfo.Protected = true; + beatmaps.Delete(setInfo); } } - beatmaps.Database.GetChildren(setInfo); Beatmap.Value = beatmaps.GetWorkingBeatmap(setInfo.Beatmaps[0]); track = Beatmap.Value.Track; diff --git a/osu.Game/osu.Game.csproj b/osu.Game/osu.Game.csproj index 43c300dc30..13c4056bba 100644 --- a/osu.Game/osu.Game.csproj +++ b/osu.Game/osu.Game.csproj @@ -75,9 +75,11 @@ + + @@ -89,6 +91,8 @@ + +