Introduce a reference counting file store

osu.Desktop.VisualTests/Tests/TestCasePlaySongSelect.cs

@@ -31,7 +31,7 @@ namespace osu.Desktop.VisualTests.Tests
                 var backingDatabase = storage.GetDatabase(@"client");
 
                 rulesets = new RulesetDatabase(backingDatabase);
-                store = new BeatmapStore(storage, backingDatabase, rulesets);
+                store = new BeatmapStore(storage, null, backingDatabase, rulesets);
 
                 var sets = new List<BeatmapSetInfo>();
 

osu.Desktop/Beatmaps/IO/LegacyFilesystemReader.cs

@@ -1,43 +0,0 @@
-// Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
-// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
-
-using System.IO;
-using System.Linq;
-using osu.Game.Beatmaps.IO;
-
-namespace osu.Desktop.Beatmaps.IO
-{
-    /// <summary>
-    /// Reads an extracted legacy beatmap from disk.
-    /// </summary>
-    public class LegacyFilesystemReader : ArchiveReader
-    {
-        public static void Register() => AddReader<LegacyFilesystemReader>((storage, path) => Directory.Exists(path));
-
-        private readonly string basePath;
-
-        public LegacyFilesystemReader(string path)
-        {
-            basePath = path;
-
-            BeatmapFilenames = Directory.GetFiles(basePath, @"*.osu").Select(Path.GetFileName).ToArray();
-
-            if (BeatmapFilenames.Length == 0)
-                throw new FileNotFoundException(@"This directory contains no beatmaps");
-
-            StoryboardFilename = Directory.GetFiles(basePath, @"*.osb").Select(Path.GetFileName).FirstOrDefault();
-        }
-
-        public override Stream GetStream(string name)
-        {
-            return File.OpenRead(Path.Combine(basePath, name));
-        }
-
-        public override void Dispose()
-        {
-            // no-op
-        }
-
-        public override Stream GetUnderlyingStream() => null;
-    }
-}

osu.Desktop/Program.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.IO;
-using osu.Desktop.Beatmaps.IO;
 using osu.Framework.Desktop;
 using osu.Framework.Desktop.Platform;
 using osu.Game.IPC;
@@ -15,8 +14,6 @@ namespace osu.Desktop
         [STAThread]
         public static int Main(string[] args)
         {
-            LegacyFilesystemReader.Register();
-
             // Back up the cwd before DesktopGameHost changes it
             var cwd = Environment.CurrentDirectory;
 

osu.Desktop/osu.Desktop.csproj

@@ -228,7 +228,6 @@
     <Compile Include="OsuGameDesktop.cs" />
     <Compile Include="Overlays\VersionManager.cs" />
     <Compile Include="Program.cs" />
-    <Compile Include="Beatmaps\IO\LegacyFilesystemReader.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>

osu.Game.Tests/Beatmaps/Formats/OsuLegacyDecoderTest.cs

@@ -16,12 +16,6 @@ namespace osu.Game.Tests.Beatmaps.Formats
     [TestFixture]
     public class OsuLegacyDecoderTest
     {
-        [OneTimeSetUpAttribute]
-        public void SetUp()
-        {
-            OsuLegacyDecoder.Register();
-        }
-
         [Test]
         public void TestDecodeMetadata()
         {

osu.Game.Tests/Beatmaps/IO/OszArchiveReaderTest.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
 using System.IO;
+using System.Linq;
 using NUnit.Framework;
 using osu.Game.Beatmaps;
 using osu.Game.Beatmaps.IO;
@@ -13,12 +14,6 @@ namespace osu.Game.Tests.Beatmaps.IO
     [TestFixture]
     public class OszArchiveReaderTest
     {
-        [OneTimeSetUpAttribute]
-        public void SetUp()
-        {
-            OszArchiveReader.Register();
-        }
-
         [Test]
         public void TestReadBeatmaps()
         {
@@ -40,7 +35,7 @@ namespace osu.Game.Tests.Beatmaps.IO
                     "Soleily - Renatus (MMzz) [Muzukashii].osu",
                     "Soleily - Renatus (MMzz) [Oni].osu"
                 };
-                var maps = reader.BeatmapFilenames;
+                var maps = reader.Filenames.ToArray();
                 foreach (var map in expected)
                     Assert.Contains(map, maps);
             }

osu.Game/Beatmaps/BeatmapDatabase.cs

@@ -5,7 +5,6 @@ using System;
 using System.Collections.Generic;
 using osu.Framework.Logging;
 using osu.Game.Database;
-using osu.Game.Screens.Menu;
 using SQLite.Net;
 using SQLiteNetExtensions.Extensions;
 
@@ -19,11 +18,13 @@ namespace osu.Game.Beatmaps
         public event Action<BeatmapSetInfo> BeatmapSetAdded;
         public event Action<BeatmapSetInfo> BeatmapSetRemoved;
 
-        public BeatmapDatabase(SQLiteConnection connection) : base(connection)
+        public BeatmapDatabase(SQLiteConnection connection)
+            : base(connection)
         {
         }
 
-        protected override Type[] ValidTypes => new[] {
+        protected override Type[] ValidTypes => new[]
+        {
             typeof(BeatmapSetInfo),
             typeof(BeatmapInfo),
             typeof(BeatmapMetadata),
@@ -37,17 +38,21 @@ namespace osu.Game.Beatmaps
                 Connection.DropTable<BeatmapMetadata>();
                 Connection.DropTable<BeatmapDifficulty>();
                 Connection.DropTable<BeatmapSetInfo>();
+                Connection.DropTable<BeatmapSetFileInfo>();
                 Connection.DropTable<BeatmapInfo>();
             }
 
             Connection.CreateTable<BeatmapMetadata>();
             Connection.CreateTable<BeatmapDifficulty>();
             Connection.CreateTable<BeatmapSetInfo>();
+            Connection.CreateTable<BeatmapSetFileInfo>();
             Connection.CreateTable<BeatmapInfo>();
 
             deletePending();
         }
 
+        public void Update(BeatmapSetInfo setInfo) => Connection.Update(setInfo);
+
         public void Import(IEnumerable<BeatmapSetInfo> beatmapSets)
         {
             lock (Connection)
@@ -64,24 +69,32 @@ namespace osu.Game.Beatmaps
             }
         }
 
-        public void Delete(IEnumerable<BeatmapSetInfo> beatmapSets)
+        public bool Delete(BeatmapSetInfo set)
         {
-            foreach (var s in beatmapSets)
+            if (set.DeletePending) return false;
-            {
+
-                s.DeletePending = true;
+            set.DeletePending = true;
-                Update(s, false);
+            Connection.Update(set);
-                BeatmapSetRemoved?.Invoke(s);
+
-            }
+            BeatmapSetRemoved?.Invoke(set);
+            return true;
+        }
+
+        public bool Undelete(BeatmapSetInfo set)
+        {
+            if (!set.DeletePending) return false;
+
+            set.DeletePending = false;
+            Connection.Update(set);
+
+            BeatmapSetAdded?.Invoke(set);
+            return true;
         }
 
         private void deletePending()
         {
-            foreach (var b in GetAllWithChildren<BeatmapSetInfo>(b => b.DeletePending))
+            foreach (var b in GetAllWithChildren<BeatmapSetInfo>(b => b.DeletePending && !b.Protected))
             {
-                if (b.Hash == Intro.MENU_MUSIC_BEATMAP_HASH)
-                    // this is a bit hacky, but will do for now.
-                    continue;
-
                 try
                 {
                     foreach (var i in b.Beatmaps)

osu.Game/Beatmaps/BeatmapInfo.cs

@@ -95,11 +95,11 @@ namespace osu.Game.Beatmaps
         }
 
         public bool AudioEquals(BeatmapInfo other) => other != null && BeatmapSet != null && other.BeatmapSet != null &&
-                                                      BeatmapSet.Path == other.BeatmapSet.Path &&
+                                                      BeatmapSet.Hash == other.BeatmapSet.Hash &&
                                                       (Metadata ?? BeatmapSet.Metadata).AudioFile == (other.Metadata ?? other.BeatmapSet.Metadata).AudioFile;
 
         public bool BackgroundEquals(BeatmapInfo other) => other != null && BeatmapSet != null && other.BeatmapSet != null &&
-                                                      BeatmapSet.Path == other.BeatmapSet.Path &&
+                                                      BeatmapSet.Hash == other.BeatmapSet.Hash &&
                                                       (Metadata ?? BeatmapSet.Metadata).BackgroundFile == (other.Metadata ?? other.BeatmapSet.Metadata).BackgroundFile;
     }
 }

osu.Game/Beatmaps/BeatmapSetFileInfo.cs

@@ -0,0 +1,17 @@
+// Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
+// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
+
+using osu.Game.IO;
+using SQLiteNetExtensions.Attributes;
+
+namespace osu.Game.Beatmaps
+{
+    public class BeatmapSetFileInfo
+    {
+        [ForeignKey(typeof(BeatmapSetInfo))]
+        public int BeatmapSetInfoID { get; set; }
+
+        [ForeignKey(typeof(FileInfo))]
+        public int FileInfoID { get; set; }
+    }
+}

osu.Game/Beatmaps/BeatmapSetInfo.cs

@@ -3,6 +3,7 @@
 
 using System.Collections.Generic;
 using System.Linq;
+using osu.Game.IO;
 using SQLite.Net.Attributes;
 using SQLiteNetExtensions.Attributes;
 
@@ -34,8 +35,11 @@ namespace osu.Game.Beatmaps
 
         public string Hash { get; set; }
 
-        public string Path { get; set; }
+        public string StoryboardFile => Files.FirstOrDefault(f => f.Filename.EndsWith(".osb"))?.Filename;
 
-        public string StoryboardFile { get; set; }
+        [ManyToMany(typeof(BeatmapSetFileInfo))]
+        public List<FileInfo> Files { get; set; }
+
+        public bool Protected { get; set; }
     }
 }

osu.Game/Beatmaps/BeatmapStore.cs

@@ -5,14 +5,17 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Ionic.Zip;
 using osu.Framework.Extensions;
 using osu.Framework.Logging;
 using osu.Framework.Platform;
 using osu.Game.Beatmaps.Formats;
 using osu.Game.Beatmaps.IO;
+using osu.Game.IO;
 using osu.Game.IPC;
 using osu.Game.Rulesets;
 using SQLite.Net;
+using FileInfo = osu.Game.IO.FileInfo;
 
 namespace osu.Game.Beatmaps
 {
@@ -25,6 +28,7 @@ namespace osu.Game.Beatmaps
         public readonly BeatmapDatabase Database;
 
         private readonly Storage storage;
+        private readonly FileDatabase files;
 
         private readonly RulesetDatabase rulesets;
 
@@ -39,14 +43,16 @@ namespace osu.Game.Beatmaps
         /// </summary>
         public WorkingBeatmap DefaultBeatmap { private get; set; }
 
-        public BeatmapStore(Storage storage, SQLiteConnection connection, RulesetDatabase rulesets, IIpcHost importHost = null)
+        public BeatmapStore(Storage storage, FileDatabase files, SQLiteConnection connection, RulesetDatabase rulesets, IIpcHost importHost = null)
         {
             Database = new BeatmapDatabase(connection);
             Database.BeatmapSetAdded += s => BeatmapSetAdded?.Invoke(s);
             Database.BeatmapSetRemoved += s => BeatmapSetRemoved?.Invoke(s);
 
             this.storage = storage;
+            this.files = files;
             this.rulesets = rulesets;
+
             if (importHost != null)
                 ipc = new BeatmapIPCChannel(importHost, this);
         }
@@ -61,7 +67,8 @@ namespace osu.Game.Beatmaps
             {
                 try
                 {
-                    Import(ArchiveReader.GetReader(storage, path));
+                    using (ArchiveReader reader = getReaderFrom(path))
+                        Import(reader);
 
                     // We may or may not want to delete the file depending on where it is stored.
                     //  e.g. reconstructing/repairing database with beatmaps from default storage.
@@ -88,28 +95,44 @@ namespace osu.Game.Beatmaps
         /// Import a beatmap from an <see cref="ArchiveReader"/>.
         /// </summary>
         /// <param name="archiveReader">The beatmap to be imported.</param>
-        public void Import(ArchiveReader archiveReader)
+        public BeatmapSetInfo Import(ArchiveReader archiveReader)
         {
             BeatmapSetInfo set = importToStorage(archiveReader);
 
             //If we have an ID then we already exist in the database.
             if (set.ID == 0)
                 Database.Import(new[] { set });
+
+            return set;
         }
 
         /// <summary>
         /// Delete a beatmap from the store.
         /// </summary>
         /// <param name="beatmapSet">The beatmap to delete.</param>
-        public void Delete(BeatmapSetInfo beatmapSet) => Database.Delete(new[] { beatmapSet });
+        public void Delete(BeatmapSetInfo beatmapSet)
+        {
+            if (!Database.Delete(beatmapSet)) return;
+
+            if (!beatmapSet.Protected)
+                files.Dereference(beatmapSet.Files);
+        }
+
+        public void Undelete(BeatmapSetInfo beatmapSet)
+        {
+            if (!Database.Undelete(beatmapSet)) return;
+
+            files.Reference(beatmapSet.Files);
+        }
 
         public WorkingBeatmap GetWorkingBeatmap(BeatmapInfo beatmapInfo, WorkingBeatmap previous = null)
         {
             if (beatmapInfo == null || beatmapInfo == DefaultBeatmap?.BeatmapInfo)
                 return DefaultBeatmap;
 
-            if (beatmapInfo.BeatmapSet == null || beatmapInfo.Ruleset == null)
+            beatmapInfo = Database.GetChildren(beatmapInfo, true);
-                beatmapInfo = Database.GetChildren(beatmapInfo, true);
+
+            Database.GetChildren(beatmapInfo.BeatmapSet, true);
 
             if (beatmapInfo.BeatmapSet == null)
                 throw new InvalidOperationException($@"Beatmap set {beatmapInfo.BeatmapSetInfoID} is not in the local database.");
@@ -117,7 +140,7 @@ namespace osu.Game.Beatmaps
             if (beatmapInfo.Metadata == null)
                 beatmapInfo.Metadata = beatmapInfo.BeatmapSet.Metadata;
 
-            WorkingBeatmap working = new BeatmapStoreWorkingBeatmap(() => string.IsNullOrEmpty(beatmapInfo.BeatmapSet.Path) ? null : ArchiveReader.GetReader(storage, beatmapInfo.BeatmapSet.Path), beatmapInfo);
+            WorkingBeatmap working = new BeatmapStoreWorkingBeatmap(files.Store, beatmapInfo);
 
             previous?.TransferTo(working);
 
@@ -132,38 +155,43 @@ namespace osu.Game.Beatmaps
             Database.Reset();
         }
 
+        private ArchiveReader getReaderFrom(string path)
+        {
+            if (ZipFile.IsZipFile(path))
+                return new OszArchiveReader(storage.GetStream(path));
+            else
+                return new LegacyFilesystemReader(path);
+        }
+
         private BeatmapSetInfo importToStorage(ArchiveReader archiveReader)
         {
             BeatmapMetadata metadata;
 
-            using (var stream = new StreamReader(archiveReader.GetStream(archiveReader.BeatmapFilenames[0])))
+            using (var stream = new StreamReader(archiveReader.GetStream(archiveReader.Filenames.First(f => f.EndsWith(".osu")))))
                 metadata = BeatmapDecoder.GetDecoder(stream).Decode(stream).Metadata;
 
-            string hash;
+            MemoryStream hashable = new MemoryStream();
-            string path;
 
-            using (var input = archiveReader.GetUnderlyingStream())
+            List<FileInfo> fileInfos = new List<FileInfo>();
+
+            foreach (string file in archiveReader.Filenames)
             {
-                hash = input.GetMd5Hash();
+                using (Stream s = archiveReader.GetStream(file))
-                input.Seek(0, SeekOrigin.Begin);
+                {
-                path = Path.Combine(@"beatmaps", hash.Remove(1), hash.Remove(2), hash);
+                    fileInfos.Add(files.Add(s, file));
-                if (!storage.Exists(path))
+                    s.CopyTo(hashable);
-                    using (var output = storage.GetStream(path, FileAccess.Write))
+                }
-                        input.CopyTo(output);
             }
 
-            var existing = Database.Query<BeatmapSetInfo>().FirstOrDefault(b => b.Hash == hash);
+            var overallHash = hashable.GetMd5Hash();
+
+            var existing = Database.Query<BeatmapSetInfo>().FirstOrDefault(b => b.Hash == overallHash);
 
             if (existing != null)
             {
                 Database.GetChildren(existing);
 
-                if (existing.DeletePending)
+                Undelete(existing);
-                {
-                    existing.DeletePending = false;
-                    Database.Update(existing, false);
-                    BeatmapSetAdded?.Invoke(existing);
-                }
 
                 return existing;
             }
@@ -172,41 +200,51 @@ namespace osu.Game.Beatmaps
             {
                 OnlineBeatmapSetID = metadata.OnlineBeatmapSetID,
                 Beatmaps = new List<BeatmapInfo>(),
-                Path = path,
+                Hash = overallHash,
-                Hash = hash,
+                Files = fileInfos,
                 Metadata = metadata
             };
 
-            using (var archive = ArchiveReader.GetReader(storage, path))
+            var mapNames = archiveReader.Filenames.Where(f => f.EndsWith(".osu"));
+
+            foreach (var name in mapNames)
             {
-                string[] mapNames = archive.BeatmapFilenames;
+                using (var raw = archiveReader.GetStream(name))
-                foreach (var name in mapNames)
+                using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit
-                    using (var raw = archive.GetStream(name))
+                using (var sr = new StreamReader(ms))
-                    using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit
+                {
-                    using (var sr = new StreamReader(ms))
+                    raw.CopyTo(ms);
-                    {
+                    ms.Position = 0;
-                        raw.CopyTo(ms);
-                        ms.Position = 0;
 
-                        var decoder = BeatmapDecoder.GetDecoder(sr);
+                    var decoder = BeatmapDecoder.GetDecoder(sr);
-                        Beatmap beatmap = decoder.Decode(sr);
+                    Beatmap beatmap = decoder.Decode(sr);
 
-                        beatmap.BeatmapInfo.Path = name;
+                    beatmap.BeatmapInfo.Path = name;
-                        beatmap.BeatmapInfo.Hash = ms.GetMd5Hash();
+                    beatmap.BeatmapInfo.Hash = ms.GetMd5Hash();
 
-                        // TODO: Diff beatmap metadata with set metadata and leave it here if necessary
+                    // TODO: Diff beatmap metadata with set metadata and leave it here if necessary
-                        beatmap.BeatmapInfo.Metadata = null;
+                    beatmap.BeatmapInfo.Metadata = null;
 
-                        // TODO: this should be done in a better place once we actually need to dynamically update it.
+                    // TODO: this should be done in a better place once we actually need to dynamically update it.
-                        beatmap.BeatmapInfo.Ruleset = rulesets.Query<RulesetInfo>().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID);
+                    beatmap.BeatmapInfo.Ruleset = rulesets.Query<RulesetInfo>().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID);
-                        beatmap.BeatmapInfo.StarDifficulty = rulesets.Query<RulesetInfo>().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID)?.CreateInstance()?.CreateDifficultyCalculator(beatmap).Calculate() ?? 0;
+                    beatmap.BeatmapInfo.StarDifficulty = rulesets.Query<RulesetInfo>().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID)?.CreateInstance()?.CreateDifficultyCalculator(beatmap)
+                                                                 .Calculate() ?? 0;
 
-                        beatmapSet.Beatmaps.Add(beatmap.BeatmapInfo);
+                    beatmapSet.Beatmaps.Add(beatmap.BeatmapInfo);
-                    }
+                }
-                beatmapSet.StoryboardFile = archive.StoryboardFilename;
             }
 
             return beatmapSet;
         }
+
+        public BeatmapSetInfo QueryBeatmapSet(Func<BeatmapSetInfo, bool> func)
+        {
+            BeatmapSetInfo set = Database.Query<BeatmapSetInfo>().FirstOrDefault(func);
+
+            if (set != null)
+                Database.GetChildren(set, true);
+
+            return set;
+        }
     }
 }

osu.Game/Beatmaps/BeatmapStoreWorkingBeatmap.cs

@@ -1,8 +1,8 @@
 // Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
-using System;
 using System.IO;
+using System.Linq;
 using osu.Framework.Audio.Track;
 using osu.Framework.Graphics.Textures;
 using osu.Framework.IO.Stores;
@@ -12,12 +12,12 @@ namespace osu.Game.Beatmaps
 {
     internal class BeatmapStoreWorkingBeatmap : WorkingBeatmap
     {
-        private readonly Func<IResourceStore<byte[]>> getStore;
+        private readonly IResourceStore<byte[]> store;
 
-        public BeatmapStoreWorkingBeatmap(Func<IResourceStore<byte[]>> getStore, BeatmapInfo beatmapInfo)
+        public BeatmapStoreWorkingBeatmap(IResourceStore<byte[]> store, BeatmapInfo beatmapInfo)
             : base(beatmapInfo)
         {
-            this.getStore = getStore;
+            this.store = store;
         }
 
         protected override Beatmap GetBeatmap()
@@ -27,7 +27,7 @@ namespace osu.Game.Beatmaps
                 Beatmap beatmap;
 
                 BeatmapDecoder decoder;
-                using (var stream = new StreamReader(getStore().GetStream(BeatmapInfo.Path)))
+                using (var stream = new StreamReader(store.GetStream(getPathForFile(BeatmapInfo.Path))))
                 {
                     decoder = BeatmapDecoder.GetDecoder(stream);
                     beatmap = decoder.Decode(stream);
@@ -36,7 +36,7 @@ namespace osu.Game.Beatmaps
                 if (beatmap == null || BeatmapSetInfo.StoryboardFile == null)
                     return beatmap;
 
-                using (var stream = new StreamReader(getStore().GetStream(BeatmapSetInfo.StoryboardFile)))
+                using (var stream = new StreamReader(store.GetStream(getPathForFile(BeatmapSetInfo.StoryboardFile))))
                     decoder.Decode(stream, beatmap);
 
 
@@ -45,6 +45,8 @@ namespace osu.Game.Beatmaps
             catch { return null; }
         }
 
+        private string getPathForFile(string filename) => BeatmapSetInfo.Files.First(f => f.Filename == filename).StoragePath;
+
         protected override Texture GetBackground()
         {
             if (Metadata?.BackgroundFile == null)
@@ -52,7 +54,7 @@ namespace osu.Game.Beatmaps
 
             try
             {
-                return new TextureStore(new RawTextureLoaderStore(getStore()), false).Get(Metadata.BackgroundFile);
+                return new TextureStore(new RawTextureLoaderStore(store), false).Get(getPathForFile(Metadata.BackgroundFile));
             }
             catch { return null; }
         }
@@ -61,7 +63,7 @@ namespace osu.Game.Beatmaps
         {
             try
             {
-                var trackData = getStore().GetStream(Metadata.AudioFile);
+                var trackData = store.GetStream(getPathForFile(Metadata.AudioFile));
                 return trackData == null ? null : new TrackBass(trackData);
             }
             catch { return new TrackVirtual(); }

osu.Game/Beatmaps/Formats/BeatmapDecoder.cs

@@ -12,6 +12,11 @@ namespace osu.Game.Beatmaps.Formats
     {
         private static readonly Dictionary<string, Type> decoders = new Dictionary<string, Type>();
 
+        static BeatmapDecoder()
+        {
+            OsuLegacyDecoder.Register();
+        }
+
         public static BeatmapDecoder GetDecoder(StreamReader stream)
         {
             string line = stream.ReadLine()?.Trim();

osu.Game/Beatmaps/IO/ArchiveReader.cs

@@ -5,45 +5,11 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using osu.Framework.IO.Stores;
-using osu.Framework.Platform;
 
 namespace osu.Game.Beatmaps.IO
 {
     public abstract class ArchiveReader : IDisposable, IResourceStore<byte[]>
     {
-        private class Reader
-        {
-            public Func<Storage, string, bool> Test;
-            public Type Type;
-        }
-
-        private static readonly List<Reader> readers = new List<Reader>();
-
-        public static ArchiveReader GetReader(Storage storage, string path)
-        {
-            foreach (var reader in readers)
-            {
-                if (reader.Test(storage, path))
-                    return (ArchiveReader)Activator.CreateInstance(reader.Type, storage.GetStream(path));
-            }
-            throw new IOException(@"Unknown file format");
-        }
-
-        protected static void AddReader<T>(Func<Storage, string, bool> test) where T : ArchiveReader
-        {
-            readers.Add(new Reader { Test = test, Type = typeof(T) });
-        }
-
-        /// <summary>
-        /// Gets a list of beatmap file names.
-        /// </summary>
-        public string[] BeatmapFilenames { get; protected set; }
-
-        /// <summary>
-        /// The storyboard filename. Null if no storyboard is present.
-        /// </summary>
-        public string StoryboardFilename { get; protected set; }
-
         /// <summary>
         /// Opens a stream for reading a specific file from this archive.
         /// </summary>
@@ -51,6 +17,8 @@ namespace osu.Game.Beatmaps.IO
 
         public abstract void Dispose();
 
+        public abstract IEnumerable<string> Filenames { get; }
+
         public virtual byte[] Get(string name)
         {
             using (Stream input = GetStream(name))

osu.Game/Beatmaps/IO/LegacyFilesystemReader.cs

@@ -0,0 +1,33 @@
+// Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
+// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
+
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace osu.Game.Beatmaps.IO
+{
+    /// <summary>
+    /// Reads an extracted legacy beatmap from disk.
+    /// </summary>
+    public class LegacyFilesystemReader : ArchiveReader
+    {
+        private readonly string path;
+
+        public LegacyFilesystemReader(string path)
+        {
+            this.path = path;
+        }
+
+        public override Stream GetStream(string name) => File.OpenRead(Path.Combine(path, name));
+
+        public override void Dispose()
+        {
+            // no-op
+        }
+
+        public override IEnumerable<string> Filenames => Directory.GetFiles(path).Select(Path.GetFileName).ToArray();
+
+        public override Stream GetUnderlyingStream() => null;
+    }
+}

osu.Game/Beatmaps/IO/OszArchiveReader.cs

@@ -1,25 +1,15 @@
 // Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
 // Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
 
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Ionic.Zip;
-using osu.Game.Beatmaps.Formats;
 
 namespace osu.Game.Beatmaps.IO
 {
     public sealed class OszArchiveReader : ArchiveReader
     {
-        public static void Register()
-        {
-            AddReader<OszArchiveReader>((storage, path) =>
-            {
-                using (var stream = storage.GetStream(path))
-                    return stream != null && ZipFile.IsZipFile(stream, false);
-            });
-            OsuLegacyDecoder.Register();
-        }
-
         private readonly Stream archiveStream;
         private readonly ZipFile archive;
 
@@ -27,13 +17,6 @@ namespace osu.Game.Beatmaps.IO
         {
             this.archiveStream = archiveStream;
             archive = ZipFile.Read(archiveStream);
-
-            BeatmapFilenames = archive.Entries.Where(e => e.FileName.EndsWith(@".osu")).Select(e => e.FileName).ToArray();
-
-            if (BeatmapFilenames.Length == 0)
-                throw new FileNotFoundException(@"This directory contains no beatmaps");
-
-            StoryboardFilename = archive.Entries.Where(e => e.FileName.EndsWith(@".osb")).Select(e => e.FileName).FirstOrDefault();
         }
 
         public override Stream GetStream(string name)
@@ -41,7 +24,16 @@ namespace osu.Game.Beatmaps.IO
             ZipEntry entry = archive.Entries.SingleOrDefault(e => e.FileName == name);
             if (entry == null)
                 throw new FileNotFoundException();
-            return entry.OpenReader();
+
+            // allow seeking
+            MemoryStream copy = new MemoryStream();
+
+            using (Stream s = entry.OpenReader())
+                s.CopyTo(copy);
+
+            copy.Position = 0;
+
+            return copy;
         }
 
         public override void Dispose()
@@ -50,6 +42,8 @@ namespace osu.Game.Beatmaps.IO
             archiveStream.Dispose();
         }
 
+        public override IEnumerable<string> Filenames => archive.Entries.Select(e => e.FileName).ToArray();
+
         public override Stream GetUnderlyingStream() => archiveStream;
     }
 }

osu.Game/Database/DatabaseStore.cs

@@ -3,9 +3,9 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Linq.Expressions;
 using osu.Framework.Logging;
+using osu.Framework.Platform;
 using SQLite.Net;
 using SQLiteNetExtensions.Extensions;
 
@@ -13,10 +13,12 @@ namespace osu.Game.Database
 {
     public abstract class DatabaseStore
     {
-        protected SQLiteConnection Connection { get; }
+        protected readonly Storage Storage;
+        protected readonly SQLiteConnection Connection;
 
-        protected DatabaseStore(SQLiteConnection connection)
+        protected DatabaseStore(SQLiteConnection connection, Storage storage = null)
         {
+            Storage = storage;
             Connection = connection;
 
             try
@@ -63,15 +65,5 @@ namespace osu.Game.Database
         }
 
         protected abstract Type[] ValidTypes { get; }
-
-        public void Update<T>(T record, bool cascade = true) where T : class
-        {
-            if (ValidTypes.All(t => t != typeof(T)))
-                throw new ArgumentException("Must be a type managed by BeatmapDatabase", nameof(T));
-            if (cascade)
-                Connection.UpdateWithChildren(record);
-            else
-                Connection.Update(record);
-        }
     }
 }

osu.Game/IO/FileDatabase.cs

@@ -0,0 +1,114 @@
+// Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
+// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using osu.Framework.Extensions;
+using osu.Framework.IO.Stores;
+using osu.Framework.Logging;
+using osu.Framework.Platform;
+using osu.Game.Database;
+using SQLite.Net;
+
+namespace osu.Game.IO
+{
+    /// <summary>
+    /// Handles the Store and retrieval of Files/FileSets to the database backing
+    /// </summary>
+    public class FileDatabase : DatabaseStore
+    {
+        private const string prefix = "files";
+
+        public readonly ResourceStore<byte[]> Store;
+
+        public FileDatabase(SQLiteConnection connection, Storage storage) : base(connection, storage)
+        {
+            Store = new NamespacedResourceStore<byte[]>(new StorageBackedResourceStore(storage), prefix);
+        }
+
+        protected override Type[] ValidTypes => new[] {
+            typeof(FileInfo),
+        };
+
+        protected override void Prepare(bool reset = false)
+        {
+            if (reset)
+                Connection.DropTable<FileInfo>();
+
+            Connection.CreateTable<FileInfo>();
+
+            deletePending();
+        }
+
+        public FileInfo Add(Stream data, string filename = null)
+        {
+            string hash = data.GetMd5Hash();
+
+            var info = new FileInfo
+            {
+                Filename = filename,
+                Hash = hash,
+            };
+
+            var existing = Connection.Table<FileInfo>().FirstOrDefault(f => f.Hash == info.Hash);
+
+            if (existing != null)
+            {
+                info = existing;
+            }
+            else
+            {
+                string path = Path.Combine(prefix, info.StoragePath);
+
+                data.Seek(0, SeekOrigin.Begin);
+
+                if (!Storage.Exists(path))
+                    using (var output = Storage.GetStream(path, FileAccess.Write))
+                        data.CopyTo(output);
+
+                data.Seek(0, SeekOrigin.Begin);
+
+                Connection.Insert(info);
+            }
+
+            Reference(new[] { info });
+            return info;
+        }
+
+        public void Reference(IEnumerable<FileInfo> files)
+        {
+            foreach (var f in files)
+            {
+                f.ReferenceCount++;
+                Connection.Update(f);
+            }
+        }
+
+        public void Dereference(IEnumerable<FileInfo> files)
+        {
+            foreach (var f in files)
+            {
+                f.ReferenceCount--;
+                Connection.Update(f);
+            }
+        }
+
+        private void deletePending()
+        {
+            foreach (var f in GetAllWithChildren<FileInfo>(f => f.ReferenceCount < 1))
+            {
+                try
+                {
+                    Connection.Delete(f);
+                    Storage.Delete(Path.Combine(prefix, f.Hash));
+                }
+                catch (Exception e)
+                {
+                    Logger.Error(e, $@"Could not delete beatmap {f}");
+                }
+            }
+        }
+    }
+}

osu.Game/IO/FileInfo.cs

@@ -0,0 +1,24 @@
+// Copyright (c) 2007-2017 ppy Pty Ltd <contact@ppy.sh>.
+// Licensed under the MIT Licence - https://raw.githubusercontent.com/ppy/osu/master/LICENCE
+
+using System.IO;
+using SQLite.Net.Attributes;
+
+namespace osu.Game.IO
+{
+    public class FileInfo
+    {
+        [PrimaryKey, AutoIncrement]
+        public int ID { get; set; }
+
+        public string Filename { get; set; }
+
+        [Indexed(Unique = true)]
+        public string Hash { get; set; }
+
+        public string StoragePath => Path.Combine(Hash.Remove(1), Hash.Remove(2), Hash);
+
+        [Indexed]
+        public int ReferenceCount { get; set; }
+    }
+}

osu.Game/OsuGameBase.cs

@@ -11,7 +11,6 @@ using osu.Framework.Graphics.Containers;
 using osu.Framework.IO.Stores;
 using osu.Framework.Platform;
 using osu.Game.Beatmaps;
-using osu.Game.Beatmaps.IO;
 using osu.Game.Configuration;
 using osu.Game.Graphics;
 using osu.Game.Graphics.Cursor;
@@ -19,6 +18,7 @@ using osu.Game.Graphics.Processing;
 using osu.Game.Online.API;
 using SQLite.Net;
 using osu.Framework.Graphics.Performance;
+using osu.Game.IO;
 using osu.Game.Rulesets;
 using osu.Game.Rulesets.Scoring;
 
@@ -32,6 +32,8 @@ namespace osu.Game
 
         protected RulesetDatabase RulesetDatabase;
 
+        protected FileDatabase FileDatabase;
+
         protected ScoreDatabase ScoreDatabase;
 
         protected override string MainResourceFile => @"osu.Game.Resources.dll";
@@ -96,7 +98,8 @@ namespace osu.Game
             SQLiteConnection connection = Host.Storage.GetDatabase(@"client");
 
             dependencies.Cache(RulesetDatabase = new RulesetDatabase(connection));
-            dependencies.Cache(BeatmapStore = new BeatmapStore(Host.Storage, connection, RulesetDatabase, Host));
+            dependencies.Cache(FileDatabase = new FileDatabase(connection, Host.Storage));
+            dependencies.Cache(BeatmapStore = new BeatmapStore(Host.Storage, FileDatabase, connection, RulesetDatabase, Host));
             dependencies.Cache(ScoreDatabase = new ScoreDatabase(Host.Storage, connection, Host, BeatmapStore));
             dependencies.Cache(new OsuColour());
 
@@ -131,8 +134,6 @@ namespace osu.Game
             Beatmap = new NonNullableBindable<WorkingBeatmap>(defaultBeatmap);
             BeatmapStore.DefaultBeatmap = defaultBeatmap;
 
-            OszArchiveReader.Register();
-
             dependencies.Cache(API = new APIAccess
             {
                 Username = LocalConfig.Get<string>(OsuSetting.Username),

osu.Game/Screens/Menu/Intro.cs

@@ -22,7 +22,7 @@ namespace osu.Game.Screens.Menu
     {
         private readonly OsuLogo logo;
 
-        public const string MENU_MUSIC_BEATMAP_HASH = "21c1271b91234385978b5418881fdd88";
+        private const string menu_music_beatmap_hash = "715a09144f885d746644c1983e285044";
 
         /// <summary>
         /// Whether we have loaded the menu previously.
@@ -84,23 +84,18 @@ namespace osu.Game.Screens.Menu
 
             if (setInfo == null)
             {
-                var query = beatmaps.Database.Query<BeatmapSetInfo>().Where(b => b.Hash == MENU_MUSIC_BEATMAP_HASH);
+                setInfo = beatmaps.QueryBeatmapSet(b => b.Hash == menu_music_beatmap_hash);
-
-                setInfo = query.FirstOrDefault();
 
                 if (setInfo == null)
                 {
                     // we need to import the default menu background beatmap
-                    beatmaps.Import(new OszArchiveReader(game.Resources.GetStream(@"Tracks/circles.osz")));
+                    setInfo = beatmaps.Import(new OszArchiveReader(game.Resources.GetStream(@"Tracks/circles.osz")));
 
-                    setInfo = query.First();
+                    setInfo.Protected = true;
-
+                    beatmaps.Delete(setInfo);
-                    setInfo.DeletePending = true;
-                    beatmaps.Database.Update(setInfo, false);
                 }
             }
 
-            beatmaps.Database.GetChildren(setInfo);
             Beatmap.Value = beatmaps.GetWorkingBeatmap(setInfo.Beatmaps[0]);
 
             track = Beatmap.Value.Track;

osu.Game/osu.Game.csproj

@@ -75,9 +75,11 @@
     <Compile Include="Audio\SampleInfo.cs" />
     <Compile Include="Audio\SampleInfoList.cs" />
     <Compile Include="Beatmaps\BeatmapDatabase.cs" />
+    <Compile Include="Beatmaps\BeatmapSetFileInfo.cs" />
     <Compile Include="Beatmaps\Drawables\BeatmapBackgroundSprite.cs" />
     <Compile Include="Beatmaps\DifficultyCalculator.cs" />
     <Compile Include="Beatmaps\DummyWorkingBeatmap.cs" />
+    <Compile Include="Beatmaps\IO\LegacyFilesystemReader.cs" />
     <Compile Include="Graphics\Containers\OsuClickableContainer.cs" />
     <Compile Include="Graphics\Containers\OsuFocusedOverlayContainer.cs" />
     <Compile Include="Graphics\Containers\OsuScrollContainer.cs" />
@@ -89,6 +91,8 @@
     <Compile Include="Graphics\UserInterface\MenuItemType.cs" />
     <Compile Include="Graphics\UserInterface\OsuContextMenu.cs" />
     <Compile Include="Graphics\UserInterface\OsuContextMenuItem.cs" />
+    <Compile Include="IO\FileDatabase.cs" />
+    <Compile Include="IO\FileInfo.cs" />
     <Compile Include="Online\API\Requests\GetUsersRequest.cs" />
     <Compile Include="Online\API\Requests\PostMessageRequest.cs" />
     <Compile Include="Online\Chat\ErrorMessage.cs" />