Add realm FileStore and test coverage

osu.Game.Tests/Database/FileStoreTests.cs

@@ -0,0 +1,114 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+using osu.Framework.Logging;
+using osu.Game.Models;
+using osu.Game.Stores;
+
+#nullable enable
+
+namespace osu.Game.Tests.Database
+{
+    public class FileStoreTests : RealmTest
+    {
+        [Test]
+        public void TestImportFile()
+        {
+            RunTestWithRealm((realmFactory, storage) =>
+            {
+                var realm = realmFactory.Context;
+                var files = new RealmFileStore(realmFactory, storage);
+
+                var testData = new MemoryStream(new byte[] { 0, 1, 2, 3 });
+
+                realm.Write(() => files.Add(testData, realm));
+
+                Assert.True(files.Storage.Exists("0/05/054edec1d0211f624fed0cbca9d4f9400b0e491c43742af2c5b0abebf0c990d8"));
+                Assert.True(files.Storage.Exists(realm.All<RealmFile>().First().StoragePath));
+            });
+        }
+
+        [Test]
+        public void TestImportSameFileTwice()
+        {
+            RunTestWithRealm((realmFactory, storage) =>
+            {
+                var realm = realmFactory.Context;
+                var files = new RealmFileStore(realmFactory, storage);
+
+                var testData = new MemoryStream(new byte[] { 0, 1, 2, 3 });
+
+                realm.Write(() => files.Add(testData, realm));
+                realm.Write(() => files.Add(testData, realm));
+
+                Assert.AreEqual(1, realm.All<RealmFile>().Count());
+            });
+        }
+
+        [Test]
+        public void TestDontPurgeReferenced()
+        {
+            RunTestWithRealm((realmFactory, storage) =>
+            {
+                var realm = realmFactory.Context;
+                var files = new RealmFileStore(realmFactory, storage);
+
+                var file = realm.Write(() => files.Add(new MemoryStream(new byte[] { 0, 1, 2, 3 }), realm));
+
+                var timer = new Stopwatch();
+                timer.Start();
+
+                realm.Write(() =>
+                {
+                    // attach the file to an arbitrary beatmap
+                    var beatmapSet = CreateBeatmapSet(CreateRuleset());
+
+                    beatmapSet.Files.Add(new RealmNamedFileUsage(file, "arbitrary.resource"));
+
+                    realm.Add(beatmapSet);
+                });
+
+                Logger.Log($"Import complete at {timer.ElapsedMilliseconds}");
+
+                string path = file.StoragePath;
+
+                Assert.True(realm.All<RealmFile>().Any());
+                Assert.True(files.Storage.Exists(path));
+
+                files.Cleanup();
+                Logger.Log($"Cleanup complete at {timer.ElapsedMilliseconds}");
+
+                Assert.True(realm.All<RealmFile>().Any());
+                Assert.True(file.IsValid);
+                Assert.True(files.Storage.Exists(path));
+            });
+        }
+
+        [Test]
+        public void TestPurgeUnreferenced()
+        {
+            RunTestWithRealm((realmFactory, storage) =>
+            {
+                var realm = realmFactory.Context;
+                var files = new RealmFileStore(realmFactory, storage);
+
+                var file = realm.Write(() => files.Add(new MemoryStream(new byte[] { 0, 1, 2, 3 }), realm));
+
+                string path = file.StoragePath;
+
+                Assert.True(realm.All<RealmFile>().Any());
+                Assert.True(files.Storage.Exists(path));
+
+                files.Cleanup();
+
+                Assert.False(realm.All<RealmFile>().Any());
+                Assert.False(file.IsValid);
+                Assert.False(files.Storage.Exists(path));
+            });
+        }
+    }
+}

osu.Game.Tests/Database/RealmTest.cs

@@ -5,10 +5,12 @@ using System;
 using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 using NUnit.Framework;
+using osu.Framework.Extensions;
 using osu.Framework.Logging;
 using osu.Framework.Platform;
 using osu.Framework.Testing;
 using osu.Game.Database;
+using osu.Game.Models;
 
 #nullable enable
 
@@ -70,6 +72,46 @@ namespace osu.Game.Tests.Database
             }
         }
 
+        protected static RealmBeatmapSet CreateBeatmapSet(RealmRuleset ruleset)
+        {
+            RealmFile createRealmFile() => new RealmFile { Hash = Guid.NewGuid().ToString().ComputeSHA2Hash() };
+
+            var metadata = new RealmBeatmapMetadata
+            {
+                Title = "My Love",
+                Artist = "Kuba Oms"
+            };
+
+            var beatmapSet = new RealmBeatmapSet
+            {
+                Beatmaps =
+                {
+                    new RealmBeatmap(ruleset, new RealmBeatmapDifficulty(), metadata) { DifficultyName = "Easy", },
+                    new RealmBeatmap(ruleset, new RealmBeatmapDifficulty(), metadata) { DifficultyName = "Normal", },
+                    new RealmBeatmap(ruleset, new RealmBeatmapDifficulty(), metadata) { DifficultyName = "Hard", },
+                    new RealmBeatmap(ruleset, new RealmBeatmapDifficulty(), metadata) { DifficultyName = "Insane", }
+                },
+                Files =
+                {
+                    new RealmNamedFileUsage(createRealmFile(), "test [easy].osu"),
+                    new RealmNamedFileUsage(createRealmFile(), "test [normal].osu"),
+                    new RealmNamedFileUsage(createRealmFile(), "test [hard].osu"),
+                    new RealmNamedFileUsage(createRealmFile(), "test [insane].osu"),
+                }
+            };
+
+            for (int i = 0; i < 8; i++)
+                beatmapSet.Files.Add(new RealmNamedFileUsage(createRealmFile(), $"hitsound{i}.mp3"));
+
+            foreach (var b in beatmapSet.Beatmaps)
+                b.BeatmapSet = beatmapSet;
+
+            return beatmapSet;
+        }
+
+        protected static RealmRuleset CreateRuleset() =>
+            new RealmRuleset(0, "osu!", "osu", true);
+
         private class RealmTestGame : Framework.Game
         {
             public RealmTestGame(Func<Task> work)

osu.Game/Stores/RealmFileStore.cs

@@ -0,0 +1,113 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.IO;
+using System.Linq;
+using osu.Framework.Extensions;
+using osu.Framework.IO.Stores;
+using osu.Framework.Logging;
+using osu.Framework.Platform;
+using osu.Game.Database;
+using osu.Game.Models;
+using Realms;
+
+#nullable enable
+
+namespace osu.Game.Stores
+{
+    /// <summary>
+    /// Handles the Store and retrieval of Files/FileSets to the database backing
+    /// </summary>
+    public class RealmFileStore
+    {
+        private readonly RealmContextFactory realmFactory;
+        public readonly IResourceStore<byte[]> Store;
+
+        public Storage Storage;
+
+        public RealmFileStore(RealmContextFactory realmFactory, Storage storage)
+        {
+            this.realmFactory = realmFactory;
+
+            Storage = storage.GetStorageForDirectory(@"files");
+            Store = new StorageBackedResourceStore(Storage);
+        }
+
+        /// <summary>
+        /// Add a new file to the game-wide database, copying it to permanent storage if not already present.
+        /// </summary>
+        /// <param name="data">The file data stream.</param>
+        /// <param name="realm">The realm instance to add to. Should already be in a transaction.</param>
+        /// <returns></returns>
+        public RealmFile Add(Stream data, Realm realm)
+        {
+            string hash = data.ComputeSHA2Hash();
+
+            var existing = realm.Find<RealmFile>(hash);
+
+            var file = existing ?? new RealmFile { Hash = hash };
+
+            if (!checkFileExistsAndMatchesHash(file))
+                copyToStore(file, data);
+
+            if (!file.IsManaged)
+                realm.Add(file);
+
+            return file;
+        }
+
+        private void copyToStore(RealmFile file, Stream data)
+        {
+            data.Seek(0, SeekOrigin.Begin);
+
+            using (var output = Storage.GetStream(file.StoragePath, FileAccess.Write))
+                data.CopyTo(output);
+
+            data.Seek(0, SeekOrigin.Begin);
+        }
+
+        private bool checkFileExistsAndMatchesHash(RealmFile file)
+        {
+            string path = file.StoragePath;
+
+            // we may be re-adding a file to fix missing store entries.
+            if (!Storage.Exists(path))
+                return false;
+
+            // even if the file already exists, check the existing checksum for safety.
+            using (var stream = Storage.GetStream(path))
+                return stream.ComputeSHA2Hash() == file.Hash;
+        }
+
+        public void Cleanup()
+        {
+            var realm = realmFactory.Context;
+
+            // can potentially be run asynchronously, although we will need to consider operation order for disk deletion vs realm removal.
+            using (var transaction = realm.BeginWrite())
+            {
+                // TODO: consider using a realm native query to avoid iterating all files (https://github.com/realm/realm-dotnet/issues/2659#issuecomment-927823707)
+                var files = realm.All<RealmFile>().ToList();
+
+                foreach (var file in files)
+                {
+                    if (file.BacklinksCount > 0)
+                        continue;
+
+                    try
+                    {
+                        Storage.Delete(file.StoragePath);
+                        realm.Remove(file);
+                    }
+                    catch (Exception e)
+                    {
+                        Logger.Error(e, $@"Could not delete databased file {file.Hash}");
+                    }
+                }
+
+                transaction.Commit();
+            }
+        }
+    }
+}