Merge pull request #8962 from peppy/migration-backend

Storage directory migration support

osu.Game.Tests/NonVisual/CustomDataDirectoryTest.cs

@@ -8,14 +8,23 @@ using System.Threading;
 using System.Threading.Tasks;
 using NUnit.Framework;
 using osu.Framework.Allocation;
+using osu.Framework.Configuration;
 using osu.Framework.Platform;
 using osu.Game.Configuration;
+using osu.Game.IO;
 
 namespace osu.Game.Tests.NonVisual
 {
     [TestFixture]
     public class CustomDataDirectoryTest
     {
+        [SetUp]
+        public void SetUp()
+        {
+            if (Directory.Exists(customPath))
+                Directory.Delete(customPath, true);
+        }
+
         [Test]
         public void TestDefaultDirectory()
         {
@@ -108,6 +117,109 @@ namespace osu.Game.Tests.NonVisual
             }
         }
 
+        [Test]
+        public void TestMigration()
+        {
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestMigration)))
+            {
+                try
+                {
+                    var osu = loadOsu(host);
+                    var storage = osu.Dependencies.Get<Storage>();
+
+                    // ensure we perform a save
+                    host.Dependencies.Get<FrameworkConfigManager>().Save();
+
+                    // ensure we "use" cache
+                    host.Storage.GetStorageForDirectory("cache");
+
+                    // for testing nested files are not ignored (only top level)
+                    host.Storage.GetStorageForDirectory("test-nested").GetStorageForDirectory("cache");
+
+                    string defaultStorageLocation = Path.Combine(Environment.CurrentDirectory, "headless", nameof(TestMigration));
+
+                    Assert.That(storage.GetFullPath("."), Is.EqualTo(defaultStorageLocation));
+
+                    osu.Migrate(customPath);
+
+                    Assert.That(storage.GetFullPath("."), Is.EqualTo(customPath));
+
+                    // ensure cache was not moved
+                    Assert.That(host.Storage.ExistsDirectory("cache"));
+
+                    // ensure nested cache was moved
+                    Assert.That(!host.Storage.ExistsDirectory(Path.Combine("test-nested", "cache")));
+                    Assert.That(storage.ExistsDirectory(Path.Combine("test-nested", "cache")));
+
+                    foreach (var file in OsuStorage.IGNORE_FILES)
+                    {
+                        Assert.That(host.Storage.Exists(file), Is.True);
+                        Assert.That(storage.Exists(file), Is.False);
+                    }
+
+                    foreach (var dir in OsuStorage.IGNORE_DIRECTORIES)
+                    {
+                        Assert.That(host.Storage.ExistsDirectory(dir), Is.True);
+                        Assert.That(storage.ExistsDirectory(dir), Is.False);
+                    }
+
+                    Assert.That(new StreamReader(host.Storage.GetStream("storage.ini")).ReadToEnd().Contains($"FullPath = {customPath}"));
+                }
+                finally
+                {
+                    host.Exit();
+                }
+            }
+        }
+
+        [Test]
+        public void TestMigrationBetweenTwoTargets()
+        {
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestMigrationBetweenTwoTargets)))
+            {
+                try
+                {
+                    var osu = loadOsu(host);
+
+                    string customPath2 = $"{customPath}-2";
+
+                    const string database_filename = "client.db";
+
+                    Assert.DoesNotThrow(() => osu.Migrate(customPath));
+                    Assert.That(File.Exists(Path.Combine(customPath, database_filename)));
+
+                    Assert.DoesNotThrow(() => osu.Migrate(customPath2));
+                    Assert.That(File.Exists(Path.Combine(customPath2, database_filename)));
+
+                    Assert.DoesNotThrow(() => osu.Migrate(customPath));
+                    Assert.That(File.Exists(Path.Combine(customPath, database_filename)));
+                }
+                finally
+                {
+                    host.Exit();
+                }
+            }
+        }
+
+        [Test]
+        public void TestMigrationToSameTargetFails()
+        {
+            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestMigrationToSameTargetFails)))
+            {
+                try
+                {
+                    var osu = loadOsu(host);
+
+                    Assert.DoesNotThrow(() => osu.Migrate(customPath));
+                    Assert.Throws<InvalidOperationException>(() => osu.Migrate(customPath));
+                }
+                finally
+                {
+                    host.Exit();
+                }
+            }
+        }
+
         private OsuGameBase loadOsu(GameHost host)
         {
             var osu = new OsuGameBase();

osu.Game/Database/DatabaseContextFactory.cs

@@ -160,5 +160,13 @@ namespace osu.Game.Database
                 }
             }
         }
+
+        public void FlushConnections()
+        {
+            foreach (var context in threadContexts.Values)
+                context.Dispose();
+
+            recycleThreadContexts();
+        }
     }
 }

osu.Game/IO/OsuStorage.cs

@@ -1,6 +1,10 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
 using osu.Framework.Logging;
 using osu.Framework.Platform;
 using osu.Game.Configuration;
@@ -9,17 +13,119 @@ namespace osu.Game.IO
 {
     public class OsuStorage : WrappedStorage
     {
+        private readonly GameHost host;
+        private readonly StorageConfigManager storageConfig;
+
+        internal static readonly string[] IGNORE_DIRECTORIES = { "cache" };
+
+        internal static readonly string[] IGNORE_FILES =
+        {
+            "framework.ini",
+            "storage.ini"
+        };
+
         public OsuStorage(GameHost host)
             : base(host.Storage, string.Empty)
         {
-            var storageConfig = new StorageConfigManager(host.Storage);
+            this.host = host;
+
+            storageConfig = new StorageConfigManager(host.Storage);
 
             var customStoragePath = storageConfig.Get<string>(StorageConfig.FullPath);
 
             if (!string.IsNullOrEmpty(customStoragePath))
-            {
                 ChangeTargetStorage(host.GetStorage(customStoragePath));
-                Logger.Storage = UnderlyingStorage.GetStorageForDirectory("logs");
+        }
+
+        protected override void ChangeTargetStorage(Storage newStorage)
+        {
+            base.ChangeTargetStorage(newStorage);
+            Logger.Storage = UnderlyingStorage.GetStorageForDirectory("logs");
+        }
+
+        public void Migrate(string newLocation)
+        {
+            var source = new DirectoryInfo(GetFullPath("."));
+            var destination = new DirectoryInfo(newLocation);
+
+            // ensure the new location has no files present, else hard abort
+            if (destination.Exists)
+            {
+                if (destination.GetFiles().Length > 0 || destination.GetDirectories().Length > 0)
+                    throw new InvalidOperationException("Migration destination already has files present");
+
+                deleteRecursive(destination);
+            }
+
+            copyRecursive(source, destination);
+
+            ChangeTargetStorage(host.GetStorage(newLocation));
+
+            storageConfig.Set(StorageConfig.FullPath, newLocation);
+            storageConfig.Save();
+
+            deleteRecursive(source);
+        }
+
+        private static void deleteRecursive(DirectoryInfo target, bool topLevelExcludes = true)
+        {
+            foreach (System.IO.FileInfo fi in target.GetFiles())
+            {
+                if (topLevelExcludes && IGNORE_FILES.Contains(fi.Name))
+                    continue;
+
+                fi.Delete();
+            }
+
+            foreach (DirectoryInfo dir in target.GetDirectories())
+            {
+                if (topLevelExcludes && IGNORE_DIRECTORIES.Contains(dir.Name))
+                    continue;
+
+                dir.Delete(true);
+            }
+        }
+
+        private static void copyRecursive(DirectoryInfo source, DirectoryInfo destination, bool topLevelExcludes = true)
+        {
+            // based off example code https://docs.microsoft.com/en-us/dotnet/api/system.io.directoryinfo
+            Directory.CreateDirectory(destination.FullName);
+
+            foreach (System.IO.FileInfo fi in source.GetFiles())
+            {
+                if (topLevelExcludes && IGNORE_FILES.Contains(fi.Name))
+                    continue;
+
+                attemptCopy(fi, Path.Combine(destination.FullName, fi.Name));
+            }
+
+            foreach (DirectoryInfo dir in source.GetDirectories())
+            {
+                if (topLevelExcludes && IGNORE_DIRECTORIES.Contains(dir.Name))
+                    continue;
+
+                copyRecursive(dir, destination.CreateSubdirectory(dir.Name), false);
+            }
+        }
+
+        private static void attemptCopy(System.IO.FileInfo fileInfo, string destination)
+        {
+            int tries = 5;
+
+            while (true)
+            {
+                try
+                {
+                    fileInfo.CopyTo(destination, true);
+                    return;
+                }
+                catch (Exception)
+                {
+                    if (tries-- == 0)
+                        throw;
+                }
+
+                Thread.Sleep(50);
             }
         }
     }

osu.Game/IO/WrappedStorage.cs

@@ -27,7 +27,7 @@ namespace osu.Game.IO
 
         protected virtual string MutatePath(string path) => !string.IsNullOrEmpty(subPath) ? Path.Combine(subPath, path) : path;
 
-        protected void ChangeTargetStorage(Storage newStorage)
+        protected virtual void ChangeTargetStorage(Storage newStorage)
         {
             UnderlyingStorage = newStorage;
         }

osu.Game/OsuGameBase.cs

@@ -328,6 +328,8 @@ namespace osu.Game
         {
             base.Dispose(isDisposing);
             RulesetStore?.Dispose();
+
+            contextFactory.FlushConnections();
         }
 
         private class OsuUserInputManager : UserInputManager
@@ -355,5 +357,11 @@ namespace osu.Game
                 public override bool ChangeFocusOnClick => false;
             }
         }
+
+        public void Migrate(string path)
+        {
+            contextFactory.FlushConnections();
+            (Storage as OsuStorage)?.Migrate(path);
+        }
     }
 }