diff --git a/osu.Game.Tests/NonVisual/CustomDataDirectoryTest.cs b/osu.Game.Tests/NonVisual/CustomDataDirectoryTest.cs
index 8ea0e34214..199e69a19d 100644
--- a/osu.Game.Tests/NonVisual/CustomDataDirectoryTest.cs
+++ b/osu.Game.Tests/NonVisual/CustomDataDirectoryTest.cs
@@ -19,24 +19,18 @@ namespace osu.Game.Tests.NonVisual
     [TestFixture]
     public class CustomDataDirectoryTest
     {
-        [SetUp]
-        public void SetUp()
-        {
-            if (Directory.Exists(customPath))
-                Directory.Delete(customPath, true);
-        }
-
         [Test]
         public void TestDefaultDirectory()
         {
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestDefaultDirectory)))
+            using (HeadlessGameHost host = new CustomTestHeadlessGameHost(nameof(TestDefaultDirectory)))
             {
                 try
                 {
+                    string defaultStorageLocation = getDefaultLocationFor(nameof(TestDefaultDirectory));
+
                     var osu = loadOsu(host);
                     var storage = osu.Dependencies.Get<Storage>();
 
-                    string defaultStorageLocation = Path.Combine(RuntimeInfo.StartupDirectory, "headless", nameof(TestDefaultDirectory));
                     Assert.That(storage.GetFullPath("."), Is.EqualTo(defaultStorageLocation));
                 }
                 finally
@@ -46,21 +40,14 @@ namespace osu.Game.Tests.NonVisual
             }
         }
 
-        private string customPath => Path.Combine(RuntimeInfo.StartupDirectory, "custom-path");
-
         [Test]
         public void TestCustomDirectory()
         {
-            using (var host = new HeadlessGameHost(nameof(TestCustomDirectory)))
+            string customPath = prepareCustomPath();
+
+            using (var host = new CustomTestHeadlessGameHost(nameof(TestCustomDirectory)))
             {
-                string defaultStorageLocation = Path.Combine(RuntimeInfo.StartupDirectory, "headless", nameof(TestCustomDirectory));
-
-                // need access before the game has constructed its own storage yet.
-                Storage storage = new DesktopStorage(defaultStorageLocation, host);
-                // manual cleaning so we can prepare a config file.
-                storage.DeleteDirectory(string.Empty);
-
-                using (var storageConfig = new StorageConfigManager(storage))
+                using (var storageConfig = new StorageConfigManager(host.InitialStorage))
                     storageConfig.Set(StorageConfig.FullPath, customPath);
 
                 try
@@ -68,7 +55,7 @@ namespace osu.Game.Tests.NonVisual
                     var osu = loadOsu(host);
 
                     // switch to DI'd storage
-                    storage = osu.Dependencies.Get<Storage>();
+                    var storage = osu.Dependencies.Get<Storage>();
 
                     Assert.That(storage.GetFullPath("."), Is.EqualTo(customPath));
                 }
@@ -82,16 +69,11 @@ namespace osu.Game.Tests.NonVisual
         [Test]
         public void TestSubDirectoryLookup()
         {
-            using (var host = new HeadlessGameHost(nameof(TestSubDirectoryLookup)))
+            string customPath = prepareCustomPath();
+
+            using (var host = new CustomTestHeadlessGameHost(nameof(TestSubDirectoryLookup)))
             {
-                string defaultStorageLocation = Path.Combine(RuntimeInfo.StartupDirectory, "headless", nameof(TestSubDirectoryLookup));
-
-                // need access before the game has constructed its own storage yet.
-                Storage storage = new DesktopStorage(defaultStorageLocation, host);
-                // manual cleaning so we can prepare a config file.
-                storage.DeleteDirectory(string.Empty);
-
-                using (var storageConfig = new StorageConfigManager(storage))
+                using (var storageConfig = new StorageConfigManager(host.InitialStorage))
                     storageConfig.Set(StorageConfig.FullPath, customPath);
 
                 try
@@ -99,7 +81,7 @@ namespace osu.Game.Tests.NonVisual
                     var osu = loadOsu(host);
 
                     // switch to DI'd storage
-                    storage = osu.Dependencies.Get<Storage>();
+                    var storage = osu.Dependencies.Get<Storage>();
 
                     string actualTestFile = Path.Combine(customPath, "rulesets", "test");
 
@@ -120,10 +102,14 @@ namespace osu.Game.Tests.NonVisual
         [Test]
         public void TestMigration()
         {
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestMigration)))
+            string customPath = prepareCustomPath();
+
+            using (HeadlessGameHost host = new CustomTestHeadlessGameHost(nameof(TestMigration)))
             {
                 try
                 {
+                    string defaultStorageLocation = getDefaultLocationFor(nameof(TestMigration));
+
                     var osu = loadOsu(host);
                     var storage = osu.Dependencies.Get<Storage>();
 
@@ -139,8 +125,6 @@ namespace osu.Game.Tests.NonVisual
                     // for testing nested files are not ignored (only top level)
                     host.Storage.GetStorageForDirectory("test-nested").GetStorageForDirectory("cache");
 
-                    string defaultStorageLocation = Path.Combine(RuntimeInfo.StartupDirectory, "headless", nameof(TestMigration));
-
                     Assert.That(storage.GetFullPath("."), Is.EqualTo(defaultStorageLocation));
 
                     osu.Migrate(customPath);
@@ -178,14 +162,15 @@ namespace osu.Game.Tests.NonVisual
         [Test]
         public void TestMigrationBetweenTwoTargets()
         {
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestMigrationBetweenTwoTargets)))
+            string customPath = prepareCustomPath();
+            string customPath2 = prepareCustomPath("-2");
+
+            using (HeadlessGameHost host = new CustomTestHeadlessGameHost(nameof(TestMigrationBetweenTwoTargets)))
             {
                 try
                 {
                     var osu = loadOsu(host);
 
-                    string customPath2 = $"{customPath}-2";
-
                     const string database_filename = "client.db";
 
                     Assert.DoesNotThrow(() => osu.Migrate(customPath));
@@ -207,7 +192,9 @@ namespace osu.Game.Tests.NonVisual
         [Test]
         public void TestMigrationToSameTargetFails()
         {
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestMigrationToSameTargetFails)))
+            string customPath = prepareCustomPath();
+
+            using (HeadlessGameHost host = new CustomTestHeadlessGameHost(nameof(TestMigrationToSameTargetFails)))
             {
                 try
                 {
@@ -226,7 +213,9 @@ namespace osu.Game.Tests.NonVisual
         [Test]
         public void TestMigrationToNestedTargetFails()
         {
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestMigrationToNestedTargetFails)))
+            string customPath = prepareCustomPath();
+
+            using (HeadlessGameHost host = new CustomTestHeadlessGameHost(nameof(TestMigrationToNestedTargetFails)))
             {
                 try
                 {
@@ -253,7 +242,9 @@ namespace osu.Game.Tests.NonVisual
         [Test]
         public void TestMigrationToSeeminglyNestedTarget()
         {
-            using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestMigrationToSeeminglyNestedTarget)))
+            string customPath = prepareCustomPath();
+
+            using (HeadlessGameHost host = new CustomTestHeadlessGameHost(nameof(TestMigrationToSeeminglyNestedTarget)))
             {
                 try
                 {
@@ -282,6 +273,7 @@ namespace osu.Game.Tests.NonVisual
             var osu = new OsuGameBase();
             Task.Run(() => host.Run(osu));
             waitForOrAssert(() => osu.IsLoaded, @"osu! failed to start in a reasonable amount of time");
+
             return osu;
         }
 
@@ -294,5 +286,39 @@ namespace osu.Game.Tests.NonVisual
 
             Assert.IsTrue(task.Wait(timeout), failureMessage);
         }
+
+        private static string getDefaultLocationFor(string testTypeName)
+        {
+            string path = Path.Combine(RuntimeInfo.StartupDirectory, "headless", testTypeName);
+
+            if (Directory.Exists(path))
+                Directory.Delete(path, true);
+
+            return path;
+        }
+
+        private string prepareCustomPath(string suffix = "")
+        {
+            string path = Path.Combine(RuntimeInfo.StartupDirectory, $"custom-path{suffix}");
+
+            if (Directory.Exists(path))
+                Directory.Delete(path, true);
+
+            return path;
+        }
+
+        public class CustomTestHeadlessGameHost : HeadlessGameHost
+        {
+            public Storage InitialStorage { get; }
+
+            public CustomTestHeadlessGameHost(string name)
+                : base(name)
+            {
+                string defaultStorageLocation = getDefaultLocationFor(name);
+
+                InitialStorage = new DesktopStorage(defaultStorageLocation, this);
+                InitialStorage.DeleteDirectory(string.Empty);
+            }
+        }
     }
 }