From 306e13fa7b11ab242300d3a4a4c314dd014d098a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 25 Nov 2021 14:06:02 +0900 Subject: [PATCH 1/9] Remove all usages of `FileStore.QueryFiles` --- osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs | 2 +- osu.Game/IO/FileStore.cs | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index cac1bf05a6..6e2b9d20a8 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -1050,7 +1050,7 @@ namespace osu.Game.Tests.Beatmaps.IO private static void checkSingleReferencedFileCount(OsuGameBase osu, int expected) { - Assert.AreEqual(expected, osu.Dependencies.Get().QueryFiles(f => f.ReferenceCount == 1).Count()); + Assert.AreEqual(expected, osu.Dependencies.Get().Get().FileInfo.Count(f => f.ReferenceCount == 1)); } private static void ensureLoaded(OsuGameBase osu, int timeout = 60000) diff --git a/osu.Game/IO/FileStore.cs b/osu.Game/IO/FileStore.cs index 165cf756aa..ebe1ebfe69 100644 --- a/osu.Game/IO/FileStore.cs +++ b/osu.Game/IO/FileStore.cs @@ -2,11 +2,8 @@ // See the LICENCE file in the repository root for full licence text. using System; -using System.Collections.Generic; using System.IO; using System.Linq; -using System.Linq.Expressions; -using Microsoft.EntityFrameworkCore; using osu.Framework.Extensions; using osu.Framework.IO.Stores; using osu.Framework.Logging; @@ -31,13 +28,6 @@ namespace osu.Game.IO Store = new StorageBackedResourceStore(Storage); } - /// - /// Perform a lookup query on available s. - /// - /// The query. - /// Results from the provided query. - public IEnumerable QueryFiles(Expression> query) => ContextFactory.Get().Set().AsNoTracking().Where(f => f.ReferenceCount > 0).Where(query); - public FileInfo Add(Stream data, bool reference = true) { using (var usage = ContextFactory.GetForWrite()) From fd321109da8f9f621c7ae5b0e3dfb974154a1fc6 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 25 Nov 2021 14:06:12 +0900 Subject: [PATCH 2/9] Remove unnecessary `virtual` specification on `Refresh` --- osu.Game/Database/DatabaseBackedStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Database/DatabaseBackedStore.cs b/osu.Game/Database/DatabaseBackedStore.cs index a11efba54b..03e1c014b2 100644 --- a/osu.Game/Database/DatabaseBackedStore.cs +++ b/osu.Game/Database/DatabaseBackedStore.cs @@ -19,7 +19,7 @@ namespace osu.Game.Database /// The object to use as a reference when negotiating a local instance. /// An optional lookup source which will be used to query and populate a freshly retrieved replacement. If not provided, the refreshed object will still be returned but will not have any includes. /// A valid EF-stored type. - protected virtual void Refresh(ref T obj, IQueryable lookupSource = null) where T : class, IHasPrimaryKey + protected void Refresh(ref T obj, IQueryable lookupSource = null) where T : class, IHasPrimaryKey { using (var usage = ContextFactory.GetForWrite()) { From 204bd2b604a30f13c899d14a804f1a1582b1463e Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 25 Nov 2021 14:17:42 +0900 Subject: [PATCH 3/9] Ensure `Cleanup` can run from non-update thread and add basic log output --- osu.Game/Stores/RealmFileStore.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/osu.Game/Stores/RealmFileStore.cs b/osu.Game/Stores/RealmFileStore.cs index aebf330443..f9abbda4c0 100644 --- a/osu.Game/Stores/RealmFileStore.cs +++ b/osu.Game/Stores/RealmFileStore.cs @@ -86,9 +86,13 @@ namespace osu.Game.Stores public void Cleanup() { - var realm = realmFactory.Context; + Logger.Log(@"Beginning realm file store cleanup"); + + int totalFiles = 0; + int removedFiles = 0; // can potentially be run asynchronously, although we will need to consider operation order for disk deletion vs realm removal. + using (var realm = realmFactory.CreateContext()) using (var transaction = realm.BeginWrite()) { // TODO: consider using a realm native query to avoid iterating all files (https://github.com/realm/realm-dotnet/issues/2659#issuecomment-927823707) @@ -96,11 +100,14 @@ namespace osu.Game.Stores foreach (var file in files) { + totalFiles++; + if (file.BacklinksCount > 0) continue; try { + removedFiles++; Storage.Delete(file.GetStoragePath()); realm.Remove(file); } @@ -112,6 +119,8 @@ namespace osu.Game.Stores transaction.Commit(); } + + Logger.Log($@"Finished realm file store cleanup ({removedFiles} of {totalFiles} deleted)"); } } } From 315e126497db5d6c343b57662d5e66ea4fefa834 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 25 Nov 2021 14:12:49 +0900 Subject: [PATCH 4/9] Add automatic clean-up call for `RealmFileStore` --- osu.Game/Database/RealmContextFactory.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 738ecaea7c..0cffb2b0ae 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -14,6 +14,7 @@ using osu.Framework.Statistics; using osu.Game.Configuration; using osu.Game.Input.Bindings; using osu.Game.Models; +using osu.Game.Stores; using Realms; #nullable enable @@ -106,6 +107,8 @@ namespace osu.Game.Database private void cleanupPendingDeletions() { + new RealmFileStore(this, storage).Cleanup(); + using (var realm = CreateContext()) using (var transaction = realm.BeginWrite()) { From bcdb73e1e847cc2a2d099f4c24600fbae3b5d228 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 25 Nov 2021 14:28:27 +0900 Subject: [PATCH 5/9] Clean up files last --- osu.Game/Database/RealmContextFactory.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/osu.Game/Database/RealmContextFactory.cs b/osu.Game/Database/RealmContextFactory.cs index 0cffb2b0ae..3c5dfeafe8 100644 --- a/osu.Game/Database/RealmContextFactory.cs +++ b/osu.Game/Database/RealmContextFactory.cs @@ -107,8 +107,6 @@ namespace osu.Game.Database private void cleanupPendingDeletions() { - new RealmFileStore(this, storage).Cleanup(); - using (var realm = CreateContext()) using (var transaction = realm.BeginWrite()) { @@ -124,6 +122,10 @@ namespace osu.Game.Database transaction.Commit(); } + + // clean up files after dropping any pending deletions. + // in the future we may want to only do this when the game is idle, rather than on every startup. + new RealmFileStore(this, storage).Cleanup(); } /// From 60b207f20aa34f31aefd723716a347d6248aa574 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 25 Nov 2021 14:34:13 +0900 Subject: [PATCH 6/9] Reduce interface exposure of `ScoreManager` --- osu.Game/Scoring/ScoreManager.cs | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/osu.Game/Scoring/ScoreManager.cs b/osu.Game/Scoring/ScoreManager.cs index 29144e7bdc..abc0bfdda7 100644 --- a/osu.Game/Scoring/ScoreManager.cs +++ b/osu.Game/Scoring/ScoreManager.cs @@ -25,7 +25,7 @@ using osu.Game.Rulesets.Scoring; namespace osu.Game.Scoring { - public class ScoreManager : IModelManager, IModelImporter, IModelFileManager, IModelDownloader + public class ScoreManager : IModelManager, IModelImporter, IModelDownloader { private readonly Scheduler scheduler; private readonly Func difficulties; @@ -342,25 +342,6 @@ namespace osu.Game.Scoring #endregion - #region Implementation of IModelFileManager - - public void ReplaceFile(ScoreInfo model, ScoreFileInfo file, Stream contents, string filename = null) - { - scoreModelManager.ReplaceFile(model, file, contents, filename); - } - - public void DeleteFile(ScoreInfo model, ScoreFileInfo file) - { - scoreModelManager.DeleteFile(model, file); - } - - public void AddFile(ScoreInfo model, Stream contents, string filename) - { - scoreModelManager.AddFile(model, contents, filename); - } - - #endregion - #region Implementation of IModelDownloader public event Action> DownloadBegan From e6cfe44652ac475f733135863458ac569487b7d9 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 25 Nov 2021 16:27:55 +0900 Subject: [PATCH 7/9] Fix occasional test failure due to default value oversight in `TestSceneBeatmapCarousel` --- .../Visual/SongSelect/TestSceneBeatmapCarousel.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs index 5906979bc4..f637c715a1 100644 --- a/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs +++ b/osu.Game.Tests/Visual/SongSelect/TestSceneBeatmapCarousel.cs @@ -572,7 +572,7 @@ namespace osu.Game.Tests.Visual.SongSelect AddStep("add mixed ruleset beatmapset", () => { - testMixed = TestResources.CreateTestBeatmapSetInfo(); + testMixed = TestResources.CreateTestBeatmapSetInfo(3); for (int i = 0; i <= 2; i++) { @@ -595,7 +595,7 @@ namespace osu.Game.Tests.Visual.SongSelect BeatmapSetInfo testSingle = null; AddStep("add single ruleset beatmapset", () => { - testSingle = TestResources.CreateTestBeatmapSetInfo(); + testSingle = TestResources.CreateTestBeatmapSetInfo(3); testSingle.Beatmaps.ForEach(b => { b.Ruleset = rulesets.AvailableRulesets.ElementAt(1); @@ -615,7 +615,7 @@ namespace osu.Game.Tests.Visual.SongSelect List manySets = new List(); for (int i = 1; i <= 50; i++) - manySets.Add(TestResources.CreateTestBeatmapSetInfo(i)); + manySets.Add(TestResources.CreateTestBeatmapSetInfo(3)); loadBeatmaps(manySets); From ec9a09d5a42a0a2818c369faf22e134802e73917 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 25 Nov 2021 16:56:19 +0900 Subject: [PATCH 8/9] Add missing licence headers --- osu.Game/Database/LegacyBeatmapImporter.cs | 3 +++ osu.Game/Database/LegacyScoreImporter.cs | 3 +++ osu.Game/Database/LegacySkinImporter.cs | 5 ++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/osu.Game/Database/LegacyBeatmapImporter.cs b/osu.Game/Database/LegacyBeatmapImporter.cs index 041a692655..97f6eba6c2 100644 --- a/osu.Game/Database/LegacyBeatmapImporter.cs +++ b/osu.Game/Database/LegacyBeatmapImporter.cs @@ -1,3 +1,6 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + using osu.Framework.Platform; using osu.Game.Beatmaps; using osu.Game.IO; diff --git a/osu.Game/Database/LegacyScoreImporter.cs b/osu.Game/Database/LegacyScoreImporter.cs index 984aa8a1ac..48445b7bdb 100644 --- a/osu.Game/Database/LegacyScoreImporter.cs +++ b/osu.Game/Database/LegacyScoreImporter.cs @@ -1,3 +1,6 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + using System; using System.Collections.Generic; using System.IO; diff --git a/osu.Game/Database/LegacySkinImporter.cs b/osu.Game/Database/LegacySkinImporter.cs index b102e26bc6..2f05ccae45 100644 --- a/osu.Game/Database/LegacySkinImporter.cs +++ b/osu.Game/Database/LegacySkinImporter.cs @@ -1,3 +1,6 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + using osu.Game.Skinning; namespace osu.Game.Database @@ -11,4 +14,4 @@ namespace osu.Game.Database { } } -} \ No newline at end of file +} From 79459c1aebc968ac4821161399126075b6301c93 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Thu, 25 Nov 2021 17:12:15 +0900 Subject: [PATCH 9/9] Fix typo in class and variable names --- ...ImportManager.cs => LegacyImportManager.cs} | 2 +- osu.Game/OsuGame.cs | 4 ++-- .../Sections/Maintenance/GeneralSettings.cs | 18 +++++++++--------- osu.Game/Screens/Select/SongSelect.cs | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) rename osu.Game/Database/{LeagcyImportManager.cs => LegacyImportManager.cs} (98%) diff --git a/osu.Game/Database/LeagcyImportManager.cs b/osu.Game/Database/LegacyImportManager.cs similarity index 98% rename from osu.Game/Database/LeagcyImportManager.cs rename to osu.Game/Database/LegacyImportManager.cs index 8ee63429e2..4dc26b18bb 100644 --- a/osu.Game/Database/LeagcyImportManager.cs +++ b/osu.Game/Database/LegacyImportManager.cs @@ -22,7 +22,7 @@ namespace osu.Game.Database /// /// Handles migration of legacy user data from osu-stable. /// - public class LeagcyImportManager : Component + public class LegacyImportManager : Component { [Resolved] private SkinManager skins { get; set; } diff --git a/osu.Game/OsuGame.cs b/osu.Game/OsuGame.cs index 9b9ec585e8..0e050304f0 100644 --- a/osu.Game/OsuGame.cs +++ b/osu.Game/OsuGame.cs @@ -116,7 +116,7 @@ namespace osu.Game private readonly DifficultyRecommender difficultyRecommender = new DifficultyRecommender(); [Cached] - private readonly LeagcyImportManager leagcyImportManager = new LeagcyImportManager(); + private readonly LegacyImportManager legacyImportManager = new LegacyImportManager(); [Cached] private readonly ScreenshotManager screenshotManager = new ScreenshotManager(); @@ -782,7 +782,7 @@ namespace osu.Game PostNotification = n => Notifications.Post(n), }, Add, true); - loadComponentSingleFile(leagcyImportManager, Add); + loadComponentSingleFile(legacyImportManager, Add); loadComponentSingleFile(screenshotManager, Add); diff --git a/osu.Game/Overlays/Settings/Sections/Maintenance/GeneralSettings.cs b/osu.Game/Overlays/Settings/Sections/Maintenance/GeneralSettings.cs index 9dfd0d8ff4..5bc89ec77c 100644 --- a/osu.Game/Overlays/Settings/Sections/Maintenance/GeneralSettings.cs +++ b/osu.Game/Overlays/Settings/Sections/Maintenance/GeneralSettings.cs @@ -31,9 +31,9 @@ namespace osu.Game.Overlays.Settings.Sections.Maintenance private SettingsButton undeleteButton; [BackgroundDependencyLoader(permitNulls: true)] - private void load(BeatmapManager beatmaps, ScoreManager scores, SkinManager skins, [CanBeNull] CollectionManager collectionManager, [CanBeNull] LeagcyImportManager leagcyImportManager, DialogOverlay dialogOverlay) + private void load(BeatmapManager beatmaps, ScoreManager scores, SkinManager skins, [CanBeNull] CollectionManager collectionManager, [CanBeNull] LegacyImportManager legacyImportManager, DialogOverlay dialogOverlay) { - if (leagcyImportManager?.SupportsImportFromStable == true) + if (legacyImportManager?.SupportsImportFromStable == true) { Add(importBeatmapsButton = new SettingsButton { @@ -41,7 +41,7 @@ namespace osu.Game.Overlays.Settings.Sections.Maintenance Action = () => { importBeatmapsButton.Enabled.Value = false; - leagcyImportManager.ImportFromStableAsync(StableContent.Beatmaps).ContinueWith(t => Schedule(() => importBeatmapsButton.Enabled.Value = true)); + legacyImportManager.ImportFromStableAsync(StableContent.Beatmaps).ContinueWith(t => Schedule(() => importBeatmapsButton.Enabled.Value = true)); } }); } @@ -59,7 +59,7 @@ namespace osu.Game.Overlays.Settings.Sections.Maintenance } }); - if (leagcyImportManager?.SupportsImportFromStable == true) + if (legacyImportManager?.SupportsImportFromStable == true) { Add(importScoresButton = new SettingsButton { @@ -67,7 +67,7 @@ namespace osu.Game.Overlays.Settings.Sections.Maintenance Action = () => { importScoresButton.Enabled.Value = false; - leagcyImportManager.ImportFromStableAsync(StableContent.Scores).ContinueWith(t => Schedule(() => importScoresButton.Enabled.Value = true)); + legacyImportManager.ImportFromStableAsync(StableContent.Scores).ContinueWith(t => Schedule(() => importScoresButton.Enabled.Value = true)); } }); } @@ -85,7 +85,7 @@ namespace osu.Game.Overlays.Settings.Sections.Maintenance } }); - if (leagcyImportManager?.SupportsImportFromStable == true) + if (legacyImportManager?.SupportsImportFromStable == true) { Add(importSkinsButton = new SettingsButton { @@ -93,7 +93,7 @@ namespace osu.Game.Overlays.Settings.Sections.Maintenance Action = () => { importSkinsButton.Enabled.Value = false; - leagcyImportManager.ImportFromStableAsync(StableContent.Skins).ContinueWith(t => Schedule(() => importSkinsButton.Enabled.Value = true)); + legacyImportManager.ImportFromStableAsync(StableContent.Skins).ContinueWith(t => Schedule(() => importSkinsButton.Enabled.Value = true)); } }); } @@ -113,7 +113,7 @@ namespace osu.Game.Overlays.Settings.Sections.Maintenance if (collectionManager != null) { - if (leagcyImportManager?.SupportsImportFromStable == true) + if (legacyImportManager?.SupportsImportFromStable == true) { Add(importCollectionsButton = new SettingsButton { @@ -121,7 +121,7 @@ namespace osu.Game.Overlays.Settings.Sections.Maintenance Action = () => { importCollectionsButton.Enabled.Value = false; - leagcyImportManager.ImportFromStableAsync(StableContent.Collections).ContinueWith(t => Schedule(() => importCollectionsButton.Enabled.Value = true)); + legacyImportManager.ImportFromStableAsync(StableContent.Collections).ContinueWith(t => Schedule(() => importCollectionsButton.Enabled.Value = true)); } }); } diff --git a/osu.Game/Screens/Select/SongSelect.cs b/osu.Game/Screens/Select/SongSelect.cs index b6933a9830..25efe22892 100644 --- a/osu.Game/Screens/Select/SongSelect.cs +++ b/osu.Game/Screens/Select/SongSelect.cs @@ -51,7 +51,7 @@ namespace osu.Game.Screens.Select protected virtual bool ShowFooter => true; - protected virtual bool DisplayStableImportPrompt => leagcyImportManager?.SupportsImportFromStable == true; + protected virtual bool DisplayStableImportPrompt => legacyImportManager?.SupportsImportFromStable == true; public override bool? AllowTrackAdjustments => true; @@ -90,7 +90,7 @@ namespace osu.Game.Screens.Select private BeatmapManager beatmaps { get; set; } [Resolved(CanBeNull = true)] - private LeagcyImportManager leagcyImportManager { get; set; } + private LegacyImportManager legacyImportManager { get; set; } protected ModSelectOverlay ModSelect { get; private set; } @@ -297,7 +297,7 @@ namespace osu.Game.Screens.Select { dialogOverlay.Push(new ImportFromStablePopup(() => { - Task.Run(() => leagcyImportManager.ImportFromStableAsync(StableContent.All)); + Task.Run(() => legacyImportManager.ImportFromStableAsync(StableContent.All)); })); } });