From 1ccdfd736429a43f913491a6d562086c97e5133d Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Tue, 2 Jun 2020 14:03:13 +0900 Subject: [PATCH 01/30] Pull playlist beatmap checksum from api --- osu.Game/Online/API/APIPlaylistBeatmap.cs | 23 +++++++++++++++++++ .../API/Requests/Responses/APIBeatmap.cs | 2 +- osu.Game/Online/Multiplayer/PlaylistItem.cs | 3 +-- 3 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 osu.Game/Online/API/APIPlaylistBeatmap.cs diff --git a/osu.Game/Online/API/APIPlaylistBeatmap.cs b/osu.Game/Online/API/APIPlaylistBeatmap.cs new file mode 100644 index 0000000000..4f7786e880 --- /dev/null +++ b/osu.Game/Online/API/APIPlaylistBeatmap.cs @@ -0,0 +1,23 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using Newtonsoft.Json; +using osu.Game.Beatmaps; +using osu.Game.Online.API.Requests.Responses; +using osu.Game.Rulesets; + +namespace osu.Game.Online.API +{ + public class APIPlaylistBeatmap : APIBeatmap + { + [JsonProperty("checksum")] + public string Checksum { get; set; } + + public override BeatmapInfo ToBeatmap(RulesetStore rulesets) + { + var b = base.ToBeatmap(rulesets); + b.MD5Hash = Checksum; + return b; + } + } +} diff --git a/osu.Game/Online/API/Requests/Responses/APIBeatmap.cs b/osu.Game/Online/API/Requests/Responses/APIBeatmap.cs index e023a2502f..ae65ac09b2 100644 --- a/osu.Game/Online/API/Requests/Responses/APIBeatmap.cs +++ b/osu.Game/Online/API/Requests/Responses/APIBeatmap.cs @@ -64,7 +64,7 @@ namespace osu.Game.Online.API.Requests.Responses [JsonProperty(@"max_combo")] private int? maxCombo { get; set; } - public BeatmapInfo ToBeatmap(RulesetStore rulesets) + public virtual BeatmapInfo ToBeatmap(RulesetStore rulesets) { var set = BeatmapSet?.ToBeatmapSet(rulesets); diff --git a/osu.Game/Online/Multiplayer/PlaylistItem.cs b/osu.Game/Online/Multiplayer/PlaylistItem.cs index 9d6e8eb8e3..416091a1aa 100644 --- a/osu.Game/Online/Multiplayer/PlaylistItem.cs +++ b/osu.Game/Online/Multiplayer/PlaylistItem.cs @@ -7,7 +7,6 @@ using Newtonsoft.Json; using osu.Framework.Bindables; using osu.Game.Beatmaps; using osu.Game.Online.API; -using osu.Game.Online.API.Requests.Responses; using osu.Game.Rulesets; using osu.Game.Rulesets.Mods; @@ -37,7 +36,7 @@ namespace osu.Game.Online.Multiplayer public readonly BindableList RequiredMods = new BindableList(); [JsonProperty("beatmap")] - private APIBeatmap apiBeatmap { get; set; } + private APIPlaylistBeatmap apiBeatmap { get; set; } private APIMod[] allowedModsBacking; From 68fbe9f4c144ad8be6be89286053fb08ccd5f50d Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Tue, 2 Jun 2020 14:03:50 +0900 Subject: [PATCH 02/30] Add checksum validation to the ready/start button --- .../Multi/Match/Components/ReadyButton.cs | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/osu.Game/Screens/Multi/Match/Components/ReadyButton.cs b/osu.Game/Screens/Multi/Match/Components/ReadyButton.cs index e1f86fcc97..a64f24dd7e 100644 --- a/osu.Game/Screens/Multi/Match/Components/ReadyButton.cs +++ b/osu.Game/Screens/Multi/Match/Components/ReadyButton.cs @@ -3,6 +3,7 @@ using System; using System.Linq; +using System.Linq.Expressions; using osu.Framework.Allocation; using osu.Framework.Bindables; using osu.Game.Beatmaps; @@ -52,24 +53,14 @@ namespace osu.Game.Screens.Multi.Match.Components private void updateSelectedItem(PlaylistItem item) { - hasBeatmap = false; - - int? beatmapId = SelectedItem.Value?.Beatmap.Value?.OnlineBeatmapID; - if (beatmapId == null) - return; - - hasBeatmap = beatmaps.QueryBeatmap(b => b.OnlineBeatmapID == beatmapId) != null; + hasBeatmap = findBeatmap(expr => beatmaps.QueryBeatmap(expr)); } private void beatmapUpdated(ValueChangedEvent> weakSet) { if (weakSet.NewValue.TryGetTarget(out var set)) { - int? beatmapId = SelectedItem.Value?.Beatmap.Value?.OnlineBeatmapID; - if (beatmapId == null) - return; - - if (set.Beatmaps.Any(b => b.OnlineBeatmapID == beatmapId)) + if (findBeatmap(expr => set.Beatmaps.AsQueryable().FirstOrDefault(expr))) Schedule(() => hasBeatmap = true); } } @@ -78,15 +69,22 @@ namespace osu.Game.Screens.Multi.Match.Components { if (weakSet.NewValue.TryGetTarget(out var set)) { - int? beatmapId = SelectedItem.Value?.Beatmap.Value?.OnlineBeatmapID; - if (beatmapId == null) - return; - - if (set.Beatmaps.Any(b => b.OnlineBeatmapID == beatmapId)) + if (findBeatmap(expr => set.Beatmaps.AsQueryable().FirstOrDefault(expr))) Schedule(() => hasBeatmap = false); } } + private bool findBeatmap(Func>, BeatmapInfo> expression) + { + int? beatmapId = SelectedItem.Value?.Beatmap.Value?.OnlineBeatmapID; + string checksum = SelectedItem.Value?.Beatmap.Value?.MD5Hash; + + if (beatmapId == null || checksum == null) + return false; + + return expression(b => b.OnlineBeatmapID == beatmapId && b.MD5Hash == checksum) != null; + } + protected override void Update() { base.Update(); From b41bb5a6824d8f4467b4178708cce88ace77011c Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Tue, 2 Jun 2020 14:04:00 +0900 Subject: [PATCH 03/30] Update databased MD5 hash on save --- osu.Game/Beatmaps/BeatmapManager.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index f626b45e42..e5907809f3 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -201,7 +201,9 @@ namespace osu.Game.Beatmaps using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true)) new LegacyBeatmapEncoder(beatmapContent).Encode(sw); - stream.Seek(0, SeekOrigin.Begin); + var attachedInfo = setInfo.Beatmaps.Single(b => b.ID == info.ID); + var md5Hash = stream.ComputeMD5Hash(); + attachedInfo.MD5Hash = md5Hash; UpdateFile(setInfo, setInfo.Files.Single(f => string.Equals(f.Filename, info.Path, StringComparison.OrdinalIgnoreCase)), stream); } From 17e91695e0cf982b76b34db1184aca8be4a7020b Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Tue, 2 Jun 2020 14:04:51 +0900 Subject: [PATCH 04/30] Add checksum validation to the panel download buttons --- .../Screens/Multi/DrawableRoomPlaylistItem.cs | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/osu.Game/Screens/Multi/DrawableRoomPlaylistItem.cs b/osu.Game/Screens/Multi/DrawableRoomPlaylistItem.cs index c024304856..414c1f5748 100644 --- a/osu.Game/Screens/Multi/DrawableRoomPlaylistItem.cs +++ b/osu.Game/Screens/Multi/DrawableRoomPlaylistItem.cs @@ -188,7 +188,7 @@ namespace osu.Game.Screens.Multi X = -18, Children = new Drawable[] { - new PlaylistDownloadButton(item.Beatmap.Value.BeatmapSet) + new PlaylistDownloadButton(item) { Size = new Vector2(50, 30) }, @@ -212,9 +212,15 @@ namespace osu.Game.Screens.Multi private class PlaylistDownloadButton : BeatmapPanelDownloadButton { - public PlaylistDownloadButton(BeatmapSetInfo beatmapSet) - : base(beatmapSet) + private readonly PlaylistItem playlistItem; + + [Resolved] + private BeatmapManager beatmapManager { get; set; } + + public PlaylistDownloadButton(PlaylistItem playlistItem) + : base(playlistItem.Beatmap.Value.BeatmapSet) { + this.playlistItem = playlistItem; Alpha = 0; } @@ -223,11 +229,26 @@ namespace osu.Game.Screens.Multi base.LoadComplete(); State.BindValueChanged(stateChanged, true); + FinishTransforms(true); } private void stateChanged(ValueChangedEvent state) { - this.FadeTo(state.NewValue == DownloadState.LocallyAvailable ? 0 : 1, 500); + switch (state.NewValue) + { + case DownloadState.LocallyAvailable: + // Perform a local query of the beatmap by beatmap checksum, and reset the state if not matching. + if (beatmapManager.QueryBeatmap(b => b.MD5Hash == playlistItem.Beatmap.Value.MD5Hash) == null) + State.Value = DownloadState.NotDownloaded; + else + this.FadeTo(0, 500); + + break; + + default: + this.FadeTo(1, 500); + break; + } } } From 3c85561cdce09b7531aac9ea14bd8a681c159d8c Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Tue, 2 Jun 2020 14:31:43 +0900 Subject: [PATCH 05/30] Add tests --- .../TestSceneDrawableRoomPlaylist.cs | 73 +++++++++++++++++++ osu.Game/Tests/Beatmaps/TestBeatmap.cs | 21 +++++- 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneDrawableRoomPlaylist.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneDrawableRoomPlaylist.cs index 5ef4dd6773..55b026eff6 100644 --- a/osu.Game.Tests/Visual/Multiplayer/TestSceneDrawableRoomPlaylist.cs +++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneDrawableRoomPlaylist.cs @@ -4,12 +4,18 @@ using System.Collections.Generic; using System.Linq; using NUnit.Framework; +using osu.Framework.Allocation; +using osu.Framework.Audio; using osu.Framework.Graphics; using osu.Framework.Graphics.Containers; +using osu.Framework.Platform; using osu.Framework.Testing; +using osu.Game.Beatmaps; using osu.Game.Graphics.Containers; using osu.Game.Graphics.UserInterface; using osu.Game.Online.Multiplayer; +using osu.Game.Overlays; +using osu.Game.Rulesets; using osu.Game.Rulesets.Osu; using osu.Game.Rulesets.Osu.Mods; using osu.Game.Screens.Multi; @@ -23,6 +29,18 @@ namespace osu.Game.Tests.Visual.Multiplayer { private TestPlaylist playlist; + private BeatmapManager manager; + private RulesetStore rulesets; + + [BackgroundDependencyLoader] + private void load(GameHost host, AudioManager audio) + { + Dependencies.Cache(rulesets = new RulesetStore(ContextFactory)); + Dependencies.Cache(manager = new BeatmapManager(LocalStorage, ContextFactory, rulesets, null, audio, host, Beatmap.Default)); + + manager.Import(new TestBeatmap(new OsuRuleset().RulesetInfo).BeatmapInfo.BeatmapSet).Wait(); + } + [Test] public void TestNonEditableNonSelectable() { @@ -182,6 +200,28 @@ namespace osu.Game.Tests.Visual.Multiplayer AddStep("click delete button", () => InputManager.Click(MouseButton.Left)); } + [Test] + public void TestDownloadButtonHiddenInitiallyWhenBeatmapExists() + { + createPlaylist(new TestBeatmap(new OsuRuleset().RulesetInfo).BeatmapInfo); + + AddAssert("download button hidden", () => !playlist.ChildrenOfType().Single().IsPresent); + } + + [Test] + public void TestDownloadButtonVisibleInitiallyWhenBeatmapDoesNotExist() + { + var byOnlineId = new TestBeatmap(new OsuRuleset().RulesetInfo).BeatmapInfo; + byOnlineId.BeatmapSet.OnlineBeatmapSetID = 1337; // Some random ID that does not exist locally. + + var byChecksum = new TestBeatmap(new OsuRuleset().RulesetInfo).BeatmapInfo; + byChecksum.MD5Hash = "1337"; // Some random checksum that does not exist locally. + + createPlaylist(byOnlineId, byChecksum); + + AddAssert("download buttons shown", () => playlist.ChildrenOfType().All(d => d.IsPresent)); + } + private void moveToItem(int index, Vector2? offset = null) => AddStep($"move mouse to item {index}", () => InputManager.MoveMouseTo(playlist.ChildrenOfType>().ElementAt(index), offset)); @@ -235,6 +275,39 @@ namespace osu.Game.Tests.Visual.Multiplayer AddUntilStep("wait for items to load", () => playlist.ItemMap.Values.All(i => i.IsLoaded)); } + private void createPlaylist(params BeatmapInfo[] beatmaps) + { + AddStep("create playlist", () => + { + Child = playlist = new TestPlaylist(false, false) + { + Anchor = Anchor.Centre, + Origin = Anchor.Centre, + Size = new Vector2(500, 300) + }; + + int index = 0; + + foreach (var b in beatmaps) + { + playlist.Items.Add(new PlaylistItem + { + ID = index++, + Beatmap = { Value = b }, + Ruleset = { Value = new OsuRuleset().RulesetInfo }, + RequiredMods = + { + new OsuModHardRock(), + new OsuModDoubleTime(), + new OsuModAutoplay() + } + }); + } + }); + + AddUntilStep("wait for items to load", () => playlist.ItemMap.Values.All(i => i.IsLoaded)); + } + private class TestPlaylist : DrawableRoomPlaylist { public new IReadOnlyDictionary> ItemMap => base.ItemMap; diff --git a/osu.Game/Tests/Beatmaps/TestBeatmap.cs b/osu.Game/Tests/Beatmaps/TestBeatmap.cs index a7c84bf692..9fc20fd0f2 100644 --- a/osu.Game/Tests/Beatmaps/TestBeatmap.cs +++ b/osu.Game/Tests/Beatmaps/TestBeatmap.cs @@ -1,9 +1,11 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System; using System.Collections.Generic; using System.IO; using System.Text; +using osu.Framework.Extensions; using osu.Game.Beatmaps; using osu.Game.IO; using osu.Game.Rulesets; @@ -43,10 +45,25 @@ namespace osu.Game.Tests.Beatmaps private static Beatmap createTestBeatmap() { using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(test_beatmap_data))) - using (var reader = new LineBufferedReader(stream)) - return Decoder.GetDecoder(reader).Decode(reader); + { + using (var reader = new LineBufferedReader(stream)) + { + var b = Decoder.GetDecoder(reader).Decode(reader); + + b.BeatmapInfo.MD5Hash = test_beatmap_hash.Value.md5; + b.BeatmapInfo.Hash = test_beatmap_hash.Value.sha2; + + return b; + } + } } + private static readonly Lazy<(string md5, string sha2)> test_beatmap_hash = new Lazy<(string md5, string sha2)>(() => + { + using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(test_beatmap_data))) + return (stream.ComputeMD5Hash(), stream.ComputeSHA2Hash()); + }); + private const string test_beatmap_data = @"osu file format v14 [General] From fac96f6dddf9dba724fa0b298444694310c508af Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Tue, 2 Jun 2020 17:02:01 +0900 Subject: [PATCH 06/30] Fix match beatmap not updating after re-download --- .../Multiplayer/TestSceneMatchSubScreen.cs | 57 +++++++++++++++++-- .../Match/Components/MatchSettingsOverlay.cs | 2 +- .../Screens/Multi/Match/MatchSubScreen.cs | 13 +---- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/osu.Game.Tests/Visual/Multiplayer/TestSceneMatchSubScreen.cs b/osu.Game.Tests/Visual/Multiplayer/TestSceneMatchSubScreen.cs index d678d5a814..6154e646f8 100644 --- a/osu.Game.Tests/Visual/Multiplayer/TestSceneMatchSubScreen.cs +++ b/osu.Game.Tests/Visual/Multiplayer/TestSceneMatchSubScreen.cs @@ -5,7 +5,9 @@ using System; using System.Linq; using NUnit.Framework; using osu.Framework.Allocation; +using osu.Framework.Audio; using osu.Framework.Bindables; +using osu.Framework.Platform; using osu.Framework.Screens; using osu.Framework.Testing; using osu.Game.Beatmaps; @@ -29,14 +31,20 @@ namespace osu.Game.Tests.Visual.Multiplayer [Cached(typeof(IRoomManager))] private readonly TestRoomManager roomManager = new TestRoomManager(); - [Resolved] - private BeatmapManager beatmaps { get; set; } - - [Resolved] - private RulesetStore rulesets { get; set; } + private BeatmapManager manager; + private RulesetStore rulesets; private TestMatchSubScreen match; + [BackgroundDependencyLoader] + private void load(GameHost host, AudioManager audio) + { + Dependencies.Cache(rulesets = new RulesetStore(ContextFactory)); + Dependencies.Cache(manager = new BeatmapManager(LocalStorage, ContextFactory, rulesets, null, audio, host, Beatmap.Default)); + + manager.Import(new TestBeatmap(new OsuRuleset().RulesetInfo).BeatmapInfo.BeatmapSet).Wait(); + } + [SetUp] public void Setup() => Schedule(() => { @@ -75,10 +83,49 @@ namespace osu.Game.Tests.Visual.Multiplayer AddAssert("first playlist item selected", () => match.SelectedItem.Value == Room.Playlist[0]); } + [Test] + public void TestBeatmapUpdatedOnReImport() + { + BeatmapSetInfo importedSet = null; + + AddStep("import altered beatmap", () => + { + var beatmap = new TestBeatmap(new OsuRuleset().RulesetInfo); + beatmap.BeatmapInfo.BaseDifficulty.CircleSize = 1; + + importedSet = manager.Import(beatmap.BeatmapInfo.BeatmapSet).Result; + }); + + AddStep("load room", () => + { + Room.Name.Value = "my awesome room"; + Room.Host.Value = new User { Id = 2, Username = "peppy" }; + Room.Playlist.Add(new PlaylistItem + { + Beatmap = { Value = importedSet.Beatmaps[0] }, + Ruleset = { Value = new OsuRuleset().RulesetInfo } + }); + }); + + AddStep("create room", () => + { + InputManager.MoveMouseTo(match.ChildrenOfType().Single()); + InputManager.Click(MouseButton.Left); + }); + + AddAssert("match has altered beatmap", () => match.Beatmap.Value.Beatmap.BeatmapInfo.BaseDifficulty.CircleSize == 1); + + AddStep("re-import original beatmap", () => manager.Import(new TestBeatmap(new OsuRuleset().RulesetInfo).BeatmapInfo.BeatmapSet).Wait()); + + AddAssert("match has original beatmap", () => match.Beatmap.Value.Beatmap.BeatmapInfo.BaseDifficulty.CircleSize != 1); + } + private class TestMatchSubScreen : MatchSubScreen { public new Bindable SelectedItem => base.SelectedItem; + public new Bindable Beatmap => base.Beatmap; + public TestMatchSubScreen(Room room) : base(room) { diff --git a/osu.Game/Screens/Multi/Match/Components/MatchSettingsOverlay.cs b/osu.Game/Screens/Multi/Match/Components/MatchSettingsOverlay.cs index 54c4f8f7c7..49a0fc434b 100644 --- a/osu.Game/Screens/Multi/Match/Components/MatchSettingsOverlay.cs +++ b/osu.Game/Screens/Multi/Match/Components/MatchSettingsOverlay.cs @@ -433,7 +433,7 @@ namespace osu.Game.Screens.Multi.Match.Components } } - private class CreateRoomButton : TriangleButton + public class CreateRoomButton : TriangleButton { public CreateRoomButton() { diff --git a/osu.Game/Screens/Multi/Match/MatchSubScreen.cs b/osu.Game/Screens/Multi/Match/MatchSubScreen.cs index e1d72d9600..bbfbaf81af 100644 --- a/osu.Game/Screens/Multi/Match/MatchSubScreen.cs +++ b/osu.Game/Screens/Multi/Match/MatchSubScreen.cs @@ -207,6 +207,8 @@ namespace osu.Game.Screens.Multi.Match Ruleset.Value = item.Ruleset.Value; } + private void beatmapUpdated(ValueChangedEvent> weakSet) => Schedule(updateWorkingBeatmap); + private void updateWorkingBeatmap() { var beatmap = SelectedItem.Value?.Beatmap.Value; @@ -217,17 +219,6 @@ namespace osu.Game.Screens.Multi.Match Beatmap.Value = beatmapManager.GetWorkingBeatmap(localBeatmap); } - private void beatmapUpdated(ValueChangedEvent> weakSet) - { - Schedule(() => - { - if (Beatmap.Value != beatmapManager.DefaultBeatmap) - return; - - updateWorkingBeatmap(); - }); - } - private void onStart() { switch (type.Value) From dfb9687fb5bfc47670ea6b017410e46c76c5905c Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Tue, 2 Jun 2020 17:22:09 +0900 Subject: [PATCH 07/30] Extract update into PreUpdate(), add test --- osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs | 3 +++ osu.Game/Beatmaps/BeatmapManager.cs | 17 +++++++++++++---- osu.Game/Database/ArchiveModelManager.cs | 10 ++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 5eb11a3264..88bb39a521 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -602,6 +602,8 @@ namespace osu.Game.Tests.Beatmaps.IO Beatmap beatmapToUpdate = (Beatmap)manager.GetWorkingBeatmap(setToUpdate.Beatmaps.First(b => b.RulesetID == 0)).Beatmap; BeatmapSetFileInfo fileToUpdate = setToUpdate.Files.First(f => beatmapToUpdate.BeatmapInfo.Path.Contains(f.Filename)); + string oldMd5Hash = beatmapToUpdate.BeatmapInfo.MD5Hash; + using (var stream = new MemoryStream()) { using (var writer = new StreamWriter(stream, Encoding.UTF8, 1024, true)) @@ -624,6 +626,7 @@ namespace osu.Game.Tests.Beatmaps.IO Beatmap updatedBeatmap = (Beatmap)manager.GetWorkingBeatmap(manager.QueryBeatmap(b => b.ID == beatmapToUpdate.BeatmapInfo.ID)).Beatmap; Assert.That(updatedBeatmap.HitObjects.Count, Is.EqualTo(1)); Assert.That(updatedBeatmap.HitObjects[0].StartTime, Is.EqualTo(5000)); + Assert.That(updatedBeatmap.BeatmapInfo.MD5Hash, Is.Not.EqualTo(oldMd5Hash)); } finally { diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index e5907809f3..668ac6ee10 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -201,10 +201,6 @@ namespace osu.Game.Beatmaps using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true)) new LegacyBeatmapEncoder(beatmapContent).Encode(sw); - var attachedInfo = setInfo.Beatmaps.Single(b => b.ID == info.ID); - var md5Hash = stream.ComputeMD5Hash(); - attachedInfo.MD5Hash = md5Hash; - UpdateFile(setInfo, setInfo.Files.Single(f => string.Equals(f.Filename, info.Path, StringComparison.OrdinalIgnoreCase)), stream); } @@ -213,6 +209,19 @@ namespace osu.Game.Beatmaps workingCache.Remove(working); } + protected override void PreUpdate(BeatmapSetInfo item) + { + base.PreUpdate(item); + + foreach (var info in item.Beatmaps) + { + var file = item.Files.FirstOrDefault(f => string.Equals(f.Filename, info.Path, StringComparison.OrdinalIgnoreCase))?.FileInfo.StoragePath; + + using (var stream = Files.Store.GetStream(file)) + info.MD5Hash = stream.ComputeMD5Hash(); + } + } + private readonly WeakList workingCache = new WeakList(); /// diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index ae55a7b14a..f7e81ae4bc 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -430,10 +430,20 @@ namespace osu.Game.Database { item.Hash = computeHash(item); + PreUpdate(item); + ModelStore.Update(item); } } + /// + /// Perform any final actions before the update to database executes. + /// + /// The that is being updated. + protected virtual void PreUpdate(TModel item) + { + } + /// /// Delete an item from the manager. /// Is a no-op for already deleted items. From 40e64eed475b7c09de60643671035e5cd0ec9967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Tue, 2 Jun 2020 23:15:14 +0200 Subject: [PATCH 08/30] Add contributing guidelines --- CONTRIBUTING.md | 121 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..441521f9ef --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,121 @@ +# Contributing Guidelines + +Thank you for showing interest in the development of osu!lazer! We aim to provide a good collaborating environment for everyone involved, and as such have decided to list some of the most important things to keep in mind in the process. The guidelines below have been chosen based on past experience. + +These are not "official rules" *per se*, but following them will help everyone deal with things in the most efficient manner. + +## Table of contents + +1. [I would like to submit an issue!](#i-would-like-to-submit-an-issue) +2. [I would like to submit a pull request!](#i-would-like-to-submit-a-pull-request) + +## I would like to submit an issue! + +When it comes to issues, bug reports and feature suggestions are welcomed, although please keep in mind that at any point in time, hundreds of issues are open, which vary in severity and the amount of time needed to address them. As such it's not uncommon for issues to remain unresolved for a long time or even closed outright if they are deemed not important enough to fix in the foreseeable future. Issues that are required to "go live" or otherwise achieve parity with stable are prioritised the most. + +* **Before submitting an issue, try searching existing issues first.** + + For housekeeping purposes, we close issues that overlap with or duplicate other pre-existing issues - you can help us not have to do that by searching existing issues yourself first. The issue search box, as well as the issue tag system, are tools you can use to check if an issue has been reported before. + +* **When submitting a bug report, please try to include as much detail as possible.** + + Bugs are not equal - some of them will be reproducible every time on pretty much all hardware, while others will be hard to track down due to being specific to particular hardware or even somewhat random in nature. As such, providing as much detail as possible when reporting a bug is hugely appreciated. A good starting set of information contains of: + + * the in-game logs, which are located at: + * `%AppData%/osu/logs` (on Windows), + * `~/.local/share/osu/logs` (on Linux and macOS), + * `Android/Data/sh.ppy.osulazer/logs` (on Android), + * on iOS they can be obtained by connecting your device to your desktop and copying the `logs` directory from the app's own document storage using iTunes, + * your system specifications (including the operating system and platform you are playing on), + * a reproduction scenario (list of steps you have performed leading up to the occurrence of the bug), + * a video or picture of the bug, if at all possible. + +* **Provide more information when asked to do so.** + + Sometimes when a bug is more elusive or complicated, none of the information listed above will pinpoint a concrete cause of the problem. In this case we will most likely ask you for additional info, such as a Windows Event Log dump or a copy of your local lazer database (`client.db`). Providing that information is beneficial to both parties - we can track down the problem better, and hopefully fix it for you at some point once we know where it is! + +* **When submitting a feature proposal, please describe it in the most understandable way you can.** + + Communicating your idea for a feature can often be hard, and we would like to avoid any misunderstandings. As such, please try to explain your idea in a short, but understandable manner - it's best to avoid jargon or terms and references that could be considered obscure. A mock-up picture (doesn't have to be good!) of the feature can also go a long way in explaining. + +* **Refrain from posting "+1" comments.** + + If an issue has already been created, saying that you also experience it without providing any additional details doesn't really help us in any way. To express support for a proposal or indicate that you are also affected by a particular bug, you can use comment reactions instead. + +* **Refrain from asking if an issue has been resolved yet.** + + As mentioned above, the issue tracker has hundreds of issues open at any given time. Currently the game is being worked on by two members of the core team, and a handful of outside contributors who offer their free time to help out. As such, it can happen that an issue gets placed on the backburner due to being less important; generally posting a comment demanding its resolution some months or years after it is reported is not very likely to increase its priority. + +* **Avoid long discussions about non-development topics.** + + GitHub is mostly a developer space, and as such isn't really fit for lengthened discussions about gameplay mechanics (which might not even be in any way confirmed for the final release) and similar non-technical matters. Such matters are probably best addressed at the osu! forums. + +## I would like to submit a pull request! + +We also welcome pull requests from unaffiliated contributors. The issue tracker should provide plenty of issues that you can work on; we also mark issues that we think would be good for newcomers with the [`good-first-issue`](https://github.com/ppy/osu/issues?q=is%3Aissue+is%3Aopen+label%3Agood-first-issue) label. + +However, do keep in mind that the core team is committed to bringing osu!lazer up to par with stable first and foremost, so depending on what your contribution concerns, it might not be merged and released right away. Our approach to managing issues and their priorities is described [in the wiki](https://github.com/ppy/osu/wiki/Project-management). + +Here are some key things to note before jumping in: + +* **Make sure you are comfortable with C\# and your development environment.** + + While we are accepting of all kinds of contributions, we also have a certain quality standard we'd like to uphold and limited time to review your code. Therefore, we would like to avoid providing entry-level advice, and as such if you're not very familiar with C\# as a programming language, we'd recommend that you start off with a few personal projects to get acquainted with the language's syntax, toolchain and principles of object-oriented programming first. + +* **Make sure you are familiar with git and the pull request workflow.** + + [git](https://git-scm.com/) is a distributed version control system that might not be very intuitive at the beginning if you're not familiar with version control. In particular, projects using git have a particular workflow for submitting code changes, which is called the pull request workflow. + + To make things run more smoothly, we recommend that you look up some online resources to familiarise yourself with the git vocabulary and commands, and practice working with forks and submitting pull requests at your own pace. A high-level overview of the process can be found in [this article by GitHub](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/proposing-changes-to-your-work-with-pull-requests). + +* **Make sure to submit pull requests off of a topic branch.** + + As described in the article linked in the previous point, topic branches help you parallelise your work and separate it from the main `master` branch, and additionally are easier for maintainers to work with. Working with multiple `master` branches across many remotes is difficult to keep track of, and it's easy to make a mistake and push to the wrong `master` branch by accident. + +* **Refrain from making changes through the GitHub web interface.** + + Even though GitHub provides an option to edit code or replace files in the repository using the web interface, we strongly discourage using it in most scenarios. Editing files this way is inefficient and likely to introduce whitespace or file encoding changes that make it more difficult to review the code. + + Code written through the web interface will also very likely be questioned outright by the reviewers, as it is likely that it has not been properly tested or that it will fail continuous integration checks. We strongly encourage using an IDE like [Visual Studio](https://visualstudio.microsoft.com/), [Visual Studio Code](https://code.visualstudio.com/) or [JetBrains Rider](https://www.jetbrains.com/rider/) instead. + +* **Add tests for your code whenever possible.** + + Automated tests are an essential part of a quality and reliable codebase. They help to make the code more maintainable by ensuring it is safe to reorganise (or refactor) the code in various ways, and also prevent regressions - bugs that resurface after having been fixed at some point in the past. If it is viable, please put in the time to add tests, so that the changes you make can last for a (hopefully) very long time. + +* **Run tests before opening a pull request.** + + Tying into the previous point, sometimes changes in one part of the codebase can result in unpredictable changes in behaviour in other pieces of the code. This is why it is best to always try to run tests before opening a PR. + + Continuous integration will always run the tests for you (and us), too, but it is best not to rely on it, as there might be many builds queued at any time. Running tests on your own will help you be more certain that at the point of clicking the "Create pull request" button, your changes are as ready as can be. + +* **Run code style analysis before opening a pull request.** + + As part of continuous integration, we also run code style analysis, which is supposed to make sure that your code is formatted the same way as all the pre-existing code in the repository. The reason we enforce a particular code style everywhere is to make sure the codebase is consistent in that regard - having one whitespace convention in one place and another one elsewhere causes disorganisation. + +* **Make sure to keep the *Allow edits from maintainers* check box checked.** + + To speed up the merging process, collaborators and team members will sometimes want to push changes to your branch themselves, to make minor code style adjustments or to otherwise refactor the code without having to describe how they'd like the code to look like in painstaking detail. Having the *Allow edits from maintainers* check box checked lets them do that; without it they are forced to report issues back to you and wait for you to address them. + +* **Refrain from continually merging the master branch back to the PR.** + + Unless there are merge conflicts that need resolution, there is no need to keep merging `master` back to a branch over and over again. One of the maintainers will merge `master` themselves before merging the PR itself anyway, and continual merge commits can cause CI to get overwhelmed due to queueing up too many builds. + +* **Refrain from force-pushing to the PR branch.** + + Force-pushing should be avoided, as it can lead to accidentally overwriting a maintainer's changes or CI building wrong commits. We value all history in the project, so there is no need to squash or amend commits in most cases. + + The cases in which force-pushing is warranted are very rare (such as accidentally leaking sensitive info in one of the files committed, adding unrelated files, or mis-merging a dependent PR). + +* **Be patient when waiting for the code to be reviewed and merged.** + + As much as we'd like to review all contributions as fast as possible, our time is limited, as team members have to work on their own tasks in addition to reviewing code. As such, work needs to be prioritised, and it can unfortunately take weeks or months for your PR to be merged, depending on how important it is deemed to be. + +* **Don't mistake criticism of code for criticism of your person.** + + As mentioned before, we are highly committed to quality when it comes to the lazer project. This means that contributions from less experienced community members can take multiple rounds of review to get to a mergeable state. We try our utmost best to never conflate a person with the code they authored, and to keep the discussion focused on the code at all times. Please consider our comments and requests a learning experience, and don't treat it as a personal attack. + +* **Feel free to reach out for help.** + + If you're uncertain about some part of the codebase or some inner workings of the game and framework, please reach out either by leaving a comment in the relevant issue or PR thread, or by posting a message in the [development Discord server](https://discord.gg/ppy). We will try to help you as much as we can. + + When it comes to which form of communication is best, GitHub generally lends better to longer-form discussions, while Discord is better for snappy call-and-response answers. Use your best discretion when deciding, and try to keep a single discussion in one place instead of moving back and forth. From 74875f9b629715a1e5a13e65704af277fb2c8c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 3 Jun 2020 06:47:10 +0200 Subject: [PATCH 09/30] Apply review suggestions & other cleanups --- CONTRIBUTING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 441521f9ef..331534ad73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,21 +11,21 @@ These are not "official rules" *per se*, but following them will help everyone d ## I would like to submit an issue! -When it comes to issues, bug reports and feature suggestions are welcomed, although please keep in mind that at any point in time, hundreds of issues are open, which vary in severity and the amount of time needed to address them. As such it's not uncommon for issues to remain unresolved for a long time or even closed outright if they are deemed not important enough to fix in the foreseeable future. Issues that are required to "go live" or otherwise achieve parity with stable are prioritised the most. +Issues, bug reports and feature suggestions are welcomed, though please keep in mind that at any point in time, hundreds of issues are open, which vary in severity and the amount of time needed to address them. As such it's not uncommon for issues to remain unresolved for a long time or even closed outright if they are deemed not important enough to fix in the foreseeable future. Issues that are required to "go live" or otherwise achieve parity with stable are prioritised the most. * **Before submitting an issue, try searching existing issues first.** - For housekeeping purposes, we close issues that overlap with or duplicate other pre-existing issues - you can help us not have to do that by searching existing issues yourself first. The issue search box, as well as the issue tag system, are tools you can use to check if an issue has been reported before. + For housekeeping purposes, we close issues that overlap with or duplicate other pre-existing issues - you can help us not to have to do that by searching existing issues yourself first. The issue search box, as well as the issue tag system, are tools you can use to check if an issue has been reported before. * **When submitting a bug report, please try to include as much detail as possible.** - Bugs are not equal - some of them will be reproducible every time on pretty much all hardware, while others will be hard to track down due to being specific to particular hardware or even somewhat random in nature. As such, providing as much detail as possible when reporting a bug is hugely appreciated. A good starting set of information contains of: + Bugs are not equal - some of them will be reproducible every time on pretty much all hardware, while others will be hard to track down due to being specific to particular hardware or even somewhat random in nature. As such, providing as much detail as possible when reporting a bug is hugely appreciated. A good starting set of information consists of: * the in-game logs, which are located at: * `%AppData%/osu/logs` (on Windows), * `~/.local/share/osu/logs` (on Linux and macOS), * `Android/Data/sh.ppy.osulazer/logs` (on Android), - * on iOS they can be obtained by connecting your device to your desktop and copying the `logs` directory from the app's own document storage using iTunes, + * on iOS they can be obtained by connecting your device to your desktop and [copying the `logs` directory from the app's own document storage using iTunes](https://support.apple.com/en-us/HT201301#copy-to-computer), * your system specifications (including the operating system and platform you are playing on), * a reproduction scenario (list of steps you have performed leading up to the occurrence of the bug), * a video or picture of the bug, if at all possible. From c155ab83399a51bdab44a48d5805463c8520318a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 3 Jun 2020 18:03:10 +0900 Subject: [PATCH 10/30] Check filenames and timestamps before reusing an already imported model --- osu.Game/Beatmaps/BeatmapManager.cs | 4 ++-- osu.Game/Database/ArchiveModelManager.cs | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index f626b45e42..e7cef13c68 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -258,9 +258,9 @@ namespace osu.Game.Beatmaps /// The first result for the provided query, or null if no results were found. public BeatmapSetInfo QueryBeatmapSet(Expression> query) => beatmaps.ConsumableItems.AsNoTracking().FirstOrDefault(query); - protected override bool CanUndelete(BeatmapSetInfo existing, BeatmapSetInfo import) + protected override bool CanReuseExisting(BeatmapSetInfo existing, BeatmapSetInfo import) { - if (!base.CanUndelete(existing, import)) + if (!base.CanReuseExisting(existing, import)) return false; var existingIds = existing.Beatmaps.Select(b => b.OnlineBeatmapID).OrderBy(i => i); diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index ae55a7b14a..4d7d3e96e6 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -332,7 +332,7 @@ namespace osu.Game.Database if (existing != null) { - if (CanUndelete(existing, item)) + if (CanReuseExisting(existing, item)) { Undelete(existing); LogForModel(item, $"Found existing {HumanisedModelName} for {item} (ID {existing.ID}) – skipping import."); @@ -660,13 +660,29 @@ namespace osu.Game.Database protected TModel CheckForExisting(TModel model) => model.Hash == null ? null : ModelStore.ConsumableItems.FirstOrDefault(b => b.Hash == model.Hash); /// - /// After an existing is found during an import process, the default behaviour is to restore the existing + /// After an existing is found during an import process, the default behaviour is to use/restore the existing /// item and skip the import. This method allows changing that behaviour. /// /// The existing model. /// The newly imported model. /// Whether the existing model should be restored and used. Returning false will delete the existing and force a re-import. - protected virtual bool CanUndelete(TModel existing, TModel import) => true; + protected virtual bool CanReuseExisting(TModel existing, TModel import) => + getFilenames(existing.Files).SequenceEqual(getFilenames(import.Files)) && + // poor-man's (cheap) equality comparison, avoiding hashing unnecessarily. + // can switch to full hash checks on a per-case basis (or for all) if we decide this is not a performance issue. + getTimestamps(existing.Files).SequenceEqual(getTimestamps(import.Files)); + + private IEnumerable getFilenames(List files) + { + foreach (var f in files.OrderBy(f => f.Filename)) + yield return f.Filename; + } + + private IEnumerable getTimestamps(List files) + { + foreach (var f in files.OrderBy(f => f.Filename)) + yield return File.GetLastWriteTimeUtc(Files.Storage.GetFullPath(f.FileInfo.StoragePath)).ToFileTime(); + } private DbSet queryModel() => ContextFactory.Get().Set(); From 012933545eccd4510dc3c0a70b040610dad0bf8d Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 3 Jun 2020 18:30:27 +0900 Subject: [PATCH 11/30] Add test coverage --- .../Beatmaps/IO/ImportBeatmapTest.cs | 161 ++++++++++++++++++ osu.Game/Database/ArchiveModelManager.cs | 2 +- 2 files changed, 162 insertions(+), 1 deletion(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 5eb11a3264..12c9c92e90 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -12,6 +12,7 @@ using NUnit.Framework; using osu.Framework.Platform; using osu.Game.IPC; using osu.Framework.Allocation; +using osu.Framework.Extensions; using osu.Framework.Logging; using osu.Game.Beatmaps; using osu.Game.Beatmaps.Formats; @@ -22,6 +23,7 @@ using SharpCompress.Archives; using SharpCompress.Archives.Zip; using SharpCompress.Common; using SharpCompress.Writers.Zip; +using FileInfo = System.IO.FileInfo; namespace osu.Game.Tests.Beatmaps.IO { @@ -93,6 +95,165 @@ namespace osu.Game.Tests.Beatmaps.IO } } + [Test] + public async Task TestImportThenImportWithReZip() + { + using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImportWithNewerTimestamp))) + { + try + { + var osu = loadOsu(host); + + var temp = TestResources.GetTestBeatmapForImport(); + + string extractedFolder = $"{temp}_extracted"; + Directory.CreateDirectory(extractedFolder); + + try + { + var imported = await LoadOszIntoOsu(osu); + + string hashBefore = hashFile(temp); + + using (var zip = ZipArchive.Open(temp)) + zip.WriteToDirectory(extractedFolder); + + using (var zip = ZipArchive.Create()) + { + zip.AddAllFromDirectory(extractedFolder); + zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate)); + } + + // zip files differ because different compression or encoder. + Assert.AreNotEqual(hashBefore, hashFile(temp)); + + var importedSecondTime = await osu.Dependencies.Get().Import(temp); + + ensureLoaded(osu); + + // but contents doesn't, so existing should still be used. + Assert.IsTrue(imported.ID == importedSecondTime.ID); + Assert.IsTrue(imported.Beatmaps.First().ID == importedSecondTime.Beatmaps.First().ID); + } + finally + { + Directory.Delete(extractedFolder, true); + } + } + finally + { + host.Exit(); + } + } + } + + private string hashFile(string filename) + { + using (var s = File.OpenRead(filename)) + return s.ComputeMD5Hash(); + } + + [Test] + public async Task TestImportThenImportWithNewerTimestamp() + { + using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImportWithNewerTimestamp))) + { + try + { + var osu = loadOsu(host); + + var temp = TestResources.GetTestBeatmapForImport(); + + string extractedFolder = $"{temp}_extracted"; + Directory.CreateDirectory(extractedFolder); + + try + { + var imported = await LoadOszIntoOsu(osu); + + using (var zip = ZipArchive.Open(temp)) + zip.WriteToDirectory(extractedFolder); + + // change timestamp + new FileInfo(Directory.GetFiles(extractedFolder).First()).LastWriteTime = DateTime.Now; + + using (var zip = ZipArchive.Create()) + { + zip.AddAllFromDirectory(extractedFolder); + zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate)); + } + + var importedSecondTime = await osu.Dependencies.Get().Import(temp); + + ensureLoaded(osu); + + // check the newly "imported" beatmap is not the original. + Assert.IsTrue(imported.ID != importedSecondTime.ID); + Assert.IsTrue(imported.Beatmaps.First().ID != importedSecondTime.Beatmaps.First().ID); + } + finally + { + Directory.Delete(extractedFolder, true); + } + } + finally + { + host.Exit(); + } + } + } + + [Test] + public async Task TestImportThenImportWithDifferentFilename() + { + using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImportWithDifferentFilename))) + { + try + { + var osu = loadOsu(host); + + var temp = TestResources.GetTestBeatmapForImport(); + + string extractedFolder = $"{temp}_extracted"; + Directory.CreateDirectory(extractedFolder); + + try + { + var imported = await LoadOszIntoOsu(osu); + + using (var zip = ZipArchive.Open(temp)) + zip.WriteToDirectory(extractedFolder); + + // change filename + var firstFile = new FileInfo(Directory.GetFiles(extractedFolder).First()); + firstFile.MoveTo(Path.Combine(firstFile.DirectoryName, $"{firstFile.Name}-changed{firstFile.Extension}")); + + using (var zip = ZipArchive.Create()) + { + zip.AddAllFromDirectory(extractedFolder); + zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate)); + } + + var importedSecondTime = await osu.Dependencies.Get().Import(temp); + + ensureLoaded(osu); + + // check the newly "imported" beatmap is not the original. + Assert.IsTrue(imported.ID != importedSecondTime.ID); + Assert.IsTrue(imported.Beatmaps.First().ID != importedSecondTime.Beatmaps.First().ID); + } + finally + { + Directory.Delete(extractedFolder, true); + } + } + finally + { + host.Exit(); + } + } + } + [Test] public async Task TestImportCorruptThenImport() { diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 4d7d3e96e6..5ca9423de2 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -276,7 +276,7 @@ namespace osu.Game.Database // for now, concatenate all .osu files in the set to create a unique hash. MemoryStream hashable = new MemoryStream(); - foreach (TFileModel file in item.Files.Where(f => HashableFileTypes.Any(f.Filename.EndsWith))) + foreach (TFileModel file in item.Files.Where(f => HashableFileTypes.Any(f.Filename.EndsWith)).OrderBy(f => f.Filename)) { using (Stream s = Files.Store.GetStream(file.FileInfo.StoragePath)) s.CopyTo(hashable); From 25160dc220d9f2f0bde4125f0bafd7446e3ad354 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 3 Jun 2020 19:15:52 +0900 Subject: [PATCH 12/30] Fix test name --- osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 12c9c92e90..12f06059f7 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -98,7 +98,7 @@ namespace osu.Game.Tests.Beatmaps.IO [Test] public async Task TestImportThenImportWithReZip() { - using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImportWithNewerTimestamp))) + using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImportWithReZip))) { try { From 5ed3cd205f068d572caddc0ae01aa7ab8a580a6a Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 3 Jun 2020 22:35:01 +0900 Subject: [PATCH 13/30] Simplify reuse check using FileInfo IDs --- .../Beatmaps/IO/ImportBeatmapTest.cs | 9 +++++---- osu.Game/Database/ArchiveModelManager.cs | 20 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 12f06059f7..9b34eece5f 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -154,9 +154,9 @@ namespace osu.Game.Tests.Beatmaps.IO } [Test] - public async Task TestImportThenImportWithNewerTimestamp() + public async Task TestImportThenImportWithChangedFile() { - using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImportWithNewerTimestamp))) + using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImportWithChangedFile))) { try { @@ -174,8 +174,9 @@ namespace osu.Game.Tests.Beatmaps.IO using (var zip = ZipArchive.Open(temp)) zip.WriteToDirectory(extractedFolder); - // change timestamp - new FileInfo(Directory.GetFiles(extractedFolder).First()).LastWriteTime = DateTime.Now; + // arbitrary write to non-hashed file + using (var sw = new FileInfo(Directory.GetFiles(extractedFolder, "*.mp3").First()).AppendText()) + sw.WriteLine("text"); using (var zip = ZipArchive.Create()) { diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index 5ca9423de2..0fe8dd1268 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -667,10 +667,16 @@ namespace osu.Game.Database /// The newly imported model. /// Whether the existing model should be restored and used. Returning false will delete the existing and force a re-import. protected virtual bool CanReuseExisting(TModel existing, TModel import) => - getFilenames(existing.Files).SequenceEqual(getFilenames(import.Files)) && - // poor-man's (cheap) equality comparison, avoiding hashing unnecessarily. - // can switch to full hash checks on a per-case basis (or for all) if we decide this is not a performance issue. - getTimestamps(existing.Files).SequenceEqual(getTimestamps(import.Files)); + // for the best or worst, we copy and import files of a new import before checking whether + // it is a duplicate. so to check if anything has changed, we can just compare all FileInfo IDs. + getIDs(existing.Files).SequenceEqual(getIDs(import.Files)) && + getFilenames(existing.Files).SequenceEqual(getFilenames(import.Files)); + + private IEnumerable getIDs(List files) + { + foreach (var f in files.OrderBy(f => f.Filename)) + yield return f.FileInfo.ID; + } private IEnumerable getFilenames(List files) { @@ -678,12 +684,6 @@ namespace osu.Game.Database yield return f.Filename; } - private IEnumerable getTimestamps(List files) - { - foreach (var f in files.OrderBy(f => f.Filename)) - yield return File.GetLastWriteTimeUtc(Files.Storage.GetFullPath(f.FileInfo.StoragePath)).ToFileTime(); - } - private DbSet queryModel() => ContextFactory.Get().Set(); protected virtual string HumanisedModelName => $"{typeof(TModel).Name.Replace("Info", "").ToLower()}"; From 66ec2afe5cdcd9eb77adf5015966e5dcb652c1d1 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Wed, 3 Jun 2020 23:38:40 +0900 Subject: [PATCH 14/30] Remove broken import test --- .../Beatmaps/IO/ImportBeatmapTest.cs | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 9b34eece5f..546bf758c1 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -1,4 +1,4 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. using System; @@ -374,37 +374,6 @@ namespace osu.Game.Tests.Beatmaps.IO } } - [Test] - public async Task TestImportThenImportDifferentHash() - { - // unfortunately for the time being we need to reference osu.Framework.Desktop for a game host here. - using (HeadlessGameHost host = new CleanRunHeadlessGameHost(nameof(TestImportThenImportDifferentHash))) - { - try - { - var osu = loadOsu(host); - var manager = osu.Dependencies.Get(); - - var imported = await LoadOszIntoOsu(osu); - - imported.Hash += "-changed"; - manager.Update(imported); - - var importedSecondTime = await LoadOszIntoOsu(osu); - - Assert.IsTrue(imported.ID != importedSecondTime.ID); - Assert.IsTrue(imported.Beatmaps.First().ID < importedSecondTime.Beatmaps.First().ID); - - // only one beatmap will exist as the online set ID matched, causing purging of the first import. - checkBeatmapSetCount(osu, 1); - } - finally - { - host.Exit(); - } - } - } - [Test] public async Task TestImportThenDeleteThenImport() { From c2fd2b861642a6fcac8412891d0365104c6ed6b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 3 Jun 2020 23:20:43 +0200 Subject: [PATCH 15/30] Add notes about draft PRs & pushing --- CONTRIBUTING.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 331534ad73..9666f249e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,6 +92,16 @@ Here are some key things to note before jumping in: As part of continuous integration, we also run code style analysis, which is supposed to make sure that your code is formatted the same way as all the pre-existing code in the repository. The reason we enforce a particular code style everywhere is to make sure the codebase is consistent in that regard - having one whitespace convention in one place and another one elsewhere causes disorganisation. +* **Make sure that the pull request is complete before opening it.** + + Whether it's fixing a bug or implementing new functionality, it's best that you make sure that the change you want to submit as a pull request is as complete as it can be before clicking the *Create pull request* button. Having to track if a pull request is ready for review or not places additional burden on reviewers. + + Draft pull requests are an option, but use them sparingly and within reason. They are best suited to discuss code changes that cannot be easily described in natural language or have a potential large impact on the future direction of the project. When in doubt, don't open drafts unless a maintainer asks you to do so. + +* **Only push code when it's ready.** + + As an extension of the above, when making changes to an already-open PR, please try to only push changes you are reasonably certain of. Pushing after every commit causes the continuous integration build queue to grow in size, slowing down work and taking up time that could be spent verifying other changes. + * **Make sure to keep the *Allow edits from maintainers* check box checked.** To speed up the merging process, collaborators and team members will sometimes want to push changes to your branch themselves, to make minor code style adjustments or to otherwise refactor the code without having to describe how they'd like the code to look like in painstaking detail. Having the *Allow edits from maintainers* check box checked lets them do that; without it they are forced to report issues back to you and wait for you to address them. From ddf5282d0e24d798a157d2c9704f8b30a7944731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 3 Jun 2020 23:33:49 +0200 Subject: [PATCH 16/30] Move items from README.md to contributing guidelines --- CONTRIBUTING.md | 8 +++++++- README.md | 6 ------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9666f249e2..6c327f01b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,7 +52,7 @@ Issues, bug reports and feature suggestions are welcomed, though please keep in ## I would like to submit a pull request! -We also welcome pull requests from unaffiliated contributors. The issue tracker should provide plenty of issues that you can work on; we also mark issues that we think would be good for newcomers with the [`good-first-issue`](https://github.com/ppy/osu/issues?q=is%3Aissue+is%3Aopen+label%3Agood-first-issue) label. +We also welcome pull requests from unaffiliated contributors. The [issue tracker](https://github.com/ppy/osu/issues) should provide plenty of issues that you can work on; we also mark issues that we think would be good for newcomers with the [`good-first-issue`](https://github.com/ppy/osu/issues?q=is%3Aissue+is%3Aopen+label%3Agood-first-issue) label. However, do keep in mind that the core team is committed to bringing osu!lazer up to par with stable first and foremost, so depending on what your contribution concerns, it might not be merged and released right away. Our approach to managing issues and their priorities is described [in the wiki](https://github.com/ppy/osu/wiki/Project-management). @@ -62,12 +62,18 @@ Here are some key things to note before jumping in: While we are accepting of all kinds of contributions, we also have a certain quality standard we'd like to uphold and limited time to review your code. Therefore, we would like to avoid providing entry-level advice, and as such if you're not very familiar with C\# as a programming language, we'd recommend that you start off with a few personal projects to get acquainted with the language's syntax, toolchain and principles of object-oriented programming first. + In addition, please take the time to take a look at and get acquainted with the [development and testing](https://github.com/ppy/osu-framework/wiki/Development-and-Testing) procedure we have set up. + * **Make sure you are familiar with git and the pull request workflow.** [git](https://git-scm.com/) is a distributed version control system that might not be very intuitive at the beginning if you're not familiar with version control. In particular, projects using git have a particular workflow for submitting code changes, which is called the pull request workflow. To make things run more smoothly, we recommend that you look up some online resources to familiarise yourself with the git vocabulary and commands, and practice working with forks and submitting pull requests at your own pace. A high-level overview of the process can be found in [this article by GitHub](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/proposing-changes-to-your-work-with-pull-requests). +* **Double-check designs before starting work on new functionality.** + + When implementing new features, keep in mind that we already have a lot of the UI designed. If you wish to work on something with the intention of having it included in the official distribution, please open an issue for discussion and we will give you what you need from a design perspective to proceed. If you want to make *changes* to the design, we recommend you open an issue with your intentions before spending too much time to ensure no effort is wasted. + * **Make sure to submit pull requests off of a topic branch.** As described in the article linked in the previous point, topic branches help you parallelise your work and separate it from the main `master` branch, and additionally are easier for maintainers to work with. Working with multiple `master` branches across many remotes is difficult to keep track of, and it's easy to make a mistake and push to the wrong `master` branch by accident. diff --git a/README.md b/README.md index 336bf33f7e..9e1cc20c8b 100644 --- a/README.md +++ b/README.md @@ -93,12 +93,6 @@ JetBrains ReSharper InspectCode is also used for wider rule sets. You can run it ## Contributing -We welcome all contributions, but keep in mind that we already have a lot of the UI designed. If you wish to work on something with the intention of having it included in the official distribution, please open an issue for discussion and we will give you what you need from a design perspective to proceed. If you want to make *changes* to the design, we recommend you open an issue with your intentions before spending too much time to ensure no effort is wasted. - -If you're unsure of what you can help with, check out the [list of open issues](https://github.com/ppy/osu/issues) (especially those with the ["good first issue"](https://github.com/ppy/osu/issues?q=is%3Aopen+label%3Agood-first-issue+sort%3Aupdated-desc) label). - -Before starting, please make sure you are familiar with the [development and testing](https://github.com/ppy/osu-framework/wiki/Development-and-Testing) procedure we have set up. New component development, and where possible, bug fixing and debugging existing components **should always be done under VisualTests**. - Note that while we already have certain standards in place, nothing is set in stone. If you have an issue with the way code is structured, with any libraries we are using, or with any processes involved with contributing, *please* bring it up. We welcome all feedback so we can make contributing to this project as painless as possible. For those interested, we love to reward quality contributions via [bounties](https://docs.google.com/spreadsheets/d/1jNXfj_S3Pb5PErA-czDdC9DUu4IgUbe1Lt8E7CYUJuE/view?&rm=minimal#gid=523803337), paid out via PayPal or osu!supporter tags. Don't hesitate to [request a bounty](https://docs.google.com/forms/d/e/1FAIpQLSet_8iFAgPMG526pBZ2Kic6HSh7XPM3fE8xPcnWNkMzINDdYg/viewform) for your work on this project. From af3daaaeafd294a740d9586ee56e13858b75b5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 3 Jun 2020 23:39:29 +0200 Subject: [PATCH 17/30] Add reference to contributing guidelines in README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9e1cc20c8b..dc3ee63844 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,8 @@ JetBrains ReSharper InspectCode is also used for wider rule sets. You can run it ## Contributing +When it comes to contributing to the project, the two main things you can do to help out are reporting issues and submitting pull requests. Based on past experiences, we have prepared a [list of contributing guidelines](CONTRIBUTING.md) that should hopefully ease you into our collaboration process and answer the most frequently-asked questions. + Note that while we already have certain standards in place, nothing is set in stone. If you have an issue with the way code is structured, with any libraries we are using, or with any processes involved with contributing, *please* bring it up. We welcome all feedback so we can make contributing to this project as painless as possible. For those interested, we love to reward quality contributions via [bounties](https://docs.google.com/spreadsheets/d/1jNXfj_S3Pb5PErA-czDdC9DUu4IgUbe1Lt8E7CYUJuE/view?&rm=minimal#gid=523803337), paid out via PayPal or osu!supporter tags. Don't hesitate to [request a bounty](https://docs.google.com/forms/d/e/1FAIpQLSet_8iFAgPMG526pBZ2Kic6HSh7XPM3fE8xPcnWNkMzINDdYg/viewform) for your work on this project. From 6b88141e58b6d3863b1aeb9db41d39225cd00bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Thu, 4 Jun 2020 21:30:59 +0200 Subject: [PATCH 18/30] Add mania sample conversion test --- .../ManiaBeatmapSampleConversionTest.cs | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs diff --git a/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs b/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs new file mode 100644 index 0000000000..dbf1cf5f72 --- /dev/null +++ b/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs @@ -0,0 +1,72 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using osu.Framework.Utils; +using osu.Game.Audio; +using osu.Game.Rulesets.Mania.Objects; +using osu.Game.Rulesets.Objects; +using osu.Game.Tests.Beatmaps; + +namespace osu.Game.Rulesets.Mania.Tests +{ + [TestFixture] + public class ManiaBeatmapSampleConversionTest : BeatmapConversionTest, SampleConvertValue> + { + protected override string ResourceAssembly => "osu.Game.Rulesets.Mania"; + + public void Test(string name) => base.Test(name); + + protected override IEnumerable CreateConvertValue(HitObject hitObject) + { + yield return new SampleConvertValue + { + StartTime = hitObject.StartTime, + EndTime = hitObject.GetEndTime(), + Column = ((ManiaHitObject)hitObject).Column, + NodeSamples = getSampleNames((hitObject as HoldNote)?.NodeSamples) + }; + } + + private IList> getSampleNames(List> hitSampleInfo) + => hitSampleInfo?.Select(samples => + (IList)samples.Select(sample => sample.LookupNames.First()).ToList()) + .ToList(); + + protected override Ruleset CreateRuleset() => new ManiaRuleset(); + } + + public struct SampleConvertValue : IEquatable + { + /// + /// A sane value to account for osu!stable using ints everywhere. + /// + private const float conversion_lenience = 2; + + public double StartTime; + public double EndTime; + public int Column; + public IList> NodeSamples; + + public bool Equals(SampleConvertValue other) + => Precision.AlmostEquals(StartTime, other.StartTime, conversion_lenience) + && Precision.AlmostEquals(EndTime, other.EndTime, conversion_lenience) + && samplesEqual(NodeSamples, other.NodeSamples); + + private static bool samplesEqual(ICollection> first, ICollection> second) + { + if (first == null && second == null) + return true; + + // both items can't be null now, so if any single one is, then they're not equal + if (first == null || second == null) + return false; + + return first.Count == second.Count + && first.Zip(second).All(samples => samples.First.SequenceEqual(samples.Second)); + } + } +} From 35544ede50069851ff7cfa0fecdf141fe94345db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Thu, 4 Jun 2020 21:54:19 +0200 Subject: [PATCH 19/30] Add failing test cases --- .../ManiaBeatmapSampleConversionTest.cs | 2 ++ .../convert-samples-expected-conversion.json | 30 +++++++++++++++++++ .../Testing/Beatmaps/convert-samples.osu | 16 ++++++++++ .../mania-samples-expected-conversion.json | 25 ++++++++++++++++ .../Testing/Beatmaps/mania-samples.osu | 19 ++++++++++++ 5 files changed, 92 insertions(+) create mode 100644 osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/convert-samples-expected-conversion.json create mode 100644 osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/convert-samples.osu create mode 100644 osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/mania-samples-expected-conversion.json create mode 100644 osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/mania-samples.osu diff --git a/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs b/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs index dbf1cf5f72..2f6918d263 100644 --- a/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs +++ b/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs @@ -18,6 +18,8 @@ namespace osu.Game.Rulesets.Mania.Tests { protected override string ResourceAssembly => "osu.Game.Rulesets.Mania"; + [TestCase("convert-samples")] + [TestCase("mania-samples")] public void Test(string name) => base.Test(name); protected override IEnumerable CreateConvertValue(HitObject hitObject) diff --git a/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/convert-samples-expected-conversion.json b/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/convert-samples-expected-conversion.json new file mode 100644 index 0000000000..b8ce85eef5 --- /dev/null +++ b/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/convert-samples-expected-conversion.json @@ -0,0 +1,30 @@ +{ + "Mappings": [{ + "StartTime": 1000.0, + "Objects": [{ + "StartTime": 1000.0, + "EndTime": 2750.0, + "Column": 1, + "NodeSamples": [ + ["normal-hitnormal"], + ["soft-hitnormal"], + ["drum-hitnormal"] + ] + }, { + "StartTime": 1875.0, + "EndTime": 2750.0, + "Column": 0, + "NodeSamples": [ + ["soft-hitnormal"], + ["drum-hitnormal"] + ] + }] + }, { + "StartTime": 3750.0, + "Objects": [{ + "StartTime": 3750.0, + "EndTime": 3750.0, + "Column": 3 + }] + }] +} \ No newline at end of file diff --git a/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/convert-samples.osu b/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/convert-samples.osu new file mode 100644 index 0000000000..16b73992d2 --- /dev/null +++ b/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/convert-samples.osu @@ -0,0 +1,16 @@ +osu file format v14 + +[Difficulty] +HPDrainRate:5 +CircleSize:5 +OverallDifficulty:5 +ApproachRate:5 +SliderMultiplier:1.4 +SliderTickRate:1 + +[TimingPoints] +0,500,4,1,0,100,1,0 + +[HitObjects] +88,99,1000,6,0,L|306:259,2,245,0|0|0,1:0|2:0|3:0,0:0:0:0: +259,118,3750,1,0,0:0:0:0: diff --git a/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/mania-samples-expected-conversion.json b/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/mania-samples-expected-conversion.json new file mode 100644 index 0000000000..e22540614d --- /dev/null +++ b/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/mania-samples-expected-conversion.json @@ -0,0 +1,25 @@ +{ + "Mappings": [{ + "StartTime": 500.0, + "Objects": [{ + "StartTime": 500.0, + "EndTime": 1500.0, + "Column": 0, + "NodeSamples": [ + ["normal-hitnormal"], + [] + ] + }] + }, { + "StartTime": 2000.0, + "Objects": [{ + "StartTime": 2000.0, + "EndTime": 3000.0, + "Column": 2, + "NodeSamples": [ + ["drum-hitnormal"], + [] + ] + }] + }] +} \ No newline at end of file diff --git a/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/mania-samples.osu b/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/mania-samples.osu new file mode 100644 index 0000000000..7c75b45e5f --- /dev/null +++ b/osu.Game.Rulesets.Mania/Resources/Testing/Beatmaps/mania-samples.osu @@ -0,0 +1,19 @@ +osu file format v14 + +[General] +Mode: 3 + +[Difficulty] +HPDrainRate:5 +CircleSize:5 +OverallDifficulty:5 +ApproachRate:5 +SliderMultiplier:1.4 +SliderTickRate:1 + +[TimingPoints] +0,500,4,1,0,100,1,0 + +[HitObjects] +51,192,500,128,0,1500:1:0:0:0: +256,192,2000,128,0,3000:3:0:0:0: From ac019bddd61b798f86c0f9e545a5d1e45de5a746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Thu, 4 Jun 2020 22:28:55 +0200 Subject: [PATCH 20/30] Only play samples at start of hold note in mania maps --- .../Beatmaps/ManiaBeatmapConverter.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs b/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs index 32abf5e7f9..b025ac7992 100644 --- a/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs +++ b/osu.Game.Rulesets.Mania/Beatmaps/ManiaBeatmapConverter.cs @@ -6,6 +6,7 @@ using System; using System.Linq; using System.Collections.Generic; using osu.Framework.Utils; +using osu.Game.Audio; using osu.Game.Beatmaps; using osu.Game.Rulesets.Objects; using osu.Game.Rulesets.Objects.Types; @@ -239,7 +240,7 @@ namespace osu.Game.Rulesets.Mania.Beatmaps Duration = endTimeData.Duration, Column = column, Samples = HitObject.Samples, - NodeSamples = (HitObject as IHasRepeats)?.NodeSamples + NodeSamples = (HitObject as IHasRepeats)?.NodeSamples ?? defaultNodeSamples }); } else if (HitObject is IHasXPosition) @@ -254,6 +255,16 @@ namespace osu.Game.Rulesets.Mania.Beatmaps return pattern; } + + /// + /// osu!mania-specific beatmaps in stable only play samples at the start of the hold note. + /// + private List> defaultNodeSamples + => new List> + { + HitObject.Samples, + new List() + }; } } } From c4cae006aa800e78f29b46dfcdccda0220838a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Thu, 4 Jun 2020 22:47:14 +0200 Subject: [PATCH 21/30] Correctly slice node sample list when converting --- .../Legacy/DistanceObjectPatternGenerator.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/osu.Game.Rulesets.Mania/Beatmaps/Patterns/Legacy/DistanceObjectPatternGenerator.cs b/osu.Game.Rulesets.Mania/Beatmaps/Patterns/Legacy/DistanceObjectPatternGenerator.cs index 1bd796511b..b49b881656 100644 --- a/osu.Game.Rulesets.Mania/Beatmaps/Patterns/Legacy/DistanceObjectPatternGenerator.cs +++ b/osu.Game.Rulesets.Mania/Beatmaps/Patterns/Legacy/DistanceObjectPatternGenerator.cs @@ -472,15 +472,21 @@ namespace osu.Game.Rulesets.Mania.Beatmaps.Patterns.Legacy /// /// The time to retrieve the sample info list from. /// - private IList sampleInfoListAt(double time) + private IList sampleInfoListAt(double time) => nodeSamplesAt(time)?.First() ?? HitObject.Samples; + + /// + /// Retrieves the list of node samples that occur at time greater than or equal to . + /// + /// The time to retrieve node samples at. + private IEnumerable> nodeSamplesAt(double time) { if (!(HitObject is IHasPathWithRepeats curveData)) - return HitObject.Samples; + return null; double segmentTime = (EndTime - HitObject.StartTime) / spanCount; int index = (int)(segmentTime == 0 ? 0 : (time - HitObject.StartTime) / segmentTime); - return curveData.NodeSamples[index]; + return curveData.NodeSamples.Skip(index); } /// @@ -511,7 +517,7 @@ namespace osu.Game.Rulesets.Mania.Beatmaps.Patterns.Legacy Duration = endTime - startTime, Column = column, Samples = HitObject.Samples, - NodeSamples = (HitObject as IHasRepeats)?.NodeSamples + NodeSamples = nodeSamplesAt(startTime)?.ToList() }; } From 4c6116e6e7c9aa1300430954ef4e6cbcc793f39b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Thu, 4 Jun 2020 23:50:58 +0200 Subject: [PATCH 22/30] Fix compilation failure in Android test project --- .../ManiaBeatmapSampleConversionTest.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs b/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs index 2f6918d263..d8f87195d1 100644 --- a/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs +++ b/osu.Game.Rulesets.Mania.Tests/ManiaBeatmapSampleConversionTest.cs @@ -58,17 +58,19 @@ namespace osu.Game.Rulesets.Mania.Tests && Precision.AlmostEquals(EndTime, other.EndTime, conversion_lenience) && samplesEqual(NodeSamples, other.NodeSamples); - private static bool samplesEqual(ICollection> first, ICollection> second) + private static bool samplesEqual(ICollection> firstSampleList, ICollection> secondSampleList) { - if (first == null && second == null) + if (firstSampleList == null && secondSampleList == null) return true; // both items can't be null now, so if any single one is, then they're not equal - if (first == null || second == null) + if (firstSampleList == null || secondSampleList == null) return false; - return first.Count == second.Count - && first.Zip(second).All(samples => samples.First.SequenceEqual(samples.Second)); + return firstSampleList.Count == secondSampleList.Count + // cannot use .Zip() without the selector function as it doesn't compile in android test project + && firstSampleList.Zip(secondSampleList, (first, second) => (first, second)) + .All(samples => samples.first.SequenceEqual(samples.second)); } } } From 896177801a57e0bd2161309d05775be4d0d087bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Fri, 5 Jun 2020 00:07:27 +0200 Subject: [PATCH 23/30] Avoid creating copies of node samples every time --- .../Patterns/Legacy/DistanceObjectPatternGenerator.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/osu.Game.Rulesets.Mania/Beatmaps/Patterns/Legacy/DistanceObjectPatternGenerator.cs b/osu.Game.Rulesets.Mania/Beatmaps/Patterns/Legacy/DistanceObjectPatternGenerator.cs index b49b881656..9fbdf58e21 100644 --- a/osu.Game.Rulesets.Mania/Beatmaps/Patterns/Legacy/DistanceObjectPatternGenerator.cs +++ b/osu.Game.Rulesets.Mania/Beatmaps/Patterns/Legacy/DistanceObjectPatternGenerator.cs @@ -478,7 +478,7 @@ namespace osu.Game.Rulesets.Mania.Beatmaps.Patterns.Legacy /// Retrieves the list of node samples that occur at time greater than or equal to . /// /// The time to retrieve node samples at. - private IEnumerable> nodeSamplesAt(double time) + private List> nodeSamplesAt(double time) { if (!(HitObject is IHasPathWithRepeats curveData)) return null; @@ -486,7 +486,9 @@ namespace osu.Game.Rulesets.Mania.Beatmaps.Patterns.Legacy double segmentTime = (EndTime - HitObject.StartTime) / spanCount; int index = (int)(segmentTime == 0 ? 0 : (time - HitObject.StartTime) / segmentTime); - return curveData.NodeSamples.Skip(index); + + // avoid slicing the list & creating copies, if at all possible. + return index == 0 ? curveData.NodeSamples : curveData.NodeSamples.Skip(index).ToList(); } /// @@ -517,7 +519,7 @@ namespace osu.Game.Rulesets.Mania.Beatmaps.Patterns.Legacy Duration = endTime - startTime, Column = column, Samples = HitObject.Samples, - NodeSamples = nodeSamplesAt(startTime)?.ToList() + NodeSamples = nodeSamplesAt(startTime) }; } From 0107e9ba16deb94cd8f04c5dcf36b7ca2a781adc Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Fri, 5 Jun 2020 19:18:00 +0900 Subject: [PATCH 24/30] Change lookups to use SingleOrDefault() --- osu.Game/Beatmaps/BeatmapManager.cs | 2 +- osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 668ac6ee10..1f92d5461f 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -215,7 +215,7 @@ namespace osu.Game.Beatmaps foreach (var info in item.Beatmaps) { - var file = item.Files.FirstOrDefault(f => string.Equals(f.Filename, info.Path, StringComparison.OrdinalIgnoreCase))?.FileInfo.StoragePath; + var file = item.Files.SingleOrDefault(f => string.Equals(f.Filename, info.Path, StringComparison.OrdinalIgnoreCase))?.FileInfo.StoragePath; using (var stream = Files.Store.GetStream(file)) info.MD5Hash = stream.ComputeMD5Hash(); diff --git a/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs b/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs index e62a9bb39d..39c5ccab27 100644 --- a/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs +++ b/osu.Game/Beatmaps/BeatmapManager_WorkingBeatmap.cs @@ -42,7 +42,7 @@ namespace osu.Game.Beatmaps } } - private string getPathForFile(string filename) => BeatmapSetInfo.Files.FirstOrDefault(f => string.Equals(f.Filename, filename, StringComparison.OrdinalIgnoreCase))?.FileInfo.StoragePath; + private string getPathForFile(string filename) => BeatmapSetInfo.Files.SingleOrDefault(f => string.Equals(f.Filename, filename, StringComparison.OrdinalIgnoreCase))?.FileInfo.StoragePath; private TextureStore textureStore; From dfed27bd4633e5b2d1268f8851fec97a698d61e6 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Mon, 8 Jun 2020 14:24:21 +0900 Subject: [PATCH 25/30] Add back stream seeking for sanity --- osu.Game/Beatmaps/BeatmapManager.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index f11e94e63d..4e3714a582 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -201,6 +201,8 @@ namespace osu.Game.Beatmaps using (var sw = new StreamWriter(stream, Encoding.UTF8, 1024, true)) new LegacyBeatmapEncoder(beatmapContent).Encode(sw); + stream.Seek(0, SeekOrigin.Begin); + UpdateFile(setInfo, setInfo.Files.Single(f => string.Equals(f.Filename, info.Path, StringComparison.OrdinalIgnoreCase)), stream); } From 443977aa8d71071a7566a4be643ffea72b77fee1 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Mon, 8 Jun 2020 14:40:17 +0900 Subject: [PATCH 26/30] Remove PreUpdate, update hash in Save() --- osu.Game/Beatmaps/BeatmapManager.cs | 22 ++++++++-------------- osu.Game/Database/ArchiveModelManager.cs | 11 ----------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/osu.Game/Beatmaps/BeatmapManager.cs b/osu.Game/Beatmaps/BeatmapManager.cs index 4e3714a582..cbcdf51551 100644 --- a/osu.Game/Beatmaps/BeatmapManager.cs +++ b/osu.Game/Beatmaps/BeatmapManager.cs @@ -203,7 +203,14 @@ namespace osu.Game.Beatmaps stream.Seek(0, SeekOrigin.Begin); - UpdateFile(setInfo, setInfo.Files.Single(f => string.Equals(f.Filename, info.Path, StringComparison.OrdinalIgnoreCase)), stream); + using (ContextFactory.GetForWrite()) + { + var beatmapInfo = setInfo.Beatmaps.Single(b => b.ID == info.ID); + beatmapInfo.MD5Hash = stream.ComputeMD5Hash(); + + stream.Seek(0, SeekOrigin.Begin); + UpdateFile(setInfo, setInfo.Files.Single(f => string.Equals(f.Filename, info.Path, StringComparison.OrdinalIgnoreCase)), stream); + } } var working = workingCache.FirstOrDefault(w => w.BeatmapInfo?.ID == info.ID); @@ -211,19 +218,6 @@ namespace osu.Game.Beatmaps workingCache.Remove(working); } - protected override void PreUpdate(BeatmapSetInfo item) - { - base.PreUpdate(item); - - foreach (var info in item.Beatmaps) - { - var file = item.Files.SingleOrDefault(f => string.Equals(f.Filename, info.Path, StringComparison.OrdinalIgnoreCase))?.FileInfo.StoragePath; - - using (var stream = Files.Store.GetStream(file)) - info.MD5Hash = stream.ComputeMD5Hash(); - } - } - private readonly WeakList workingCache = new WeakList(); /// diff --git a/osu.Game/Database/ArchiveModelManager.cs b/osu.Game/Database/ArchiveModelManager.cs index b9479af623..915d980d24 100644 --- a/osu.Game/Database/ArchiveModelManager.cs +++ b/osu.Game/Database/ArchiveModelManager.cs @@ -429,21 +429,10 @@ namespace osu.Game.Database using (ContextFactory.GetForWrite()) { item.Hash = computeHash(item); - - PreUpdate(item); - ModelStore.Update(item); } } - /// - /// Perform any final actions before the update to database executes. - /// - /// The that is being updated. - protected virtual void PreUpdate(TModel item) - { - } - /// /// Delete an item from the manager. /// Is a no-op for already deleted items. From ff555c41c6b667ebd91ba46f166cbd247ffeece3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2020 08:57:44 +0000 Subject: [PATCH 27/30] Bump Sentry from 2.1.1 to 2.1.3 Bumps [Sentry](https://github.com/getsentry/sentry-dotnet) from 2.1.1 to 2.1.3. - [Release notes](https://github.com/getsentry/sentry-dotnet/releases) - [Commits](https://github.com/getsentry/sentry-dotnet/compare/2.1.1...2.1.3) Signed-off-by: dependabot-preview[bot] --- osu.Game/osu.Game.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/osu.Game.csproj b/osu.Game/osu.Game.csproj index 4d6358575b..c41d0a0cf6 100644 --- a/osu.Game/osu.Game.csproj +++ b/osu.Game/osu.Game.csproj @@ -26,7 +26,7 @@ - + From bbf8864f1478d609fe2eb7184cbd303e0cc9a14b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2020 09:45:31 +0000 Subject: [PATCH 28/30] Bump Microsoft.Build.Traversal from 2.0.34 to 2.0.48 Bumps [Microsoft.Build.Traversal](https://github.com/Microsoft/MSBuildSdks) from 2.0.34 to 2.0.48. - [Release notes](https://github.com/Microsoft/MSBuildSdks/releases) - [Changelog](https://github.com/microsoft/MSBuildSdks/blob/master/RELEASE.md) - [Commits](https://github.com/Microsoft/MSBuildSdks/compare/Microsoft.Build.Traversal.2.0.34...Microsoft.Build.Traversal.2.0.48) Signed-off-by: dependabot-preview[bot] --- global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global.json b/global.json index 6c793a3f1d..bdb90eb0e9 100644 --- a/global.json +++ b/global.json @@ -5,6 +5,6 @@ "version": "3.1.100" }, "msbuild-sdks": { - "Microsoft.Build.Traversal": "2.0.34" + "Microsoft.Build.Traversal": "2.0.48" } } \ No newline at end of file From e0c94304c79637c86e9304e0471ce37bb139f223 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2020 09:45:31 +0000 Subject: [PATCH 29/30] Bump Humanizer from 2.8.11 to 2.8.26 Bumps [Humanizer](https://github.com/Humanizr/Humanizer) from 2.8.11 to 2.8.26. - [Release notes](https://github.com/Humanizr/Humanizer/releases) - [Changelog](https://github.com/Humanizr/Humanizer/blob/master/release_notes.md) - [Commits](https://github.com/Humanizr/Humanizer/compare/v2.8.11...v2.8.26) Signed-off-by: dependabot-preview[bot] --- osu.Game/osu.Game.csproj | 2 +- osu.iOS.props | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Game/osu.Game.csproj b/osu.Game/osu.Game.csproj index c41d0a0cf6..8213719c01 100644 --- a/osu.Game/osu.Game.csproj +++ b/osu.Game/osu.Game.csproj @@ -20,7 +20,7 @@ - + diff --git a/osu.iOS.props b/osu.iOS.props index 6b55fa51ff..fd13455c63 100644 --- a/osu.iOS.props +++ b/osu.iOS.props @@ -76,7 +76,7 @@ - + From 8a021e0beb39a897816d8da99983eb9de6e4b419 Mon Sep 17 00:00:00 2001 From: smoogipoo Date: Mon, 8 Jun 2020 22:35:01 +0900 Subject: [PATCH 30/30] Use save method in test --- .../Beatmaps/IO/ImportBeatmapTest.cs | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs index 55368f6676..249a8caba9 100644 --- a/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs +++ b/osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs @@ -1,11 +1,10 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. using System; using System.IO; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; @@ -15,7 +14,6 @@ using osu.Framework.Allocation; using osu.Framework.Extensions; using osu.Framework.Logging; using osu.Game.Beatmaps; -using osu.Game.Beatmaps.Formats; using osu.Game.IO; using osu.Game.Rulesets.Osu.Objects; using osu.Game.Tests.Resources; @@ -730,25 +728,17 @@ namespace osu.Game.Tests.Beatmaps.IO await osu.Dependencies.Get().Import(temp); BeatmapSetInfo setToUpdate = manager.GetAllUsableBeatmapSets()[0]; + + var beatmapInfo = setToUpdate.Beatmaps.First(b => b.RulesetID == 0); Beatmap beatmapToUpdate = (Beatmap)manager.GetWorkingBeatmap(setToUpdate.Beatmaps.First(b => b.RulesetID == 0)).Beatmap; BeatmapSetFileInfo fileToUpdate = setToUpdate.Files.First(f => beatmapToUpdate.BeatmapInfo.Path.Contains(f.Filename)); string oldMd5Hash = beatmapToUpdate.BeatmapInfo.MD5Hash; - using (var stream = new MemoryStream()) - { - using (var writer = new StreamWriter(stream, Encoding.UTF8, 1024, true)) - { - beatmapToUpdate.HitObjects.Clear(); - beatmapToUpdate.HitObjects.Add(new HitCircle { StartTime = 5000 }); + beatmapToUpdate.HitObjects.Clear(); + beatmapToUpdate.HitObjects.Add(new HitCircle { StartTime = 5000 }); - new LegacyBeatmapEncoder(beatmapToUpdate).Encode(writer); - } - - stream.Seek(0, SeekOrigin.Begin); - - manager.UpdateFile(setToUpdate, fileToUpdate, stream); - } + manager.Save(beatmapInfo, beatmapToUpdate); // Check that the old file reference has been removed Assert.That(manager.QueryBeatmapSet(s => s.ID == setToUpdate.ID).Files.All(f => f.ID != fileToUpdate.ID));