mirror of
https://github.com/ppy/osu.git
synced 2024-09-22 00:47:24 +08:00
bbcc149e2e
This is one step closer to sanity in terms of physical files. As per the comment I have left in place, we really should be checking file sizes or hashes, but to keep things simple and fast I've opted to just cover the "missing file" scenario for now. Ran into this when testing against a foreign `client.realm` by: - Noticing a beatmap doesn't load - Deleting said beatmap - Downloading via beatmap overlay - Beatmap is restored but still doesn't work Note that I've kept the logic where this will undelete an existing import rather than create one from fresh, as I think that is beneficial to the user (ie. it will still keep any linked scores on restore).
1048 lines
42 KiB
C#
1048 lines
42 KiB
C#
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
|
|
// See the LICENCE file in the repository root for full licence text.
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using NUnit.Framework;
|
|
using osu.Framework.Extensions;
|
|
using osu.Framework.Extensions.ObjectExtensions;
|
|
using osu.Framework.Logging;
|
|
using osu.Framework.Platform;
|
|
using osu.Game.Beatmaps;
|
|
using osu.Game.Database;
|
|
using osu.Game.Extensions;
|
|
using osu.Game.IO.Archives;
|
|
using osu.Game.Models;
|
|
using osu.Game.Overlays.Notifications;
|
|
using osu.Game.Rulesets;
|
|
using osu.Game.Stores;
|
|
using osu.Game.Tests.Resources;
|
|
using Realms;
|
|
using SharpCompress.Archives;
|
|
using SharpCompress.Archives.Zip;
|
|
using SharpCompress.Common;
|
|
using SharpCompress.Writers.Zip;
|
|
|
|
#nullable enable
|
|
|
|
namespace osu.Game.Tests.Database
|
|
{
|
|
[TestFixture]
|
|
public class BeatmapImporterTests : RealmTest
|
|
{
|
|
[Test]
|
|
public void TestDetachBeatmapSet()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using (var importer = new BeatmapModelManager(realmFactory, storage))
|
|
using (new RulesetStore(realmFactory, storage))
|
|
{
|
|
ILive<BeatmapSetInfo>? beatmapSet;
|
|
|
|
using (var reader = new ZipArchiveReader(TestResources.GetTestBeatmapStream()))
|
|
beatmapSet = await importer.Import(reader);
|
|
|
|
Assert.NotNull(beatmapSet);
|
|
Debug.Assert(beatmapSet != null);
|
|
|
|
BeatmapSetInfo? detachedBeatmapSet = null;
|
|
|
|
beatmapSet.PerformRead(live =>
|
|
{
|
|
detachedBeatmapSet = live.Detach();
|
|
|
|
// files are omitted
|
|
Assert.AreEqual(0, detachedBeatmapSet.Files.Count);
|
|
|
|
Assert.AreEqual(live.Beatmaps.Count, detachedBeatmapSet.Beatmaps.Count);
|
|
Assert.AreEqual(live.Beatmaps.Select(f => f.Difficulty).Count(), detachedBeatmapSet.Beatmaps.Select(f => f.Difficulty).Count());
|
|
Assert.AreEqual(live.Metadata, detachedBeatmapSet.Metadata);
|
|
});
|
|
|
|
Debug.Assert(detachedBeatmapSet != null);
|
|
|
|
// Check detached instances can all be accessed without throwing.
|
|
Assert.AreEqual(0, detachedBeatmapSet.Files.Count);
|
|
Assert.NotNull(detachedBeatmapSet.Beatmaps.Count);
|
|
Assert.NotZero(detachedBeatmapSet.Beatmaps.Select(f => f.Difficulty).Count());
|
|
Assert.NotNull(detachedBeatmapSet.Metadata);
|
|
|
|
// Check cyclic reference to beatmap set
|
|
Assert.AreEqual(detachedBeatmapSet, detachedBeatmapSet.Beatmaps.First().BeatmapSet);
|
|
}
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestUpdateDetachedBeatmapSet()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using (var importer = new BeatmapModelManager(realmFactory, storage))
|
|
using (new RulesetStore(realmFactory, storage))
|
|
{
|
|
ILive<BeatmapSetInfo>? beatmapSet;
|
|
|
|
using (var reader = new ZipArchiveReader(TestResources.GetTestBeatmapStream()))
|
|
beatmapSet = await importer.Import(reader);
|
|
|
|
Assert.NotNull(beatmapSet);
|
|
Debug.Assert(beatmapSet != null);
|
|
|
|
// Detach at the BeatmapInfo point, similar to what GetWorkingBeatmap does.
|
|
BeatmapInfo? detachedBeatmap = null;
|
|
|
|
beatmapSet.PerformRead(s => detachedBeatmap = s.Beatmaps.First().Detach());
|
|
|
|
BeatmapSetInfo? detachedBeatmapSet = detachedBeatmap?.BeatmapSet;
|
|
|
|
Debug.Assert(detachedBeatmapSet != null);
|
|
|
|
var newUser = new RealmUser { Username = "peppy", OnlineID = 2 };
|
|
|
|
detachedBeatmapSet.Beatmaps.First().Metadata.Artist = "New Artist";
|
|
detachedBeatmapSet.Beatmaps.First().Metadata.Author = newUser;
|
|
|
|
Assert.AreNotEqual(detachedBeatmapSet.Status, BeatmapOnlineStatus.Ranked);
|
|
detachedBeatmapSet.Status = BeatmapOnlineStatus.Ranked;
|
|
|
|
beatmapSet.PerformWrite(s =>
|
|
{
|
|
detachedBeatmapSet.CopyChangesToRealm(s);
|
|
});
|
|
|
|
beatmapSet.PerformRead(s =>
|
|
{
|
|
// Check above changes explicitly.
|
|
Assert.AreEqual(BeatmapOnlineStatus.Ranked, s.Status);
|
|
Assert.AreEqual("New Artist", s.Beatmaps.First().Metadata.Artist);
|
|
Assert.AreEqual(newUser, s.Beatmaps.First().Metadata.Author);
|
|
Assert.NotZero(s.Files.Count);
|
|
|
|
// Check nothing was lost in the copy operation.
|
|
Assert.AreEqual(s.Files.Count, detachedBeatmapSet.Files.Count);
|
|
Assert.AreEqual(s.Files.Select(f => f.File).Count(), detachedBeatmapSet.Files.Select(f => f.File).Count());
|
|
Assert.AreEqual(s.Beatmaps.Count, detachedBeatmapSet.Beatmaps.Count);
|
|
Assert.AreEqual(s.Beatmaps.Select(f => f.Difficulty).Count(), detachedBeatmapSet.Beatmaps.Select(f => f.Difficulty).Count());
|
|
Assert.AreEqual(s.Metadata, detachedBeatmapSet.Metadata);
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportBeatmapThenCleanup()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using (var importer = new BeatmapModelManager(realmFactory, storage))
|
|
using (new RulesetStore(realmFactory, storage))
|
|
{
|
|
ILive<BeatmapSetInfo>? imported;
|
|
|
|
using (var reader = new ZipArchiveReader(TestResources.GetTestBeatmapStream()))
|
|
imported = await importer.Import(reader);
|
|
|
|
Assert.AreEqual(1, realmFactory.Context.All<BeatmapSetInfo>().Count());
|
|
|
|
Assert.NotNull(imported);
|
|
Debug.Assert(imported != null);
|
|
|
|
imported.PerformWrite(s => s.DeletePending = true);
|
|
|
|
Assert.AreEqual(1, realmFactory.Context.All<BeatmapSetInfo>().Count(s => s.DeletePending));
|
|
}
|
|
});
|
|
|
|
Logger.Log("Running with no work to purge pending deletions");
|
|
|
|
RunTestWithRealm((realmFactory, _) => { Assert.AreEqual(0, realmFactory.Context.All<BeatmapSetInfo>().Count()); });
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportWhenClosed()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
await LoadOszIntoStore(importer, realmFactory.Context);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestAccessFileAfterImport()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
var beatmap = imported.Beatmaps.First();
|
|
var file = beatmap.File;
|
|
|
|
Assert.NotNull(file);
|
|
Assert.AreEqual(beatmap.Hash, file!.File.Hash);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenDelete()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
deleteBeatmapSet(imported, realmFactory.Context);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenDeleteFromStream()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? tempPath = TestResources.GetTestBeatmapForImport();
|
|
|
|
ILive<BeatmapSetInfo>? importedSet;
|
|
|
|
using (var stream = File.OpenRead(tempPath))
|
|
{
|
|
importedSet = await importer.Import(new ImportTask(stream, Path.GetFileName(tempPath)));
|
|
EnsureLoaded(realmFactory.Context);
|
|
}
|
|
|
|
Assert.NotNull(importedSet);
|
|
Debug.Assert(importedSet != null);
|
|
|
|
Assert.IsTrue(File.Exists(tempPath), "Stream source file somehow went missing");
|
|
File.Delete(tempPath);
|
|
|
|
var imported = realmFactory.Context.All<BeatmapSetInfo>().First(beatmapSet => beatmapSet.ID == importedSet.ID);
|
|
|
|
deleteBeatmapSet(imported, realmFactory.Context);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenImport()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
var importedSecondTime = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
// check the newly "imported" beatmap is actually just the restored previous import. since it matches hash.
|
|
Assert.IsTrue(imported.ID == importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID == importedSecondTime.Beatmaps.First().ID);
|
|
|
|
checkBeatmapSetCount(realmFactory.Context, 1);
|
|
checkSingleReferencedFileCount(realmFactory.Context, 18);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenImportWithReZip()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? temp = TestResources.GetTestBeatmapForImport();
|
|
|
|
string extractedFolder = $"{temp}_extracted";
|
|
Directory.CreateDirectory(extractedFolder);
|
|
|
|
try
|
|
{
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
string hashBefore = hashFile(temp);
|
|
|
|
using (var zip = ZipArchive.Open(temp))
|
|
zip.WriteToDirectory(extractedFolder);
|
|
|
|
using (var zip = ZipArchive.Create())
|
|
{
|
|
zip.AddAllFromDirectory(extractedFolder);
|
|
zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate));
|
|
}
|
|
|
|
// zip files differ because different compression or encoder.
|
|
Assert.AreNotEqual(hashBefore, hashFile(temp));
|
|
|
|
var importedSecondTime = await importer.Import(new ImportTask(temp));
|
|
|
|
EnsureLoaded(realmFactory.Context);
|
|
|
|
Assert.NotNull(importedSecondTime);
|
|
Debug.Assert(importedSecondTime != null);
|
|
|
|
// but contents doesn't, so existing should still be used.
|
|
Assert.IsTrue(imported.ID == importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID == importedSecondTime.PerformRead(s => s.Beatmaps.First().ID));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(extractedFolder, true);
|
|
}
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenImportWithChangedHashedFile()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? temp = TestResources.GetTestBeatmapForImport();
|
|
|
|
string extractedFolder = $"{temp}_extracted";
|
|
Directory.CreateDirectory(extractedFolder);
|
|
|
|
try
|
|
{
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
await createScoreForBeatmap(realmFactory.Context, imported.Beatmaps.First());
|
|
|
|
using (var zip = ZipArchive.Open(temp))
|
|
zip.WriteToDirectory(extractedFolder);
|
|
|
|
// arbitrary write to hashed file
|
|
// this triggers the special BeatmapManager.PreImport deletion/replacement flow.
|
|
using (var sw = new FileInfo(Directory.GetFiles(extractedFolder, "*.osu").First()).AppendText())
|
|
await sw.WriteLineAsync("// changed");
|
|
|
|
using (var zip = ZipArchive.Create())
|
|
{
|
|
zip.AddAllFromDirectory(extractedFolder);
|
|
zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate));
|
|
}
|
|
|
|
var importedSecondTime = await importer.Import(new ImportTask(temp));
|
|
|
|
EnsureLoaded(realmFactory.Context);
|
|
|
|
// check the newly "imported" beatmap is not the original.
|
|
Assert.NotNull(importedSecondTime);
|
|
Debug.Assert(importedSecondTime != null);
|
|
|
|
Assert.IsTrue(imported.ID != importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID != importedSecondTime.PerformRead(s => s.Beatmaps.First().ID));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(extractedFolder, true);
|
|
}
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
[Ignore("intentionally broken by import optimisations")]
|
|
public void TestImportThenImportWithChangedFile()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? temp = TestResources.GetTestBeatmapForImport();
|
|
|
|
string extractedFolder = $"{temp}_extracted";
|
|
Directory.CreateDirectory(extractedFolder);
|
|
|
|
try
|
|
{
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
using (var zip = ZipArchive.Open(temp))
|
|
zip.WriteToDirectory(extractedFolder);
|
|
|
|
// arbitrary write to non-hashed file
|
|
using (var sw = new FileInfo(Directory.GetFiles(extractedFolder, "*.mp3").First()).AppendText())
|
|
await sw.WriteLineAsync("text");
|
|
|
|
using (var zip = ZipArchive.Create())
|
|
{
|
|
zip.AddAllFromDirectory(extractedFolder);
|
|
zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate));
|
|
}
|
|
|
|
var importedSecondTime = await importer.Import(new ImportTask(temp));
|
|
|
|
EnsureLoaded(realmFactory.Context);
|
|
|
|
Assert.NotNull(importedSecondTime);
|
|
Debug.Assert(importedSecondTime != null);
|
|
|
|
// check the newly "imported" beatmap is not the original.
|
|
Assert.IsTrue(imported.ID != importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID != importedSecondTime.PerformRead(s => s.Beatmaps.First().ID));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(extractedFolder, true);
|
|
}
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenImportWithDifferentFilename()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? temp = TestResources.GetTestBeatmapForImport();
|
|
|
|
string extractedFolder = $"{temp}_extracted";
|
|
Directory.CreateDirectory(extractedFolder);
|
|
|
|
try
|
|
{
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
using (var zip = ZipArchive.Open(temp))
|
|
zip.WriteToDirectory(extractedFolder);
|
|
|
|
// change filename
|
|
var firstFile = new FileInfo(Directory.GetFiles(extractedFolder).First());
|
|
firstFile.MoveTo(Path.Combine(firstFile.DirectoryName.AsNonNull(), $"{firstFile.Name}-changed{firstFile.Extension}"));
|
|
|
|
using (var zip = ZipArchive.Create())
|
|
{
|
|
zip.AddAllFromDirectory(extractedFolder);
|
|
zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate));
|
|
}
|
|
|
|
var importedSecondTime = await importer.Import(new ImportTask(temp));
|
|
|
|
EnsureLoaded(realmFactory.Context);
|
|
|
|
Assert.NotNull(importedSecondTime);
|
|
Debug.Assert(importedSecondTime != null);
|
|
|
|
// check the newly "imported" beatmap is not the original.
|
|
Assert.IsTrue(imported.ID != importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID != importedSecondTime.PerformRead(s => s.Beatmaps.First().ID));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(extractedFolder, true);
|
|
}
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
[Ignore("intentionally broken by import optimisations")]
|
|
public void TestImportCorruptThenImport()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
var firstFile = imported.Files.First();
|
|
|
|
long originalLength;
|
|
using (var stream = storage.GetStream(firstFile.File.GetStoragePath()))
|
|
originalLength = stream.Length;
|
|
|
|
using (var stream = storage.GetStream(firstFile.File.GetStoragePath(), FileAccess.Write, FileMode.Create))
|
|
stream.WriteByte(0);
|
|
|
|
var importedSecondTime = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
using (var stream = storage.GetStream(firstFile.File.GetStoragePath()))
|
|
Assert.AreEqual(stream.Length, originalLength, "Corruption was not fixed on second import");
|
|
|
|
// check the newly "imported" beatmap is actually just the restored previous import. since it matches hash.
|
|
Assert.IsTrue(imported.ID == importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID == importedSecondTime.Beatmaps.First().ID);
|
|
|
|
checkBeatmapSetCount(realmFactory.Context, 1);
|
|
checkSingleReferencedFileCount(realmFactory.Context, 18);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestModelCreationFailureDoesntReturn()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var progressNotification = new ImportProgressNotification();
|
|
|
|
var zipStream = new MemoryStream();
|
|
|
|
using (var zip = ZipArchive.Create())
|
|
zip.SaveTo(zipStream, new ZipWriterOptions(CompressionType.Deflate));
|
|
|
|
var imported = await importer.Import(
|
|
progressNotification,
|
|
new ImportTask(zipStream, string.Empty)
|
|
);
|
|
|
|
checkBeatmapSetCount(realmFactory.Context, 0);
|
|
checkBeatmapCount(realmFactory.Context, 0);
|
|
|
|
Assert.IsEmpty(imported);
|
|
Assert.AreEqual(ProgressNotificationState.Cancelled, progressNotification.State);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestRollbackOnFailure()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
int loggedExceptionCount = 0;
|
|
|
|
Logger.NewEntry += l =>
|
|
{
|
|
if (l.Target == LoggingTarget.Database && l.Exception != null)
|
|
Interlocked.Increment(ref loggedExceptionCount);
|
|
};
|
|
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
realmFactory.Context.Write(() => imported.Hash += "-changed");
|
|
|
|
checkBeatmapSetCount(realmFactory.Context, 1);
|
|
checkBeatmapCount(realmFactory.Context, 12);
|
|
checkSingleReferencedFileCount(realmFactory.Context, 18);
|
|
|
|
string? brokenTempFilename = TestResources.GetTestBeatmapForImport();
|
|
|
|
MemoryStream brokenOsu = new MemoryStream();
|
|
MemoryStream brokenOsz = new MemoryStream(await File.ReadAllBytesAsync(brokenTempFilename));
|
|
|
|
File.Delete(brokenTempFilename);
|
|
|
|
using (var outStream = File.Open(brokenTempFilename, FileMode.CreateNew))
|
|
using (var zip = ZipArchive.Open(brokenOsz))
|
|
{
|
|
zip.AddEntry("broken.osu", brokenOsu, false);
|
|
zip.SaveTo(outStream, CompressionType.Deflate);
|
|
}
|
|
|
|
// this will trigger purging of the existing beatmap (online set id match) but should rollback due to broken osu.
|
|
try
|
|
{
|
|
await importer.Import(new ImportTask(brokenTempFilename));
|
|
}
|
|
catch
|
|
{
|
|
}
|
|
|
|
checkBeatmapSetCount(realmFactory.Context, 1);
|
|
checkBeatmapCount(realmFactory.Context, 12);
|
|
|
|
checkSingleReferencedFileCount(realmFactory.Context, 18);
|
|
|
|
Assert.AreEqual(1, loggedExceptionCount);
|
|
|
|
File.Delete(brokenTempFilename);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenDeleteThenImportOptimisedPath()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
deleteBeatmapSet(imported, realmFactory.Context);
|
|
|
|
Assert.IsTrue(imported.DeletePending);
|
|
|
|
var importedSecondTime = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
// check the newly "imported" beatmap is actually just the restored previous import. since it matches hash.
|
|
Assert.IsTrue(imported.ID == importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID == importedSecondTime.Beatmaps.First().ID);
|
|
Assert.IsFalse(imported.DeletePending);
|
|
Assert.IsFalse(importedSecondTime.DeletePending);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenReimportAfterMissingFiles()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
deleteBeatmapSet(imported, realmFactory.Context);
|
|
|
|
Assert.IsTrue(imported.DeletePending);
|
|
|
|
// intentionally nuke all files
|
|
storage.DeleteDirectory("files");
|
|
|
|
Assert.That(imported.Files.All(f => !storage.GetStorageForDirectory("files").Exists(f.File.GetStoragePath())));
|
|
|
|
var importedSecondTime = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
// check the newly "imported" beatmap is actually just the restored previous import. since it matches hash.
|
|
Assert.IsTrue(imported.ID == importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID == importedSecondTime.Beatmaps.First().ID);
|
|
Assert.IsFalse(imported.DeletePending);
|
|
Assert.IsFalse(importedSecondTime.DeletePending);
|
|
|
|
// check that the files now exist, even though they were deleted above.
|
|
Assert.That(importedSecondTime.Files.All(f => storage.GetStorageForDirectory("files").Exists(f.File.GetStoragePath())));
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenDeleteThenImportNonOptimisedPath()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new NonOptimisedBeatmapImporter(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
deleteBeatmapSet(imported, realmFactory.Context);
|
|
|
|
Assert.IsTrue(imported.DeletePending);
|
|
|
|
var importedSecondTime = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
// check the newly "imported" beatmap is actually just the restored previous import. since it matches hash.
|
|
Assert.IsTrue(imported.ID == importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID == importedSecondTime.Beatmaps.First().ID);
|
|
Assert.IsFalse(imported.DeletePending);
|
|
Assert.IsFalse(importedSecondTime.DeletePending);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportThenDeleteThenImportWithOnlineIDsMissing()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var imported = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
realmFactory.Context.Write(() =>
|
|
{
|
|
foreach (var b in imported.Beatmaps)
|
|
b.OnlineID = -1;
|
|
});
|
|
|
|
deleteBeatmapSet(imported, realmFactory.Context);
|
|
|
|
var importedSecondTime = await LoadOszIntoStore(importer, realmFactory.Context);
|
|
|
|
// check the newly "imported" beatmap has been reimported due to mismatch (even though hashes matched)
|
|
Assert.IsTrue(imported.ID != importedSecondTime.ID);
|
|
Assert.IsTrue(imported.Beatmaps.First().ID != importedSecondTime.Beatmaps.First().ID);
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportWithDuplicateBeatmapIDs()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
var metadata = new BeatmapMetadata
|
|
{
|
|
Artist = "SomeArtist",
|
|
Author =
|
|
{
|
|
Username = "SomeAuthor"
|
|
}
|
|
};
|
|
|
|
var ruleset = realmFactory.Context.All<RulesetInfo>().First();
|
|
|
|
var toImport = new BeatmapSetInfo
|
|
{
|
|
OnlineID = 1,
|
|
Beatmaps =
|
|
{
|
|
new BeatmapInfo(ruleset, new BeatmapDifficulty(), metadata)
|
|
{
|
|
OnlineID = 2,
|
|
},
|
|
new BeatmapInfo(ruleset, new BeatmapDifficulty(), metadata)
|
|
{
|
|
OnlineID = 2,
|
|
Status = BeatmapOnlineStatus.Loved,
|
|
}
|
|
}
|
|
};
|
|
|
|
var imported = await importer.Import(toImport);
|
|
|
|
Assert.NotNull(imported);
|
|
Debug.Assert(imported != null);
|
|
|
|
Assert.AreEqual(-1, imported.PerformRead(s => s.Beatmaps[0].OnlineID));
|
|
Assert.AreEqual(-1, imported.PerformRead(s => s.Beatmaps[1].OnlineID));
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportWhenFileOpen()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? temp = TestResources.GetTestBeatmapForImport();
|
|
using (File.OpenRead(temp))
|
|
await importer.Import(temp);
|
|
EnsureLoaded(realmFactory.Context);
|
|
File.Delete(temp);
|
|
Assert.IsFalse(File.Exists(temp), "We likely held a read lock on the file when we shouldn't");
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportWithDuplicateHashes()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? temp = TestResources.GetTestBeatmapForImport();
|
|
|
|
string extractedFolder = $"{temp}_extracted";
|
|
Directory.CreateDirectory(extractedFolder);
|
|
|
|
try
|
|
{
|
|
using (var zip = ZipArchive.Open(temp))
|
|
zip.WriteToDirectory(extractedFolder);
|
|
|
|
using (var zip = ZipArchive.Create())
|
|
{
|
|
zip.AddAllFromDirectory(extractedFolder);
|
|
zip.AddEntry("duplicate.osu", Directory.GetFiles(extractedFolder, "*.osu").First());
|
|
zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate));
|
|
}
|
|
|
|
await importer.Import(temp);
|
|
|
|
EnsureLoaded(realmFactory.Context);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(extractedFolder, true);
|
|
}
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportNestedStructure()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? temp = TestResources.GetTestBeatmapForImport();
|
|
|
|
string extractedFolder = $"{temp}_extracted";
|
|
string subfolder = Path.Combine(extractedFolder, "subfolder");
|
|
|
|
Directory.CreateDirectory(subfolder);
|
|
|
|
try
|
|
{
|
|
using (var zip = ZipArchive.Open(temp))
|
|
zip.WriteToDirectory(subfolder);
|
|
|
|
using (var zip = ZipArchive.Create())
|
|
{
|
|
zip.AddAllFromDirectory(extractedFolder);
|
|
zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate));
|
|
}
|
|
|
|
var imported = await importer.Import(new ImportTask(temp));
|
|
|
|
Assert.NotNull(imported);
|
|
Debug.Assert(imported != null);
|
|
|
|
EnsureLoaded(realmFactory.Context);
|
|
|
|
Assert.IsFalse(imported.PerformRead(s => s.Files.Any(f => f.Filename.Contains("subfolder"))), "Files contain common subfolder");
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(extractedFolder, true);
|
|
}
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestImportWithIgnoredDirectoryInArchive()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? temp = TestResources.GetTestBeatmapForImport();
|
|
|
|
string extractedFolder = $"{temp}_extracted";
|
|
string dataFolder = Path.Combine(extractedFolder, "actual_data");
|
|
string resourceForkFolder = Path.Combine(extractedFolder, "__MACOSX");
|
|
string resourceForkFilePath = Path.Combine(resourceForkFolder, ".extracted");
|
|
|
|
Directory.CreateDirectory(dataFolder);
|
|
Directory.CreateDirectory(resourceForkFolder);
|
|
|
|
using (var resourceForkFile = File.CreateText(resourceForkFilePath))
|
|
{
|
|
await resourceForkFile.WriteLineAsync("adding content so that it's not empty");
|
|
}
|
|
|
|
try
|
|
{
|
|
using (var zip = ZipArchive.Open(temp))
|
|
zip.WriteToDirectory(dataFolder);
|
|
|
|
using (var zip = ZipArchive.Create())
|
|
{
|
|
zip.AddAllFromDirectory(extractedFolder);
|
|
zip.SaveTo(temp, new ZipWriterOptions(CompressionType.Deflate));
|
|
}
|
|
|
|
var imported = await importer.Import(new ImportTask(temp));
|
|
|
|
Assert.NotNull(imported);
|
|
Debug.Assert(imported != null);
|
|
|
|
EnsureLoaded(realmFactory.Context);
|
|
|
|
Assert.IsFalse(imported.PerformRead(s => s.Files.Any(f => f.Filename.Contains("__MACOSX"))), "Files contain resource fork folder, which should be ignored");
|
|
Assert.IsFalse(imported.PerformRead(s => s.Files.Any(f => f.Filename.Contains("actual_data"))), "Files contain common subfolder");
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(extractedFolder, true);
|
|
}
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
public void TestUpdateBeatmapInfo()
|
|
{
|
|
RunTestWithRealmAsync(async (realmFactory, storage) =>
|
|
{
|
|
using var importer = new BeatmapModelManager(realmFactory, storage);
|
|
using var store = new RulesetStore(realmFactory, storage);
|
|
|
|
string? temp = TestResources.GetTestBeatmapForImport();
|
|
await importer.Import(temp);
|
|
|
|
// Update via the beatmap, not the beatmap info, to ensure correct linking
|
|
BeatmapSetInfo setToUpdate = realmFactory.Context.All<BeatmapSetInfo>().First();
|
|
|
|
var beatmapToUpdate = setToUpdate.Beatmaps.First();
|
|
|
|
realmFactory.Context.Write(() => beatmapToUpdate.DifficultyName = "updated");
|
|
|
|
BeatmapInfo updatedInfo = realmFactory.Context.All<BeatmapInfo>().First(b => b.ID == beatmapToUpdate.ID);
|
|
Assert.That(updatedInfo.DifficultyName, Is.EqualTo("updated"));
|
|
});
|
|
}
|
|
|
|
public static async Task<BeatmapSetInfo?> LoadQuickOszIntoOsu(BeatmapImporter importer, Realm realm)
|
|
{
|
|
string? temp = TestResources.GetQuickTestBeatmapForImport();
|
|
|
|
var importedSet = await importer.Import(new ImportTask(temp));
|
|
|
|
Assert.NotNull(importedSet);
|
|
|
|
EnsureLoaded(realm);
|
|
|
|
waitForOrAssert(() => !File.Exists(temp), "Temporary file still exists after standard import", 5000);
|
|
|
|
return realm.All<BeatmapSetInfo>().FirstOrDefault(beatmapSet => beatmapSet.ID == importedSet!.ID);
|
|
}
|
|
|
|
public static async Task<BeatmapSetInfo> LoadOszIntoStore(BeatmapImporter importer, Realm realm, string? path = null, bool virtualTrack = false)
|
|
{
|
|
string? temp = path ?? TestResources.GetTestBeatmapForImport(virtualTrack);
|
|
|
|
var importedSet = await importer.Import(new ImportTask(temp));
|
|
|
|
Assert.NotNull(importedSet);
|
|
Debug.Assert(importedSet != null);
|
|
|
|
EnsureLoaded(realm);
|
|
|
|
waitForOrAssert(() => !File.Exists(temp), "Temporary file still exists after standard import", 5000);
|
|
|
|
return realm.All<BeatmapSetInfo>().First(beatmapSet => beatmapSet.ID == importedSet.ID);
|
|
}
|
|
|
|
private void deleteBeatmapSet(BeatmapSetInfo imported, Realm realm)
|
|
{
|
|
realm.Write(() => imported.DeletePending = true);
|
|
|
|
checkBeatmapSetCount(realm, 0);
|
|
checkBeatmapSetCount(realm, 1, true);
|
|
|
|
Assert.IsTrue(realm.All<BeatmapSetInfo>().First(_ => true).DeletePending);
|
|
}
|
|
|
|
private static Task createScoreForBeatmap(Realm realm, BeatmapInfo beatmap)
|
|
{
|
|
// TODO: reimplement when we have score support in realm.
|
|
// return ImportScoreTest.LoadScoreIntoOsu(osu, new ScoreInfo
|
|
// {
|
|
// OnlineID = 2,
|
|
// Beatmap = beatmap,
|
|
// BeatmapInfoID = beatmap.ID
|
|
// }, new ImportScoreTest.TestArchiveReader());
|
|
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
private static void checkBeatmapSetCount(Realm realm, int expected, bool includeDeletePending = false)
|
|
{
|
|
Assert.AreEqual(expected, includeDeletePending
|
|
? realm.All<BeatmapSetInfo>().Count()
|
|
: realm.All<BeatmapSetInfo>().Count(s => !s.DeletePending));
|
|
}
|
|
|
|
private static string hashFile(string filename)
|
|
{
|
|
using (var s = File.OpenRead(filename))
|
|
return s.ComputeMD5Hash();
|
|
}
|
|
|
|
private static void checkBeatmapCount(Realm realm, int expected)
|
|
{
|
|
Assert.AreEqual(expected, realm.All<BeatmapInfo>().Where(_ => true).ToList().Count);
|
|
}
|
|
|
|
private static void checkSingleReferencedFileCount(Realm realm, int expected)
|
|
{
|
|
int singleReferencedCount = 0;
|
|
|
|
foreach (var f in realm.All<RealmFile>())
|
|
{
|
|
if (f.BacklinksCount == 1)
|
|
singleReferencedCount++;
|
|
}
|
|
|
|
Assert.AreEqual(expected, singleReferencedCount);
|
|
}
|
|
|
|
internal static void EnsureLoaded(Realm realm, int timeout = 60000)
|
|
{
|
|
IQueryable<BeatmapSetInfo>? resultSets = null;
|
|
|
|
waitForOrAssert(() =>
|
|
{
|
|
realm.Refresh();
|
|
return (resultSets = realm.All<BeatmapSetInfo>().Where(s => !s.DeletePending && s.OnlineID == 241526)).Any();
|
|
}, @"BeatmapSet did not import to the database in allocated time.", timeout);
|
|
|
|
// ensure we were stored to beatmap database backing...
|
|
Assert.IsTrue(resultSets?.Count() == 1, $@"Incorrect result count found ({resultSets?.Count()} but should be 1).");
|
|
|
|
IEnumerable<BeatmapSetInfo> queryBeatmapSets() => realm.All<BeatmapSetInfo>().Where(s => !s.DeletePending && s.OnlineID == 241526);
|
|
|
|
var set = queryBeatmapSets().First();
|
|
|
|
// ReSharper disable once PossibleUnintendedReferenceComparison
|
|
IEnumerable<BeatmapInfo> queryBeatmaps() => realm.All<BeatmapInfo>().Where(s => s.BeatmapSet != null && s.BeatmapSet == set);
|
|
|
|
Assert.AreEqual(12, queryBeatmaps().Count(), @"Beatmap count was not correct");
|
|
Assert.AreEqual(1, queryBeatmapSets().Count(), @"Beatmapset count was not correct");
|
|
|
|
int countBeatmapSetBeatmaps;
|
|
int countBeatmaps;
|
|
|
|
Assert.AreEqual(
|
|
countBeatmapSetBeatmaps = queryBeatmapSets().First().Beatmaps.Count,
|
|
countBeatmaps = queryBeatmaps().Count(),
|
|
$@"Incorrect database beatmap count post-import ({countBeatmaps} but should be {countBeatmapSetBeatmaps}).");
|
|
|
|
foreach (BeatmapInfo b in set.Beatmaps)
|
|
Assert.IsTrue(set.Beatmaps.Any(c => c.OnlineID == b.OnlineID));
|
|
Assert.IsTrue(set.Beatmaps.Count > 0);
|
|
}
|
|
|
|
private static void waitForOrAssert(Func<bool> result, string failureMessage, int timeout = 60000)
|
|
{
|
|
const int sleep = 200;
|
|
|
|
while (timeout > 0)
|
|
{
|
|
Thread.Sleep(sleep);
|
|
timeout -= sleep;
|
|
|
|
if (result())
|
|
return;
|
|
}
|
|
|
|
Assert.Fail(failureMessage);
|
|
}
|
|
|
|
public class NonOptimisedBeatmapImporter : BeatmapImporter
|
|
{
|
|
public NonOptimisedBeatmapImporter(RealmContextFactory realmFactory, Storage storage)
|
|
: base(realmFactory, storage)
|
|
{
|
|
}
|
|
|
|
protected override bool HasCustomHashFunction => true;
|
|
}
|
|
}
|
|
}
|