diff --git a/osu.Game.Tests/Resources/Archives/japanese-filename.osz b/osu.Game.Tests/Resources/Archives/japanese-filename.osz new file mode 100644 index 0000000000..4825c88179 Binary files /dev/null and b/osu.Game.Tests/Resources/Archives/japanese-filename.osz differ diff --git a/osu.Game.Tests/Skins/TestSceneBeatmapSkinResources.cs b/osu.Game.Tests/Skins/TestSceneBeatmapSkinResources.cs index d9212386c3..5086b64433 100644 --- a/osu.Game.Tests/Skins/TestSceneBeatmapSkinResources.cs +++ b/osu.Game.Tests/Skins/TestSceneBeatmapSkinResources.cs @@ -1,6 +1,7 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. +using System.IO; using NUnit.Framework; using osu.Framework.Allocation; using osu.Framework.Audio.Track; @@ -12,6 +13,7 @@ using osu.Game.Beatmaps; using osu.Game.Database; using osu.Game.Tests.Resources; using osu.Game.Tests.Visual; +using MemoryStream = System.IO.MemoryStream; namespace osu.Game.Tests.Skins { @@ -21,6 +23,52 @@ namespace osu.Game.Tests.Skins [Resolved] private BeatmapManager beatmaps { get; set; } = null!; + [Test] + public void TestRetrieveAndLegacyExportJapaneseFilename() + { + IWorkingBeatmap beatmap = null!; + MemoryStream outStream = null!; + + // Ensure importer encoding is correct + AddStep("import beatmap", () => beatmap = importBeatmapFromArchives(@"japanese-filename.osz")); + AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null); + + // Ensure exporter encoding is correct (round trip) + AddStep("export", () => + { + outStream = new MemoryStream(); + + new LegacyBeatmapExporter(LocalStorage) + .ExportToStream((BeatmapSetInfo)beatmap.BeatmapInfo.BeatmapSet!, outStream, null); + }); + + AddStep("import beatmap again", () => beatmap = importBeatmapFromStream(outStream)); + AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null); + } + + [Test] + public void TestRetrieveAndNonLegacyExportJapaneseFilename() + { + IWorkingBeatmap beatmap = null!; + MemoryStream outStream = null!; + + // Ensure importer encoding is correct + AddStep("import beatmap", () => beatmap = importBeatmapFromArchives(@"japanese-filename.osz")); + AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null); + + // Ensure exporter encoding is correct (round trip) + AddStep("export", () => + { + outStream = new MemoryStream(); + + new BeatmapExporter(LocalStorage) + .ExportToStream((BeatmapSetInfo)beatmap.BeatmapInfo.BeatmapSet!, outStream, null); + }); + + AddStep("import beatmap again", () => beatmap = importBeatmapFromStream(outStream)); + AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null); + } + [Test] public void TestRetrieveOggAudio() { @@ -45,6 +93,12 @@ namespace osu.Game.Tests.Skins AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"spinner-osu")) != null); } + private IWorkingBeatmap importBeatmapFromStream(Stream stream) + { + var imported = beatmaps.Import(new ImportTask(stream, "filename.osz")).GetResultSafely(); + return imported.AsNonNull().PerformRead(s => beatmaps.GetWorkingBeatmap(s.Beatmaps[0])); + } + private IWorkingBeatmap importBeatmapFromArchives(string filename) { var imported = beatmaps.Import(new ImportTask(TestResources.OpenResource($@"Archives/{filename}"), filename)).GetResultSafely(); diff --git a/osu.Game/Database/BeatmapExporter.cs b/osu.Game/Database/BeatmapExporter.cs index f37c57dea5..01ef09d3d7 100644 --- a/osu.Game/Database/BeatmapExporter.cs +++ b/osu.Game/Database/BeatmapExporter.cs @@ -17,6 +17,8 @@ namespace osu.Game.Database { } + protected override bool UseFixedEncoding => false; + protected override string FileExtension => @".olz"; } } diff --git a/osu.Game/Database/LegacyArchiveExporter.cs b/osu.Game/Database/LegacyArchiveExporter.cs index 9805207591..e4d3ed4681 100644 --- a/osu.Game/Database/LegacyArchiveExporter.cs +++ b/osu.Game/Database/LegacyArchiveExporter.cs @@ -3,10 +3,12 @@ using System.IO; using System.Linq; +using System.Text; using System.Threading; using osu.Framework.Logging; using osu.Framework.Platform; using osu.Game.Extensions; +using osu.Game.IO.Archives; using osu.Game.Overlays.Notifications; using Realms; using SharpCompress.Common; @@ -22,6 +24,11 @@ namespace osu.Game.Database public abstract class LegacyArchiveExporter : LegacyExporter where TModel : RealmObject, IHasNamedFiles, IHasGuidPrimaryKey { + /// + /// Whether to always use Shift-JIS encoding for archive filenames (like osu!stable did). + /// + protected virtual bool UseFixedEncoding => true; + protected LegacyArchiveExporter(Storage storage) : base(storage) { @@ -29,7 +36,12 @@ namespace osu.Game.Database public override void ExportToStream(TModel model, Stream outputStream, ProgressNotification? notification, CancellationToken cancellationToken = default) { - using (var writer = new ZipWriter(outputStream, new ZipWriterOptions(CompressionType.Deflate))) + var zipWriterOptions = new ZipWriterOptions(CompressionType.Deflate) + { + ArchiveEncoding = UseFixedEncoding ? ZipArchiveReader.DEFAULT_ENCODING : new ArchiveEncoding(Encoding.UTF8, Encoding.UTF8) + }; + + using (var writer = new ZipWriter(outputStream, zipWriterOptions)) { int i = 0; int fileCount = model.Files.Count(); diff --git a/osu.Game/Database/RealmArchiveModelImporter.cs b/osu.Game/Database/RealmArchiveModelImporter.cs index bc4954c6ea..0014e246dc 100644 --- a/osu.Game/Database/RealmArchiveModelImporter.cs +++ b/osu.Game/Database/RealmArchiveModelImporter.cs @@ -449,16 +449,6 @@ namespace osu.Game.Database return reader.Name.ComputeSHA2Hash(); } - /// - /// Create all required s for the provided archive, adding them to the global file store. - /// - private List createFileInfos(ArchiveReader reader, RealmFileStore files, Realm realm) - { - var fileInfos = new List(); - - return fileInfos; - } - private IEnumerable<(string original, string shortened)> getShortenedFilenames(ArchiveReader reader) { string prefix = reader.Filenames.GetCommonPrefix(); diff --git a/osu.Game/IO/Archives/ZipArchiveReader.cs b/osu.Game/IO/Archives/ZipArchiveReader.cs index cc5c65d184..6bb2a314e7 100644 --- a/osu.Game/IO/Archives/ZipArchiveReader.cs +++ b/osu.Game/IO/Archives/ZipArchiveReader.cs @@ -7,23 +7,45 @@ using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using Microsoft.Toolkit.HighPerformance; using osu.Framework.IO.Stores; using SharpCompress.Archives.Zip; +using SharpCompress.Common; +using SharpCompress.Readers; using SixLabors.ImageSharp.Memory; namespace osu.Game.IO.Archives { public sealed class ZipArchiveReader : ArchiveReader { + /// + /// Archives created by osu!stable still write out as Shift-JIS. + /// We want to force this fallback rather than leave it up to the library/system. + /// In the future we may want to change exports to set the zip UTF-8 flag and use that instead. + /// + public static readonly ArchiveEncoding DEFAULT_ENCODING; + private readonly Stream archiveStream; private readonly ZipArchive archive; + static ZipArchiveReader() + { + // Required to support rare code pages. + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + + DEFAULT_ENCODING = new ArchiveEncoding(Encoding.GetEncoding(932), Encoding.GetEncoding(932)); + } + public ZipArchiveReader(Stream archiveStream, string name = null) : base(name) { this.archiveStream = archiveStream; - archive = ZipArchive.Open(archiveStream); + + archive = ZipArchive.Open(archiveStream, new ReaderOptions + { + ArchiveEncoding = DEFAULT_ENCODING + }); } public override Stream GetStream(string name)