Merge pull request #28034 from peppy/fix-zip-encoding

Fix filenames containing japanese not being read correctly

osu.Game.Tests/Skins/TestSceneBeatmapSkinResources.cs

@@ -1,6 +1,7 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System.IO;
 using NUnit.Framework;
 using osu.Framework.Allocation;
 using osu.Framework.Audio.Track;
@@ -12,6 +13,7 @@ using osu.Game.Beatmaps;
 using osu.Game.Database;
 using osu.Game.Tests.Resources;
 using osu.Game.Tests.Visual;
+using MemoryStream = System.IO.MemoryStream;
 
 namespace osu.Game.Tests.Skins
 {
@@ -21,6 +23,52 @@ namespace osu.Game.Tests.Skins
         [Resolved]
         private BeatmapManager beatmaps { get; set; } = null!;
 
+        [Test]
+        public void TestRetrieveAndLegacyExportJapaneseFilename()
+        {
+            IWorkingBeatmap beatmap = null!;
+            MemoryStream outStream = null!;
+
+            // Ensure importer encoding is correct
+            AddStep("import beatmap", () => beatmap = importBeatmapFromArchives(@"japanese-filename.osz"));
+            AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null);
+
+            // Ensure exporter encoding is correct (round trip)
+            AddStep("export", () =>
+            {
+                outStream = new MemoryStream();
+
+                new LegacyBeatmapExporter(LocalStorage)
+                    .ExportToStream((BeatmapSetInfo)beatmap.BeatmapInfo.BeatmapSet!, outStream, null);
+            });
+
+            AddStep("import beatmap again", () => beatmap = importBeatmapFromStream(outStream));
+            AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null);
+        }
+
+        [Test]
+        public void TestRetrieveAndNonLegacyExportJapaneseFilename()
+        {
+            IWorkingBeatmap beatmap = null!;
+            MemoryStream outStream = null!;
+
+            // Ensure importer encoding is correct
+            AddStep("import beatmap", () => beatmap = importBeatmapFromArchives(@"japanese-filename.osz"));
+            AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null);
+
+            // Ensure exporter encoding is correct (round trip)
+            AddStep("export", () =>
+            {
+                outStream = new MemoryStream();
+
+                new BeatmapExporter(LocalStorage)
+                    .ExportToStream((BeatmapSetInfo)beatmap.BeatmapInfo.BeatmapSet!, outStream, null);
+            });
+
+            AddStep("import beatmap again", () => beatmap = importBeatmapFromStream(outStream));
+            AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null);
+        }
+
         [Test]
         public void TestRetrieveOggAudio()
         {
@@ -45,6 +93,12 @@ namespace osu.Game.Tests.Skins
             AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"spinner-osu")) != null);
         }
 
+        private IWorkingBeatmap importBeatmapFromStream(Stream stream)
+        {
+            var imported = beatmaps.Import(new ImportTask(stream, "filename.osz")).GetResultSafely();
+            return imported.AsNonNull().PerformRead(s => beatmaps.GetWorkingBeatmap(s.Beatmaps[0]));
+        }
+
         private IWorkingBeatmap importBeatmapFromArchives(string filename)
         {
             var imported = beatmaps.Import(new ImportTask(TestResources.OpenResource($@"Archives/{filename}"), filename)).GetResultSafely();

osu.Game/Database/BeatmapExporter.cs

@@ -17,6 +17,8 @@ namespace osu.Game.Database
         {
         }
 
+        protected override bool UseFixedEncoding => false;
+
         protected override string FileExtension => @".olz";
     }
 }

osu.Game/Database/LegacyArchiveExporter.cs

@@ -3,10 +3,12 @@
 
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using osu.Framework.Logging;
 using osu.Framework.Platform;
 using osu.Game.Extensions;
+using osu.Game.IO.Archives;
 using osu.Game.Overlays.Notifications;
 using Realms;
 using SharpCompress.Common;
@@ -22,6 +24,11 @@ namespace osu.Game.Database
     public abstract class LegacyArchiveExporter<TModel> : LegacyExporter<TModel>
         where TModel : RealmObject, IHasNamedFiles, IHasGuidPrimaryKey
     {
+        /// <summary>
+        /// Whether to always use Shift-JIS encoding for archive filenames (like osu!stable did).
+        /// </summary>
+        protected virtual bool UseFixedEncoding => true;
+
         protected LegacyArchiveExporter(Storage storage)
             : base(storage)
         {
@@ -29,7 +36,12 @@ namespace osu.Game.Database
 
         public override void ExportToStream(TModel model, Stream outputStream, ProgressNotification? notification, CancellationToken cancellationToken = default)
         {
-            using (var writer = new ZipWriter(outputStream, new ZipWriterOptions(CompressionType.Deflate)))
+            var zipWriterOptions = new ZipWriterOptions(CompressionType.Deflate)
+            {
+                ArchiveEncoding = UseFixedEncoding ? ZipArchiveReader.DEFAULT_ENCODING : new ArchiveEncoding(Encoding.UTF8, Encoding.UTF8)
+            };
+
+            using (var writer = new ZipWriter(outputStream, zipWriterOptions))
             {
                 int i = 0;
                 int fileCount = model.Files.Count();

osu.Game/Database/RealmArchiveModelImporter.cs

@@ -449,16 +449,6 @@ namespace osu.Game.Database
             return reader.Name.ComputeSHA2Hash();
         }
 
-        /// <summary>
-        /// Create all required <see cref="File"/>s for the provided archive, adding them to the global file store.
-        /// </summary>
-        private List<RealmNamedFileUsage> createFileInfos(ArchiveReader reader, RealmFileStore files, Realm realm)
-        {
-            var fileInfos = new List<RealmNamedFileUsage>();
-
-            return fileInfos;
-        }
-
         private IEnumerable<(string original, string shortened)> getShortenedFilenames(ArchiveReader reader)
         {
             string prefix = reader.Filenames.GetCommonPrefix();

osu.Game/IO/Archives/ZipArchiveReader.cs

@@ -7,23 +7,45 @@ using System.Buffers;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using Microsoft.Toolkit.HighPerformance;
 using osu.Framework.IO.Stores;
 using SharpCompress.Archives.Zip;
+using SharpCompress.Common;
+using SharpCompress.Readers;
 using SixLabors.ImageSharp.Memory;
 
 namespace osu.Game.IO.Archives
 {
     public sealed class ZipArchiveReader : ArchiveReader
     {
+        /// <summary>
+        /// Archives created by osu!stable still write out as Shift-JIS.
+        /// We want to force this fallback rather than leave it up to the library/system.
+        /// In the future we may want to change exports to set the zip UTF-8 flag and use that instead.
+        /// </summary>
+        public static readonly ArchiveEncoding DEFAULT_ENCODING;
+
         private readonly Stream archiveStream;
         private readonly ZipArchive archive;
 
+        static ZipArchiveReader()
+        {
+            // Required to support rare code pages.
+            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+
+            DEFAULT_ENCODING = new ArchiveEncoding(Encoding.GetEncoding(932), Encoding.GetEncoding(932));
+        }
+
         public ZipArchiveReader(Stream archiveStream, string name = null)
             : base(name)
         {
             this.archiveStream = archiveStream;
-            archive = ZipArchive.Open(archiveStream);
+
+            archive = ZipArchive.Open(archiveStream, new ReaderOptions
+            {
+                ArchiveEncoding = DEFAULT_ENCODING
+            });
         }
 
         public override Stream GetStream(string name)