From c8f7f2215b4c9b3bcb5df8fdc46b7f906890dd2c Mon Sep 17 00:00:00 2001
From: Dean Herbert <pe@ppy.sh>
Date: Mon, 29 Apr 2024 18:49:17 +0800
Subject: [PATCH] Force encoding to Shift-JIS for archive filenames
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

After way too much time investigating this, the encoding situation is
not great right now.

- Stable sets the "default code page" to be used for encoding filenames
to Shift-JIS (932):
https://github.com/peppy/osu-stable-reference/blob/c29ebd7fc52113013fb4ac2db230699d81e1fe2c/osu!/GameBase.cs#L3099
- Lazer does nothing (therefore using UTF-8).

When importing to lazer, stable files are assumed to be UTF-8. This
means that the linked beatmaps don't work correctly. Forcing lazer to
decompress *and* compress using Shift-JIS will fix this.

Here's a rough idea of how things look for japanese character filenames
in current `master`:

| | stable | lazer |
|--------|--------|--------|
| export encoding | shift-jis | utf8 |
| utf8 [bit flag](https://superuser.com/a/1507988) set | ❌ | ❌ |
| import stable export osz | ✅  | ❌ |
| import lazer export osz | ❌ | ✅ |
| windows unzip | ❌ | ❌ |
| macos unzip | ✅ | ✅ |

and after this change

| | stable | lazer |
|--------|--------|--------|
| export encoding | shift-jis | shift-jis |
| utf8 [bit flag](https://superuser.com/a/1507988) set | ❌ | ❌ |
| import stable export osz | ✅  | ✅ |
| import lazer export osz | ✅ | ✅ |
| windows unzip | ❌ | ❌ |
| macos unzip | ✅ | ✅ |

A future endeavour to improve compatibility would be to look at setting
the utf8 flag in lazer, switching the default to utf8, and ensuring the
stable supports this flag (I don't believe it does right now).
---
 osu.Game/Database/LegacyArchiveExporter.cs |  6 +++++-
 osu.Game/IO/Archives/ZipArchiveReader.cs   | 24 +++++++++++++++++++++-
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/osu.Game/Database/LegacyArchiveExporter.cs b/osu.Game/Database/LegacyArchiveExporter.cs
index 9805207591..1d9d252220 100644
--- a/osu.Game/Database/LegacyArchiveExporter.cs
+++ b/osu.Game/Database/LegacyArchiveExporter.cs
@@ -7,6 +7,7 @@ using System.Threading;
 using osu.Framework.Logging;
 using osu.Framework.Platform;
 using osu.Game.Extensions;
+using osu.Game.IO.Archives;
 using osu.Game.Overlays.Notifications;
 using Realms;
 using SharpCompress.Common;
@@ -29,7 +30,10 @@ namespace osu.Game.Database
 
         public override void ExportToStream(TModel model, Stream outputStream, ProgressNotification? notification, CancellationToken cancellationToken = default)
         {
-            using (var writer = new ZipWriter(outputStream, new ZipWriterOptions(CompressionType.Deflate)))
+            using (var writer = new ZipWriter(outputStream, new ZipWriterOptions(CompressionType.Deflate)
+                   {
+                       ArchiveEncoding = ZipArchiveReader.DEFAULT_ENCODING
+                   }))
             {
                 int i = 0;
                 int fileCount = model.Files.Count();
diff --git a/osu.Game/IO/Archives/ZipArchiveReader.cs b/osu.Game/IO/Archives/ZipArchiveReader.cs
index cc5c65d184..6bb2a314e7 100644
--- a/osu.Game/IO/Archives/ZipArchiveReader.cs
+++ b/osu.Game/IO/Archives/ZipArchiveReader.cs
@@ -7,23 +7,45 @@ using System.Buffers;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using Microsoft.Toolkit.HighPerformance;
 using osu.Framework.IO.Stores;
 using SharpCompress.Archives.Zip;
+using SharpCompress.Common;
+using SharpCompress.Readers;
 using SixLabors.ImageSharp.Memory;
 
 namespace osu.Game.IO.Archives
 {
     public sealed class ZipArchiveReader : ArchiveReader
     {
+        /// <summary>
+        /// Archives created by osu!stable still write out as Shift-JIS.
+        /// We want to force this fallback rather than leave it up to the library/system.
+        /// In the future we may want to change exports to set the zip UTF-8 flag and use that instead.
+        /// </summary>
+        public static readonly ArchiveEncoding DEFAULT_ENCODING;
+
         private readonly Stream archiveStream;
         private readonly ZipArchive archive;
 
+        static ZipArchiveReader()
+        {
+            // Required to support rare code pages.
+            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+
+            DEFAULT_ENCODING = new ArchiveEncoding(Encoding.GetEncoding(932), Encoding.GetEncoding(932));
+        }
+
         public ZipArchiveReader(Stream archiveStream, string name = null)
             : base(name)
         {
             this.archiveStream = archiveStream;
-            archive = ZipArchive.Open(archiveStream);
+
+            archive = ZipArchive.Open(archiveStream, new ReaderOptions
+            {
+                ArchiveEncoding = DEFAULT_ENCODING
+            });
         }
 
         public override Stream GetStream(string name)