Merge branch 'master' into preserve-storyboard

osu.Game.Tests/Skins/TestSceneBeatmapSkinResources.cs

@@ -1,6 +1,7 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using System.IO;
 using NUnit.Framework;
 using osu.Framework.Allocation;
 using osu.Framework.Audio.Track;
@@ -12,6 +13,7 @@ using osu.Game.Beatmaps;
 using osu.Game.Database;
 using osu.Game.Tests.Resources;
 using osu.Game.Tests.Visual;
+using MemoryStream = System.IO.MemoryStream;
 
 namespace osu.Game.Tests.Skins
 {
@@ -21,6 +23,52 @@ namespace osu.Game.Tests.Skins
         [Resolved]
         private BeatmapManager beatmaps { get; set; } = null!;
 
+        [Test]
+        public void TestRetrieveAndLegacyExportJapaneseFilename()
+        {
+            IWorkingBeatmap beatmap = null!;
+            MemoryStream outStream = null!;
+
+            // Ensure importer encoding is correct
+            AddStep("import beatmap", () => beatmap = importBeatmapFromArchives(@"japanese-filename.osz"));
+            AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null);
+
+            // Ensure exporter encoding is correct (round trip)
+            AddStep("export", () =>
+            {
+                outStream = new MemoryStream();
+
+                new LegacyBeatmapExporter(LocalStorage)
+                    .ExportToStream((BeatmapSetInfo)beatmap.BeatmapInfo.BeatmapSet!, outStream, null);
+            });
+
+            AddStep("import beatmap again", () => beatmap = importBeatmapFromStream(outStream));
+            AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null);
+        }
+
+        [Test]
+        public void TestRetrieveAndNonLegacyExportJapaneseFilename()
+        {
+            IWorkingBeatmap beatmap = null!;
+            MemoryStream outStream = null!;
+
+            // Ensure importer encoding is correct
+            AddStep("import beatmap", () => beatmap = importBeatmapFromArchives(@"japanese-filename.osz"));
+            AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null);
+
+            // Ensure exporter encoding is correct (round trip)
+            AddStep("export", () =>
+            {
+                outStream = new MemoryStream();
+
+                new BeatmapExporter(LocalStorage)
+                    .ExportToStream((BeatmapSetInfo)beatmap.BeatmapInfo.BeatmapSet!, outStream, null);
+            });
+
+            AddStep("import beatmap again", () => beatmap = importBeatmapFromStream(outStream));
+            AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"見本")) != null);
+        }
+
         [Test]
         public void TestRetrieveOggAudio()
         {
@@ -45,6 +93,12 @@ namespace osu.Game.Tests.Skins
             AddAssert("sample is non-null", () => beatmap.Skin.GetSample(new SampleInfo(@"spinner-osu")) != null);
         }
 
+        private IWorkingBeatmap importBeatmapFromStream(Stream stream)
+        {
+            var imported = beatmaps.Import(new ImportTask(stream, "filename.osz")).GetResultSafely();
+            return imported.AsNonNull().PerformRead(s => beatmaps.GetWorkingBeatmap(s.Beatmaps[0]));
+        }
+
         private IWorkingBeatmap importBeatmapFromArchives(string filename)
         {
             var imported = beatmaps.Import(new ImportTask(TestResources.OpenResource($@"Archives/{filename}"), filename)).GetResultSafely();

osu.Game/Database/BeatmapExporter.cs

@@ -17,6 +17,8 @@ namespace osu.Game.Database
         {
         }
 
+        protected override bool UseFixedEncoding => false;
+
         protected override string FileExtension => @".olz";
     }
 }

osu.Game/Database/LegacyArchiveExporter.cs

@@ -3,10 +3,12 @@
 
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using osu.Framework.Logging;
 using osu.Framework.Platform;
 using osu.Game.Extensions;
+using osu.Game.IO.Archives;
 using osu.Game.Overlays.Notifications;
 using Realms;
 using SharpCompress.Common;
@@ -22,6 +24,11 @@ namespace osu.Game.Database
     public abstract class LegacyArchiveExporter<TModel> : LegacyExporter<TModel>
         where TModel : RealmObject, IHasNamedFiles, IHasGuidPrimaryKey
     {
+        /// <summary>
+        /// Whether to always use Shift-JIS encoding for archive filenames (like osu!stable did).
+        /// </summary>
+        protected virtual bool UseFixedEncoding => true;
+
         protected LegacyArchiveExporter(Storage storage)
             : base(storage)
         {
@@ -29,7 +36,12 @@ namespace osu.Game.Database
 
         public override void ExportToStream(TModel model, Stream outputStream, ProgressNotification? notification, CancellationToken cancellationToken = default)
         {
-            using (var writer = new ZipWriter(outputStream, new ZipWriterOptions(CompressionType.Deflate)))
+            var zipWriterOptions = new ZipWriterOptions(CompressionType.Deflate)
+            {
+                ArchiveEncoding = UseFixedEncoding ? ZipArchiveReader.DEFAULT_ENCODING : new ArchiveEncoding(Encoding.UTF8, Encoding.UTF8)
+            };
+
+            using (var writer = new ZipWriter(outputStream, zipWriterOptions))
             {
                 int i = 0;
                 int fileCount = model.Files.Count();

osu.Game/Database/RealmAccess.cs

@@ -35,6 +35,7 @@ using osu.Game.Rulesets.Mods;
 using osu.Game.Scoring;
 using osu.Game.Scoring.Legacy;
 using osu.Game.Skinning;
+using osu.Game.Utils;
 using osuTK.Input;
 using Realms;
 using Realms.Exceptions;
@@ -321,12 +322,32 @@ namespace osu.Game.Database
                 {
                     Logger.Error(e, "Your local database is too new to work with this version of osu!. Please close osu! and install the latest release to recover your data.");
 
-                    // If a newer version database already exists, don't backup again. We can presume that the first backup is the one we care about.
+                    // If a newer version database already exists, don't create another backup. We can presume that the first backup is the one we care about.
                     if (!storage.Exists(newerVersionFilename))
                         createBackup(newerVersionFilename);
                 }
                 else
                 {
+                    // This error can occur due to file handles still being open by a previous instance.
+                    // If this is the case, rather than assuming the realm file is corrupt, block game startup.
+                    if (e.Message.StartsWith("SetEndOfFile() failed", StringComparison.Ordinal))
+                    {
+                        // This will throw if the realm file is not available for write access after 5 seconds.
+                        FileUtils.AttemptOperation(() =>
+                        {
+                            if (storage.Exists(Filename))
+                            {
+                                using (var _ = storage.GetStream(Filename, FileAccess.ReadWrite))
+                                {
+                                }
+                            }
+                        }, 20);
+
+                        // If the above eventually succeeds, try and continue startup as per normal.
+                        // This may throw again but let's allow it to, and block startup.
+                        return getRealmInstance();
+                    }
+
                     Logger.Error(e, "Realm startup failed with unrecoverable error; starting with a fresh database. A backup of your database has been made.");
                     createBackup($"{Filename.Replace(realm_extension, string.Empty)}_{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}_corrupt{realm_extension}");
                 }
@@ -1142,33 +1163,18 @@ namespace osu.Game.Database
         {
             Logger.Log($"Creating full realm database backup at {backupFilename}", LoggingTarget.Database);
 
-            int attempts = 10;
+            FileUtils.AttemptOperation(() =>
-
-            while (true)
             {
-                try
+                using (var source = storage.GetStream(Filename, mode: FileMode.Open))
                 {
-                    using (var source = storage.GetStream(Filename, mode: FileMode.Open))
+                    // source may not exist.
-                    {
+                    if (source == null)
-                        // source may not exist.
+                        return;
-                        if (source == null)
-                            return;
 
-                        using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew))
+                    using (var destination = storage.GetStream(backupFilename, FileAccess.Write, FileMode.CreateNew))
-                            source.CopyTo(destination);
+                        source.CopyTo(destination);
-                    }
-
-                    return;
                 }
-                catch (IOException)
+            }, 20);
-                {
-                    if (attempts-- <= 0)
-                        throw;
-
-                    // file may be locked during use.
-                    Thread.Sleep(500);
-                }
-            }
         }
 
         /// <summary>

osu.Game/Database/RealmArchiveModelImporter.cs

@@ -449,16 +449,6 @@ namespace osu.Game.Database
             return reader.Name.ComputeSHA2Hash();
         }
 
-        /// <summary>
-        /// Create all required <see cref="File"/>s for the provided archive, adding them to the global file store.
-        /// </summary>
-        private List<RealmNamedFileUsage> createFileInfos(ArchiveReader reader, RealmFileStore files, Realm realm)
-        {
-            var fileInfos = new List<RealmNamedFileUsage>();
-
-            return fileInfos;
-        }
-
         private IEnumerable<(string original, string shortened)> getShortenedFilenames(ArchiveReader reader)
         {
             string prefix = reader.Filenames.GetCommonPrefix();

osu.Game/IO/Archives/ZipArchiveReader.cs

@@ -7,23 +7,45 @@ using System.Buffers;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using Microsoft.Toolkit.HighPerformance;
 using osu.Framework.IO.Stores;
 using SharpCompress.Archives.Zip;
+using SharpCompress.Common;
+using SharpCompress.Readers;
 using SixLabors.ImageSharp.Memory;
 
 namespace osu.Game.IO.Archives
 {
     public sealed class ZipArchiveReader : ArchiveReader
     {
+        /// <summary>
+        /// Archives created by osu!stable still write out as Shift-JIS.
+        /// We want to force this fallback rather than leave it up to the library/system.
+        /// In the future we may want to change exports to set the zip UTF-8 flag and use that instead.
+        /// </summary>
+        public static readonly ArchiveEncoding DEFAULT_ENCODING;
+
         private readonly Stream archiveStream;
         private readonly ZipArchive archive;
 
+        static ZipArchiveReader()
+        {
+            // Required to support rare code pages.
+            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+
+            DEFAULT_ENCODING = new ArchiveEncoding(Encoding.GetEncoding(932), Encoding.GetEncoding(932));
+        }
+
         public ZipArchiveReader(Stream archiveStream, string name = null)
             : base(name)
         {
             this.archiveStream = archiveStream;
-            archive = ZipArchive.Open(archiveStream);
+
+            archive = ZipArchive.Open(archiveStream, new ReaderOptions
+            {
+                ArchiveEncoding = DEFAULT_ENCODING
+            });
         }
 
         public override Stream GetStream(string name)

osu.Game/OsuGame.cs

@@ -841,7 +841,10 @@ namespace osu.Game
             {
                 // General expectation that osu! starts in fullscreen by default (also gives the most predictable performance).
                 // However, macOS is bound to have issues when using exclusive fullscreen as it takes full control away from OS, therefore borderless is default there.
-                { FrameworkSetting.WindowMode, RuntimeInfo.OS == RuntimeInfo.Platform.macOS ? WindowMode.Borderless : WindowMode.Fullscreen }
+                { FrameworkSetting.WindowMode, RuntimeInfo.OS == RuntimeInfo.Platform.macOS ? WindowMode.Borderless : WindowMode.Fullscreen },
+                { FrameworkSetting.VolumeUniversal, 0.6 },
+                { FrameworkSetting.VolumeMusic, 0.6 },
+                { FrameworkSetting.VolumeEffect, 0.6 },
             };
         }
 

osu.Game/Screens/Backgrounds/BackgroundScreenDefault.cs

@@ -56,10 +56,6 @@ namespace osu.Game.Screens.Backgrounds
             introSequence = config.GetBindable<IntroSequence>(OsuSetting.IntroSequence);
 
             AddInternal(seasonalBackgroundLoader);
-
-            // Load first background asynchronously as part of BDL load.
-            currentDisplay = RNG.Next(0, background_count);
-            Next();
         }
 
         protected override void LoadComplete()
@@ -73,6 +69,9 @@ namespace osu.Game.Screens.Backgrounds
             introSequence.ValueChanged += _ => Scheduler.AddOnce(next);
             seasonalBackgroundLoader.SeasonalBackgroundChanged += () => Scheduler.AddOnce(next);
 
+            currentDisplay = RNG.Next(0, background_count);
+            Next();
+
             // helper function required for AddOnce usage.
             void next() => Next();
         }