Implement fallback decoder registration

After the preparatory introduction of LineBufferedReader, it is now
possible to introduce registration of fallback decoders that won't drop
input supplied in the first line of the file.

A fallback decoder is used when the magic in the first line of the file
does not match any of the other known decoders. In such a case,
the fallback decoder is constructed and provided a LineBufferedReader
instance. The process of matching magic only peeks the first non-empty
line, so it is available for re-reading in Decode() using ReadLine().

There can be only one fallback decoder per type; a second attempt of
registering a fallback will result in an exception to avoid bugs.

To address the issue of parsing failing on badly or non-headered files,
set the legacy decoders for Beatmaps and Storyboards as the fallbacks.

Due to non-trivial logic, several new, passing unit tests with possible
edge cases also included.

osu.Game.Tests/Beatmaps/Formats/LegacyBeatmapDecoderTest.cs

@@ -2,6 +2,7 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System;
+using System.IO;
 using NUnit.Framework;
 using osuTK;
 using osuTK.Graphics;
@@ -490,5 +491,105 @@ namespace osu.Game.Tests.Beatmaps.Formats
                 Assert.DoesNotThrow(() => decoder.Decode(badStream));
             }
         }
+
+        [Test]
+        public void TestFallbackDecoderForCorruptedHeader()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("corrupted-header.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("Beatmap with corrupted header", beatmap.Metadata.Title);
+                Assert.AreEqual("Evil Hacker", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestFallbackDecoderForMissingHeader()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("missing-header.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("Beatmap with no header", beatmap.Metadata.Title);
+                Assert.AreEqual("Incredibly Evil Hacker", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestDecodeFileWithEmptyLinesAtStart()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("empty-lines-at-start.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("Empty lines at start", beatmap.Metadata.Title);
+                Assert.AreEqual("Edge Case Hunter", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestDecodeFileWithEmptyLinesAndNoHeader()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("empty-line-instead-of-header.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("The dog ate the file header", beatmap.Metadata.Title);
+                Assert.AreEqual("Why does this keep happening", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestDecodeFileWithContentImmediatelyAfterHeader()
+        {
+            Decoder<Beatmap> decoder = null;
+            Beatmap beatmap = null;
+
+            using (var resStream = TestResources.OpenResource("no-empty-line-after-header.osu"))
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.DoesNotThrow(() => decoder = Decoder.GetDecoder<Beatmap>(stream));
+                Assert.IsInstanceOf<LegacyBeatmapDecoder>(decoder);
+                Assert.DoesNotThrow(() => beatmap = decoder.Decode(stream));
+                Assert.IsNotNull(beatmap);
+                Assert.AreEqual("No empty line delimiting header from contents", beatmap.Metadata.Title);
+                Assert.AreEqual("Edge Case Hunter", beatmap.Metadata.AuthorString);
+            }
+        }
+
+        [Test]
+        public void TestDecodeEmptyFile()
+        {
+            using (var resStream = new MemoryStream())
+            using (var stream = new LineBufferedReader(resStream))
+            {
+                Assert.Throws<IOException>(() => Decoder.GetDecoder<Beatmap>(stream));
+            }
+        }
     }
 }

osu.Game.Tests/Beatmaps/IO/ImportBeatmapTest.cs

@@ -126,7 +126,7 @@ namespace osu.Game.Tests.Beatmaps.IO
 
                     var breakTemp = TestResources.GetTestBeatmapForImport();
 
-                    MemoryStream brokenOsu = new MemoryStream(new byte[] { 1, 3, 3, 7 });
+                    MemoryStream brokenOsu = new MemoryStream();
                     MemoryStream brokenOsz = new MemoryStream(File.ReadAllBytes(breakTemp));
 
                     File.Delete(breakTemp);

osu.Game.Tests/Resources/corrupted-header.osu

@@ -0,0 +1,5 @@
+ow computerosu file format v14
+
+[Metadata]
+Title: Beatmap with corrupted header
+Creator: Evil Hacker

osu.Game.Tests/Resources/empty-line-instead-of-header.osu

@@ -0,0 +1,5 @@
+
+
+[Metadata]
+Title: The dog ate the file header
+Creator: Why does this keep happening

osu.Game.Tests/Resources/empty-lines-at-start.osu

@@ -0,0 +1,8 @@
+
+
+
+osu file format v14
+
+[Metadata]
+Title: Empty lines at start
+Creator: Edge Case Hunter

osu.Game.Tests/Resources/missing-header.osu

@@ -0,0 +1,4 @@
+[Metadata]
+
+Title: Beatmap with no header
+Creator: Incredibly Evil Hacker

osu.Game.Tests/Resources/no-empty-line-after-header.osu

@@ -0,0 +1,4 @@
+osu file format v14
+[Metadata]
+Title: No empty line delimiting header from contents
+Creator: Edge Case Hunter

osu.Game.Tests/osu.Game.Tests.csproj

@@ -1,5 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="..\osu.TestProject.props" />
+  <ItemGroup>
+    <None Remove="Resources\corrupted-header.osu" />
+    <None Remove="Resources\empty-line-instead-of-header.osu" />
+    <None Remove="Resources\empty-lines-at-start.osu" />
+    <None Remove="Resources\missing-header.osu" />
+    <None Remove="Resources\no-empty-line-after-header.osu" />
+  </ItemGroup>
   <ItemGroup Label="Package References">
     <PackageReference Include="Appveyor.TestLogger" Version="2.0.0" />
     <PackageReference Include="DeepEqual" Version="2.0.0" />

osu.Game/Beatmaps/Formats/Decoder.cs

@@ -28,6 +28,7 @@ namespace osu.Game.Beatmaps.Formats
     public abstract class Decoder
     {
         private static readonly Dictionary<Type, Dictionary<string, Func<string, Decoder>>> decoders = new Dictionary<Type, Dictionary<string, Func<string, Decoder>>>();
+        private static readonly Dictionary<Type, Func<Decoder>> fallback_decoders = new Dictionary<Type, Func<Decoder>>();
 
         static Decoder()
         {
@@ -49,21 +50,31 @@ namespace osu.Game.Beatmaps.Formats
             if (!decoders.TryGetValue(typeof(T), out var typedDecoders))
                 throw new IOException(@"Unknown decoder type");
 
-            string line;
+            // start off with the first line of the file
+            string line = stream.PeekLine()?.Trim();
 
-            do
+            while (line != null && line.Length == 0)
             {
-                line = stream.ReadLine()?.Trim();
-            } while (line != null && line.Length == 0);
+                // consume the previously peeked empty line and advance to the next one
+                stream.ReadLine();
+                line = stream.PeekLine()?.Trim();
+            }
 
             if (line == null)
-                throw new IOException(@"Unknown file format (null)");
+                throw new IOException("Unknown file format (null)");
 
             var decoder = typedDecoders.Select(d => line.StartsWith(d.Key, StringComparison.InvariantCulture) ? d.Value : null).FirstOrDefault();
-            if (decoder == null)
-                throw new IOException($@"Unknown file format ({line})");
 
-            return (Decoder<T>)decoder.Invoke(line);
+            // it's important the magic does NOT get consumed here, since sometimes it's part of the structure
+            // (see JsonBeatmapDecoder - the magic string is the opening brace)
+            // decoder implementations should therefore not die on receiving their own magic
+            if (decoder != null)
+                return (Decoder<T>)decoder.Invoke(line);
+
+            if (!fallback_decoders.TryGetValue(typeof(T), out var fallbackDecoder))
+                throw new IOException($"Unknown file format ({line})");
+
+            return (Decoder<T>)fallbackDecoder.Invoke();
         }
 
         /// <summary>
@@ -78,5 +89,19 @@ namespace osu.Game.Beatmaps.Formats
 
             typedDecoders[magic] = constructor;
         }
+
+        /// <summary>
+        /// Registers a fallback decoder instantiation function.
+        /// The fallback will be returned if the first line of the decoded stream does not match any known magic.
+        /// </summary>
+        /// <typeparam name="T">Type of object being decoded.</typeparam>
+        /// <param name="constructor">A function that constructs the fallback<see cref="Decoder"/>.</param>
+        protected static void SetFallbackDecoder<T>(Func<Decoder> constructor)
+        {
+            if (fallback_decoders.ContainsKey(typeof(T)))
+                throw new InvalidOperationException($"A fallback decoder was already added for type {typeof(T)}.");
+
+            fallback_decoders[typeof(T)] = constructor;
+        }
     }
 }

osu.Game/Beatmaps/Formats/LegacyBeatmapDecoder.cs

@@ -26,6 +26,7 @@ namespace osu.Game.Beatmaps.Formats
         public static void Register()
         {
             AddDecoder<Beatmap>(@"osu file format v", m => new LegacyBeatmapDecoder(Parsing.ParseInt(m.Split('v').Last())));
+            SetFallbackDecoder<Beatmap>(() => new LegacyBeatmapDecoder());
         }
 
         /// <summary>

osu.Game/Beatmaps/Formats/LegacyStoryboardDecoder.cs

@@ -34,6 +34,7 @@ namespace osu.Game.Beatmaps.Formats
             // note that this isn't completely correct
             AddDecoder<Storyboard>(@"osu file format v", m => new LegacyStoryboardDecoder());
             AddDecoder<Storyboard>(@"[Events]", m => new LegacyStoryboardDecoder());
+            SetFallbackDecoder<Storyboard>(() => new LegacyStoryboardDecoder());
         }
 
         protected override void ParseStreamInto(LineBufferedReader stream, Storyboard storyboard)

osu.Game/IO/LineBufferedReader.cs

@@ -25,6 +25,7 @@ namespace osu.Game.IO
 
         /// <summary>
         /// Reads the next line from the stream without consuming it.
+        /// Subsequent calls to <see cref="PeekLine"/> without a <see cref="ReadLine"/> will return the same string.
         /// </summary>
         public string PeekLine()
         {
@@ -39,6 +40,7 @@ namespace osu.Game.IO
 
         /// <summary>
         /// Reads the next line from the stream and consumes it.
+        /// If a line was peeked, that same line will then be consumed and returned.
         /// </summary>
         public string ReadLine() => lineBuffer.Count > 0 ? lineBuffer.Dequeue() : streamReader.ReadLine();