Merge pull request #10582 from peppy/spectator

Add spectator replay streaming support

osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecorder.cs

@@ -166,6 +166,12 @@ namespace osu.Game.Tests.Visual.Gameplay
             playbackManager?.ReplayInputHandler.SetFrameFromTime(Time.Current - 100);
         }
 
+        [TearDownSteps]
+        public void TearDown()
+        {
+            AddStep("stop recorder", () => recorder.Expire());
+        }
+
         public class TestFramedReplayInputHandler : FramedReplayInputHandler<TestReplayFrame>
         {
             public TestFramedReplayInputHandler(Replay replay)

osu.Game.Tests/Visual/Gameplay/TestSceneSpectatorPlayback.cs

@@ -0,0 +1,357 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Linq;
+using NUnit.Framework;
+using osu.Framework.Allocation;
+using osu.Framework.Bindables;
+using osu.Framework.Graphics;
+using osu.Framework.Graphics.Containers;
+using osu.Framework.Graphics.Shapes;
+using osu.Framework.Input.Bindings;
+using osu.Framework.Input.Events;
+using osu.Framework.Input.StateChanges;
+using osu.Framework.Logging;
+using osu.Framework.Testing;
+using osu.Framework.Timing;
+using osu.Game.Beatmaps;
+using osu.Game.Graphics.Sprites;
+using osu.Game.Online.API;
+using osu.Game.Online.Spectator;
+using osu.Game.Replays;
+using osu.Game.Replays.Legacy;
+using osu.Game.Rulesets;
+using osu.Game.Rulesets.Replays;
+using osu.Game.Rulesets.Replays.Types;
+using osu.Game.Rulesets.UI;
+using osu.Game.Tests.Visual.UserInterface;
+using osuTK;
+using osuTK.Graphics;
+
+namespace osu.Game.Tests.Visual.Gameplay
+{
+    public class TestSceneSpectatorPlayback : OsuManualInputManagerTestScene
+    {
+        protected override bool UseOnlineAPI => true;
+
+        private TestRulesetInputManager playbackManager;
+        private TestRulesetInputManager recordingManager;
+
+        private Replay replay;
+
+        private IBindableList<int> users;
+
+        private TestReplayRecorder recorder;
+
+        private readonly ManualClock manualClock = new ManualClock();
+
+        private OsuSpriteText latencyDisplay;
+
+        private TestFramedReplayInputHandler replayHandler;
+
+        [Resolved]
+        private IAPIProvider api { get; set; }
+
+        [Resolved]
+        private SpectatorStreamingClient streamingClient { get; set; }
+
+        [SetUp]
+        public void SetUp() => Schedule(() =>
+        {
+            replay = new Replay();
+
+            users = streamingClient.PlayingUsers.GetBoundCopy();
+            users.BindCollectionChanged((obj, args) =>
+            {
+                switch (args.Action)
+                {
+                    case NotifyCollectionChangedAction.Add:
+                        foreach (int user in args.NewItems)
+                        {
+                            if (user == api.LocalUser.Value.Id)
+                                streamingClient.WatchUser(user);
+                        }
+
+                        break;
+
+                    case NotifyCollectionChangedAction.Remove:
+                        foreach (int user in args.OldItems)
+                        {
+                            if (user == api.LocalUser.Value.Id)
+                                streamingClient.StopWatchingUser(user);
+                        }
+
+                        break;
+                }
+            }, true);
+
+            streamingClient.OnNewFrames += onNewFrames;
+
+            Add(new GridContainer
+            {
+                RelativeSizeAxes = Axes.Both,
+                Content = new[]
+                {
+                    new Drawable[]
+                    {
+                        recordingManager = new TestRulesetInputManager(new TestSceneModSettings.TestRulesetInfo(), 0, SimultaneousBindingMode.Unique)
+                        {
+                            Recorder = recorder = new TestReplayRecorder
+                            {
+                                ScreenSpaceToGamefield = pos => recordingManager.ToLocalSpace(pos),
+                            },
+                            Child = new Container
+                            {
+                                RelativeSizeAxes = Axes.Both,
+                                Children = new Drawable[]
+                                {
+                                    new Box
+                                    {
+                                        Colour = Color4.Brown,
+                                        RelativeSizeAxes = Axes.Both,
+                                    },
+                                    new OsuSpriteText
+                                    {
+                                        Text = "Sending",
+                                        Scale = new Vector2(3),
+                                        Anchor = Anchor.Centre,
+                                        Origin = Anchor.Centre,
+                                    },
+                                    new TestInputConsumer()
+                                }
+                            },
+                        }
+                    },
+                    new Drawable[]
+                    {
+                        playbackManager = new TestRulesetInputManager(new TestSceneModSettings.TestRulesetInfo(), 0, SimultaneousBindingMode.Unique)
+                        {
+                            Clock = new FramedClock(manualClock),
+                            ReplayInputHandler = replayHandler = new TestFramedReplayInputHandler(replay)
+                            {
+                                GamefieldToScreenSpace = pos => playbackManager.ToScreenSpace(pos),
+                            },
+                            Child = new Container
+                            {
+                                RelativeSizeAxes = Axes.Both,
+                                Children = new Drawable[]
+                                {
+                                    new Box
+                                    {
+                                        Colour = Color4.DarkBlue,
+                                        RelativeSizeAxes = Axes.Both,
+                                    },
+                                    new OsuSpriteText
+                                    {
+                                        Text = "Receiving",
+                                        Scale = new Vector2(3),
+                                        Anchor = Anchor.Centre,
+                                        Origin = Anchor.Centre,
+                                    },
+                                    new TestInputConsumer()
+                                }
+                            },
+                        }
+                    }
+                }
+            });
+
+            Add(latencyDisplay = new OsuSpriteText());
+        });
+
+        private void onNewFrames(int userId, FrameDataBundle frames)
+        {
+            Logger.Log($"Received {frames.Frames.Count()} new frames ({string.Join(',', frames.Frames.Select(f => ((int)f.Time).ToString()))})");
+
+            foreach (var legacyFrame in frames.Frames)
+            {
+                var frame = new TestReplayFrame();
+                frame.FromLegacy(legacyFrame, null, null);
+                replay.Frames.Add(frame);
+            }
+        }
+
+        [Test]
+        public void TestBasic()
+        {
+        }
+
+        private double latency = SpectatorStreamingClient.TIME_BETWEEN_SENDS;
+
+        protected override void Update()
+        {
+            base.Update();
+
+            if (latencyDisplay == null) return;
+
+            // propagate initial time value
+            if (manualClock.CurrentTime == 0)
+            {
+                manualClock.CurrentTime = Time.Current;
+                return;
+            }
+
+            if (replayHandler.NextFrame != null)
+            {
+                var lastFrame = replay.Frames.LastOrDefault();
+
+                // this isn't perfect as we basically can't be aware of the rate-of-send here (the streamer is not sending data when not being moved).
+                // in gameplay playback, the case where NextFrame is null would pause gameplay and handle this correctly; it's strictly a test limitation / best effort implementation.
+                if (lastFrame != null)
+                    latency = Math.Max(latency, Time.Current - lastFrame.Time);
+
+                latencyDisplay.Text = $"latency: {latency:N1}";
+
+                double proposedTime = Time.Current - latency + Time.Elapsed;
+
+                // this will either advance by one or zero frames.
+                double? time = replayHandler.SetFrameFromTime(proposedTime);
+
+                if (time == null)
+                    return;
+
+                manualClock.CurrentTime = time.Value;
+            }
+        }
+
+        [TearDownSteps]
+        public void TearDown()
+        {
+            AddStep("stop recorder", () =>
+            {
+                recorder.Expire();
+                streamingClient.OnNewFrames -= onNewFrames;
+            });
+        }
+
+        public class TestFramedReplayInputHandler : FramedReplayInputHandler<TestReplayFrame>
+        {
+            public TestFramedReplayInputHandler(Replay replay)
+                : base(replay)
+            {
+            }
+
+            public override void CollectPendingInputs(List<IInput> inputs)
+            {
+                inputs.Add(new MousePositionAbsoluteInput { Position = GamefieldToScreenSpace(CurrentFrame?.Position ?? Vector2.Zero) });
+                inputs.Add(new ReplayState<TestAction> { PressedActions = CurrentFrame?.Actions ?? new List<TestAction>() });
+            }
+        }
+
+        public class TestInputConsumer : CompositeDrawable, IKeyBindingHandler<TestAction>
+        {
+            public override bool ReceivePositionalInputAt(Vector2 screenSpacePos) => Parent.ReceivePositionalInputAt(screenSpacePos);
+
+            private readonly Box box;
+
+            public TestInputConsumer()
+            {
+                Size = new Vector2(30);
+
+                Origin = Anchor.Centre;
+
+                InternalChildren = new Drawable[]
+                {
+                    box = new Box
+                    {
+                        Colour = Color4.Black,
+                        RelativeSizeAxes = Axes.Both,
+                    },
+                };
+            }
+
+            protected override bool OnMouseMove(MouseMoveEvent e)
+            {
+                Position = e.MousePosition;
+                return base.OnMouseMove(e);
+            }
+
+            public bool OnPressed(TestAction action)
+            {
+                box.Colour = Color4.White;
+                return true;
+            }
+
+            public void OnReleased(TestAction action)
+            {
+                box.Colour = Color4.Black;
+            }
+        }
+
+        public class TestRulesetInputManager : RulesetInputManager<TestAction>
+        {
+            public TestRulesetInputManager(RulesetInfo ruleset, int variant, SimultaneousBindingMode unique)
+                : base(ruleset, variant, unique)
+            {
+            }
+
+            protected override KeyBindingContainer<TestAction> CreateKeyBindingContainer(RulesetInfo ruleset, int variant, SimultaneousBindingMode unique)
+                => new TestKeyBindingContainer();
+
+            internal class TestKeyBindingContainer : KeyBindingContainer<TestAction>
+            {
+                public override IEnumerable<KeyBinding> DefaultKeyBindings => new[]
+                {
+                    new KeyBinding(InputKey.MouseLeft, TestAction.Down),
+                };
+            }
+        }
+
+        public class TestReplayFrame : ReplayFrame, IConvertibleReplayFrame
+        {
+            public Vector2 Position;
+
+            public List<TestAction> Actions = new List<TestAction>();
+
+            public TestReplayFrame(double time, Vector2 position, params TestAction[] actions)
+                : base(time)
+            {
+                Position = position;
+                Actions.AddRange(actions);
+            }
+
+            public TestReplayFrame()
+            {
+            }
+
+            public void FromLegacy(LegacyReplayFrame currentFrame, IBeatmap beatmap, ReplayFrame lastFrame = null)
+            {
+                Position = currentFrame.Position;
+                Time = currentFrame.Time;
+                if (currentFrame.MouseLeft)
+                    Actions.Add(TestAction.Down);
+            }
+
+            public LegacyReplayFrame ToLegacy(IBeatmap beatmap)
+            {
+                ReplayButtonState state = ReplayButtonState.None;
+
+                if (Actions.Contains(TestAction.Down))
+                    state |= ReplayButtonState.Left1;
+
+                return new LegacyReplayFrame(Time, Position.X, Position.Y, state);
+            }
+        }
+
+        public enum TestAction
+        {
+            Down,
+        }
+
+        internal class TestReplayRecorder : ReplayRecorder<TestAction>
+        {
+            public TestReplayRecorder()
+                : base(new Replay())
+            {
+            }
+
+            protected override ReplayFrame HandleFrame(Vector2 mousePosition, List<TestAction> actions, ReplayFrame previousFrame)
+            {
+                return new TestReplayFrame(Time.Current, mousePosition, actions.ToArray());
+            }
+        }
+    }
+}

osu.Game/Online/API/APIMod.cs

@@ -53,5 +53,13 @@ namespace osu.Game.Online.API
         }
 
         public bool Equals(IMod other) => Acronym == other?.Acronym;
+
+        public override string ToString()
+        {
+            if (Settings.Count > 0)
+                return $"{Acronym} ({string.Join(',', Settings.Select(kvp => $"{kvp.Key}:{kvp.Value}"))})";
+
+            return $"{Acronym}";
+        }
     }
 }

osu.Game/Online/API/DummyAPIAccess.cs

@@ -20,6 +20,8 @@ namespace osu.Game.Online.API
 
         public Bindable<UserActivity> Activity { get; } = new Bindable<UserActivity>();
 
+        public string AccessToken => "token";
+
         public bool IsLoggedIn => State.Value == APIState.Online;
 
         public string ProvidedUsername => LocalUser.Value.Username;

osu.Game/Online/API/IAPIProvider.cs

@@ -21,6 +21,11 @@ namespace osu.Game.Online.API
         /// </summary>
         Bindable<UserActivity> Activity { get; }
 
+        /// <summary>
+        /// Retrieve the OAuth access token.
+        /// </summary>
+        string AccessToken { get; }
+
         /// <summary>
         /// Returns whether the local user is logged in.
         /// </summary>

osu.Game/Online/Spectator/FrameDataBundle.cs

@@ -0,0 +1,20 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.Collections.Generic;
+using osu.Game.Replays.Legacy;
+
+namespace osu.Game.Online.Spectator
+{
+    [Serializable]
+    public class FrameDataBundle
+    {
+        public IEnumerable<LegacyReplayFrame> Frames { get; set; }
+
+        public FrameDataBundle(IEnumerable<LegacyReplayFrame> frames)
+        {
+            Frames = frames;
+        }
+    }
+}

osu.Game/Online/Spectator/ISpectatorClient.cs

@@ -0,0 +1,34 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Threading.Tasks;
+
+namespace osu.Game.Online.Spectator
+{
+    /// <summary>
+    /// An interface defining a spectator client instance.
+    /// </summary>
+    public interface ISpectatorClient
+    {
+        /// <summary>
+        /// Signals that a user has begun a new play session.
+        /// </summary>
+        /// <param name="userId">The user.</param>
+        /// <param name="state">The state of gameplay.</param>
+        Task UserBeganPlaying(int userId, SpectatorState state);
+
+        /// <summary>
+        /// Signals that a user has finished a play session.
+        /// </summary>
+        /// <param name="userId">The user.</param>
+        /// <param name="state">The state of gameplay.</param>
+        Task UserFinishedPlaying(int userId, SpectatorState state);
+
+        /// <summary>
+        /// Called when new frames are available for a subscribed user's play session.
+        /// </summary>
+        /// <param name="userId">The user.</param>
+        /// <param name="data">The frame data.</param>
+        Task UserSentFrames(int userId, FrameDataBundle data);
+    }
+}

osu.Game/Online/Spectator/ISpectatorServer.cs

@@ -0,0 +1,44 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System.Threading.Tasks;
+
+namespace osu.Game.Online.Spectator
+{
+    /// <summary>
+    /// An interface defining the spectator server instance.
+    /// </summary>
+    public interface ISpectatorServer
+    {
+        /// <summary>
+        /// Signal the start of a new play session.
+        /// </summary>
+        /// <param name="state">The state of gameplay.</param>
+        Task BeginPlaySession(SpectatorState state);
+
+        /// <summary>
+        /// Send a bundle of frame data for the current play session.
+        /// </summary>
+        /// <param name="data">The frame data.</param>
+        Task SendFrameData(FrameDataBundle data);
+
+        /// <summary>
+        /// Signal the end of a play session.
+        /// </summary>
+        /// <param name="state">The state of gameplay.</param>
+        Task EndPlaySession(SpectatorState state);
+
+        /// <summary>
+        /// Request spectating data for the specified user. May be called on multiple users and offline users.
+        /// For offline users, a subscription will be created and data will begin streaming on next play.
+        /// </summary>
+        /// <param name="userId">The user to subscribe to.</param>
+        Task StartWatchingUser(int userId);
+
+        /// <summary>
+        /// Stop requesting spectating data for the specified user. Unsubscribes from receiving further data.
+        /// </summary>
+        /// <param name="userId">The user to unsubscribe from.</param>
+        Task EndWatchingUser(int userId);
+    }
+}

osu.Game/Online/Spectator/SpectatorState.cs

@@ -0,0 +1,26 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using osu.Game.Online.API;
+
+namespace osu.Game.Online.Spectator
+{
+    [Serializable]
+    public class SpectatorState : IEquatable<SpectatorState>
+    {
+        public int? BeatmapID { get; set; }
+
+        public int? RulesetID { get; set; }
+
+        [NotNull]
+        public IEnumerable<APIMod> Mods { get; set; } = Enumerable.Empty<APIMod>();
+
+        public bool Equals(SpectatorState other) => BeatmapID == other?.BeatmapID && Mods.SequenceEqual(other?.Mods) && RulesetID == other?.RulesetID;
+
+        public override string ToString() => $"Beatmap:{BeatmapID} Mods:{string.Join(',', Mods)} Ruleset:{RulesetID}";
+    }
+}

osu.Game/Online/Spectator/SpectatorStreamingClient.cs

@@ -0,0 +1,270 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.SignalR.Client;
+using Microsoft.Extensions.DependencyInjection;
+using Newtonsoft.Json;
+using osu.Framework.Allocation;
+using osu.Framework.Bindables;
+using osu.Framework.Graphics;
+using osu.Game.Beatmaps;
+using osu.Game.Online.API;
+using osu.Game.Replays.Legacy;
+using osu.Game.Rulesets;
+using osu.Game.Rulesets.Mods;
+using osu.Game.Rulesets.Replays;
+using osu.Game.Rulesets.Replays.Types;
+
+namespace osu.Game.Online.Spectator
+{
+    public class SpectatorStreamingClient : Component, ISpectatorClient
+    {
+        /// <summary>
+        /// The maximum milliseconds between frame bundle sends.
+        /// </summary>
+        public const double TIME_BETWEEN_SENDS = 200;
+
+        private HubConnection connection;
+
+        private readonly List<int> watchingUsers = new List<int>();
+
+        public IBindableList<int> PlayingUsers => playingUsers;
+
+        private readonly BindableList<int> playingUsers = new BindableList<int>();
+
+        private readonly IBindable<APIState> apiState = new Bindable<APIState>();
+
+        private bool isConnected;
+
+        [Resolved]
+        private IAPIProvider api { get; set; }
+
+        [Resolved]
+        private IBindable<WorkingBeatmap> beatmap { get; set; }
+
+        [Resolved]
+        private IBindable<RulesetInfo> ruleset { get; set; }
+
+        [Resolved]
+        private IBindable<IReadOnlyList<Mod>> mods { get; set; }
+
+        private readonly SpectatorState currentState = new SpectatorState();
+
+        private bool isPlaying;
+
+        /// <summary>
+        /// Called whenever new frames arrive from the server.
+        /// </summary>
+        public event Action<int, FrameDataBundle> OnNewFrames;
+
+        [BackgroundDependencyLoader]
+        private void load()
+        {
+            apiState.BindTo(api.State);
+            apiState.BindValueChanged(apiStateChanged, true);
+        }
+
+        private void apiStateChanged(ValueChangedEvent<APIState> state)
+        {
+            switch (state.NewValue)
+            {
+                case APIState.Failing:
+                case APIState.Offline:
+                    connection?.StopAsync();
+                    connection = null;
+                    break;
+
+                case APIState.Online:
+                    Task.Run(connect);
+                    break;
+            }
+        }
+
+        private const string endpoint = "https://spectator.ppy.sh/spectator";
+
+        private async Task connect()
+        {
+            if (connection != null)
+                return;
+
+            connection = new HubConnectionBuilder()
+                         .WithUrl(endpoint, options =>
+                         {
+                             options.Headers.Add("Authorization", $"Bearer {api.AccessToken}");
+                         })
+                         .AddNewtonsoftJsonProtocol(options => { options.PayloadSerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; })
+                         .Build();
+
+            // until strong typed client support is added, each method must be manually bound (see https://github.com/dotnet/aspnetcore/issues/15198)
+            connection.On<int, SpectatorState>(nameof(ISpectatorClient.UserBeganPlaying), ((ISpectatorClient)this).UserBeganPlaying);
+            connection.On<int, FrameDataBundle>(nameof(ISpectatorClient.UserSentFrames), ((ISpectatorClient)this).UserSentFrames);
+            connection.On<int, SpectatorState>(nameof(ISpectatorClient.UserFinishedPlaying), ((ISpectatorClient)this).UserFinishedPlaying);
+
+            connection.Closed += async ex =>
+            {
+                isConnected = false;
+                playingUsers.Clear();
+
+                if (ex != null) await tryUntilConnected();
+            };
+
+            await tryUntilConnected();
+
+            async Task tryUntilConnected()
+            {
+                while (api.State.Value == APIState.Online)
+                {
+                    try
+                    {
+                        // reconnect on any failure
+                        await connection.StartAsync();
+
+                        // success
+                        isConnected = true;
+
+                        // resubscribe to watched users
+                        var users = watchingUsers.ToArray();
+                        watchingUsers.Clear();
+                        foreach (var userId in users)
+                            WatchUser(userId);
+
+                        // re-send state in case it wasn't received
+                        if (isPlaying)
+                            beginPlaying();
+
+                        break;
+                    }
+                    catch
+                    {
+                        await Task.Delay(5000);
+                    }
+                }
+            }
+        }
+
+        Task ISpectatorClient.UserBeganPlaying(int userId, SpectatorState state)
+        {
+            if (!playingUsers.Contains(userId))
+                playingUsers.Add(userId);
+
+            return Task.CompletedTask;
+        }
+
+        Task ISpectatorClient.UserFinishedPlaying(int userId, SpectatorState state)
+        {
+            playingUsers.Remove(userId);
+            return Task.CompletedTask;
+        }
+
+        Task ISpectatorClient.UserSentFrames(int userId, FrameDataBundle data)
+        {
+            OnNewFrames?.Invoke(userId, data);
+            return Task.CompletedTask;
+        }
+
+        public void BeginPlaying()
+        {
+            if (isPlaying)
+                throw new InvalidOperationException($"Cannot invoke {nameof(BeginPlaying)} when already playing");
+
+            isPlaying = true;
+
+            // transfer state at point of beginning play
+            currentState.BeatmapID = beatmap.Value.BeatmapInfo.OnlineBeatmapID;
+            currentState.RulesetID = ruleset.Value.ID;
+            currentState.Mods = mods.Value.Select(m => new APIMod(m));
+
+            beginPlaying();
+        }
+
+        private void beginPlaying()
+        {
+            Debug.Assert(isPlaying);
+
+            if (!isConnected) return;
+
+            connection.SendAsync(nameof(ISpectatorServer.BeginPlaySession), currentState);
+        }
+
+        public void SendFrames(FrameDataBundle data)
+        {
+            if (!isConnected) return;
+
+            lastSend = connection.SendAsync(nameof(ISpectatorServer.SendFrameData), data);
+        }
+
+        public void EndPlaying()
+        {
+            isPlaying = false;
+
+            if (!isConnected) return;
+
+            connection.SendAsync(nameof(ISpectatorServer.EndPlaySession), currentState);
+        }
+
+        public void WatchUser(int userId)
+        {
+            if (watchingUsers.Contains(userId))
+                return;
+
+            watchingUsers.Add(userId);
+
+            if (!isConnected) return;
+
+            connection.SendAsync(nameof(ISpectatorServer.StartWatchingUser), userId);
+        }
+
+        public void StopWatchingUser(int userId)
+        {
+            watchingUsers.Remove(userId);
+
+            if (!isConnected) return;
+
+            connection.SendAsync(nameof(ISpectatorServer.EndWatchingUser), userId);
+        }
+
+        private readonly Queue<LegacyReplayFrame> pendingFrames = new Queue<LegacyReplayFrame>();
+
+        private double lastSendTime;
+
+        private Task lastSend;
+
+        private const int max_pending_frames = 30;
+
+        protected override void Update()
+        {
+            base.Update();
+
+            if (pendingFrames.Count > 0 && Time.Current - lastSendTime > TIME_BETWEEN_SENDS)
+                purgePendingFrames();
+        }
+
+        public void HandleFrame(ReplayFrame frame)
+        {
+            if (frame is IConvertibleReplayFrame convertible)
+                pendingFrames.Enqueue(convertible.ToLegacy(beatmap.Value.Beatmap));
+
+            if (pendingFrames.Count > max_pending_frames)
+                purgePendingFrames();
+        }
+
+        private void purgePendingFrames()
+        {
+            if (lastSend?.IsCompleted == false)
+                return;
+
+            var frames = pendingFrames.ToArray();
+
+            pendingFrames.Clear();
+
+            SendFrames(new FrameDataBundle(frames));
+
+            lastSendTime = Time.Current;
+        }
+    }
+}

osu.Game/OsuGameBase.cs

@@ -30,6 +30,7 @@ using osu.Game.Database;
 using osu.Game.Input;
 using osu.Game.Input.Bindings;
 using osu.Game.IO;
+using osu.Game.Online.Spectator;
 using osu.Game.Overlays;
 using osu.Game.Resources;
 using osu.Game.Rulesets;
@@ -74,6 +75,8 @@ namespace osu.Game
 
         protected IAPIProvider API;
 
+        private SpectatorStreamingClient spectatorStreaming;
+
         protected MenuCursorContainer MenuCursorContainer;
 
         protected MusicController MusicController;
@@ -189,9 +192,9 @@ namespace osu.Game
             dependencies.Cache(SkinManager = new SkinManager(Storage, contextFactory, Host, Audio, new NamespacedResourceStore<byte[]>(Resources, "Skins/Legacy")));
             dependencies.CacheAs<ISkinSource>(SkinManager);
 
-            API ??= new APIAccess(LocalConfig);
+            dependencies.CacheAs(API ??= new APIAccess(LocalConfig));
 
-            dependencies.CacheAs(API);
+            dependencies.CacheAs(spectatorStreaming = new SpectatorStreamingClient());
 
             var defaultBeatmap = new DummyWorkingBeatmap(Audio, Textures);
 
@@ -247,8 +250,11 @@ namespace osu.Game
 
             FileStore.Cleanup();
 
+            // add api components to hierarchy.
             if (API is APIAccess apiAccess)
                 AddInternal(apiAccess);
+            AddInternal(spectatorStreaming);
+
             AddInternal(RulesetConfigCache);
 
             MenuCursorContainer = new MenuCursorContainer { RelativeSizeAxes = Axes.Both };

osu.Game/Replays/Legacy/LegacyReplayFrame.cs

@@ -1,6 +1,7 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+using Newtonsoft.Json;
 using osu.Game.Rulesets.Replays;
 using osuTK;
 
@@ -8,17 +9,28 @@ namespace osu.Game.Replays.Legacy
 {
     public class LegacyReplayFrame : ReplayFrame
     {
+        [JsonIgnore]
         public Vector2 Position => new Vector2(MouseX ?? 0, MouseY ?? 0);
 
         public float? MouseX;
         public float? MouseY;
 
+        [JsonIgnore]
         public bool MouseLeft => MouseLeft1 || MouseLeft2;
+
+        [JsonIgnore]
         public bool MouseRight => MouseRight1 || MouseRight2;
 
+        [JsonIgnore]
         public bool MouseLeft1 => ButtonState.HasFlag(ReplayButtonState.Left1);
+
+        [JsonIgnore]
         public bool MouseRight1 => ButtonState.HasFlag(ReplayButtonState.Right1);
+
+        [JsonIgnore]
         public bool MouseLeft2 => ButtonState.HasFlag(ReplayButtonState.Left2);
+
+        [JsonIgnore]
         public bool MouseRight2 => ButtonState.HasFlag(ReplayButtonState.Right2);
 
         public ReplayButtonState ButtonState;

osu.Game/Rulesets/UI/ReplayRecorder.cs

@@ -4,10 +4,14 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using osu.Framework.Allocation;
+using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Input;
 using osu.Framework.Input.Bindings;
 using osu.Framework.Input.Events;
+using osu.Game.Beatmaps;
+using osu.Game.Online.Spectator;
 using osu.Game.Replays;
 using osu.Game.Rulesets.Replays;
 using osuTK;
@@ -25,6 +29,12 @@ namespace osu.Game.Rulesets.UI
 
         public int RecordFrameRate = 60;
 
+        [Resolved(canBeNull: true)]
+        private SpectatorStreamingClient spectatorStreaming { get; set; }
+
+        [Resolved]
+        private IBindable<WorkingBeatmap> beatmap { get; set; }
+
         protected ReplayRecorder(Replay target)
         {
             this.target = target;
@@ -39,6 +49,14 @@ namespace osu.Game.Rulesets.UI
             base.LoadComplete();
 
             inputManager = GetContainingInputManager();
+
+            spectatorStreaming?.BeginPlaying();
+        }
+
+        protected override void Dispose(bool isDisposing)
+        {
+            base.Dispose(isDisposing);
+            spectatorStreaming?.EndPlaying();
         }
 
         protected override bool OnMouseMove(MouseMoveEvent e)
@@ -72,7 +90,11 @@ namespace osu.Game.Rulesets.UI
             var frame = HandleFrame(position, pressedActions, last);
 
             if (frame != null)
+            {
                 target.Frames.Add(frame);
+
+                spectatorStreaming?.HandleFrame(frame);
+            }
         }
 
         protected abstract ReplayFrame HandleFrame(Vector2 mousePosition, List<T> actions, ReplayFrame previousFrame);

osu.Game/Screens/Play/Player.cs

@@ -152,7 +152,9 @@ namespace osu.Game.Screens.Play
         {
             base.LoadComplete();
 
-            PrepareReplay();
+            // replays should never be recorded or played back when autoplay is enabled
+            if (!Mods.Value.Any(m => m is ModAutoplay))
+                PrepareReplay();
         }
 
         private Replay recordingReplay;

osu.Game/osu.Game.csproj

@@ -21,6 +21,8 @@
     <PackageReference Include="Dapper" Version="2.0.35" />
     <PackageReference Include="DiffPlex" Version="1.6.3" />
     <PackageReference Include="Humanizer" Version="2.8.26" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="3.1.9" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="3.1.9" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="2.2.6" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite.Core" Version="2.2.6" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />