diff --git a/osu.Game.Rulesets.Catch/UI/CatchReplayRecorder.cs b/osu.Game.Rulesets.Catch/UI/CatchReplayRecorder.cs
index 9a4d1f9585..1ddb5ac630 100644
--- a/osu.Game.Rulesets.Catch/UI/CatchReplayRecorder.cs
+++ b/osu.Game.Rulesets.Catch/UI/CatchReplayRecorder.cs
@@ -2,10 +2,10 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
-using osu.Game.Replays;
 using osu.Game.Rulesets.Catch.Replays;
 using osu.Game.Rulesets.Replays;
 using osu.Game.Rulesets.UI;
+using osu.Game.Scoring;
 using osuTK;
 
 namespace osu.Game.Rulesets.Catch.UI
@@ -14,7 +14,7 @@ namespace osu.Game.Rulesets.Catch.UI
     {
         private readonly CatchPlayfield playfield;
 
-        public CatchReplayRecorder(Replay target, CatchPlayfield playfield)
+        public CatchReplayRecorder(Score target, CatchPlayfield playfield)
             : base(target)
         {
             this.playfield = playfield;
diff --git a/osu.Game.Rulesets.Catch/UI/DrawableCatchRuleset.cs b/osu.Game.Rulesets.Catch/UI/DrawableCatchRuleset.cs
index 46733181e3..9389fa803b 100644
--- a/osu.Game.Rulesets.Catch/UI/DrawableCatchRuleset.cs
+++ b/osu.Game.Rulesets.Catch/UI/DrawableCatchRuleset.cs
@@ -13,6 +13,7 @@ using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Objects.Drawables;
 using osu.Game.Rulesets.UI;
 using osu.Game.Rulesets.UI.Scrolling;
+using osu.Game.Scoring;
 
 namespace osu.Game.Rulesets.Catch.UI
 {
@@ -31,7 +32,7 @@ namespace osu.Game.Rulesets.Catch.UI
 
         protected override ReplayInputHandler CreateReplayInputHandler(Replay replay) => new CatchFramedReplayInputHandler(replay);
 
-        protected override ReplayRecorder CreateReplayRecorder(Replay replay) => new CatchReplayRecorder(replay, (CatchPlayfield)Playfield);
+        protected override ReplayRecorder CreateReplayRecorder(Score score) => new CatchReplayRecorder(score, (CatchPlayfield)Playfield);
 
         protected override Playfield CreatePlayfield() => new CatchPlayfield(Beatmap.BeatmapInfo.BaseDifficulty, CreateDrawableRepresentation);
 
diff --git a/osu.Game.Rulesets.Mania/UI/DrawableManiaRuleset.cs b/osu.Game.Rulesets.Mania/UI/DrawableManiaRuleset.cs
index 7f5b9a6ee0..941ac9816c 100644
--- a/osu.Game.Rulesets.Mania/UI/DrawableManiaRuleset.cs
+++ b/osu.Game.Rulesets.Mania/UI/DrawableManiaRuleset.cs
@@ -23,6 +23,7 @@ using osu.Game.Rulesets.Objects;
 using osu.Game.Rulesets.Objects.Drawables;
 using osu.Game.Rulesets.UI;
 using osu.Game.Rulesets.UI.Scrolling;
+using osu.Game.Scoring;
 
 namespace osu.Game.Rulesets.Mania.UI
 {
@@ -132,6 +133,6 @@ namespace osu.Game.Rulesets.Mania.UI
 
         protected override ReplayInputHandler CreateReplayInputHandler(Replay replay) => new ManiaFramedReplayInputHandler(replay);
 
-        protected override ReplayRecorder CreateReplayRecorder(Replay replay) => new ManiaReplayRecorder(replay);
+        protected override ReplayRecorder CreateReplayRecorder(Score score) => new ManiaReplayRecorder(score);
     }
 }
diff --git a/osu.Game.Rulesets.Mania/UI/ManiaReplayRecorder.cs b/osu.Game.Rulesets.Mania/UI/ManiaReplayRecorder.cs
index 18275000a2..b502d1f9e5 100644
--- a/osu.Game.Rulesets.Mania/UI/ManiaReplayRecorder.cs
+++ b/osu.Game.Rulesets.Mania/UI/ManiaReplayRecorder.cs
@@ -2,18 +2,18 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
-using osu.Game.Replays;
 using osu.Game.Rulesets.Mania.Replays;
 using osu.Game.Rulesets.Replays;
 using osu.Game.Rulesets.UI;
+using osu.Game.Scoring;
 using osuTK;
 
 namespace osu.Game.Rulesets.Mania.UI
 {
     public class ManiaReplayRecorder : ReplayRecorder<ManiaAction>
     {
-        public ManiaReplayRecorder(Replay replay)
-            : base(replay)
+        public ManiaReplayRecorder(Score score)
+            : base(score)
         {
         }
 
diff --git a/osu.Game.Rulesets.Osu/UI/DrawableOsuRuleset.cs b/osu.Game.Rulesets.Osu/UI/DrawableOsuRuleset.cs
index 69179137a6..df3f7c64e4 100644
--- a/osu.Game.Rulesets.Osu/UI/DrawableOsuRuleset.cs
+++ b/osu.Game.Rulesets.Osu/UI/DrawableOsuRuleset.cs
@@ -14,6 +14,7 @@ using osu.Game.Rulesets.Osu.Configuration;
 using osu.Game.Rulesets.Osu.Objects;
 using osu.Game.Rulesets.Osu.Replays;
 using osu.Game.Rulesets.UI;
+using osu.Game.Scoring;
 using osu.Game.Screens.Play;
 using osuTK;
 
@@ -44,7 +45,7 @@ namespace osu.Game.Rulesets.Osu.UI
 
         protected override ReplayInputHandler CreateReplayInputHandler(Replay replay) => new OsuFramedReplayInputHandler(replay);
 
-        protected override ReplayRecorder CreateReplayRecorder(Replay replay) => new OsuReplayRecorder(replay);
+        protected override ReplayRecorder CreateReplayRecorder(Score score) => new OsuReplayRecorder(score);
 
         public override double GameplayStartTime
         {
diff --git a/osu.Game.Rulesets.Osu/UI/OsuReplayRecorder.cs b/osu.Game.Rulesets.Osu/UI/OsuReplayRecorder.cs
index b68ea136d5..1304dfe416 100644
--- a/osu.Game.Rulesets.Osu/UI/OsuReplayRecorder.cs
+++ b/osu.Game.Rulesets.Osu/UI/OsuReplayRecorder.cs
@@ -2,18 +2,18 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
-using osu.Game.Replays;
 using osu.Game.Rulesets.Osu.Replays;
 using osu.Game.Rulesets.Replays;
 using osu.Game.Rulesets.UI;
+using osu.Game.Scoring;
 using osuTK;
 
 namespace osu.Game.Rulesets.Osu.UI
 {
     public class OsuReplayRecorder : ReplayRecorder<OsuAction>
     {
-        public OsuReplayRecorder(Replay replay)
-            : base(replay)
+        public OsuReplayRecorder(Score score)
+            : base(score)
         {
         }
 
diff --git a/osu.Game.Rulesets.Taiko/UI/DrawableTaikoRuleset.cs b/osu.Game.Rulesets.Taiko/UI/DrawableTaikoRuleset.cs
index bbf8cb8de0..9cf931ee0a 100644
--- a/osu.Game.Rulesets.Taiko/UI/DrawableTaikoRuleset.cs
+++ b/osu.Game.Rulesets.Taiko/UI/DrawableTaikoRuleset.cs
@@ -17,6 +17,7 @@ using osu.Game.Replays;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Objects;
 using osu.Game.Rulesets.UI.Scrolling;
+using osu.Game.Scoring;
 using osu.Game.Skinning;
 using osuTK;
 
@@ -82,6 +83,6 @@ namespace osu.Game.Rulesets.Taiko.UI
 
         protected override ReplayInputHandler CreateReplayInputHandler(Replay replay) => new TaikoFramedReplayInputHandler(replay);
 
-        protected override ReplayRecorder CreateReplayRecorder(Replay replay) => new TaikoReplayRecorder(replay);
+        protected override ReplayRecorder CreateReplayRecorder(Score score) => new TaikoReplayRecorder(score);
     }
 }
diff --git a/osu.Game.Rulesets.Taiko/UI/TaikoReplayRecorder.cs b/osu.Game.Rulesets.Taiko/UI/TaikoReplayRecorder.cs
index 1859dabf03..e6391d1386 100644
--- a/osu.Game.Rulesets.Taiko/UI/TaikoReplayRecorder.cs
+++ b/osu.Game.Rulesets.Taiko/UI/TaikoReplayRecorder.cs
@@ -2,18 +2,18 @@
 // See the LICENCE file in the repository root for full licence text.
 
 using System.Collections.Generic;
-using osu.Game.Replays;
 using osu.Game.Rulesets.Replays;
 using osu.Game.Rulesets.Taiko.Replays;
 using osu.Game.Rulesets.UI;
+using osu.Game.Scoring;
 using osuTK;
 
 namespace osu.Game.Rulesets.Taiko.UI
 {
     public class TaikoReplayRecorder : ReplayRecorder<TaikoAction>
     {
-        public TaikoReplayRecorder(Replay replay)
-            : base(replay)
+        public TaikoReplayRecorder(Score score)
+            : base(score)
         {
         }
 
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecorder.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecorder.cs
index b72960931f..b2ad7ca5b4 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecorder.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecorder.cs
@@ -19,6 +19,7 @@ using osu.Game.Replays;
 using osu.Game.Rulesets;
 using osu.Game.Rulesets.Replays;
 using osu.Game.Rulesets.UI;
+using osu.Game.Scoring;
 using osu.Game.Screens.Play;
 using osu.Game.Tests.Visual.UserInterface;
 using osuTK;
@@ -53,7 +54,7 @@ namespace osu.Game.Tests.Visual.Gameplay
                     {
                         recordingManager = new TestRulesetInputManager(new TestSceneModSettings.TestRulesetInfo(), 0, SimultaneousBindingMode.Unique)
                         {
-                            Recorder = recorder = new TestReplayRecorder(replay)
+                            Recorder = recorder = new TestReplayRecorder(new Score { Replay = replay })
                             {
                                 ScreenSpaceToGamefield = pos => recordingManager.ToLocalSpace(pos),
                             },
@@ -271,7 +272,7 @@ namespace osu.Game.Tests.Visual.Gameplay
 
         internal class TestReplayRecorder : ReplayRecorder<TestAction>
         {
-            public TestReplayRecorder(Replay target)
+            public TestReplayRecorder(Score target)
                 : base(target)
             {
             }
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecording.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecording.cs
index 6872b6a669..40c4214749 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecording.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneReplayRecording.cs
@@ -15,6 +15,7 @@ using osu.Game.Replays;
 using osu.Game.Rulesets;
 using osu.Game.Rulesets.Replays;
 using osu.Game.Rulesets.UI;
+using osu.Game.Scoring;
 using osu.Game.Screens.Play;
 using osu.Game.Tests.Visual.UserInterface;
 using osuTK;
@@ -44,7 +45,7 @@ namespace osu.Game.Tests.Visual.Gameplay
                     {
                         recordingManager = new TestRulesetInputManager(new TestSceneModSettings.TestRulesetInfo(), 0, SimultaneousBindingMode.Unique)
                         {
-                            Recorder = new TestReplayRecorder(replay)
+                            Recorder = new TestReplayRecorder(new Score { Replay = replay })
                             {
                                 ScreenSpaceToGamefield = pos => recordingManager.ToLocalSpace(pos)
                             },
@@ -206,7 +207,7 @@ namespace osu.Game.Tests.Visual.Gameplay
 
     internal class TestReplayRecorder : ReplayRecorder<TestAction>
     {
-        public TestReplayRecorder(Replay target)
+        public TestReplayRecorder(Score target)
             : base(target)
         {
         }
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs
index 72c6fd8d44..3e5b561a6f 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectator.cs
@@ -17,6 +17,7 @@ using osu.Game.Replays.Legacy;
 using osu.Game.Rulesets.Osu;
 using osu.Game.Rulesets.Osu.Replays;
 using osu.Game.Rulesets.UI;
+using osu.Game.Scoring;
 using osu.Game.Screens.Play;
 using osu.Game.Tests.Beatmaps.IO;
 using osu.Game.Users;
@@ -272,7 +273,7 @@ namespace osu.Game.Tests.Visual.Gameplay
                     frames.Add(new LegacyReplayFrame(i * 100, RNG.Next(0, 512), RNG.Next(0, 512), buttonState));
                 }
 
-                var bundle = new FrameDataBundle(frames);
+                var bundle = new FrameDataBundle(new ScoreInfo(), frames);
                 ((ISpectatorClient)this).UserSentFrames(StreamingUser.Id, bundle);
 
                 if (!sentState)
diff --git a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectatorPlayback.cs b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectatorPlayback.cs
index 35473ee76c..e148fa381c 100644
--- a/osu.Game.Tests/Visual/Gameplay/TestSceneSpectatorPlayback.cs
+++ b/osu.Game.Tests/Visual/Gameplay/TestSceneSpectatorPlayback.cs
@@ -27,6 +27,7 @@ using osu.Game.Rulesets;
 using osu.Game.Rulesets.Replays;
 using osu.Game.Rulesets.Replays.Types;
 using osu.Game.Rulesets.UI;
+using osu.Game.Scoring;
 using osu.Game.Screens.Play;
 using osu.Game.Tests.Visual.UserInterface;
 using osuTK;
@@ -348,7 +349,7 @@ namespace osu.Game.Tests.Visual.Gameplay
         internal class TestReplayRecorder : ReplayRecorder<TestAction>
         {
             public TestReplayRecorder()
-                : base(new Replay())
+                : base(new Score())
             {
             }
 
diff --git a/osu.Game/Online/RealtimeMultiplayer/MultiplayerRoomSettings.cs b/osu.Game/Online/RealtimeMultiplayer/MultiplayerRoomSettings.cs
index d2f64235c3..60e0d1292e 100644
--- a/osu.Game/Online/RealtimeMultiplayer/MultiplayerRoomSettings.cs
+++ b/osu.Game/Online/RealtimeMultiplayer/MultiplayerRoomSettings.cs
@@ -18,13 +18,20 @@ namespace osu.Game.Online.RealtimeMultiplayer
 
         public int RulesetID { get; set; }
 
+        public string BeatmapChecksum { get; set; } = string.Empty;
+
         public string Name { get; set; } = "Unnamed room";
 
         [NotNull]
         public IEnumerable<APIMod> Mods { get; set; } = Enumerable.Empty<APIMod>();
 
-        public bool Equals(MultiplayerRoomSettings other) => BeatmapID == other.BeatmapID && Mods.SequenceEqual(other.Mods) && RulesetID == other.RulesetID && Name.Equals(other.Name, StringComparison.Ordinal);
+        public bool Equals(MultiplayerRoomSettings other)
+            => BeatmapID == other.BeatmapID
+               && BeatmapChecksum == other.BeatmapChecksum
+               && Mods.SequenceEqual(other.Mods)
+               && RulesetID == other.RulesetID
+               && Name.Equals(other.Name, StringComparison.Ordinal);
 
-        public override string ToString() => $"Name:{Name} Beatmap:{BeatmapID} Mods:{string.Join(',', Mods)} Ruleset:{RulesetID}";
+        public override string ToString() => $"Name:{Name} Beatmap:{BeatmapID} ({BeatmapChecksum}) Mods:{string.Join(',', Mods)} Ruleset:{RulesetID}";
     }
 }
diff --git a/osu.Game/Online/Spectator/FrameDataBundle.cs b/osu.Game/Online/Spectator/FrameDataBundle.cs
index 5281e61f9c..a8d0434324 100644
--- a/osu.Game/Online/Spectator/FrameDataBundle.cs
+++ b/osu.Game/Online/Spectator/FrameDataBundle.cs
@@ -1,20 +1,34 @@
 // Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
 // See the LICENCE file in the repository root for full licence text.
 
+#nullable enable
+
 using System;
 using System.Collections.Generic;
+using Newtonsoft.Json;
 using osu.Game.Replays.Legacy;
+using osu.Game.Scoring;
 
 namespace osu.Game.Online.Spectator
 {
     [Serializable]
     public class FrameDataBundle
     {
+        public FrameHeader Header { get; set; }
+
         public IEnumerable<LegacyReplayFrame> Frames { get; set; }
 
-        public FrameDataBundle(IEnumerable<LegacyReplayFrame> frames)
+        public FrameDataBundle(ScoreInfo score, IEnumerable<LegacyReplayFrame> frames)
         {
             Frames = frames;
+            Header = new FrameHeader(score);
+        }
+
+        [JsonConstructor]
+        public FrameDataBundle(FrameHeader header, IEnumerable<LegacyReplayFrame> frames)
+        {
+            Header = header;
+            Frames = frames;
         }
     }
 }
diff --git a/osu.Game/Online/Spectator/FrameHeader.cs b/osu.Game/Online/Spectator/FrameHeader.cs
new file mode 100644
index 0000000000..b4988fecf9
--- /dev/null
+++ b/osu.Game/Online/Spectator/FrameHeader.cs
@@ -0,0 +1,59 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+using osu.Game.Rulesets.Scoring;
+using osu.Game.Scoring;
+
+namespace osu.Game.Online.Spectator
+{
+    [Serializable]
+    public class FrameHeader
+    {
+        /// <summary>
+        /// The current combo of the score.
+        /// </summary>
+        public int Combo { get; set; }
+
+        /// <summary>
+        /// The maximum combo achieved up to the current point in time.
+        /// </summary>
+        public int MaxCombo { get; set; }
+
+        /// <summary>
+        /// Cumulative hit statistics.
+        /// </summary>
+        public Dictionary<HitResult, int> Statistics { get; set; }
+
+        /// <summary>
+        /// The time at which this frame was received by the server.
+        /// </summary>
+        public DateTimeOffset ReceivedTime { get; set; }
+
+        /// <summary>
+        /// Construct header summary information from a point-in-time reference to a score which is actively being played.
+        /// </summary>
+        /// <param name="score">The score for reference.</param>
+        public FrameHeader(ScoreInfo score)
+        {
+            Combo = score.Combo;
+            MaxCombo = score.MaxCombo;
+
+            // copy for safety
+            Statistics = new Dictionary<HitResult, int>(score.Statistics);
+        }
+
+        [JsonConstructor]
+        public FrameHeader(int combo, int maxCombo, Dictionary<HitResult, int> statistics, DateTimeOffset receivedTime)
+        {
+            Combo = combo;
+            MaxCombo = maxCombo;
+            Statistics = statistics;
+            ReceivedTime = receivedTime;
+        }
+    }
+}
diff --git a/osu.Game/Online/Spectator/SpectatorStreamingClient.cs b/osu.Game/Online/Spectator/SpectatorStreamingClient.cs
index 08b524087a..0167a5d025 100644
--- a/osu.Game/Online/Spectator/SpectatorStreamingClient.cs
+++ b/osu.Game/Online/Spectator/SpectatorStreamingClient.cs
@@ -21,6 +21,7 @@ using osu.Game.Rulesets;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Replays;
 using osu.Game.Rulesets.Replays.Types;
+using osu.Game.Scoring;
 using osu.Game.Screens.Play;
 
 namespace osu.Game.Online.Spectator
@@ -52,6 +53,9 @@ namespace osu.Game.Online.Spectator
         [CanBeNull]
         private IBeatmap currentBeatmap;
 
+        [CanBeNull]
+        private Score currentScore;
+
         [Resolved]
         private IBindable<RulesetInfo> currentRuleset { get; set; }
 
@@ -203,7 +207,7 @@ namespace osu.Game.Online.Spectator
             return Task.CompletedTask;
         }
 
-        public void BeginPlaying(GameplayBeatmap beatmap)
+        public void BeginPlaying(GameplayBeatmap beatmap, Score score)
         {
             if (isPlaying)
                 throw new InvalidOperationException($"Cannot invoke {nameof(BeginPlaying)} when already playing");
@@ -216,6 +220,8 @@ namespace osu.Game.Online.Spectator
             currentState.Mods = currentMods.Value.Select(m => new APIMod(m));
 
             currentBeatmap = beatmap.PlayableBeatmap;
+            currentScore = score;
+
             beginPlaying();
         }
 
@@ -308,7 +314,9 @@ namespace osu.Game.Online.Spectator
 
             pendingFrames.Clear();
 
-            SendFrames(new FrameDataBundle(frames));
+            Debug.Assert(currentScore != null);
+
+            SendFrames(new FrameDataBundle(currentScore.ScoreInfo, frames));
 
             lastSendTime = Time.Current;
         }
diff --git a/osu.Game/Rulesets/UI/DrawableRuleset.cs b/osu.Game/Rulesets/UI/DrawableRuleset.cs
index c1a601eaae..6940e43e5b 100644
--- a/osu.Game/Rulesets/UI/DrawableRuleset.cs
+++ b/osu.Game/Rulesets/UI/DrawableRuleset.cs
@@ -268,12 +268,12 @@ namespace osu.Game.Rulesets.UI
             return false;
         }
 
-        public override void SetRecordTarget(Replay recordingReplay)
+        public sealed override void SetRecordTarget(Score score)
         {
             if (!(KeyBindingInputManager is IHasRecordingHandler recordingInputManager))
                 throw new InvalidOperationException($"A {nameof(KeyBindingInputManager)} which supports recording is not available");
 
-            var recorder = CreateReplayRecorder(recordingReplay);
+            var recorder = CreateReplayRecorder(score);
 
             if (recorder == null)
                 return;
@@ -327,7 +327,7 @@ namespace osu.Game.Rulesets.UI
 
         protected virtual ReplayInputHandler CreateReplayInputHandler(Replay replay) => null;
 
-        protected virtual ReplayRecorder CreateReplayRecorder(Replay replay) => null;
+        protected virtual ReplayRecorder CreateReplayRecorder(Score score) => null;
 
         /// <summary>
         /// Creates a Playfield.
@@ -516,8 +516,8 @@ namespace osu.Game.Rulesets.UI
         /// <summary>
         /// Sets a replay to be used to record gameplay.
         /// </summary>
-        /// <param name="recordingReplay">The target to be recorded to.</param>
-        public abstract void SetRecordTarget(Replay recordingReplay);
+        /// <param name="score">The target to be recorded to.</param>
+        public abstract void SetRecordTarget(Score score);
 
         /// <summary>
         /// Invoked when the interactive user requests resuming from a paused state.
diff --git a/osu.Game/Rulesets/UI/ReplayRecorder.cs b/osu.Game/Rulesets/UI/ReplayRecorder.cs
index 1438ebd37a..a4d46e3888 100644
--- a/osu.Game/Rulesets/UI/ReplayRecorder.cs
+++ b/osu.Game/Rulesets/UI/ReplayRecorder.cs
@@ -10,8 +10,8 @@ using osu.Framework.Input;
 using osu.Framework.Input.Bindings;
 using osu.Framework.Input.Events;
 using osu.Game.Online.Spectator;
-using osu.Game.Replays;
 using osu.Game.Rulesets.Replays;
+using osu.Game.Scoring;
 using osu.Game.Screens.Play;
 using osuTK;
 
@@ -20,7 +20,7 @@ namespace osu.Game.Rulesets.UI
     public abstract class ReplayRecorder<T> : ReplayRecorder, IKeyBindingHandler<T>
         where T : struct
     {
-        private readonly Replay target;
+        private readonly Score target;
 
         private readonly List<T> pressedActions = new List<T>();
 
@@ -34,7 +34,7 @@ namespace osu.Game.Rulesets.UI
         [Resolved]
         private GameplayBeatmap gameplayBeatmap { get; set; }
 
-        protected ReplayRecorder(Replay target)
+        protected ReplayRecorder(Score target)
         {
             this.target = target;
 
@@ -49,7 +49,7 @@ namespace osu.Game.Rulesets.UI
 
             inputManager = GetContainingInputManager();
 
-            spectatorStreaming?.BeginPlaying(gameplayBeatmap);
+            spectatorStreaming?.BeginPlaying(gameplayBeatmap, target);
         }
 
         protected override void Dispose(bool isDisposing)
@@ -79,7 +79,7 @@ namespace osu.Game.Rulesets.UI
 
         private void recordFrame(bool important)
         {
-            var last = target.Frames.LastOrDefault();
+            var last = target.Replay.Frames.LastOrDefault();
 
             if (!important && last != null && Time.Current - last.Time < (1000d / RecordFrameRate))
                 return;
@@ -90,7 +90,7 @@ namespace osu.Game.Rulesets.UI
 
             if (frame != null)
             {
-                target.Frames.Add(frame);
+                target.Replay.Frames.Add(frame);
 
                 spectatorStreaming?.HandleFrame(frame);
             }
diff --git a/osu.Game/Screens/Play/Player.cs b/osu.Game/Screens/Play/Player.cs
index 7979b635aa..a54f9fc047 100644
--- a/osu.Game/Screens/Play/Player.cs
+++ b/osu.Game/Screens/Play/Player.cs
@@ -5,6 +5,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using JetBrains.Annotations;
 using osu.Framework.Allocation;
 using osu.Framework.Audio;
 using osu.Framework.Audio.Sample;
@@ -21,7 +22,6 @@ using osu.Game.Graphics.Containers;
 using osu.Game.IO.Archives;
 using osu.Game.Online.API;
 using osu.Game.Overlays;
-using osu.Game.Replays;
 using osu.Game.Rulesets;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.Scoring;
@@ -157,14 +157,17 @@ namespace osu.Game.Screens.Play
                 PrepareReplay();
         }
 
-        private Replay recordingReplay;
+        [CanBeNull]
+        private Score recordingScore;
 
         /// <summary>
         /// Run any recording / playback setup for replays.
         /// </summary>
         protected virtual void PrepareReplay()
         {
-            DrawableRuleset.SetRecordTarget(recordingReplay = new Replay());
+            DrawableRuleset.SetRecordTarget(recordingScore = new Score());
+
+            ScoreProcessor.NewJudgement += result => ScoreProcessor.PopulateScore(recordingScore.ScoreInfo);
         }
 
         [BackgroundDependencyLoader(true)]
@@ -758,9 +761,9 @@ namespace osu.Game.Screens.Play
 
             var score = new Score { ScoreInfo = CreateScore() };
 
-            if (recordingReplay?.Frames.Count > 0)
+            if (recordingScore?.Replay.Frames.Count > 0)
             {
-                score.Replay = recordingReplay;
+                score.Replay = recordingScore.Replay;
 
                 using (var stream = new MemoryStream())
                 {