From 9f89b4e6d79822f7e7eb382767818add29bda6db Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 22 Jan 2021 14:25:21 +0900 Subject: [PATCH 1/8] Rewrite connection logic to better handle failure cases The main goal here is to ensure the connection is built each connection attempt. Previously, the access token would never be updated, leading to outdated tokens failing repeatedly (in the connection retry loop) and never being able to establish a new connection as a result. Due to threading considerations, this isn't as simple as I would hope it to be. I'm open to proposals as to a better way of handling this. Also, keep in mind that this logic will need to be abstracted and (re)used in `SpectatorClient` as well. I've intentionally not done that yet until we agree that this is a good direction forward. --- .../Online/Multiplayer/MultiplayerClient.cs | 141 ++++++++++++------ 1 file changed, 93 insertions(+), 48 deletions(-) diff --git a/osu.Game/Online/Multiplayer/MultiplayerClient.cs b/osu.Game/Online/Multiplayer/MultiplayerClient.cs index 50dc8f661c..34616a45a5 100644 --- a/osu.Game/Online/Multiplayer/MultiplayerClient.cs +++ b/osu.Game/Online/Multiplayer/MultiplayerClient.cs @@ -4,7 +4,6 @@ #nullable enable using System; -using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.SignalR.Client; @@ -30,6 +29,8 @@ namespace osu.Game.Online.Multiplayer private HubConnection? connection; + private CancellationTokenSource connectCancelSource = new CancellationTokenSource(); + private readonly string endpoint; public MultiplayerClient(EndpointConfiguration endpoints) @@ -50,8 +51,7 @@ namespace osu.Game.Online.Multiplayer { case APIState.Failing: case APIState.Offline: - connection?.StopAsync(); - connection = null; + Task.Run(Disconnect); break; case APIState.Online: @@ -60,70 +60,57 @@ namespace osu.Game.Online.Multiplayer } } - protected virtual async Task Connect() + private readonly SemaphoreSlim connectionLock = new SemaphoreSlim(1); + + public Task Disconnect() => disconnect(true); + + protected async Task Connect() { - if (connection != null) - return; + cancelExistingConnect(); - connection = new HubConnectionBuilder() - .WithUrl(endpoint, options => - { - options.Headers.Add("Authorization", $"Bearer {api.AccessToken}"); - }) - .AddNewtonsoftJsonProtocol(options => { options.PayloadSerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; }) - .Build(); + await connectionLock.WaitAsync(); - // this is kind of SILLY - // https://github.com/dotnet/aspnetcore/issues/15198 - connection.On(nameof(IMultiplayerClient.RoomStateChanged), ((IMultiplayerClient)this).RoomStateChanged); - connection.On(nameof(IMultiplayerClient.UserJoined), ((IMultiplayerClient)this).UserJoined); - connection.On(nameof(IMultiplayerClient.UserLeft), ((IMultiplayerClient)this).UserLeft); - connection.On(nameof(IMultiplayerClient.HostChanged), ((IMultiplayerClient)this).HostChanged); - connection.On(nameof(IMultiplayerClient.SettingsChanged), ((IMultiplayerClient)this).SettingsChanged); - connection.On(nameof(IMultiplayerClient.UserStateChanged), ((IMultiplayerClient)this).UserStateChanged); - connection.On(nameof(IMultiplayerClient.LoadRequested), ((IMultiplayerClient)this).LoadRequested); - connection.On(nameof(IMultiplayerClient.MatchStarted), ((IMultiplayerClient)this).MatchStarted); - connection.On(nameof(IMultiplayerClient.ResultsReady), ((IMultiplayerClient)this).ResultsReady); - - connection.Closed += async ex => + try { - isConnected.Value = false; + await disconnect(false); - Logger.Log(ex != null - ? $"Multiplayer client lost connection: {ex}" - : "Multiplayer client disconnected", LoggingTarget.Network); + // this token will be valid for the scope of this connection. + // if cancelled, we can be sure that a disconnect or reconnect is handled elsewhere. + var cancellationToken = connectCancelSource.Token; - if (connection != null) - await tryUntilConnected(); - }; - - await tryUntilConnected(); - - async Task tryUntilConnected() - { - Logger.Log("Multiplayer client connecting...", LoggingTarget.Network); - - while (api.State.Value == APIState.Online) + while (api.State.Value == APIState.Online && !cancellationToken.IsCancellationRequested) { + Logger.Log("Multiplayer client connecting...", LoggingTarget.Network); + try { - Debug.Assert(connection != null); + // importantly, rebuild the connection each attempt to get an updated access token. + connection = createConnection(cancellationToken); + + await connection.StartAsync(cancellationToken); - // reconnect on any failure - await connection.StartAsync(); Logger.Log("Multiplayer client connected!", LoggingTarget.Network); - - // Success. isConnected.Value = true; - break; + return; + } + catch (OperationCanceledException) + { + //connection process was cancelled. + return; } catch (Exception e) { Logger.Log($"Multiplayer client connection error: {e}", LoggingTarget.Network); - await Task.Delay(5000); + + // retry on any failure. + await Task.Delay(5000, cancellationToken); } } } + finally + { + connectionLock.Release(); + } } protected override Task JoinRoom(long roomId) @@ -189,5 +176,63 @@ namespace osu.Game.Online.Multiplayer return connection.InvokeAsync(nameof(IMultiplayerServer.StartMatch)); } + + private async Task disconnect(bool takeLock) + { + cancelExistingConnect(); + + if (takeLock) + await connectionLock.WaitAsync(); + + try + { + if (connection != null) + await connection.StopAsync(); + } + finally + { + connection = null; + if (takeLock) + connectionLock.Release(); + } + } + + private void cancelExistingConnect() + { + connectCancelSource.Cancel(); + connectCancelSource = new CancellationTokenSource(); + } + + private HubConnection createConnection(CancellationToken cancellationToken) + { + var newConnection = new HubConnectionBuilder() + .WithUrl(endpoint, options => { options.Headers.Add("Authorization", $"Bearer {api.AccessToken}"); }) + .AddNewtonsoftJsonProtocol(options => { options.PayloadSerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; }) + .Build(); + + // this is kind of SILLY + // https://github.com/dotnet/aspnetcore/issues/15198 + newConnection.On(nameof(IMultiplayerClient.RoomStateChanged), ((IMultiplayerClient)this).RoomStateChanged); + newConnection.On(nameof(IMultiplayerClient.UserJoined), ((IMultiplayerClient)this).UserJoined); + newConnection.On(nameof(IMultiplayerClient.UserLeft), ((IMultiplayerClient)this).UserLeft); + newConnection.On(nameof(IMultiplayerClient.HostChanged), ((IMultiplayerClient)this).HostChanged); + newConnection.On(nameof(IMultiplayerClient.SettingsChanged), ((IMultiplayerClient)this).SettingsChanged); + newConnection.On(nameof(IMultiplayerClient.UserStateChanged), ((IMultiplayerClient)this).UserStateChanged); + newConnection.On(nameof(IMultiplayerClient.LoadRequested), ((IMultiplayerClient)this).LoadRequested); + newConnection.On(nameof(IMultiplayerClient.MatchStarted), ((IMultiplayerClient)this).MatchStarted); + newConnection.On(nameof(IMultiplayerClient.ResultsReady), ((IMultiplayerClient)this).ResultsReady); + + newConnection.Closed += async ex => + { + isConnected.Value = false; + + Logger.Log(ex != null ? $"Multiplayer client lost connection: {ex}" : "Multiplayer client disconnected", LoggingTarget.Network); + + // make sure a disconnect wasn't triggered (and this is still the active connection). + if (!cancellationToken.IsCancellationRequested) + await Connect(); + }; + return newConnection; + } } } From d24d23646875b0bca4755c9d5b8a0caa24f30cae Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 22 Jan 2021 14:34:58 +0900 Subject: [PATCH 2/8] Make OperationCanceledException throwing behaviour consistent --- .../Online/Multiplayer/MultiplayerClient.cs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/osu.Game/Online/Multiplayer/MultiplayerClient.cs b/osu.Game/Online/Multiplayer/MultiplayerClient.cs index 34616a45a5..aa2305c991 100644 --- a/osu.Game/Online/Multiplayer/MultiplayerClient.cs +++ b/osu.Game/Online/Multiplayer/MultiplayerClient.cs @@ -12,6 +12,7 @@ using Newtonsoft.Json; using osu.Framework.Allocation; using osu.Framework.Bindables; using osu.Framework.Logging; +using osu.Game.Extensions; using osu.Game.Online.API; using osu.Game.Online.Rooms; @@ -51,11 +52,11 @@ namespace osu.Game.Online.Multiplayer { case APIState.Failing: case APIState.Offline: - Task.Run(Disconnect); + Task.Run(Disconnect).CatchUnobservedExceptions(); break; case APIState.Online: - Task.Run(Connect); + Task.Run(Connect).CatchUnobservedExceptions(); break; } } @@ -78,8 +79,10 @@ namespace osu.Game.Online.Multiplayer // if cancelled, we can be sure that a disconnect or reconnect is handled elsewhere. var cancellationToken = connectCancelSource.Token; - while (api.State.Value == APIState.Online && !cancellationToken.IsCancellationRequested) + while (api.State.Value == APIState.Online) { + cancellationToken.ThrowIfCancellationRequested(); + Logger.Log("Multiplayer client connecting...", LoggingTarget.Network); try @@ -96,7 +99,7 @@ namespace osu.Game.Online.Multiplayer catch (OperationCanceledException) { //connection process was cancelled. - return; + throw; } catch (Exception e) { @@ -222,7 +225,7 @@ namespace osu.Game.Online.Multiplayer newConnection.On(nameof(IMultiplayerClient.MatchStarted), ((IMultiplayerClient)this).MatchStarted); newConnection.On(nameof(IMultiplayerClient.ResultsReady), ((IMultiplayerClient)this).ResultsReady); - newConnection.Closed += async ex => + newConnection.Closed += ex => { isConnected.Value = false; @@ -230,7 +233,9 @@ namespace osu.Game.Online.Multiplayer // make sure a disconnect wasn't triggered (and this is still the active connection). if (!cancellationToken.IsCancellationRequested) - await Connect(); + Task.Run(Connect, default).CatchUnobservedExceptions(); + + return Task.CompletedTask; }; return newConnection; } From c05ae3497afa8c5a7dc551496e189620ccbd11ad Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 25 Jan 2021 17:02:24 +0900 Subject: [PATCH 3/8] Make connect/disconnect private --- osu.Game/Online/Multiplayer/MultiplayerClient.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/osu.Game/Online/Multiplayer/MultiplayerClient.cs b/osu.Game/Online/Multiplayer/MultiplayerClient.cs index aa2305c991..7dc4919d23 100644 --- a/osu.Game/Online/Multiplayer/MultiplayerClient.cs +++ b/osu.Game/Online/Multiplayer/MultiplayerClient.cs @@ -25,6 +25,8 @@ namespace osu.Game.Online.Multiplayer private readonly Bindable isConnected = new Bindable(); private readonly IBindable apiState = new Bindable(); + private readonly SemaphoreSlim connectionLock = new SemaphoreSlim(1); + [Resolved] private IAPIProvider api { get; set; } = null!; @@ -52,20 +54,16 @@ namespace osu.Game.Online.Multiplayer { case APIState.Failing: case APIState.Offline: - Task.Run(Disconnect).CatchUnobservedExceptions(); + Task.Run(() => disconnect(true)).CatchUnobservedExceptions(); break; case APIState.Online: - Task.Run(Connect).CatchUnobservedExceptions(); + Task.Run(connect).CatchUnobservedExceptions(); break; } } - private readonly SemaphoreSlim connectionLock = new SemaphoreSlim(1); - - public Task Disconnect() => disconnect(true); - - protected async Task Connect() + private async Task connect() { cancelExistingConnect(); @@ -233,7 +231,7 @@ namespace osu.Game.Online.Multiplayer // make sure a disconnect wasn't triggered (and this is still the active connection). if (!cancellationToken.IsCancellationRequested) - Task.Run(Connect, default).CatchUnobservedExceptions(); + Task.Run(connect, default).CatchUnobservedExceptions(); return Task.CompletedTask; }; From 994fb2667dc22d06d0fc7aca26387ef03ec04859 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 25 Jan 2021 17:11:04 +0900 Subject: [PATCH 4/8] Call DisposeAsync instead of StopAsync --- osu.Game/Online/Multiplayer/MultiplayerClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Game/Online/Multiplayer/MultiplayerClient.cs b/osu.Game/Online/Multiplayer/MultiplayerClient.cs index 7dc4919d23..ffed2b57fe 100644 --- a/osu.Game/Online/Multiplayer/MultiplayerClient.cs +++ b/osu.Game/Online/Multiplayer/MultiplayerClient.cs @@ -188,7 +188,7 @@ namespace osu.Game.Online.Multiplayer try { if (connection != null) - await connection.StopAsync(); + await connection.DisposeAsync(); } finally { From 0f09a7feb9ee12c05c1d53464db9e6194e8818d0 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Mon, 25 Jan 2021 17:17:04 +0900 Subject: [PATCH 5/8] Avoid semaphore potentially getting held forever --- osu.Game/Online/Multiplayer/MultiplayerClient.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/osu.Game/Online/Multiplayer/MultiplayerClient.cs b/osu.Game/Online/Multiplayer/MultiplayerClient.cs index ffed2b57fe..391658f0d0 100644 --- a/osu.Game/Online/Multiplayer/MultiplayerClient.cs +++ b/osu.Game/Online/Multiplayer/MultiplayerClient.cs @@ -67,7 +67,7 @@ namespace osu.Game.Online.Multiplayer { cancelExistingConnect(); - await connectionLock.WaitAsync(); + await connectionLock.WaitAsync(10000); try { @@ -183,7 +183,7 @@ namespace osu.Game.Online.Multiplayer cancelExistingConnect(); if (takeLock) - await connectionLock.WaitAsync(); + await connectionLock.WaitAsync(10000); try { @@ -237,5 +237,12 @@ namespace osu.Game.Online.Multiplayer }; return newConnection; } + + protected override void Dispose(bool isDisposing) + { + base.Dispose(isDisposing); + + cancelExistingConnect(); + } } } From b573c96c079f9065ee4690131f4677eeeb2998be Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Tue, 26 Jan 2021 18:59:42 +0900 Subject: [PATCH 6/8] Move disconnect logic inside connection loop to ensure previous connection is disposed --- osu.Game/Online/Multiplayer/MultiplayerClient.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/osu.Game/Online/Multiplayer/MultiplayerClient.cs b/osu.Game/Online/Multiplayer/MultiplayerClient.cs index 391658f0d0..cbb91c0832 100644 --- a/osu.Game/Online/Multiplayer/MultiplayerClient.cs +++ b/osu.Game/Online/Multiplayer/MultiplayerClient.cs @@ -71,14 +71,15 @@ namespace osu.Game.Online.Multiplayer try { - await disconnect(false); - // this token will be valid for the scope of this connection. // if cancelled, we can be sure that a disconnect or reconnect is handled elsewhere. var cancellationToken = connectCancelSource.Token; while (api.State.Value == APIState.Online) { + // ensure any previous connection was disposed. + await disconnect(false); + cancellationToken.ThrowIfCancellationRequested(); Logger.Log("Multiplayer client connecting...", LoggingTarget.Network); From 4d4d97661e7a8bbe924bdf4d2d5a414f017edb3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Tue, 26 Jan 2021 21:26:50 +0100 Subject: [PATCH 7/8] Fix connection loop always getting a cancelled token --- osu.Game/Online/Multiplayer/MultiplayerClient.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/osu.Game/Online/Multiplayer/MultiplayerClient.cs b/osu.Game/Online/Multiplayer/MultiplayerClient.cs index cbb91c0832..319a8f7170 100644 --- a/osu.Game/Online/Multiplayer/MultiplayerClient.cs +++ b/osu.Game/Online/Multiplayer/MultiplayerClient.cs @@ -71,15 +71,16 @@ namespace osu.Game.Online.Multiplayer try { - // this token will be valid for the scope of this connection. - // if cancelled, we can be sure that a disconnect or reconnect is handled elsewhere. - var cancellationToken = connectCancelSource.Token; - while (api.State.Value == APIState.Online) { // ensure any previous connection was disposed. + // this will also create a new cancellation token source. await disconnect(false); + // this token will be valid for the scope of this connection. + // if cancelled, we can be sure that a disconnect or reconnect is handled elsewhere. + var cancellationToken = connectCancelSource.Token; + cancellationToken.ThrowIfCancellationRequested(); Logger.Log("Multiplayer client connecting...", LoggingTarget.Network); From c5fa818630c25ee7b3912de0d1bfeca9194bc7a3 Mon Sep 17 00:00:00 2001 From: Dean Herbert Date: Fri, 5 Feb 2021 14:08:11 +0900 Subject: [PATCH 8/8] Actually handle case of failing to achieve lock on SemaphoreSlim --- osu.Game/Online/Multiplayer/MultiplayerClient.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/osu.Game/Online/Multiplayer/MultiplayerClient.cs b/osu.Game/Online/Multiplayer/MultiplayerClient.cs index 9067c9a738..36cdf3bc8f 100644 --- a/osu.Game/Online/Multiplayer/MultiplayerClient.cs +++ b/osu.Game/Online/Multiplayer/MultiplayerClient.cs @@ -67,7 +67,8 @@ namespace osu.Game.Online.Multiplayer { cancelExistingConnect(); - await connectionLock.WaitAsync(10000); + if (!await connectionLock.WaitAsync(10000)) + throw new TimeoutException("Could not obtain a lock to connect. A previous attempt is likely stuck."); var builder = new HubConnectionBuilder() .WithUrl(endpoint, options => { options.Headers.Add("Authorization", $"Bearer {api.AccessToken}"); }); @@ -199,7 +200,10 @@ namespace osu.Game.Online.Multiplayer cancelExistingConnect(); if (takeLock) - await connectionLock.WaitAsync(10000); + { + if (!await connectionLock.WaitAsync(10000)) + throw new TimeoutException("Could not obtain a lock to disconnect. A previous attempt is likely stuck."); + } try {