| | | 1 | | // Copyright (c) ZeroC, Inc. |
| | | 2 | | |
| | | 3 | | using IceRpc.Extensions.DependencyInjection; |
| | | 4 | | using IceRpc.Features; |
| | | 5 | | using IceRpc.Retry.Internal; |
| | | 6 | | using Microsoft.Extensions.Logging; |
| | | 7 | | using Microsoft.Extensions.Logging.Abstractions; |
| | | 8 | | using System.Diagnostics; |
| | | 9 | | using System.Runtime.ExceptionServices; |
| | | 10 | | |
| | | 11 | | namespace IceRpc.Retry; |
| | | 12 | | |
| | | 13 | | /// <summary>The retry interceptor is responsible for retrying failed requests when the failure condition can be |
| | | 14 | | /// retried.</summary> |
| | | 15 | | /// <remarks>A failed request can be retried if: |
| | | 16 | | /// <list type="bullet"> |
| | | 17 | | /// <item><description><see cref="RetryOptions.MaxAttempts" /> is not reached.</description></item> |
| | | 18 | | /// <item><description><see cref="OutgoingFrame.Payload" /> can be read again.</description></item> |
| | | 19 | | /// <item><description>The failure condition can be retried.</description></item> |
| | | 20 | | /// </list> |
| | | 21 | | /// <para>In order to be able to read again the request's payload, the retry interceptor decorates the payload |
| | | 22 | | /// with <see cref="ResettablePipeReaderDecorator" />. The decorator can be reset as long as the buffered data doesn't |
| | | 23 | | /// exceed <see cref="RetryOptions.MaxPayloadSize" />.</para> |
| | | 24 | | /// <para>The request can be retried under the following failure conditions:</para> |
| | | 25 | | /// <list type="bullet"> |
| | | 26 | | /// <item><description>The status code carried by the response is <see cref="StatusCode.Unavailable" |
| | | 27 | | /// />.</description></item> |
| | | 28 | | /// <item><description>The status code carried by the response is <see cref="StatusCode.NotFound" /> and the |
| | | 29 | | /// protocol is ice.</description></item> |
| | | 30 | | /// <item><description>The request failed with an <see cref="IceRpcException" /> with one of the following error: |
| | | 31 | | /// <list type="bullet"> |
| | | 32 | | /// <item><description>The error code is <see cref="IceRpcError.InvocationCanceled" />.</description></item> |
| | | 33 | | /// <item><description>The error code is <see cref="IceRpcError.ConnectionAborted" /> or <see |
| | | 34 | | /// cref="IceRpcError.TruncatedData" /> and the request has the <see cref="RequestFieldKey.Idempotent" /> |
| | | 35 | | /// field.</description></item> |
| | | 36 | | /// </list></description></item> |
| | | 37 | | /// </list> |
| | | 38 | | /// <para>If the status code carried by the response is <see cref="StatusCode.Unavailable" /> or <see |
| | | 39 | | /// cref="StatusCode.NotFound" /> (with the ice protocol), the address of the server is removed from the set of |
| | | 40 | | /// server addresses to retry on. This ensures the request won't be retried on the unavailable server.</para></remarks> |
| | | 41 | | /// <seealso cref="RetryPipelineExtensions"/> |
| | | 42 | | /// <seealso cref="RetryInvokerBuilderExtensions"/> |
| | | 43 | | public class RetryInterceptor : IInvoker |
| | | 44 | | { |
| | | 45 | | private readonly ILogger _logger; |
| | | 46 | | private readonly int _maxAttempts; |
| | | 47 | | private readonly int _maxPayloadSize; |
| | | 48 | | private readonly IInvoker _next; |
| | | 49 | | |
| | | 50 | | /// <summary>Constructs a retry interceptor.</summary> |
| | | 51 | | /// <param name="next">The next invoker in the invocation pipeline.</param> |
| | | 52 | | /// <param name="options">The options to configure the retry interceptor.</param> |
| | | 53 | | /// <param name="logger">The logger.</param> |
| | 16 | 54 | | public RetryInterceptor(IInvoker next, RetryOptions options, ILogger logger) |
| | 16 | 55 | | { |
| | 16 | 56 | | _next = next; |
| | 16 | 57 | | _maxAttempts = options.MaxAttempts; |
| | 16 | 58 | | _maxPayloadSize = options.MaxPayloadSize; |
| | 16 | 59 | | _logger = logger; |
| | 16 | 60 | | } |
| | | 61 | | |
| | | 62 | | /// <inheritdoc/> |
| | | 63 | | public async Task<IncomingResponse> InvokeAsync(OutgoingRequest request, CancellationToken cancellationToken) |
| | 16 | 64 | | { |
| | | 65 | | // This interceptor does not support retrying requests with a payload continuation. |
| | 16 | 66 | | if (request.PayloadContinuation is not null || _maxAttempts == 1) |
| | 0 | 67 | | { |
| | 0 | 68 | | return await _next.InvokeAsync(request, cancellationToken).ConfigureAwait(false); |
| | | 69 | | } |
| | | 70 | | else |
| | 16 | 71 | | { |
| | 16 | 72 | | var decorator = new ResettablePipeReaderDecorator(request.Payload, _maxPayloadSize); |
| | 16 | 73 | | request.Payload = decorator; |
| | | 74 | | |
| | | 75 | | try |
| | 16 | 76 | | { |
| | 16 | 77 | | int attempt = 1; |
| | 16 | 78 | | IncomingResponse? response = null; |
| | 16 | 79 | | IceRpcException? exception = null; |
| | | 80 | | bool tryAgain; |
| | | 81 | | |
| | | 82 | | do |
| | 26 | 83 | | { |
| | 26 | 84 | | bool retryWithOtherReplica = false; |
| | | 85 | | |
| | | 86 | | try |
| | 26 | 87 | | { |
| | 26 | 88 | | using IDisposable? scope = CreateRetryLogScope(attempt); |
| | | 89 | | |
| | 26 | 90 | | response = await _next.InvokeAsync(request, cancellationToken).ConfigureAwait(false); |
| | | 91 | | |
| | 15 | 92 | | if (response.StatusCode == StatusCode.Unavailable || |
| | 15 | 93 | | (response.Protocol == Protocol.Ice && response.StatusCode == StatusCode.NotFound)) |
| | 4 | 94 | | { |
| | 4 | 95 | | retryWithOtherReplica = true; |
| | 4 | 96 | | } |
| | | 97 | | else |
| | 11 | 98 | | { |
| | 11 | 99 | | return response; |
| | | 100 | | } |
| | 4 | 101 | | } |
| | 9 | 102 | | catch (IceRpcException iceRpcException) when ( |
| | 9 | 103 | | iceRpcException.IceRpcError == IceRpcError.NoConnection) |
| | 1 | 104 | | { |
| | | 105 | | // NoConnection is always considered non-retryable; it typically occurs because we removed all |
| | | 106 | | // the server addresses from serverAddressFeature. Unlike other non-retryable exceptions, we |
| | | 107 | | // privilege returning the previous response (if any). |
| | 1 | 108 | | return response ?? throw RethrowException(exception ?? iceRpcException); |
| | | 109 | | } |
| | 8 | 110 | | catch (IceRpcException iceRpcException) |
| | 8 | 111 | | { |
| | 8 | 112 | | response = null; |
| | 8 | 113 | | exception = iceRpcException; |
| | 8 | 114 | | } |
| | | 115 | | |
| | 12 | 116 | | Debug.Assert(retryWithOtherReplica || exception is not null); |
| | | 117 | | |
| | | 118 | | // Check if we can retry |
| | 12 | 119 | | tryAgain = false; |
| | | 120 | | |
| | | 121 | | // The decorator is non-resettable when we've reached the last attempt (see below) or the decorator |
| | | 122 | | // caught an exception that prevents resetting, like a ReadAsync exception. |
| | 12 | 123 | | if (decorator.IsResettable) |
| | 11 | 124 | | { |
| | 11 | 125 | | if (retryWithOtherReplica) |
| | 4 | 126 | | { |
| | 4 | 127 | | if (request.Features.Get<IServerAddressFeature>() is IServerAddressFeature serverAddressFeat |
| | 4 | 128 | | serverAddressFeature.ServerAddress is ServerAddress mainServerAddress) |
| | 4 | 129 | | { |
| | | 130 | | // We don't want to retry with this server address |
| | 4 | 131 | | serverAddressFeature.RemoveServerAddress(mainServerAddress); |
| | | 132 | | |
| | 4 | 133 | | tryAgain = serverAddressFeature.ServerAddress is not null; |
| | 4 | 134 | | } |
| | | 135 | | // else there is no replica to retry with |
| | 4 | 136 | | } |
| | | 137 | | else |
| | 7 | 138 | | { |
| | 7 | 139 | | Debug.Assert(exception is not null); |
| | | 140 | | // It's always safe to retry InvocationCanceled because it's only raised before the request |
| | | 141 | | // is sent to the peer. For idempotent requests we also retry on ConnectionAborted and |
| | | 142 | | // TruncatedData. |
| | 7 | 143 | | tryAgain = exception.IceRpcError switch |
| | 7 | 144 | | { |
| | 5 | 145 | | IceRpcError.InvocationCanceled => true, |
| | 7 | 146 | | IceRpcError.ConnectionAborted or IceRpcError.TruncatedData |
| | 2 | 147 | | when request.Fields.ContainsKey(RequestFieldKey.Idempotent) => true, |
| | 1 | 148 | | _ => false |
| | 7 | 149 | | }; |
| | 7 | 150 | | } |
| | | 151 | | |
| | 11 | 152 | | if (tryAgain) |
| | 10 | 153 | | { |
| | 10 | 154 | | attempt++; |
| | 10 | 155 | | decorator.Reset(); |
| | | 156 | | |
| | | 157 | | // If this attempt is the last attempt, we make the decorator non-resettable to release the |
| | | 158 | | // memory for the request payload as soon as possible. |
| | 10 | 159 | | if (attempt == _maxAttempts) |
| | 5 | 160 | | { |
| | 5 | 161 | | decorator.IsResettable = false; |
| | 5 | 162 | | } |
| | 10 | 163 | | } |
| | 11 | 164 | | } |
| | 12 | 165 | | } |
| | 12 | 166 | | while (tryAgain); |
| | | 167 | | |
| | 2 | 168 | | Debug.Assert(response is not null || exception is not null); |
| | 2 | 169 | | Debug.Assert(response is null || response.StatusCode != StatusCode.Ok); |
| | 2 | 170 | | return response ?? throw RethrowException(exception!); |
| | | 171 | | } |
| | | 172 | | finally |
| | 16 | 173 | | { |
| | | 174 | | // We want to leave request.Payload in a correct, usable state when we exit. Usually request.Payload |
| | | 175 | | // will get completed by the caller, and we want this Complete call to flow through to the decoratee. If |
| | | 176 | | // the payload is still readable (e.g. we received a non-retryable exception before reading anything or |
| | | 177 | | // just after a Reset), an upstream interceptor may want to attempt another call that reads this payload |
| | | 178 | | // and the now non-resettable decorator will provide the correct behavior. The decorator ensures that |
| | | 179 | | // calls to AdvanceTo on the decoratee always receive ever-increasing examined values even after one or |
| | | 180 | | // more Resets. |
| | 16 | 181 | | decorator.IsResettable = false; |
| | 16 | 182 | | } |
| | | 183 | | } |
| | 11 | 184 | | } |
| | | 185 | | |
| | | 186 | | private static Exception RethrowException(Exception exception) |
| | 3 | 187 | | { |
| | 3 | 188 | | ExceptionDispatchInfo.Throw(exception); |
| | 0 | 189 | | Debug.Assert(false); |
| | 0 | 190 | | return exception; |
| | 0 | 191 | | } |
| | | 192 | | |
| | | 193 | | private IDisposable? CreateRetryLogScope(int attempt) => |
| | 26 | 194 | | _logger != NullLogger.Instance && attempt > 1 ? _logger.RetryScope(attempt, _maxAttempts) : null; |
| | | 195 | | } |