| | | 1 | | // Copyright (c) ZeroC, Inc. |
| | | 2 | | |
| | | 3 | | using IceRpc.Extensions.DependencyInjection; |
| | | 4 | | using IceRpc.Telemetry.Internal; |
| | | 5 | | using System.Buffers; |
| | | 6 | | using System.Diagnostics; |
| | | 7 | | using ZeroC.Slice.Codec; |
| | | 8 | | |
| | | 9 | | namespace IceRpc.Telemetry; |
| | | 10 | | |
| | | 11 | | /// <summary>A middleware that starts an <see cref="Activity" /> per request, following |
| | | 12 | | /// <see href="https://opentelemetry.io/">OpenTelemetry</see> conventions. The middleware restores the parent invocation |
| | | 13 | | /// activity from the request <see cref="RequestFieldKey.TraceContext" /> field before starting the dispatch activity. |
| | | 14 | | /// </summary> |
| | | 15 | | /// <remarks>The activities are only created for requests using the icerpc protocol. The activity records the outcome |
| | | 16 | | /// of the dispatch. The <c>rpc.status_code</c> tag holds the status code of the response returned by the dispatch. |
| | | 17 | | /// When the dispatch throws an exception, this tag holds the status code of the failure response the caller receives, |
| | | 18 | | /// given by <see cref="DispatchException.FromException" />. A status code that reports a problem with the request |
| | | 19 | | /// (<see cref="StatusCode.ApplicationError" />, <see cref="StatusCode.NotFound" />, |
| | | 20 | | /// <see cref="StatusCode.InvalidData" />, <see cref="StatusCode.TruncatedPayload" /> and |
| | | 21 | | /// <see cref="StatusCode.Unauthorized" />) leaves the activity status unset. Any other failure status code sets the |
| | | 22 | | /// activity status to <see cref="ActivityStatusCode.Error" /> and the <c>error.type</c> tag identifies the failure. A |
| | | 23 | | /// cancellation by the token passed to <see cref="DispatchAsync" /> is not a failure: the <c>icerpc.canceled</c> tag |
| | | 24 | | /// is set to <see langword="true" />, the activity status stays unset, and the <c>rpc.status_code</c> tag is not set |
| | | 25 | | /// since the caller receives no response.</remarks> |
| | | 26 | | /// <seealso cref="TelemetryRouterExtensions" /> |
| | | 27 | | /// <seealso cref="TelemetryDispatcherBuilderExtensions"/> |
| | | 28 | | public class TelemetryMiddleware : IDispatcher |
| | | 29 | | { |
| | | 30 | | private readonly IDispatcher _next; |
| | | 31 | | private readonly ActivitySource _activitySource; |
| | | 32 | | |
| | | 33 | | /// <summary>Constructs a telemetry middleware.</summary> |
| | | 34 | | /// <param name="next">The next dispatcher in the dispatch pipeline.</param> |
| | | 35 | | /// <param name="activitySource">The <see cref="ActivitySource" /> is used to start the request activity.</param> |
| | 31 | 36 | | public TelemetryMiddleware(IDispatcher next, ActivitySource activitySource) |
| | 31 | 37 | | { |
| | 31 | 38 | | _next = next; |
| | 31 | 39 | | _activitySource = activitySource; |
| | 31 | 40 | | } |
| | | 41 | | |
| | | 42 | | /// <inheritdoc/> |
| | | 43 | | public async ValueTask<OutgoingResponse> DispatchAsync(IncomingRequest request, CancellationToken cancellationToken) |
| | 31 | 44 | | { |
| | 31 | 45 | | if (request.Protocol.HasFields) |
| | 31 | 46 | | { |
| | 31 | 47 | | string name = $"{request.Path}/{request.Operation}"; |
| | 31 | 48 | | using Activity activity = _activitySource.CreateActivity(name, ActivityKind.Server) ?? new Activity(name); |
| | 31 | 49 | | activity.AddTag("rpc.system", "icerpc"); |
| | 31 | 50 | | activity.AddTag("rpc.service", request.Path); |
| | 31 | 51 | | activity.AddTag("rpc.method", request.Operation); |
| | 31 | 52 | | if (request.Fields.TryGetValue(RequestFieldKey.TraceContext, out ReadOnlySequence<byte> buffer)) |
| | 7 | 53 | | { |
| | 7 | 54 | | RestoreActivityContext(buffer, activity); |
| | 6 | 55 | | } |
| | 30 | 56 | | activity.Start(); |
| | | 57 | | try |
| | 30 | 58 | | { |
| | 30 | 59 | | OutgoingResponse response = await _next.DispatchAsync(request, cancellationToken).ConfigureAwait(false); |
| | 19 | 60 | | activity.SetTag("rpc.status_code", response.StatusCode.ToString()); |
| | 19 | 61 | | if (IsServerError(response.StatusCode)) |
| | 6 | 62 | | { |
| | 6 | 63 | | activity.SetTag("error.type", response.StatusCode.ToErrorType()); |
| | 6 | 64 | | activity.SetStatus(ActivityStatusCode.Error, response.ErrorMessage); |
| | 6 | 65 | | } |
| | 19 | 66 | | return response; |
| | | 67 | | } |
| | 3 | 68 | | catch (OperationCanceledException exception) when ( |
| | 3 | 69 | | cancellationToken.IsCancellationRequested && exception.CancellationToken == cancellationToken) |
| | 1 | 70 | | { |
| | 1 | 71 | | activity.SetTag("icerpc.canceled", true); |
| | 1 | 72 | | throw; |
| | | 73 | | } |
| | 10 | 74 | | catch (Exception exception) |
| | 10 | 75 | | { |
| | 10 | 76 | | DispatchException dispatchException = DispatchException.FromException(exception); |
| | 10 | 77 | | activity.SetTag("rpc.status_code", dispatchException.StatusCode.ToString()); |
| | 10 | 78 | | if (IsServerError(dispatchException.StatusCode)) |
| | 7 | 79 | | { |
| | 7 | 80 | | activity.SetTag("error.type", dispatchException.StatusCode.ToErrorType()); |
| | 7 | 81 | | activity.SetStatus(ActivityStatusCode.Error, dispatchException.ErrorMessage); |
| | 7 | 82 | | } |
| | 10 | 83 | | throw; |
| | | 84 | | } |
| | | 85 | | } |
| | | 86 | | else |
| | 0 | 87 | | { |
| | 0 | 88 | | return await _next.DispatchAsync(request, cancellationToken).ConfigureAwait(false); |
| | | 89 | | } |
| | 19 | 90 | | } |
| | | 91 | | |
| | | 92 | | internal static void RestoreActivityContext(ReadOnlySequence<byte> buffer, Activity activity) |
| | 10 | 93 | | { |
| | 10 | 94 | | var decoder = new SliceDecoder(buffer); |
| | | 95 | | |
| | | 96 | | // Read W3C traceparent binary encoding (1 byte version, 16 bytes trace-ID, 8 bytes span-ID, |
| | | 97 | | // 1 byte flags) https://www.w3.org/TR/trace-context/#traceparent-header-field-values |
| | | 98 | | |
| | 10 | 99 | | byte traceIdVersion = decoder.DecodeUInt8(); |
| | | 100 | | |
| | 9 | 101 | | using IMemoryOwner<byte> memoryOwner = MemoryPool<byte>.Shared.Rent(16); |
| | 9 | 102 | | Span<byte> traceIdSpan = memoryOwner.Memory.Span[0..16]; |
| | 9 | 103 | | decoder.CopyTo(traceIdSpan); |
| | 9 | 104 | | var traceId = ActivityTraceId.CreateFromBytes(traceIdSpan); |
| | | 105 | | |
| | 9 | 106 | | Span<byte> spanIdSpan = memoryOwner.Memory.Span[0..8]; |
| | 9 | 107 | | decoder.CopyTo(spanIdSpan); |
| | 9 | 108 | | var spanId = ActivitySpanId.CreateFromBytes(spanIdSpan); |
| | | 109 | | |
| | 9 | 110 | | var traceFlags = (ActivityTraceFlags)decoder.DecodeUInt8(); |
| | | 111 | | |
| | 9 | 112 | | activity.SetParentId(traceId, spanId, traceFlags); |
| | | 113 | | |
| | | 114 | | // Read TraceState encoded as a string |
| | 9 | 115 | | activity.TraceStateString = decoder.DecodeString(); |
| | | 116 | | |
| | | 117 | | // Decode the baggage sequence, silently clipping to MaxBaggageEntries. OpenTelemetry SDKs |
| | | 118 | | // follow a strict no-throw policy for observability operations: losing a piece of contextual |
| | | 119 | | // metadata is less damaging than failing the RPC, and silent clipping matches the behavior of |
| | | 120 | | // OpenTelemetry .NET's and Python's BaggagePropagator on incoming headers. Only the first |
| | | 121 | | // MaxBaggageEntries entries are read from the buffer; the remainder is left unconsumed. |
| | | 122 | | // |
| | | 123 | | // Activity.Baggage's enumeration order is undocumented, so duplicate-key resolution across the |
| | | 124 | | // wire is inherently undefined on both ends — we don't attempt to preserve it. |
| | 9 | 125 | | int count = decoder.DecodeSize(); |
| | 9 | 126 | | int kept = Math.Min(count, TelemetryInterceptor.MaxBaggageEntries); |
| | | 127 | | |
| | 598 | 128 | | for (int i = 0; i < kept; i++) |
| | 290 | 129 | | { |
| | 290 | 130 | | string key = decoder.DecodeString(); |
| | 290 | 131 | | string value = decoder.DecodeString(); |
| | 290 | 132 | | activity.AddBaggage(key, value); |
| | 290 | 133 | | } |
| | 18 | 134 | | } |
| | | 135 | | |
| | | 136 | | /// <summary>Checks whether a status code reports a failure of the server or the target service, as opposed to a |
| | | 137 | | /// problem with the request.</summary> |
| | | 138 | | private static bool IsServerError(StatusCode statusCode) => |
| | 29 | 139 | | statusCode switch |
| | 29 | 140 | | { |
| | 29 | 141 | | StatusCode.Ok or |
| | 29 | 142 | | StatusCode.ApplicationError or |
| | 29 | 143 | | StatusCode.NotFound or |
| | 29 | 144 | | StatusCode.InvalidData or |
| | 29 | 145 | | StatusCode.TruncatedPayload or |
| | 16 | 146 | | StatusCode.Unauthorized => false, |
| | 13 | 147 | | _ => true |
| | 29 | 148 | | }; |
| | | 149 | | } |