| | | 1 | | using System.Diagnostics; |
| | | 2 | | using System.Text.Json; |
| | | 3 | | |
| | | 4 | | using Microsoft.Extensions.AI.Evaluation; |
| | | 5 | | |
| | | 6 | | namespace NexusLabs.Needlr.AgentFramework.Langfuse; |
| | | 7 | | |
| | | 8 | | /// <summary> |
| | | 9 | | /// Active <see cref="ILangfuseScenario"/> backed by a root OpenTelemetry span and a shared |
| | | 10 | | /// <see cref="LangfuseScoreRecorder"/>. |
| | | 11 | | /// </summary> |
| | | 12 | | internal sealed class LangfuseScenario : ILangfuseScenario |
| | | 13 | | { |
| | | 14 | | private readonly Activity? _activity; |
| | | 15 | | private readonly LangfuseScoreRecorder _recorder; |
| | | 16 | | private readonly string? _sessionId; |
| | | 17 | | |
| | 12 | 18 | | public LangfuseScenario( |
| | 12 | 19 | | LangfuseScoreRecorder recorder, |
| | 12 | 20 | | string name, |
| | 12 | 21 | | string? sessionId, |
| | 12 | 22 | | string? userId, |
| | 12 | 23 | | IEnumerable<string>? tags, |
| | 12 | 24 | | IReadOnlyDictionary<string, string>? metadata) |
| | | 25 | | { |
| | 12 | 26 | | ArgumentNullException.ThrowIfNull(recorder); |
| | 12 | 27 | | ArgumentException.ThrowIfNullOrWhiteSpace(name); |
| | | 28 | | |
| | 12 | 29 | | _recorder = recorder; |
| | 12 | 30 | | _sessionId = sessionId; |
| | 12 | 31 | | _activity = LangfuseActivitySource.Source.StartActivity(name, ActivityKind.Internal); |
| | | 32 | | |
| | 12 | 33 | | ApplyTraceAttributes(_activity, name, sessionId, userId, tags, metadata); |
| | 12 | 34 | | } |
| | | 35 | | |
| | | 36 | | /// <inheritdoc /> |
| | 11 | 37 | | public string? TraceId => _activity?.TraceId.ToString(); |
| | | 38 | | |
| | | 39 | | /// <inheritdoc /> |
| | 2 | 40 | | public Activity? Activity => _activity; |
| | | 41 | | |
| | | 42 | | /// <inheritdoc /> |
| | | 43 | | public Task RecordScoreAsync(string name, double value, string? comment = null, CancellationToken cancellationToken |
| | 3 | 44 | | TraceId is { Length: > 0 } id |
| | 3 | 45 | | ? _recorder.RecordNumericAsync(id, name, value, comment, cancellationToken) |
| | 3 | 46 | | : _recorder.RecordSkippedAsync(name, cancellationToken); |
| | | 47 | | |
| | | 48 | | /// <inheritdoc /> |
| | | 49 | | public Task RecordScoreAsync(string name, bool value, string? comment = null, CancellationToken cancellationToken = |
| | 0 | 50 | | TraceId is { Length: > 0 } id |
| | 0 | 51 | | ? _recorder.RecordBooleanAsync(id, name, value, comment, cancellationToken) |
| | 0 | 52 | | : _recorder.RecordSkippedAsync(name, cancellationToken); |
| | | 53 | | |
| | | 54 | | /// <inheritdoc /> |
| | | 55 | | public Task RecordScoreAsync(string name, string value, string? comment = null, CancellationToken cancellationToken |
| | 0 | 56 | | TraceId is { Length: > 0 } id |
| | 0 | 57 | | ? _recorder.RecordCategoricalAsync(id, name, value, comment, cancellationToken) |
| | 0 | 58 | | : _recorder.RecordSkippedAsync(name, cancellationToken); |
| | | 59 | | |
| | | 60 | | /// <inheritdoc /> |
| | | 61 | | public Task RecordEvaluationAsync(EvaluationResult result, CancellationToken cancellationToken = default) |
| | | 62 | | { |
| | 1 | 63 | | ArgumentNullException.ThrowIfNull(result); |
| | | 64 | | |
| | 1 | 65 | | return TraceId is { Length: > 0 } id |
| | 1 | 66 | | ? _recorder.RecordEvaluationAsync(id, result, cancellationToken) |
| | 1 | 67 | | : _recorder.RecordSkippedAsync("evaluation", cancellationToken); |
| | | 68 | | } |
| | | 69 | | |
| | | 70 | | /// <inheritdoc /> |
| | 12 | 71 | | public void Dispose() => _activity?.Dispose(); |
| | | 72 | | |
| | | 73 | | /// <inheritdoc /> |
| | | 74 | | public void SetTracePublic(bool isPublic = true) => |
| | 1 | 75 | | _activity?.SetTag("langfuse.trace.public", isPublic); |
| | | 76 | | |
| | | 77 | | /// <inheritdoc /> |
| | | 78 | | public void SetVersion(string version) |
| | | 79 | | { |
| | 1 | 80 | | ArgumentException.ThrowIfNullOrWhiteSpace(version); |
| | 1 | 81 | | _activity?.SetTag("langfuse.version", version); |
| | 1 | 82 | | } |
| | | 83 | | |
| | | 84 | | /// <inheritdoc /> |
| | | 85 | | public void SetInput(object input) |
| | | 86 | | { |
| | 1 | 87 | | ArgumentNullException.ThrowIfNull(input); |
| | 1 | 88 | | _activity?.SetTag("langfuse.trace.input", ToAttributeValue(input)); |
| | 1 | 89 | | } |
| | | 90 | | |
| | | 91 | | /// <inheritdoc /> |
| | | 92 | | public void SetOutput(object output) |
| | | 93 | | { |
| | 1 | 94 | | ArgumentNullException.ThrowIfNull(output); |
| | 1 | 95 | | _activity?.SetTag("langfuse.trace.output", ToAttributeValue(output)); |
| | 1 | 96 | | } |
| | | 97 | | |
| | | 98 | | private static string ToAttributeValue(object value) => |
| | 2 | 99 | | value as string ?? JsonSerializer.Serialize(value); |
| | | 100 | | |
| | | 101 | | /// <inheritdoc /> |
| | | 102 | | public Task RecordSessionScoreAsync(string name, double value, string? comment = null, CancellationToken cancellatio |
| | 1 | 103 | | _sessionId is { Length: > 0 } sid |
| | 1 | 104 | | ? _recorder.RecordNumericAsync(LangfuseScoreTarget.Session(sid), name, value, comment, cancellationToken) |
| | 1 | 105 | | : SkipSessionScore(name, cancellationToken); |
| | | 106 | | |
| | | 107 | | /// <inheritdoc /> |
| | | 108 | | public Task RecordSessionScoreAsync(string name, bool value, string? comment = null, CancellationToken cancellationT |
| | 1 | 109 | | _sessionId is { Length: > 0 } sid |
| | 1 | 110 | | ? _recorder.RecordBooleanAsync(LangfuseScoreTarget.Session(sid), name, value, comment, cancellationToken) |
| | 1 | 111 | | : SkipSessionScore(name, cancellationToken); |
| | | 112 | | |
| | | 113 | | /// <inheritdoc /> |
| | | 114 | | public Task RecordSessionScoreAsync(string name, string value, string? comment = null, CancellationToken cancellatio |
| | 0 | 115 | | _sessionId is { Length: > 0 } sid |
| | 0 | 116 | | ? _recorder.RecordCategoricalAsync(LangfuseScoreTarget.Session(sid), name, value, comment, cancellationToken |
| | 0 | 117 | | : SkipSessionScore(name, cancellationToken); |
| | | 118 | | |
| | | 119 | | private Task SkipSessionScore(string name, CancellationToken cancellationToken) => |
| | 1 | 120 | | _recorder.RecordSkippedAsync( |
| | 1 | 121 | | name, |
| | 1 | 122 | | $"Cannot record session score '{name}': this scenario has no session id. " + |
| | 1 | 123 | | "Pass a sessionId when beginning the scenario to enable session-level scoring.", |
| | 1 | 124 | | cancellationToken); |
| | | 125 | | |
| | | 126 | | private static void ApplyTraceAttributes( |
| | | 127 | | Activity? activity, |
| | | 128 | | string name, |
| | | 129 | | string? sessionId, |
| | | 130 | | string? userId, |
| | | 131 | | IEnumerable<string>? tags, |
| | | 132 | | IReadOnlyDictionary<string, string>? metadata) |
| | | 133 | | { |
| | 12 | 134 | | if (activity is null) |
| | | 135 | | { |
| | 0 | 136 | | return; |
| | | 137 | | } |
| | | 138 | | |
| | 12 | 139 | | activity.SetTag("langfuse.trace.name", name); |
| | | 140 | | |
| | 12 | 141 | | if (!string.IsNullOrWhiteSpace(sessionId)) |
| | | 142 | | { |
| | 4 | 143 | | activity.SetTag("langfuse.session.id", sessionId); |
| | 4 | 144 | | activity.SetBaggage("session.id", sessionId); |
| | | 145 | | } |
| | | 146 | | |
| | 12 | 147 | | if (!string.IsNullOrWhiteSpace(userId)) |
| | | 148 | | { |
| | 3 | 149 | | activity.SetTag("langfuse.user.id", userId); |
| | 3 | 150 | | activity.SetBaggage("user.id", userId); |
| | | 151 | | } |
| | | 152 | | |
| | 12 | 153 | | if (tags is not null) |
| | | 154 | | { |
| | 3 | 155 | | var tagArray = tags.Where(t => !string.IsNullOrWhiteSpace(t)).ToArray(); |
| | 1 | 156 | | if (tagArray.Length > 0) |
| | | 157 | | { |
| | 1 | 158 | | activity.SetTag("langfuse.trace.tags", tagArray); |
| | | 159 | | } |
| | | 160 | | } |
| | | 161 | | |
| | 12 | 162 | | if (metadata is not null) |
| | | 163 | | { |
| | 4 | 164 | | foreach (var entry in metadata) |
| | | 165 | | { |
| | 1 | 166 | | if (!string.IsNullOrWhiteSpace(entry.Key)) |
| | | 167 | | { |
| | 1 | 168 | | activity.SetTag($"langfuse.trace.metadata.{entry.Key}", entry.Value); |
| | | 169 | | } |
| | | 170 | | } |
| | | 171 | | } |
| | 12 | 172 | | } |
| | | 173 | | } |