| | | 1 | | using Microsoft.Extensions.AI; |
| | | 2 | | |
| | | 3 | | using NexusLabs.Needlr.AgentFramework.Diagnostics; |
| | | 4 | | |
| | | 5 | | namespace NexusLabs.Needlr.AgentFramework.Evaluation; |
| | | 6 | | |
| | | 7 | | /// <summary> |
| | | 8 | | /// Extensions that convert <see cref="IPipelineRunResult"/> into the input shape expected |
| | | 9 | | /// by <c>Microsoft.Extensions.AI.Evaluation</c> evaluators. |
| | | 10 | | /// </summary> |
| | | 11 | | public static class PipelineEvaluationExtensions |
| | | 12 | | { |
| | | 13 | | /// <summary> |
| | | 14 | | /// Projects a pipeline run result into <see cref="EvaluationInputs"/> for the full |
| | | 15 | | /// pipeline. Input messages are collected from all stages that have diagnostics, and |
| | | 16 | | /// the response is taken from the last stage that produced a non-null |
| | | 17 | | /// <see cref="ChatResponse"/>. |
| | | 18 | | /// </summary> |
| | | 19 | | /// <param name="result">The pipeline run result to convert.</param> |
| | | 20 | | /// <returns> |
| | | 21 | | /// An <see cref="EvaluationInputs"/> whose <see cref="EvaluationInputs.Messages"/> is |
| | | 22 | | /// the union of all stage input messages, and whose |
| | | 23 | | /// <see cref="EvaluationInputs.ModelResponse"/> is the last stage's response. When no |
| | | 24 | | /// stage produced a response, the returned response contains a single empty assistant |
| | | 25 | | /// message so evaluators always receive a non-null <see cref="ChatResponse"/>. |
| | | 26 | | /// </returns> |
| | | 27 | | /// <exception cref="ArgumentNullException"><paramref name="result"/> is <see langword="null"/>.</exception> |
| | | 28 | | public static EvaluationInputs ToEvaluationInputs(this IPipelineRunResult result) |
| | | 29 | | { |
| | 4 | 30 | | ArgumentNullException.ThrowIfNull(result); |
| | | 31 | | |
| | 3 | 32 | | var allInputMessages = result.Stages |
| | 6 | 33 | | .Where(s => s.Diagnostics is not null) |
| | 6 | 34 | | .SelectMany(s => s.Diagnostics!.InputMessages) |
| | 3 | 35 | | .ToList(); |
| | | 36 | | |
| | 3 | 37 | | var lastResponse = result.Stages |
| | 8 | 38 | | .LastOrDefault(s => s.FinalResponse is not null)?.FinalResponse; |
| | 3 | 39 | | var chatResponse = lastResponse |
| | 3 | 40 | | ?? new ChatResponse(new ChatMessage(ChatRole.Assistant, string.Empty)); |
| | | 41 | | |
| | 3 | 42 | | return new EvaluationInputs(allInputMessages, chatResponse); |
| | | 43 | | } |
| | | 44 | | } |