| | | 1 | | using NexusLabs.Needlr.AgentFramework.Context; |
| | | 2 | | using NexusLabs.Needlr.AgentFramework.Diagnostics; |
| | | 3 | | using NexusLabs.Needlr.AgentFramework.Workspace; |
| | | 4 | | |
| | | 5 | | namespace NexusLabs.Needlr.AgentFramework.Testing; |
| | | 6 | | |
| | | 7 | | /// <summary> |
| | | 8 | | /// Executes <see cref="IAgentScenario"/> instances with proper workspace isolation, |
| | | 9 | | /// execution context scoping, and diagnostics capture. |
| | | 10 | | /// </summary> |
| | | 11 | | /// <remarks> |
| | | 12 | | /// <para> |
| | | 13 | | /// The runner handles the full seed → execute → verify lifecycle: |
| | | 14 | | /// </para> |
| | | 15 | | /// <list type="number"> |
| | | 16 | | /// <item>Creates an <see cref="InMemoryWorkspace"/> and calls <see cref="IAgentScenario.SeedWorkspace"/>.</item> |
| | | 17 | | /// <item>Establishes an <see cref="IAgentExecutionContext"/> with user ID and workspace in properties.</item> |
| | | 18 | | /// <item>Opens diagnostics capture.</item> |
| | | 19 | | /// <item>Invokes the agent with the scenario's system and user prompts.</item> |
| | | 20 | | /// <item>Calls <see cref="IAgentScenario.Verify"/> with the post-execution workspace and diagnostics.</item> |
| | | 21 | | /// </list> |
| | | 22 | | /// <para> |
| | | 23 | | /// <strong>Deterministic testing (no real LLM):</strong> The runner uses the <see cref="IAgentFactory"/> |
| | | 24 | | /// from DI, which respects the <c>ChatClientFactory</c> configured on the syringe. To run scenarios |
| | | 25 | | /// without real LLM calls, wire a mock <c>IChatClient</c> via the syringe: |
| | | 26 | | /// </para> |
| | | 27 | | /// <code> |
| | | 28 | | /// var sp = new Syringe() |
| | | 29 | | /// .UsingReflection() |
| | | 30 | | /// .UsingAgentFramework(af => af |
| | | 31 | | /// .Configure(opts => opts.ChatClientFactory = _ => mockChatClient.Object)) |
| | | 32 | | /// .BuildServiceProvider(config); |
| | | 33 | | /// |
| | | 34 | | /// var runner = new AgentScenarioRunner( |
| | | 35 | | /// sp.GetRequiredService<IAgentFactory>(), |
| | | 36 | | /// sp.GetRequiredService<IAgentExecutionContextAccessor>(), |
| | | 37 | | /// sp.GetRequiredService<IAgentDiagnosticsAccessor>()); |
| | | 38 | | /// </code> |
| | | 39 | | /// </remarks> |
| | | 40 | | public sealed class AgentScenarioRunner |
| | | 41 | | { |
| | | 42 | | private readonly IAgentFactory _agentFactory; |
| | | 43 | | private readonly IAgentExecutionContextAccessor _contextAccessor; |
| | | 44 | | private readonly IAgentDiagnosticsAccessor _diagnosticsAccessor; |
| | | 45 | | |
| | | 46 | | /// <param name="agentFactory">Factory for creating agents.</param> |
| | | 47 | | /// <param name="contextAccessor">Execution context accessor for scoping.</param> |
| | | 48 | | /// <param name="diagnosticsAccessor">Diagnostics accessor for capture.</param> |
| | 9 | 49 | | public AgentScenarioRunner( |
| | 9 | 50 | | IAgentFactory agentFactory, |
| | 9 | 51 | | IAgentExecutionContextAccessor contextAccessor, |
| | 9 | 52 | | IAgentDiagnosticsAccessor diagnosticsAccessor) |
| | | 53 | | { |
| | 9 | 54 | | _agentFactory = agentFactory ?? throw new ArgumentNullException(nameof(agentFactory)); |
| | 8 | 55 | | _contextAccessor = contextAccessor ?? throw new ArgumentNullException(nameof(contextAccessor)); |
| | 7 | 56 | | _diagnosticsAccessor = diagnosticsAccessor ?? throw new ArgumentNullException(nameof(diagnosticsAccessor)); |
| | 6 | 57 | | } |
| | | 58 | | |
| | | 59 | | /// <summary> |
| | | 60 | | /// Runs the scenario with full lifecycle management: seed workspace, establish context, |
| | | 61 | | /// capture diagnostics, execute agent, verify outcomes. |
| | | 62 | | /// </summary> |
| | | 63 | | /// <param name="scenario">The scenario to run.</param> |
| | | 64 | | /// <param name="cancellationToken">Cancellation token.</param> |
| | | 65 | | /// <remarks> |
| | | 66 | | /// <see cref="ScenarioRunResult.Diagnostics"/> will be <see langword="null"/> unless the |
| | | 67 | | /// <see cref="IAgentFactory"/> was configured with <c>UsingDiagnostics()</c> on the syringe. |
| | | 68 | | /// Without diagnostics middleware, <see cref="IAgentScenario.Verify"/> receives null diagnostics. |
| | | 69 | | /// </remarks> |
| | | 70 | | /// <returns>The result of the scenario run, including diagnostics and verification outcome.</returns> |
| | | 71 | | public async Task<ScenarioRunResult> RunAsync( |
| | | 72 | | IAgentScenario scenario, |
| | | 73 | | CancellationToken cancellationToken = default) |
| | | 74 | | { |
| | 6 | 75 | | ArgumentNullException.ThrowIfNull(scenario); |
| | | 76 | | |
| | | 77 | | // 1. Create and seed workspace |
| | 5 | 78 | | var workspace = new InMemoryWorkspace(); |
| | 5 | 79 | | scenario.SeedWorkspace(workspace); |
| | | 80 | | |
| | | 81 | | // 2. Create agent with scenario's prompts |
| | 5 | 82 | | var agent = _agentFactory.CreateAgent(opts => |
| | 5 | 83 | | { |
| | 5 | 84 | | opts.Name = $"Scenario-{scenario.Name}"; |
| | 5 | 85 | | opts.Instructions = scenario.SystemPrompt; |
| | 10 | 86 | | }); |
| | | 87 | | |
| | | 88 | | // 3. Establish execution context with workspace in properties |
| | 5 | 89 | | var executionContext = new AgentExecutionContext( |
| | 5 | 90 | | UserId: $"scenario-runner", |
| | 5 | 91 | | OrchestrationId: $"scenario-{scenario.Name}-{Guid.NewGuid():N}", |
| | 5 | 92 | | Properties: new Dictionary<string, object> { ["workspace"] = workspace }); |
| | | 93 | | |
| | 5 | 94 | | IAgentRunDiagnostics? diagnostics = null; |
| | 5 | 95 | | string? responseText = null; |
| | 5 | 96 | | Exception? executionError = null; |
| | | 97 | | |
| | | 98 | | // 4. Run with context + diagnostics scoping |
| | 5 | 99 | | using (_contextAccessor.BeginScope(executionContext)) |
| | 5 | 100 | | using (_diagnosticsAccessor.BeginCapture()) |
| | | 101 | | { |
| | | 102 | | try |
| | | 103 | | { |
| | 5 | 104 | | var response = await agent.RunAsync( |
| | 5 | 105 | | scenario.UserPrompt, |
| | 5 | 106 | | cancellationToken: cancellationToken); |
| | | 107 | | |
| | 5 | 108 | | responseText = response.ToString(); |
| | 5 | 109 | | } |
| | 0 | 110 | | catch (Exception ex) |
| | | 111 | | { |
| | 0 | 112 | | executionError = ex; |
| | 0 | 113 | | } |
| | | 114 | | |
| | 5 | 115 | | diagnostics = _diagnosticsAccessor.LastRunDiagnostics; |
| | 5 | 116 | | } |
| | | 117 | | |
| | | 118 | | // 5. Verify |
| | 5 | 119 | | Exception? verificationError = null; |
| | | 120 | | try |
| | | 121 | | { |
| | 5 | 122 | | scenario.Verify(workspace, diagnostics); |
| | 3 | 123 | | } |
| | | 124 | | catch (Exception ex) |
| | | 125 | | { |
| | 2 | 126 | | verificationError = ex; |
| | 2 | 127 | | } |
| | | 128 | | |
| | 5 | 129 | | return new ScenarioRunResult( |
| | 5 | 130 | | ScenarioName: scenario.Name, |
| | 5 | 131 | | Workspace: workspace, |
| | 5 | 132 | | Diagnostics: diagnostics, |
| | 5 | 133 | | ResponseText: responseText, |
| | 5 | 134 | | ExecutionError: executionError, |
| | 5 | 135 | | VerificationError: verificationError, |
| | 5 | 136 | | Succeeded: executionError is null && verificationError is null); |
| | 5 | 137 | | } |
| | | 138 | | } |
| | | 139 | | |
| | | 140 | | /// <summary> |
| | | 141 | | /// Result of running a single <see cref="IAgentScenario"/>. |
| | | 142 | | /// </summary> |
| | | 143 | | public sealed record ScenarioRunResult( |
| | | 144 | | string ScenarioName, |
| | | 145 | | IWorkspace Workspace, |
| | | 146 | | IAgentRunDiagnostics? Diagnostics, |
| | | 147 | | string? ResponseText, |
| | | 148 | | Exception? ExecutionError, |
| | | 149 | | Exception? VerificationError, |
| | | 150 | | bool Succeeded); |