| | | 1 | | namespace NexusLabs.Needlr.AgentFramework.Budget; |
| | | 2 | | |
| | | 3 | | /// <summary> |
| | | 4 | | /// <see cref="AsyncLocal{T}"/>-scoped implementation of <see cref="ITokenBudgetTracker"/> |
| | | 5 | | /// with granular input/output/total budget tracking. |
| | | 6 | | /// </summary> |
| | | 7 | | public sealed class TokenBudgetTracker : ITokenBudgetTracker |
| | | 8 | | { |
| | 1 | 9 | | private static readonly AsyncLocal<ScopeState?> _current = new(); |
| | | 10 | | |
| | | 11 | | /// <inheritdoc /> |
| | | 12 | | public IDisposable BeginScope(long maxTokens) |
| | | 13 | | { |
| | 23 | 14 | | if (maxTokens <= 0) |
| | 3 | 15 | | throw new ArgumentOutOfRangeException(nameof(maxTokens), "Budget must be greater than zero."); |
| | | 16 | | |
| | 20 | 17 | | return BeginScope(maxInputTokens: null, maxOutputTokens: null, maxTotalTokens: maxTokens); |
| | | 18 | | } |
| | | 19 | | |
| | | 20 | | /// <inheritdoc /> |
| | | 21 | | public IDisposable BeginScope(long? maxInputTokens = null, long? maxOutputTokens = null, long? maxTotalTokens = null |
| | | 22 | | { |
| | 34 | 23 | | var parent = _current.Value; |
| | 34 | 24 | | var scope = new ScopeState(maxInputTokens, maxOutputTokens, maxTotalTokens, parent); |
| | 34 | 25 | | _current.Value = scope; |
| | 34 | 26 | | return scope; |
| | | 27 | | } |
| | | 28 | | |
| | | 29 | | /// <inheritdoc /> |
| | | 30 | | public IDisposable BeginTrackingScope() |
| | | 31 | | { |
| | 5 | 32 | | return BeginScope(null, null, null); |
| | | 33 | | } |
| | | 34 | | |
| | | 35 | | /// <inheritdoc /> |
| | | 36 | | public IDisposable BeginChildScope(string name, long? maxTokens = null) |
| | | 37 | | { |
| | 5 | 38 | | ArgumentException.ThrowIfNullOrWhiteSpace(name); |
| | 5 | 39 | | if (maxTokens is <= 0) |
| | 0 | 40 | | throw new ArgumentOutOfRangeException(nameof(maxTokens), "Child scope budget must be greater than zero."); |
| | | 41 | | |
| | 5 | 42 | | var parent = _current.Value |
| | 5 | 43 | | ?? throw new InvalidOperationException("Cannot open a child scope without an active parent scope."); |
| | | 44 | | |
| | 5 | 45 | | var scope = new ScopeState( |
| | 5 | 46 | | name: name, |
| | 5 | 47 | | maxInputTokens: null, |
| | 5 | 48 | | maxOutputTokens: null, |
| | 5 | 49 | | maxTotalTokens: maxTokens, |
| | 5 | 50 | | parent: parent); |
| | 5 | 51 | | _current.Value = scope; |
| | 5 | 52 | | return scope; |
| | | 53 | | } |
| | | 54 | | |
| | | 55 | | /// <inheritdoc /> |
| | 44 | 56 | | public long CurrentTokens => _current.Value?.CurrentTotalTokens ?? 0L; |
| | | 57 | | |
| | | 58 | | /// <inheritdoc /> |
| | 9 | 59 | | public long CurrentInputTokens => _current.Value?.CurrentInputTokens ?? 0L; |
| | | 60 | | |
| | | 61 | | /// <inheritdoc /> |
| | 9 | 62 | | public long CurrentOutputTokens => _current.Value?.CurrentOutputTokens ?? 0L; |
| | | 63 | | |
| | | 64 | | /// <inheritdoc /> |
| | 54 | 65 | | public long? MaxTokens => _current.Value?.MaxTotalTokens; |
| | | 66 | | |
| | | 67 | | /// <inheritdoc /> |
| | 22 | 68 | | public long? MaxInputTokens => _current.Value?.MaxInputTokens; |
| | | 69 | | |
| | | 70 | | /// <inheritdoc /> |
| | 22 | 71 | | public long? MaxOutputTokens => _current.Value?.MaxOutputTokens; |
| | | 72 | | |
| | | 73 | | /// <inheritdoc /> |
| | | 74 | | public CancellationToken BudgetCancellationToken => |
| | 15 | 75 | | _current.Value?.CancellationToken ?? CancellationToken.None; |
| | | 76 | | |
| | | 77 | | /// <inheritdoc /> |
| | | 78 | | public void Record(long tokenCount) |
| | | 79 | | { |
| | 16 | 80 | | _current.Value?.AddTotal(tokenCount); |
| | 14 | 81 | | } |
| | | 82 | | |
| | | 83 | | /// <inheritdoc /> |
| | | 84 | | public void Record(long inputTokens, long outputTokens) |
| | | 85 | | { |
| | 42 | 86 | | _current.Value?.AddDetailed(inputTokens, outputTokens); |
| | 15 | 87 | | } |
| | | 88 | | |
| | | 89 | | private sealed class ScopeState : IDisposable |
| | | 90 | | { |
| | | 91 | | private long _currentInputTokens; |
| | | 92 | | private long _currentOutputTokens; |
| | | 93 | | private long _currentTotalTokens; |
| | | 94 | | private readonly CancellationTokenSource _cts; |
| | | 95 | | private readonly ScopeState? _parent; |
| | | 96 | | |
| | 39 | 97 | | public ScopeState(long? maxInputTokens, long? maxOutputTokens, long? maxTotalTokens, ScopeState? parent = null, |
| | | 98 | | { |
| | 39 | 99 | | MaxInputTokens = maxInputTokens; |
| | 39 | 100 | | MaxOutputTokens = maxOutputTokens; |
| | 39 | 101 | | MaxTotalTokens = maxTotalTokens; |
| | 39 | 102 | | _parent = parent; |
| | 39 | 103 | | Name = name; |
| | | 104 | | |
| | | 105 | | // Link to parent's CTS so parent cancellation cascades to children |
| | 39 | 106 | | _cts = parent is not null |
| | 39 | 107 | | ? CancellationTokenSource.CreateLinkedTokenSource(parent.CancellationToken) |
| | 39 | 108 | | : new CancellationTokenSource(); |
| | 39 | 109 | | } |
| | | 110 | | |
| | 0 | 111 | | public string? Name { get; } |
| | 46 | 112 | | public long? MaxInputTokens { get; } |
| | 45 | 113 | | public long? MaxOutputTokens { get; } |
| | 108 | 114 | | public long? MaxTotalTokens { get; } |
| | | 115 | | |
| | 25 | 116 | | public long CurrentInputTokens => Volatile.Read(ref _currentInputTokens); |
| | 25 | 117 | | public long CurrentOutputTokens => Volatile.Read(ref _currentOutputTokens); |
| | 40 | 118 | | public long CurrentTotalTokens => Volatile.Read(ref _currentTotalTokens); |
| | | 119 | | |
| | 20 | 120 | | public CancellationToken CancellationToken => _cts.Token; |
| | | 121 | | |
| | | 122 | | public void AddTotal(long tokens) |
| | | 123 | | { |
| | 17 | 124 | | var newTotal = Interlocked.Add(ref _currentTotalTokens, tokens); |
| | 17 | 125 | | CheckBudget(CurrentInputTokens, CurrentOutputTokens, newTotal); |
| | 17 | 126 | | _parent?.AddTotal(tokens); |
| | 3 | 127 | | } |
| | | 128 | | |
| | | 129 | | public void AddDetailed(long inputTokens, long outputTokens) |
| | | 130 | | { |
| | 16 | 131 | | var newInput = Interlocked.Add(ref _currentInputTokens, inputTokens); |
| | 16 | 132 | | var newOutput = Interlocked.Add(ref _currentOutputTokens, outputTokens); |
| | 16 | 133 | | var newTotal = Interlocked.Add(ref _currentTotalTokens, inputTokens + outputTokens); |
| | 16 | 134 | | CheckBudget(newInput, newOutput, newTotal); |
| | 16 | 135 | | _parent?.AddDetailed(inputTokens, outputTokens); |
| | 1 | 136 | | } |
| | | 137 | | |
| | | 138 | | private void CheckBudget(long currentInput, long currentOutput, long currentTotal) |
| | | 139 | | { |
| | 33 | 140 | | if (_cts.IsCancellationRequested) |
| | 0 | 141 | | return; |
| | | 142 | | |
| | 33 | 143 | | bool exceeded = |
| | 33 | 144 | | (MaxTotalTokens.HasValue && currentTotal >= MaxTotalTokens.Value) || |
| | 33 | 145 | | (MaxInputTokens.HasValue && currentInput >= MaxInputTokens.Value) || |
| | 33 | 146 | | (MaxOutputTokens.HasValue && currentOutput >= MaxOutputTokens.Value); |
| | | 147 | | |
| | 33 | 148 | | if (exceeded) |
| | | 149 | | { |
| | 9 | 150 | | _cts.Cancel(); |
| | | 151 | | } |
| | 33 | 152 | | } |
| | | 153 | | |
| | | 154 | | public void Dispose() |
| | | 155 | | { |
| | 39 | 156 | | _current.Value = _parent; |
| | 39 | 157 | | _cts.Dispose(); |
| | 39 | 158 | | } |
| | | 159 | | } |
| | | 160 | | } |