From 3c08272dbd31a20f82bd3af2420ec8f4488f43ee Mon Sep 17 00:00:00 2001 From: Hoang Snowy Date: Wed, 4 Nov 2026 00:35:30 +0700 Subject: [PATCH] fix(agents): specialist answers get a configurable 15-minute limit; GPT-profile roles on claude run as a Claude fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Workflow pipeline's live run on the lab never completed: every Coding and Testing answer was "stopped by the server" at exactly 5:00. LlmRequest.AgentAnswerTimeout was a hard-coded 5 minutes (since #258), and the roles' gpt-4.1 / gpt-4o-mini models were not passed to Claude Code, so the CLI's default — a slow thinking model on that machine — took 3-5+ minutes per answer. - Agents:AnswerTimeoutMinutes (Agent profiles, default 15, 1-60) replaces the constant, read per call through IAgentProfiles for the Workflow specialists, the Autopilot lead, the ticket decomposer and the retrospective. - The runner maps a non-Claude profile model on claude to REMOTE_AGENT_CLAUDE_FALLBACK_MODEL (default "sonnet"; empty keeps the CLI default) instead of silently running the machine's default model. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018WAdQ7Rm24xss5QDSvXtJB --- docs/CONFIG.md | 5 ++++ docs/LAB.md | 5 +++- src/AgentOs.Domain/Llm/LlmRequest.cs | 6 ++-- .../Agents/AgentProfiles.cs | 14 +++++++++ .../Agents/AgentSettings.cs | 15 ++++++++++ .../Agents/AutopilotOrchestratorAgent.cs | 2 +- .../Agents/BoardAgent.cs | 4 +++ .../Agents/LlmAgentBase.cs | 2 +- .../Agents/RetrospectiveAgent.cs | 2 +- .../Decomposition/TicketDecomposerAgent.cs | 2 +- src/AgentOs.RemoteAgent/CliModel.cs | 20 +++++++++++-- src/AgentOs.RemoteAgent/Program.cs | 3 +- .../Agents/AgentProfilesTests.cs | 29 +++++++++++++++++++ tests/AgentOs.Tests/Sessions/CliModelTests.cs | 16 +++++++++- 14 files changed, 113 insertions(+), 12 deletions(-) diff --git a/docs/CONFIG.md b/docs/CONFIG.md index de09be5c..7e80921b 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -182,6 +182,11 @@ aliases); `Temperature` is hidden for models that reject sampling params; `MaxTo Knowledge) before they work. `Pipeline:MaxIterations` (QA-loop cap when a story sets no `NMax`) — default 3, range 1–10. +`Agents:AnswerTimeoutMinutes` (*Answer time limit* in Agent profiles) — how long one specialist answer (a spec, a +Workflow code step, a review, an Autopilot decision, a ticket split, a retrospective) may take on a member's computer +before it is stopped — default 15, range 1–60, read per call. It was a hard-coded 5 minutes until a thinking model's +Workflow Coding answers ran past it and were killed (lab, 2026-11-04). Board runs (Coding, Review, Testing) keep their +own `Autopilot:*RunMinutes` limits. An unpriced model is charged at the fail-closed $15/$75 per 1M rate until added to `CostCalculator`. ### Settings → Autopilot diff --git a/docs/LAB.md b/docs/LAB.md index f5b7104b..5e8926e9 100644 --- a/docs/LAB.md +++ b/docs/LAB.md @@ -161,7 +161,10 @@ with `REMOTE_AGENT_AUTONOMY=off`. With the runner paired, `RUN_AGENTOS_E2E_REAL= The runner keeps two lanes: work runs (scoped: clone, build, push — `REMOTE_AGENT_MAX_PARALLEL`, default 2) and answer-only runs such as a New idea spec or ticket split (`REMOTE_AGENT_MAX_ANSWERS`, default 2), so a person waiting on -the screen never queues behind Autopilot's coding. Specialist answers get a 5-minute deadline (`LlmRequest.AgentAnswerTimeout`). +the screen never queues behind Autopilot's coding. Specialist answers get the *Answer time limit* deadline +(`Agents:AnswerTimeoutMinutes`, default 15 min, Settings › Agent profiles). A profile model claude can't serve (e.g. +`gpt-4.1`) runs as `REMOTE_AGENT_CLAUDE_FALLBACK_MODEL` (default `sonnet`; empty = the CLI's own default) instead of +silently becoming the machine's default model. ### Try the app on the lab diff --git a/src/AgentOs.Domain/Llm/LlmRequest.cs b/src/AgentOs.Domain/Llm/LlmRequest.cs index 08288627..93d4719a 100644 --- a/src/AgentOs.Domain/Llm/LlmRequest.cs +++ b/src/AgentOs.Domain/Llm/LlmRequest.cs @@ -46,9 +46,11 @@ public sealed record LlmRequest( { /// /// Deadline for one specialist answer (a spec, a ticket split, a review) on a paired computer: a CLI answer takes a - /// minute or two, more on a busy machine. Above the 120 s dispatch default, far below a coding run's budget. + /// minute or two, more on a busy machine — and a thinking model doing real Coding work in the Workflow pipeline + /// took over 5 minutes, so the old 5-minute cap killed it mid-answer (lab, 2026-11-04). The default of the + /// `Agents:AnswerTimeoutMinutes` setting; agents read the tenant's value per call. /// - public static readonly System.TimeSpan AgentAnswerTimeout = System.TimeSpan.FromMinutes(5); + public static readonly System.TimeSpan AgentAnswerTimeout = System.TimeSpan.FromMinutes(15); /// /// Optional canonical agent role behind this request ("Requirement"/"Coding"/"Testing"/ diff --git a/src/AgentOs.Modules.Pipeline/Agents/AgentProfiles.cs b/src/AgentOs.Modules.Pipeline/Agents/AgentProfiles.cs index c021fb07..7f9285f7 100644 --- a/src/AgentOs.Modules.Pipeline/Agents/AgentProfiles.cs +++ b/src/AgentOs.Modules.Pipeline/Agents/AgentProfiles.cs @@ -29,6 +29,11 @@ ValueTask RunLimitAsync(string key, CancellationToken cancellationToke /// Whether a Board agent run that ended without its JSON reply is asked once for it, for the current tenant /// (; on by default). ValueTask RecoverMissingReplyAsync(CancellationToken cancellationToken = default) => ValueTask.FromResult(true); + + /// How long one specialist answer may take on a paired computer, for the current tenant + /// (; by default). + ValueTask AnswerTimeoutAsync(CancellationToken cancellationToken = default) => + ValueTask.FromResult(AgentOs.Domain.Llm.LlmRequest.AgentAnswerTimeout); } /// over . @@ -97,6 +102,15 @@ public ValueTask RunLimitAsync(string key, CancellationToken cancellat public ValueTask RecoverMissingReplyAsync(CancellationToken cancellationToken = default) => AutopilotSettings.ReadRecoverMissingReplyAsync(_settings, CurrentTenant(), cancellationToken); + /// + public async ValueTask AnswerTimeoutAsync(CancellationToken cancellationToken = default) + { + var fallback = (int)AgentOs.Domain.Llm.LlmRequest.AgentAnswerTimeout.TotalMinutes; + var minutes = await _settings.GetIntAsync(AgentSettings.AnswerTimeoutMinutes, CurrentTenant(), fallback, cancellationToken) + .ConfigureAwait(false); + return TimeSpan.FromMinutes(Math.Clamp(minutes, AgentSettings.AnswerTimeoutMin, AgentSettings.AnswerTimeoutMax)); + } + // Ambient identity first (Blazor circuits and background runs push it); else the request's tenant context. private string? CurrentTenant() { diff --git a/src/AgentOs.Modules.Pipeline/Agents/AgentSettings.cs b/src/AgentOs.Modules.Pipeline/Agents/AgentSettings.cs index b6ab7b2b..a48fcdd5 100644 --- a/src/AgentOs.Modules.Pipeline/Agents/AgentSettings.cs +++ b/src/AgentOs.Modules.Pipeline/Agents/AgentSettings.cs @@ -25,6 +25,12 @@ public sealed class AgentSettings : ISettingsContributor /// QA-loop iteration cap. public const string MaxIterations = "Pipeline:MaxIterations"; + /// How long one specialist answer (a spec, a code step, a review, a decision) may take on a paired computer. + public const string AnswerTimeoutMinutes = "Agents:AnswerTimeoutMinutes"; + + /// Bounds of . + public const int AnswerTimeoutMin = 1, AnswerTimeoutMax = 60; + /// Largest output any current model accepts (Claude 5.x / GPT long-context: 128K). The per-model /// limit is enforced against the model catalog; this is the absolute ceiling. public const int MaxOutputTokensCeiling = 128_000; @@ -80,6 +86,15 @@ public IEnumerable Describe() Max = 10m, Help = "How many Coding → Testing → QA rounds before the run stops.", }; + + yield return new(AnswerTimeoutMinutes, SettingKind.WholeNumber, Category, "Answer time limit (minutes)") + { + DefaultValue = ((int)LlmRequest.AgentAnswerTimeout.TotalMinutes).ToString(CultureInfo.InvariantCulture), + Min = AnswerTimeoutMin, + Max = AnswerTimeoutMax, + Help = "How long one agent answer may take on a member's computer before it is stopped. Thinking models doing real " + + "work need several minutes; Board runs (Coding, Review, Testing) have their own limits under Autopilot.", + }; } // One row per registered role, each seeded from AgentsOptions (a role without a seed is a startup bug). diff --git a/src/AgentOs.Modules.Pipeline/Agents/AutopilotOrchestratorAgent.cs b/src/AgentOs.Modules.Pipeline/Agents/AutopilotOrchestratorAgent.cs index 0a1cb123..fb9e3334 100644 --- a/src/AgentOs.Modules.Pipeline/Agents/AutopilotOrchestratorAgent.cs +++ b/src/AgentOs.Modules.Pipeline/Agents/AutopilotOrchestratorAgent.cs @@ -81,7 +81,7 @@ public async Task DecideAsync(OrchestratorBrief brief, Can var request = new LlmRequest( await BoardAgent.SystemPromptAsync(prompts, AgentRoles.AutopilotOrchestrator, Instructions, null, UntrustedText.SystemRule, ct).ConfigureAwait(false), User(brief) + await PracticesAsync(brief, ct).ConfigureAwait(false), - profile.Model, profile.Temperature, profile.MaxTokens, Timeout: LlmRequest.AgentAnswerTimeout) + profile.Model, profile.Temperature, profile.MaxTokens, Timeout: await BoardAgent.AnswerTimeoutAsync(profiles, ct).ConfigureAwait(false)) { AgentRole = AgentRoles.AutopilotOrchestrator, }; diff --git a/src/AgentOs.Modules.Pipeline/Agents/BoardAgent.cs b/src/AgentOs.Modules.Pipeline/Agents/BoardAgent.cs index 3cf75120..0d239913 100644 --- a/src/AgentOs.Modules.Pipeline/Agents/BoardAgent.cs +++ b/src/AgentOs.Modules.Pipeline/Agents/BoardAgent.cs @@ -17,6 +17,10 @@ public static class BoardAgent { private static readonly AgentsOptions Seeds = new(); + /// The tenant's answer time limit (), or the default without profiles. + public static ValueTask AnswerTimeoutAsync(IAgentProfiles? profiles, CancellationToken ct) => + profiles is null ? ValueTask.FromResult(AgentOs.Domain.Llm.LlmRequest.AgentAnswerTimeout) : profiles.AnswerTimeoutAsync(ct); + /// The live profile for : the settings registry when wired, else the seed. public static async ValueTask ProfileAsync(IAgentProfiles? profiles, IOptions? options, string role, CancellationToken ct) diff --git a/src/AgentOs.Modules.Pipeline/Agents/LlmAgentBase.cs b/src/AgentOs.Modules.Pipeline/Agents/LlmAgentBase.cs index 5509c12c..ba3afdcc 100644 --- a/src/AgentOs.Modules.Pipeline/Agents/LlmAgentBase.cs +++ b/src/AgentOs.Modules.Pipeline/Agents/LlmAgentBase.cs @@ -123,7 +123,7 @@ protected async Task ExecuteAsync( // parsing the system prompt — robust against per-tenant prompt overrides — and Insights files the call // under it. AgentRole = role, - Timeout = LlmRequest.AgentAnswerTimeout, + Timeout = _profiles is null ? LlmRequest.AgentAnswerTimeout : await _profiles.AnswerTimeoutAsync(ct).ConfigureAwait(false), }; var response = await llm.SendAsync(request, ct).ConfigureAwait(false); diff --git a/src/AgentOs.Modules.Pipeline/Agents/RetrospectiveAgent.cs b/src/AgentOs.Modules.Pipeline/Agents/RetrospectiveAgent.cs index 6ddc1e9b..6817cb22 100644 --- a/src/AgentOs.Modules.Pipeline/Agents/RetrospectiveAgent.cs +++ b/src/AgentOs.Modules.Pipeline/Agents/RetrospectiveAgent.cs @@ -71,7 +71,7 @@ public async Task RunAsync(RetrospectiveInput input, Cancel } var request = new LlmRequest( await BoardAgent.SystemPromptAsync(prompts, AgentRoles.Retrospective, Instructions, null, UntrustedText.SystemRule, ct).ConfigureAwait(false), - User(input), profile.Model, profile.Temperature, profile.MaxTokens, Timeout: LlmRequest.AgentAnswerTimeout) + User(input), profile.Model, profile.Temperature, profile.MaxTokens, Timeout: await BoardAgent.AnswerTimeoutAsync(profiles, ct).ConfigureAwait(false)) { AgentRole = AgentRoles.Retrospective, }; diff --git a/src/AgentOs.Modules.Pipeline/Decomposition/TicketDecomposerAgent.cs b/src/AgentOs.Modules.Pipeline/Decomposition/TicketDecomposerAgent.cs index 3ada01cf..dde294c5 100644 --- a/src/AgentOs.Modules.Pipeline/Decomposition/TicketDecomposerAgent.cs +++ b/src/AgentOs.Modules.Pipeline/Decomposition/TicketDecomposerAgent.cs @@ -86,7 +86,7 @@ public async Task> RunAsync(RequirementSpec spec, IRe Model: profile.Model, Temperature: profile.Temperature, MaxTokens: profile.MaxTokens, - Timeout: LlmRequest.AgentAnswerTimeout) + Timeout: await BoardAgent.AnswerTimeoutAsync(_profiles, cancellationToken).ConfigureAwait(false)) { AgentRole = AgentRoles.Discovery, }; diff --git a/src/AgentOs.RemoteAgent/CliModel.cs b/src/AgentOs.RemoteAgent/CliModel.cs index 793971e8..2c6b6d37 100644 --- a/src/AgentOs.RemoteAgent/CliModel.cs +++ b/src/AgentOs.RemoteAgent/CliModel.cs @@ -1,7 +1,10 @@ // The model an agent's profile picked (Settings → Agent profiles) reaches the CLI. The server sends it on every request; // the runner used to log it and run the CLI's own default model anyway. Claude Code takes `--model `, Codex takes -// `-m `. No model, a model that CLI cannot serve (a GPT model for claude, a Claude model for codex), an unknown CLI, -// or an operator's REMOTE_AGENT_ARGS that already names a model → no flag, the CLI's default applies. +// `-m `. No model, a Claude model for codex, an unknown CLI, or an operator's REMOTE_AGENT_ARGS that already names a +// model → no flag, the CLI's default applies. A non-Claude model for claude (a profile set to gpt-4.1 run on a member's +// Claude Code) gets the configured Claude fallback (REMOTE_AGENT_CLAUDE_FALLBACK_MODEL, default "sonnet") instead of the +// CLI default: the default on a member's machine may be a slow thinking model, and Workflow Coding steps on it ran past +// the answer deadline (lab, 2026-11-04). An empty fallback keeps the CLI default. using System; using System.Collections.Generic; @@ -13,6 +16,12 @@ namespace AgentOs.RemoteAgent; /// Builds the model flag for a CLI agent. internal static partial class CliModel { + /// The Claude model a non-Claude profile model runs as on claude (an alias Claude Code resolves). + internal const string DefaultClaudeFallback = "sonnet"; + + /// The runner's environment variable that overrides (empty = CLI default). + internal const string ClaudeFallbackVariable = "REMOTE_AGENT_CLAUDE_FALLBACK_MODEL"; + // Claude Code's model aliases (besides full claude-* ids). private static readonly HashSet ClaudeAliases = new(StringComparer.OrdinalIgnoreCase) { "opus", "sonnet", "haiku", "fable", "opusplan" }; @@ -20,7 +29,10 @@ internal static partial class CliModel /// The CLI command or path (claude, C:\…\codex.cmd). /// The model id the request carries (may be empty). /// The arguments already chosen; one that names a model wins. - internal static string[] Flags(string command, string? model, IReadOnlyCollection existingArgs) + /// The Claude model claude runs when is not a Claude model + /// (empty = the CLI's default). + internal static string[] Flags(string command, string? model, IReadOnlyCollection existingArgs, + string? claudeFallback = DefaultClaudeFallback) { ArgumentNullException.ThrowIfNull(existingArgs); var id = (model ?? string.Empty).Trim(); @@ -28,9 +40,11 @@ internal static string[] Flags(string command, string? model, IReadOnlyCollectio if (existingArgs.Any(a => a is "--model" or "-m" || a.StartsWith("--model=", StringComparison.Ordinal))) { return []; } var isClaudeModel = id.StartsWith("claude", StringComparison.OrdinalIgnoreCase) || ClaudeAliases.Contains(id); + var fallback = (claudeFallback ?? string.Empty).Trim(); return CommandName(command) switch { "claude" when isClaudeModel => ["--model", id], + "claude" when fallback.Length > 0 && SafeId().IsMatch(fallback) => ["--model", fallback], "codex" when !isClaudeModel => ["-m", id], _ => [], }; diff --git a/src/AgentOs.RemoteAgent/Program.cs b/src/AgentOs.RemoteAgent/Program.cs index b1473baa..09e31096 100644 --- a/src/AgentOs.RemoteAgent/Program.cs +++ b/src/AgentOs.RemoteAgent/Program.cs @@ -349,7 +349,8 @@ static async Task RunLlmAsync(RemoteExecRequest request, Func< : scoped ? [.. profile.DefaultArgs, .. AgentOs.RemoteAgent.ScopedAutonomy.ArgsFor(cmd)] : profile.DefaultArgs; // The model the agent's profile picked (Settings → Agent profiles) — `--model` for claude, `-m` for codex (CliModel.cs). - var modelFlags = AgentOs.RemoteAgent.CliModel.Flags(cmd, request.Model, args); + var modelFlags = AgentOs.RemoteAgent.CliModel.Flags(cmd, request.Model, args, + Environment.GetEnvironmentVariable(AgentOs.RemoteAgent.CliModel.ClaudeFallbackVariable) ?? AgentOs.RemoteAgent.CliModel.DefaultClaudeFallback); args = [.. args, .. modelFlags]; // Claude with the runner's own flags streams its events, so the Board can show what it is doing (CliProgress.cs). // A custom REMOTE_AGENT_ARGS keeps exactly what the operator asked for. diff --git a/tests/AgentOs.Tests/Agents/AgentProfilesTests.cs b/tests/AgentOs.Tests/Agents/AgentProfilesTests.cs index c9375b66..0f3497db 100644 --- a/tests/AgentOs.Tests/Agents/AgentProfilesTests.cs +++ b/tests/AgentOs.Tests/Agents/AgentProfilesTests.cs @@ -79,6 +79,35 @@ public async Task Agent_ProfileEditedBetweenCalls_NextCallUsesNewProviderAndMode calls[^1].Request.Model.ShouldBe("claude-sonnet-5-5"); } + // A thinking model's Coding answer ran past the old hard-coded 5-minute cap and was killed (lab, 2026-11-04): the + // deadline is a per-tenant setting now, read per call, bounded. + [Fact] + public async Task Agent_AnswerTimeout_ComesFromTheTenantSetting_PerCall_AndIsBounded() + { + var settings = Settings(); + var (factory, calls) = RecordingFactory(); + var agent = Agent(factory, new SettingsAgentProfiles(settings)); + + using (AmbientIdentity.Push("acme", "u1")) + { + await agent.RunAsync(new UserStory("Build a todo app")); + calls[^1].Request.Timeout.ShouldBe(LlmRequest.AgentAnswerTimeout); // default + LlmRequest.AgentAnswerTimeout.ShouldBe(TimeSpan.FromMinutes(15)); + + (await settings.SetAsync(AgentSettings.AnswerTimeoutMinutes, "30", SettingScope.Tenant, "acme", "admin")).Ok.ShouldBeTrue(); + await agent.RunAsync(new UserStory("Build a todo app")); // same instance — next call + calls[^1].Request.Timeout.ShouldBe(TimeSpan.FromMinutes(30)); + } + using (AmbientIdentity.Push("globex", "u2")) + { + await agent.RunAsync(new UserStory("Build a todo app")); + calls[^1].Request.Timeout.ShouldBe(LlmRequest.AgentAnswerTimeout); // another tenant keeps the default + } + + (await settings.SetAsync(AgentSettings.AnswerTimeoutMinutes, "500", SettingScope.Tenant, "acme", "admin")).Ok + .ShouldBeFalse(); // above the descriptor's Max + } + [Fact] public async Task Agent_TenantOverride_DoesNotLeakToAnotherTenant() { diff --git a/tests/AgentOs.Tests/Sessions/CliModelTests.cs b/tests/AgentOs.Tests/Sessions/CliModelTests.cs index f9e8fcb0..b55dc6ed 100644 --- a/tests/AgentOs.Tests/Sessions/CliModelTests.cs +++ b/tests/AgentOs.Tests/Sessions/CliModelTests.cs @@ -31,7 +31,6 @@ public void Codex_GetsDashM(string command, string model) => [InlineData("claude", "")] [InlineData("claude", " ")] [InlineData("claude", null)] - [InlineData("claude", "gpt-5")] // a model claude cannot serve: its default applies [InlineData("codex", "claude-sonnet-4-6")] // a model codex cannot serve [InlineData("my-agent", "claude-sonnet-4-6")] // an unknown CLI: never guess its flags [InlineData("claude", "--dangerously-skip-permissions")] // never let a model id become a flag @@ -39,6 +38,21 @@ public void Codex_GetsDashM(string command, string model) => public void NoFlag_WhenThereIsNothingTheCliCanUse(string command, string? model) => CliModel.Flags(command, model, ClaudeDefaults).ShouldBeEmpty(); + // A non-Claude profile model on claude runs as the Claude fallback, never silently as the CLI default (a slow + // thinking model ran Workflow Coding steps past the answer deadline on the lab, 2026-11-04). + [Theory] + [InlineData("gpt-5")] + [InlineData("gpt-4.1")] + [InlineData("gpt-4o-mini")] + public void Claude_NonClaudeModel_RunsAsTheClaudeFallback(string model) + { + CliModel.Flags("claude", model, ClaudeDefaults).ShouldBe(["--model", CliModel.DefaultClaudeFallback]); + CliModel.Flags("claude", model, ClaudeDefaults, "claude-haiku-4-5").ShouldBe(["--model", "claude-haiku-4-5"]); + CliModel.Flags("claude", model, ClaudeDefaults, "").ShouldBeEmpty(); // empty = the CLI default + CliModel.Flags("claude", model, ClaudeDefaults, "--yolo").ShouldBeEmpty(); // never a flag + CliModel.Flags("claude", model, ["-p", "--model", "opus"], "sonnet").ShouldBeEmpty(); // operator args win + } + [Theory] [InlineData("--model", "opus")] [InlineData("-m", "x")]