Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/CONFIG.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@ aliases); `Temperature` is hidden for models that reject sampling params; `MaxTo
Knowledge) before they work.

`Pipeline:MaxIterations` (QA-loop cap when a story sets no `NMax`) — default 3, range 1–10.
`Agents:AnswerTimeoutMinutes` (*Answer time limit* in Agent profiles) — how long one specialist answer (a spec, a
Workflow code step, a review, an Autopilot decision, a ticket split, a retrospective) may take on a member's computer
before it is stopped — default 15, range 1–60, read per call. It was a hard-coded 5 minutes until a thinking model's
Workflow Coding answers ran past it and were killed (lab, 2026-11-04). Board runs (Coding, Review, Testing) keep their
own `Autopilot:*RunMinutes` limits.
An unpriced model is charged at the fail-closed $15/$75 per 1M rate until added to `CostCalculator`.

### Settings → Autopilot
Expand Down
5 changes: 4 additions & 1 deletion docs/LAB.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,10 @@ with `REMOTE_AGENT_AUTONOMY=off`. With the runner paired, `RUN_AGENTOS_E2E_REAL=

The runner keeps two lanes: work runs (scoped: clone, build, push — `REMOTE_AGENT_MAX_PARALLEL`, default 2) and
answer-only runs such as a New idea spec or ticket split (`REMOTE_AGENT_MAX_ANSWERS`, default 2), so a person waiting on
the screen never queues behind Autopilot's coding. Specialist answers get a 5-minute deadline (`LlmRequest.AgentAnswerTimeout`).
the screen never queues behind Autopilot's coding. Specialist answers get the *Answer time limit* deadline
(`Agents:AnswerTimeoutMinutes`, default 15 min, Settings › Agent profiles). A profile model claude can't serve (e.g.
`gpt-4.1`) runs as `REMOTE_AGENT_CLAUDE_FALLBACK_MODEL` (default `sonnet`; empty = the CLI's own default) instead of
silently becoming the machine's default model.

### Try the app on the lab

Expand Down
6 changes: 4 additions & 2 deletions src/AgentOs.Domain/Llm/LlmRequest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,11 @@ public sealed record LlmRequest(
{
/// <summary>
/// Deadline for one specialist answer (a spec, a ticket split, a review) on a paired computer: a CLI answer takes a
/// minute or two, more on a busy machine. Above the 120 s dispatch default, far below a coding run's budget.
/// minute or two, more on a busy machine — and a thinking model doing real Coding work in the Workflow pipeline
/// took over 5 minutes, so the old 5-minute cap killed it mid-answer (lab, 2026-11-04). The default of the
/// `Agents:AnswerTimeoutMinutes` setting; agents read the tenant's value per call.
/// </summary>
public static readonly System.TimeSpan AgentAnswerTimeout = System.TimeSpan.FromMinutes(5);
public static readonly System.TimeSpan AgentAnswerTimeout = System.TimeSpan.FromMinutes(15);

/// <summary>
/// Optional canonical agent role behind this request (<c>"Requirement"</c>/<c>"Coding"</c>/<c>"Testing"</c>/
Expand Down
14 changes: 14 additions & 0 deletions src/AgentOs.Modules.Pipeline/Agents/AgentProfiles.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ ValueTask<TimeSpan> RunLimitAsync(string key, CancellationToken cancellationToke
/// <summary>Whether a Board agent run that ended without its JSON reply is asked once for it, for the current tenant
/// (<see cref="AutopilotSettings.RecoverMissingReplyKey"/>; on by default).</summary>
ValueTask<bool> RecoverMissingReplyAsync(CancellationToken cancellationToken = default) => ValueTask.FromResult(true);

/// <summary>How long one specialist answer may take on a paired computer, for the current tenant
/// (<see cref="AgentSettings.AnswerTimeoutMinutes"/>; <see cref="AgentOs.Domain.Llm.LlmRequest.AgentAnswerTimeout"/> by default).</summary>
ValueTask<TimeSpan> AnswerTimeoutAsync(CancellationToken cancellationToken = default) =>
ValueTask.FromResult(AgentOs.Domain.Llm.LlmRequest.AgentAnswerTimeout);
}

/// <summary><see cref="IAgentProfiles"/> over <see cref="ISettings"/>.</summary>
Expand Down Expand Up @@ -97,6 +102,15 @@ public ValueTask<TimeSpan> RunLimitAsync(string key, CancellationToken cancellat
public ValueTask<bool> RecoverMissingReplyAsync(CancellationToken cancellationToken = default) =>
AutopilotSettings.ReadRecoverMissingReplyAsync(_settings, CurrentTenant(), cancellationToken);

/// <inheritdoc />
public async ValueTask<TimeSpan> AnswerTimeoutAsync(CancellationToken cancellationToken = default)
{
var fallback = (int)AgentOs.Domain.Llm.LlmRequest.AgentAnswerTimeout.TotalMinutes;
var minutes = await _settings.GetIntAsync(AgentSettings.AnswerTimeoutMinutes, CurrentTenant(), fallback, cancellationToken)
.ConfigureAwait(false);
return TimeSpan.FromMinutes(Math.Clamp(minutes, AgentSettings.AnswerTimeoutMin, AgentSettings.AnswerTimeoutMax));
}

// Ambient identity first (Blazor circuits and background runs push it); else the request's tenant context.
private string? CurrentTenant()
{
Expand Down
15 changes: 15 additions & 0 deletions src/AgentOs.Modules.Pipeline/Agents/AgentSettings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@ public sealed class AgentSettings : ISettingsContributor
/// <summary>QA-loop iteration cap.</summary>
public const string MaxIterations = "Pipeline:MaxIterations";

/// <summary>How long one specialist answer (a spec, a code step, a review, a decision) may take on a paired computer.</summary>
public const string AnswerTimeoutMinutes = "Agents:AnswerTimeoutMinutes";

/// <summary>Bounds of <see cref="AnswerTimeoutMinutes"/>.</summary>
public const int AnswerTimeoutMin = 1, AnswerTimeoutMax = 60;

/// <summary>Largest output any current model accepts (Claude 5.x / GPT long-context: 128K). The per-model
/// limit is enforced against the model catalog; this is the absolute ceiling.</summary>
public const int MaxOutputTokensCeiling = 128_000;
Expand Down Expand Up @@ -80,6 +86,15 @@ public IEnumerable<SettingDescriptor> Describe()
Max = 10m,
Help = "How many Coding → Testing → QA rounds before the run stops.",
};

yield return new(AnswerTimeoutMinutes, SettingKind.WholeNumber, Category, "Answer time limit (minutes)")
{
DefaultValue = ((int)LlmRequest.AgentAnswerTimeout.TotalMinutes).ToString(CultureInfo.InvariantCulture),
Min = AnswerTimeoutMin,
Max = AnswerTimeoutMax,
Help = "How long one agent answer may take on a member's computer before it is stopped. Thinking models doing real "
+ "work need several minutes; Board runs (Coding, Review, Testing) have their own limits under Autopilot.",
};
}

// One row per registered role, each seeded from AgentsOptions (a role without a seed is a startup bug).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public async Task<OrchestratorDecision> DecideAsync(OrchestratorBrief brief, Can
var request = new LlmRequest(
await BoardAgent.SystemPromptAsync(prompts, AgentRoles.AutopilotOrchestrator, Instructions, null, UntrustedText.SystemRule, ct).ConfigureAwait(false),
User(brief) + await PracticesAsync(brief, ct).ConfigureAwait(false),
profile.Model, profile.Temperature, profile.MaxTokens, Timeout: LlmRequest.AgentAnswerTimeout)
profile.Model, profile.Temperature, profile.MaxTokens, Timeout: await BoardAgent.AnswerTimeoutAsync(profiles, ct).ConfigureAwait(false))
{
AgentRole = AgentRoles.AutopilotOrchestrator,
};
Expand Down
4 changes: 4 additions & 0 deletions src/AgentOs.Modules.Pipeline/Agents/BoardAgent.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ public static class BoardAgent
{
private static readonly AgentsOptions Seeds = new();

/// <summary>The tenant's answer time limit (<see cref="AgentSettings.AnswerTimeoutMinutes"/>), or the default without profiles.</summary>
public static ValueTask<TimeSpan> AnswerTimeoutAsync(IAgentProfiles? profiles, CancellationToken ct) =>
profiles is null ? ValueTask.FromResult(AgentOs.Domain.Llm.LlmRequest.AgentAnswerTimeout) : profiles.AnswerTimeoutAsync(ct);

/// <summary>The live profile for <paramref name="role"/>: the settings registry when wired, else the seed.</summary>
public static async ValueTask<AgentOptions> ProfileAsync(IAgentProfiles? profiles, IOptions<AgentsOptions>? options, string role,
CancellationToken ct)
Expand Down
2 changes: 1 addition & 1 deletion src/AgentOs.Modules.Pipeline/Agents/LlmAgentBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ protected async Task<TResult> ExecuteAsync<TDto, TResult>(
// parsing the system prompt — robust against per-tenant prompt overrides — and Insights files the call
// under it.
AgentRole = role,
Timeout = LlmRequest.AgentAnswerTimeout,
Timeout = _profiles is null ? LlmRequest.AgentAnswerTimeout : await _profiles.AnswerTimeoutAsync(ct).ConfigureAwait(false),
};
var response = await llm.SendAsync(request, ct).ConfigureAwait(false);

Expand Down
2 changes: 1 addition & 1 deletion src/AgentOs.Modules.Pipeline/Agents/RetrospectiveAgent.cs
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public async Task<RetrospectiveResult> RunAsync(RetrospectiveInput input, Cancel
}
var request = new LlmRequest(
await BoardAgent.SystemPromptAsync(prompts, AgentRoles.Retrospective, Instructions, null, UntrustedText.SystemRule, ct).ConfigureAwait(false),
User(input), profile.Model, profile.Temperature, profile.MaxTokens, Timeout: LlmRequest.AgentAnswerTimeout)
User(input), profile.Model, profile.Temperature, profile.MaxTokens, Timeout: await BoardAgent.AnswerTimeoutAsync(profiles, ct).ConfigureAwait(false))
{
AgentRole = AgentRoles.Retrospective,
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public async Task<IReadOnlyList<TicketDraft>> RunAsync(RequirementSpec spec, IRe
Model: profile.Model,
Temperature: profile.Temperature,
MaxTokens: profile.MaxTokens,
Timeout: LlmRequest.AgentAnswerTimeout)
Timeout: await BoardAgent.AnswerTimeoutAsync(_profiles, cancellationToken).ConfigureAwait(false))
{
AgentRole = AgentRoles.Discovery,
};
Expand Down
20 changes: 17 additions & 3 deletions src/AgentOs.RemoteAgent/CliModel.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
// The model an agent's profile picked (Settings → Agent profiles) reaches the CLI. The server sends it on every request;
// the runner used to log it and run the CLI's own default model anyway. Claude Code takes `--model <id>`, Codex takes
// `-m <id>`. No model, a model that CLI cannot serve (a GPT model for claude, a Claude model for codex), an unknown CLI,
// or an operator's REMOTE_AGENT_ARGS that already names a model → no flag, the CLI's default applies.
// `-m <id>`. No model, a Claude model for codex, an unknown CLI, or an operator's REMOTE_AGENT_ARGS that already names a
// model → no flag, the CLI's default applies. A non-Claude model for claude (a profile set to gpt-4.1 run on a member's
// Claude Code) gets the configured Claude fallback (REMOTE_AGENT_CLAUDE_FALLBACK_MODEL, default "sonnet") instead of the
// CLI default: the default on a member's machine may be a slow thinking model, and Workflow Coding steps on it ran past
// the answer deadline (lab, 2026-11-04). An empty fallback keeps the CLI default.

using System;
using System.Collections.Generic;
Expand All @@ -13,24 +16,35 @@ namespace AgentOs.RemoteAgent;
/// <summary>Builds the model flag for a CLI agent.</summary>
internal static partial class CliModel
{
/// <summary>The Claude model a non-Claude profile model runs as on claude (an alias Claude Code resolves).</summary>
internal const string DefaultClaudeFallback = "sonnet";

/// <summary>The runner's environment variable that overrides <see cref="DefaultClaudeFallback"/> (empty = CLI default).</summary>
internal const string ClaudeFallbackVariable = "REMOTE_AGENT_CLAUDE_FALLBACK_MODEL";

// Claude Code's model aliases (besides full claude-* ids).
private static readonly HashSet<string> ClaudeAliases = new(StringComparer.OrdinalIgnoreCase) { "opus", "sonnet", "haiku", "fable", "opusplan" };

/// <summary>The arguments that select <paramref name="model"/> on <paramref name="command"/> (empty = none).</summary>
/// <param name="command">The CLI command or path (<c>claude</c>, <c>C:\…\codex.cmd</c>).</param>
/// <param name="model">The model id the request carries (may be empty).</param>
/// <param name="existingArgs">The arguments already chosen; one that names a model wins.</param>
internal static string[] Flags(string command, string? model, IReadOnlyCollection<string> existingArgs)
/// <param name="claudeFallback">The Claude model claude runs when <paramref name="model"/> is not a Claude model
/// (empty = the CLI's default).</param>
internal static string[] Flags(string command, string? model, IReadOnlyCollection<string> existingArgs,
string? claudeFallback = DefaultClaudeFallback)
{
ArgumentNullException.ThrowIfNull(existingArgs);
var id = (model ?? string.Empty).Trim();
if (id.Length == 0 || !SafeId().IsMatch(id)) { return []; }
if (existingArgs.Any(a => a is "--model" or "-m" || a.StartsWith("--model=", StringComparison.Ordinal))) { return []; }

var isClaudeModel = id.StartsWith("claude", StringComparison.OrdinalIgnoreCase) || ClaudeAliases.Contains(id);
var fallback = (claudeFallback ?? string.Empty).Trim();
return CommandName(command) switch
{
"claude" when isClaudeModel => ["--model", id],
"claude" when fallback.Length > 0 && SafeId().IsMatch(fallback) => ["--model", fallback],
"codex" when !isClaudeModel => ["-m", id],
_ => [],
};
Expand Down
3 changes: 2 additions & 1 deletion src/AgentOs.RemoteAgent/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,8 @@ static async Task<RemoteExecResult> RunLlmAsync(RemoteExecRequest request, Func<
: scoped ? [.. profile.DefaultArgs, .. AgentOs.RemoteAgent.ScopedAutonomy.ArgsFor(cmd)]
: profile.DefaultArgs;
// The model the agent's profile picked (Settings → Agent profiles) — `--model` for claude, `-m` for codex (CliModel.cs).
var modelFlags = AgentOs.RemoteAgent.CliModel.Flags(cmd, request.Model, args);
var modelFlags = AgentOs.RemoteAgent.CliModel.Flags(cmd, request.Model, args,
Environment.GetEnvironmentVariable(AgentOs.RemoteAgent.CliModel.ClaudeFallbackVariable) ?? AgentOs.RemoteAgent.CliModel.DefaultClaudeFallback);
args = [.. args, .. modelFlags];
// Claude with the runner's own flags streams its events, so the Board can show what it is doing (CliProgress.cs).
// A custom REMOTE_AGENT_ARGS keeps exactly what the operator asked for.
Expand Down
29 changes: 29 additions & 0 deletions tests/AgentOs.Tests/Agents/AgentProfilesTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,35 @@ public async Task Agent_ProfileEditedBetweenCalls_NextCallUsesNewProviderAndMode
calls[^1].Request.Model.ShouldBe("claude-sonnet-5-5");
}

// A thinking model's Coding answer ran past the old hard-coded 5-minute cap and was killed (lab, 2026-11-04): the
// deadline is a per-tenant setting now, read per call, bounded.
[Fact]
public async Task Agent_AnswerTimeout_ComesFromTheTenantSetting_PerCall_AndIsBounded()
{
var settings = Settings();
var (factory, calls) = RecordingFactory();
var agent = Agent(factory, new SettingsAgentProfiles(settings));

using (AmbientIdentity.Push("acme", "u1"))
{
await agent.RunAsync(new UserStory("Build a todo app"));
calls[^1].Request.Timeout.ShouldBe(LlmRequest.AgentAnswerTimeout); // default
LlmRequest.AgentAnswerTimeout.ShouldBe(TimeSpan.FromMinutes(15));

(await settings.SetAsync(AgentSettings.AnswerTimeoutMinutes, "30", SettingScope.Tenant, "acme", "admin")).Ok.ShouldBeTrue();
await agent.RunAsync(new UserStory("Build a todo app")); // same instance — next call
calls[^1].Request.Timeout.ShouldBe(TimeSpan.FromMinutes(30));
}
using (AmbientIdentity.Push("globex", "u2"))
{
await agent.RunAsync(new UserStory("Build a todo app"));
calls[^1].Request.Timeout.ShouldBe(LlmRequest.AgentAnswerTimeout); // another tenant keeps the default
}

(await settings.SetAsync(AgentSettings.AnswerTimeoutMinutes, "500", SettingScope.Tenant, "acme", "admin")).Ok
.ShouldBeFalse(); // above the descriptor's Max
}

[Fact]
public async Task Agent_TenantOverride_DoesNotLeakToAnotherTenant()
{
Expand Down
16 changes: 15 additions & 1 deletion tests/AgentOs.Tests/Sessions/CliModelTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,28 @@ public void Codex_GetsDashM(string command, string model) =>
[InlineData("claude", "")]
[InlineData("claude", " ")]
[InlineData("claude", null)]
[InlineData("claude", "gpt-5")] // a model claude cannot serve: its default applies
[InlineData("codex", "claude-sonnet-4-6")] // a model codex cannot serve
[InlineData("my-agent", "claude-sonnet-4-6")] // an unknown CLI: never guess its flags
[InlineData("claude", "--dangerously-skip-permissions")] // never let a model id become a flag
[InlineData("claude", "claude sonnet")]
public void NoFlag_WhenThereIsNothingTheCliCanUse(string command, string? model) =>
CliModel.Flags(command, model, ClaudeDefaults).ShouldBeEmpty();

// A non-Claude profile model on claude runs as the Claude fallback, never silently as the CLI default (a slow
// thinking model ran Workflow Coding steps past the answer deadline on the lab, 2026-11-04).
[Theory]
[InlineData("gpt-5")]
[InlineData("gpt-4.1")]
[InlineData("gpt-4o-mini")]
public void Claude_NonClaudeModel_RunsAsTheClaudeFallback(string model)
{
CliModel.Flags("claude", model, ClaudeDefaults).ShouldBe(["--model", CliModel.DefaultClaudeFallback]);
CliModel.Flags("claude", model, ClaudeDefaults, "claude-haiku-4-5").ShouldBe(["--model", "claude-haiku-4-5"]);
CliModel.Flags("claude", model, ClaudeDefaults, "").ShouldBeEmpty(); // empty = the CLI default
CliModel.Flags("claude", model, ClaudeDefaults, "--yolo").ShouldBeEmpty(); // never a flag
CliModel.Flags("claude", model, ["-p", "--model", "opus"], "sonnet").ShouldBeEmpty(); // operator args win
}

[Theory]
[InlineData("--model", "opus")]
[InlineData("-m", "x")]
Expand Down
Loading