From 7136fba06d147344357aa9c94835bd99bc392c7a Mon Sep 17 00:00:00 2001 From: Sai Abhishek Date: Fri, 20 Nov 2026 12:46:49 -0800 Subject: [PATCH 1/3] feat(box): Send the spans of the agent in a box to X-Ray on AgentCore Runtime ## Problem A box runtime on AgentCore Runtime sends no span of its agent to X-Ray. Thus the console shows no trace for the runtime. A live test on 2026-11-20 found four causes. That test was correct only after manual edits. 1. `agentcore create` writes `instrumentation.enableOtel: false` for a box template. The agent does not start with `opentelemetry-instrument`. 2. Box makes the environment of the agent from the profile only. The agent gets no ADOT variable. Box gives it only `OTEL_EXPORTER_OTLP_ENDPOINT`, and keeps each `OTEL_EXPORTER_OTLP_*` name for box. 3. The Runtime profile has no `cloudwatch_traces` target. Thus box sends no span to X-Ray. 4. The `OTEL_RESOURCE_ATTRIBUTES` that the Runtime sets does not get to the agent. Thus the console cannot connect a span to the runtime. Also, after this change, a warning of build is not correct: "the agent does not run under opentelemetry-instrument, so it sends no traces. Set instrumentation.enableOtel to false." ## Solution The change is only in the Runtime profile that build derives, and in the `@box/agentcore` pack that build writes. The `instrumentation.enableOtel` flag of the runtime controls each part. The local `profile.toml`, the commands of the templates, `agentcore dev`, and box do not change. When `enableOtel` is `false`, build adds nothing, and the profile and the pack are the same as before. - The additions rule starts the agent with `opentelemetry-instrument` when the command starts the Python of the venv. On a container, the command starts with `${{ workspace }}/.venv/bin/opentelemetry-instrument`. On CodeZip, the zip has `bin/opentelemetry-instrument` with a `#!/usr/bin/env python3` line, and the box cannot use that line. Thus the Python of the Runtime starts the script: `${{ inputs.python.path }} ${{ inputs.code.dir }}/bin/opentelemetry-instrument`. `RuntimePaths.otelInstrument` holds the two start commands. A command that starts with the script does not get it two times. A command of a different program does not change. - The `@box/agentcore` pack gives `[agent.env]` these ADOT variables: `OTEL_PYTHON_DISTRO`, `OTEL_PYTHON_CONFIGURATOR`, `AGENT_OBSERVABILITY_ENABLED`, `OTEL_TRACES_EXPORTER`, `OTEL_LOGS_EXPORTER`, `OTEL_METRICS_EXPORTER = "none"`, and `OTEL_PYTHON_EXCLUDED_URLS = "/ping"`. The last variable stops a span for each health check of the Runtime. The pack sets no `OTEL_EXPORTER_OTLP_*` variable. The new input `runtime.resource_attributes` reads `OTEL_RESOURCE_ATTRIBUTES`, with the default `service.name=`, and the pack gives it to the agent. Box merges the profile after the packs. Thus box uses a value that the user sets in `[agent.env]`. Build gives no warning for that value. This is the same for each other value of the user. `renderAgentCorePack` gets the runtime. - The additions rule adds `[telemetry.agentcore_traces]` (`DEFAULT_TRACES_TARGET`) when no target is `cloudwatch_traces`. It sends to X-Ray in the region of the Runtime, `${{ inputs.aws.region }}`, with `include = ["deny", "permit", "trace"]` and a bare `aws://`. Its name is `agentcore_traces_default` when a different target has the name `agentcore_traces`. The comment lines at the top of the Runtime profile name the target. - The Box network model adds this target to `BoxNetwork.telemetry` (`defaultTracesTelemetry`). Thus `boxTelemetryPolicy` gives the execution role `xray:PutSpans` and `xray:PutTraceSegments`. - `boxOtelWarnings` reads the derived profile. It gives a warning only when `enableOtel` is `true` and the agent does not start with `opentelemetry-instrument`. - `agentcore create` writes no `instrumentation` for a box template. Thus `enableOtel` is `true`, the same as for each other template. - `docs/box-local-testing.md` tells about the OTEL start and the default X-Ray target. `DISABLE_ADOT_OBSERVABILITY` stops only the default log target, as before. The default X-Ray target stays until `enableOtel` is `false`. ## Tests `bun test` gives `6000 pass`, `7 skip`, and `0 fail`. `bun run typecheck`, `bun run lint:check`, `bun run format:check`, `bun run build`, `bun run audit`, and `bun run secrets:check` have no errors. - `additions.test.ts`, "with OTEL on": the template command and a command of the user get the start command of each build. A command that starts with the script, as the program or as the script of the Python, does not change. A command of a different program does not change. The default X-Ray target comes with OTEL on and not with OTEL off. It does not come when the profile or a pack has a `cloudwatch_traces` target. A user value of an OTEL variable stays in `[agent.env]`. - `additions.test.ts`, the two tests that use `old-renderer-tables.json`: with OTEL off, each template gives the recorded profile. With OTEL on, it gives the recorded profile with only the OTEL start and the default X-Ray target. - `boxProfile.test.ts`, `renderAgentCorePack`: the ADOT variables, no `OTEL_EXPORTER_OTLP_*` variable, the resource attributes input and its default, and none of them with OTEL off. - `boxNetworkSource.test.ts`, "the default X-Ray target of a runtime with OTEL on": the grant has X-Ray with OTEL on, and build removes the grant with OTEL off. - `manager.boxNetwork.test.ts`, "build with OTEL on, as create writes a box runtime": a new project gets the start, the pack variables, the target, and the grant. With `enableOtel` set to `false`, it gets none of them. A second build changes nothing. - `cdk.test.ts`: the warning comes only for a box runtime with OTEL on, when its agent does not start with `opentelemetry-instrument`. - `runtime.test.ts`: each box template writes no `instrumentation`. For each part of the change, a mutation caused an error in one or more of these tests. Then the test script removed the mutation. There were 20 mutations. A live test used a test account in us-west-2, box 0.1.11-preview.10, and `@aws/agentcore-cdk` from the `box-integration` branch of the L3 at 08f8b23. `agentcore create` made a container project from this branch, with no manual edit. Build, deploy, and three invokes had no error, and build gave no OTEL warning. In `aws/spans`, the runtime had 32 agent spans and 46 box spans in three traces. 20 box decision spans had an agent span as their parent. There was no `GET /ping` span. Each agent span had the `service.name` and the `cloud.resource_id` of the runtime. The log stream `otel-rt-logs` of the runtime had 97 records, and 46 of them were box records. The Box gateway had 20 calls with no error, and each call was in one of the three traces. The execution role had `BoxTelemetryToXRay`. `box profile show` of the macOS preview.10 box gave the same values. It used the value of the user for `OTEL_TRACES_EXPORTER` and `OTEL_RESOURCE_ATTRIBUTES`, and the default `service.name=agent` when the Runtime sets no attributes. The live test of CodeZip did not start. The runtime stopped with "`[agent]` filesystem entry "/usr" is refused ... a system root is never a grantable tree". The Python of the Runtime has the prefix `/usr`. A second deploy with `enableOtel` set to `false` stopped with the same error. Thus this change does not cause it, and CodeZip box on the Runtime does not start today. Only unit tests examine the CodeZip start command. The test removed the two stacks, and then CloudFormation had no stack of the test. --- docs/box-local-testing.md | 15 + src/core/project/backends/boxCodeZip.test.ts | 2 +- .../project/backends/boxContainer.test.ts | 4 +- src/core/project/backends/cdk.test.ts | 124 ++++++-- src/core/project/backends/cdk.ts | 33 +- src/core/project/boxNetwork/network.ts | 14 +- src/core/project/boxNetworkSource.test.ts | 73 +++++ src/core/project/boxNetworkWriters.ts | 3 +- .../project/manager.boxCredentials.test.ts | 13 +- src/core/project/manager.boxNetwork.test.ts | 102 ++++++- src/core/project/manager.test.ts | 12 +- src/core/project/runtimeProfile/context.ts | 5 + .../project/runtimeProfile/derive.test.ts | 5 + .../project/runtimeProfile/projectPaths.ts | 6 + .../runtimeProfile/rules/additions.test.ts | 287 ++++++++++++++++-- .../project/runtimeProfile/rules/additions.ts | 104 ++++++- src/core/project/runtimeProfile/types.ts | 6 + src/core/project/templates/boxProfile.test.ts | 50 ++- src/core/project/templates/boxProfile.ts | 55 +++- src/core/project/templates/runtime.test.ts | 26 +- src/core/project/templates/runtime.ts | 5 - 21 files changed, 822 insertions(+), 122 deletions(-) diff --git a/docs/box-local-testing.md b/docs/box-local-testing.md index a677b6852..481ece1d9 100644 --- a/docs/box-local-testing.md +++ b/docs/box-local-testing.md @@ -155,6 +155,12 @@ the generated packs, and the policy files into `.box//aws/` for you to `[tool.]`; - the `@box/agentcore` packs, the project name, and the build region come in, and each CloudWatch target signs with the execution role. + - with `instrumentation.enableOtel` on, which is the default for a box template too, a command + of the Python of the venv starts with `opentelemetry-instrument`. On CodeZip, the + Python of the Runtime runs the script in the zip, `/bin/opentelemetry-instrument`. The + `@box/agentcore` pack gives the agent the ADOT variables and the `OTEL_RESOURCE_ATTRIBUTES` + that the Runtime sets. When your `[agent.env]` sets one of these variables, box uses your value. + The pack sets no `OTEL_EXPORTER_OTLP_*` variable, because box sets each one. **The default log target.** With no `cloudwatch_logs` target, the Runtime profile gets `[telemetry.agentcore]`: box sends its deny, permit, and trace records to the log group of the @@ -167,6 +173,15 @@ the generated packs, and the policy files into `.box//aws/` for you to local profile. `agentcore create` and `agentcore add runtime` give this notice for each box runtime. + **The default X-Ray target.** With `enableOtel` on and no `cloudwatch_traces` target, the + Runtime profile gets `[telemetry.agentcore_traces]`: box sends the spans of the agent, and its + own deny, permit, and trace spans, to X-Ray in the region of the Runtime. Build gives the + execution role `xray:PutSpans` and `xray:PutTraceSegments` in + `agentcore/box-telemetry-policy-.json`. To send the spans to a different target, add a + `cloudwatch_traces` target to the local profile. To send no spans, set + `instrumentation.enableOtel` to `false` in `agentcore.json`. When `enableOtel` is on and the + agent does not start with `opentelemetry-instrument`, build gives a warning. + Build stops, and names the key and its line, for a value that cannot work on the Runtime: a required grant or a command path in the project outside the code folder, in the venv, or in a home folder (`${{ home }}`, `~`, `/Users`, `/home`); a CodeZip program that `PATH` finds, such as diff --git a/src/core/project/backends/boxCodeZip.test.ts b/src/core/project/backends/boxCodeZip.test.ts index 36f99bc4d..5ea9460a3 100644 --- a/src/core/project/backends/boxCodeZip.test.ts +++ b/src/core/project/backends/boxCodeZip.test.ts @@ -103,7 +103,7 @@ describe("stageBoxCodeZip", () => { // The profile imports these generated packs, which box finds beside its binary. They are // written after box's own resources are copied, so the copy does not remove them. expect(await readFile(join(dist, AGENTCORE_PACK_PATH), "utf8")).toBe( - renderAgentCorePack(demo.spec), + renderAgentCorePack(demo.spec, demo.spec.runtimes[0]!), ); expect(await readFile(join(dist, AGENTCORE_CODEZIP_PACK_PATH), "utf8")).toBe( renderAgentCoreCodeZipPack(), diff --git a/src/core/project/backends/boxContainer.test.ts b/src/core/project/backends/boxContainer.test.ts index 2bb9f1a03..834de51ad 100644 --- a/src/core/project/backends/boxContainer.test.ts +++ b/src/core/project/backends/boxContainer.test.ts @@ -90,7 +90,9 @@ describe("stageBoxContainers", () => { `BOX_SHA256=${BOX_RELEASE.linux.sha256}\n`, ); // The container also stages the @box/agentcore pack, beside box's own packs in the image. - expect(await readFile(join(dist, PACK_PATH), "utf8")).toBe(renderAgentCorePack(demo.spec)); + expect(await readFile(join(dist, PACK_PATH), "utf8")).toBe( + renderAgentCorePack(demo.spec, demo.spec.runtimes[0]!), + ); expect((await readdir(dist)).sort()).toEqual([...STAGED, "resources"].sort()); // The runtime's .box/agent/aws/ holds a copy of what the profile writer stages, and no box. const aws = join(demo.rootPath, ".box", "agent", "aws"); diff --git a/src/core/project/backends/cdk.test.ts b/src/core/project/backends/cdk.test.ts index 8466da265..05a8f6f90 100644 --- a/src/core/project/backends/cdk.test.ts +++ b/src/core/project/backends/cdk.test.ts @@ -69,8 +69,10 @@ function unusedIdentity(): CredentialProviderCalls { const NO_BOX_NETWORKS = { boxNetworks: [] }; const OTEL_WARNING = - "Runtime 'agent' runs its agent in a box, where the agent does not run under " + - "opentelemetry-instrument, so it sends no traces. Set instrumentation.enableOtel to false."; + "Runtime 'agent' runs its agent in a box, and [agent] command in " + + ".box/agent/local/profile.toml does not run the venv's Python, so on AgentCore Runtime the " + + "agent does not start under opentelemetry-instrument and sends no traces. Run the agent with " + + "the venv's Python, or set instrumentation.enableOtel to false."; function deployInput(overrides: Partial = {}): DeployBackendInput { return { target: TARGET, confirmTeardown: async () => false, boxNetworks: [], ...overrides }; @@ -332,32 +334,100 @@ describe("CdkBackend.build", () => { expect(subject.commands).toEqual([{ command: synthCommand(input), cwd: cdkDirectory(input) }]); }); + /** The Box network model of the runtime `agent`, with its derived Runtime profile. */ + const boxNetwork = (runtimeProfile: Record) => ({ + runtime: "agent", + gateway: "agent-box-egress", + region: "us-west-2", + hosts: [], + envRoutes: [], + policyFiles: [], + runtimeProfile, + grantedActions: [], + warnings: [], + failures: [], + }); + const PLAIN = ["${{ workspace }}/.venv/bin/python", "-m", "main"]; + const UNDER_OTEL: Record = { + Container: ["${{ workspace }}/.venv/bin/opentelemetry-instrument", ...PLAIN], + CodeZip: [ + "${{ inputs.python.path }}", + "${{ inputs.code.dir }}/bin/opentelemetry-instrument", + "${{ inputs.python.path }}", + "-m", + "main", + ], + }; + // A CodeZip box build downloads box before it warns, so the cases check the warning itself. test.each([ - ["a box container", "box", "Container", true, true], - ["a box CodeZip runtime", "box", "CodeZip", true, true], - ["a box container with enableOtel false", "box", "Container", false, false], - ["a container outside a Box project", undefined, "Container", true, false], - ] as const)("warns that %s sends no traces: %p", (_label, projectType, build, otel, warns) => { + [ + "a box container that starts under opentelemetry-instrument", + "box", + "Container", + true, + true, + false, + ], + [ + "a box CodeZip runtime that starts under opentelemetry-instrument", + "box", + "CodeZip", + true, + true, + false, + ], + ["a box container that does not", "box", "Container", true, false, true], + ["a box CodeZip runtime that does not", "box", "CodeZip", true, false, true], + ["a box container with enableOtel false", "box", "Container", false, false, false], + ["a container outside a Box project", undefined, "Container", true, false, false], + ] as const)( + "warns that %s sends no traces: %p", + (_label, projectType, build, otel, started, warns) => { + const spec = ProjectSpecSchema.parse({ + name: "example", + version: 2, + ...(projectType && { projectType }), + runtimes: [ + { + name: "agent", + build, + ...(build === "CodeZip" && { runtimeVersion: "PYTHON_3_14" }), + entrypoint: "main.py", + codeLocation: "app/agent", + ...(projectType && { box: { version: BOX_RELEASE.version } }), + ...(!otel && { instrumentation: { enableOtel: false } }), + }, + ], + agentCoreGateways: projectType ? [boxGatewaySpec("agent")] : [], + }); + + const command = started ? UNDER_OTEL[build]! : PLAIN; + + expect(boxOtelWarnings(spec, [boxNetwork({ agent: { command } })])).toEqual( + warns ? [OTEL_WARNING] : [], + ); + }, + ); + + test("gives no OTEL warning for a box runtime that has no Box network model", () => { const spec = ProjectSpecSchema.parse({ name: "example", version: 2, - ...(projectType && { projectType }), + projectType: "box", runtimes: [ { name: "agent", - build, - ...(build === "CodeZip" && { runtimeVersion: "PYTHON_3_14" }), + build: "Container", entrypoint: "main.py", codeLocation: "app/agent", - ...(projectType && { box: { version: BOX_RELEASE.version } }), - ...(!otel && { instrumentation: { enableOtel: false } }), + box: { version: BOX_RELEASE.version }, }, ], - agentCoreGateways: projectType ? [boxGatewaySpec("agent")] : [], + agentCoreGateways: [boxGatewaySpec("agent")], }); - expect(boxOtelWarnings(spec)).toEqual(warns ? [OTEL_WARNING] : []); + expect(boxOtelWarnings(spec, [])).toEqual([]); }); test("build yields the OTEL warning of a box runtime", async () => { @@ -381,23 +451,17 @@ describe("CdkBackend.build", () => { await writeFile(join(input.rootPath, "app", "agent", "Dockerfile"), "FROM scratch\n"); // Build stages box from the runtime's Box network model, with its derived Runtime profile. - const boxNetworks = [ - { - runtime: "agent", - gateway: "agent-box-egress", - region: "us-west-2", - hosts: [], - envRoutes: [], - policyFiles: [], - runtimeProfile: { packs: ["@box/base", "@box/agentcore"] }, - grantedActions: [], - warnings: [], - failures: [], - }, - ]; - const events = await collect(harness().backend.build(input, { boxNetworks })); + const plain = { packs: ["@box/base", "@box/agentcore"], agent: { command: PLAIN } }; + const started = { ...plain, agent: { command: UNDER_OTEL.Container } }; + const warned = await collect( + harness().backend.build(input, { boxNetworks: [boxNetwork(plain)] }), + ); + const quiet = await collect( + harness().backend.build(input, { boxNetworks: [boxNetwork(started)] }), + ); - expect(events).toContainEqual({ type: "warning", message: OTEL_WARNING } as ProjectEvent); + expect(warned).toContainEqual({ type: "warning", message: OTEL_WARNING } as ProjectEvent); + expect(quiet).not.toContainEqual({ type: "warning", message: OTEL_WARNING } as ProjectEvent); }); test("warns but continues when the installed CDK is legacy", async () => { diff --git a/src/core/project/backends/cdk.ts b/src/core/project/backends/cdk.ts index ea65fd5ad..4bea42a52 100644 --- a/src/core/project/backends/cdk.ts +++ b/src/core/project/backends/cdk.ts @@ -12,7 +12,8 @@ import type { } from "../../../handlers/project/types"; import { stageBoxCodeZip } from "./boxCodeZip"; import { stageBoxContainers } from "./boxContainer"; -import { runtimeRunsInBox } from "../templates/boxProfile"; +import { boxLocalProfileFile, otelOn, runtimeRunsInBox } from "../templates/boxProfile"; +import { agentStartsUnderOtel } from "../runtimeProfile/rules/additions"; import { boxDownloadMessage, boxReleaseFor, resolveBoxLinuxBuild } from "./boxRelease"; import { cdkCompatibilityWarning } from "./cdk/compatibility"; import { @@ -160,18 +161,28 @@ export type CdkBackendConfig = { }; /** - * A warning for each runtime of `spec` that runs its agent in a box with OTEL on: the agent in the - * box does not run under opentelemetry-instrument, CodeZip or container, so it sends no traces. + * A warning for each runtime of `spec` that runs its agent in a box with OTEL on, and whose + * AgentCore Runtime profile in `boxNetworks` does not start the agent under opentelemetry-instrument: + * build starts only a command of the venv's Python under it, and the agent then sends no traces. */ -export function boxOtelWarnings(spec: Project["spec"]): string[] { +export function boxOtelWarnings( + spec: Project["spec"], + boxNetworks: BuildBackendInput["boxNetworks"], +): string[] { return spec.runtimes - .filter( - (runtime) => runtimeRunsInBox(spec, runtime) && (runtime.instrumentation?.enableOtel ?? true), - ) + .filter((runtime) => runtimeRunsInBox(spec, runtime) && otelOn(runtime)) + .filter((runtime) => { + const profile = boxNetworks.find( + (network) => network.runtime === runtime.name, + )?.runtimeProfile; + return profile !== undefined && !agentStartsUnderOtel(profile); + }) .map( (runtime) => - `Runtime '${runtime.name}' runs its agent in a box, where the agent does not run under ` + - "opentelemetry-instrument, so it sends no traces. Set instrumentation.enableOtel to false.", + `Runtime '${runtime.name}' runs its agent in a box, and [agent] command in ` + + `${boxLocalProfileFile(runtime.name)} does not run the venv's Python, so on AgentCore ` + + "Runtime the agent does not start under opentelemetry-instrument and sends no traces. Run " + + "the agent with the venv's Python, or set instrumentation.enableOtel to false.", ); } @@ -267,7 +278,9 @@ export class CdkBackend implements ProjectBackend { ): AsyncGenerator { yield* await this.ensureCdkDependencies(project, boxNetworks); - for (const message of boxOtelWarnings(project.spec)) yield { type: "warning", message }; + for (const message of boxOtelWarnings(project.spec, boxNetworks)) { + yield { type: "warning", message }; + } const compatibilityWarning = await cdkCompatibilityWarning(this.cdkDirectory(project)); if (compatibilityWarning) { diff --git a/src/core/project/boxNetwork/network.ts b/src/core/project/boxNetwork/network.ts index 534d8ba54..8c334237f 100644 --- a/src/core/project/boxNetwork/network.ts +++ b/src/core/project/boxNetwork/network.ts @@ -25,7 +25,7 @@ import { describeEntry, describeRule, ruleOrigin } from "./places"; import { placeOf, type ParsedPolicy } from "./policyParser"; import { runtimeProfileContext } from "../runtimeProfile/context"; import { deriveRuntimeProfile } from "../runtimeProfile/derive"; -import { runtimeProfileHeader } from "../runtimeProfile/rules/additions"; +import { defaultTracesTelemetry, runtimeProfileHeader } from "../runtimeProfile/rules/additions"; import type { TomlTable } from "../runtimeProfile/types"; import { withDeployRegion, type CloudWatchTelemetry } from "./telemetry"; import { outboundAuthWords, sameOutboundAuth, targetOutboundAuth } from "./projectSpec"; @@ -119,7 +119,10 @@ export interface BoxNetwork { * Runtime profile. */ readonly policyFiles: readonly string[]; - /** The CloudWatch targets of the profile's `[telemetry]`; absent means none. */ + /** + * The CloudWatch targets of the profile's `[telemetry]`, then the default X-Ray target of + * {@link runtimeProfile} (`DEFAULT_TRACES_TARGET`); absent means none. + */ readonly telemetry?: readonly CloudWatchTelemetry[]; /** * The AgentCore Runtime profile that build derives from the local profile (`deriveRuntimeProfile` @@ -214,6 +217,11 @@ export function buildBoxNetwork( failures.push( ...(derived?.problems ?? []).map(({ code, reason, place }) => ({ code, reason, place })), ); + const defaultTraces = derived ? defaultTracesTelemetry(derived.profile, project.region) : []; + const sent = + telemetry || defaultTraces.length > 0 + ? [...(telemetry?.telemetry ?? []), ...defaultTraces] + : undefined; return { runtime: project.runtime.name, gateway: project.gateway, @@ -221,7 +229,7 @@ export function buildBoxNetwork( hosts: targets.hosts, envRoutes: envRoutesOf(targets.hosts), policyFiles: parsed.profile.policyFiles, - ...(telemetry && { telemetry: telemetry.telemetry }), + ...(sent && { telemetry: sent }), ...(derived && context && { runtimeProfile: derived.profile, diff --git a/src/core/project/boxNetworkSource.test.ts b/src/core/project/boxNetworkSource.test.ts index 85151893e..c82097db9 100644 --- a/src/core/project/boxNetworkSource.test.ts +++ b/src/core/project/boxNetworkSource.test.ts @@ -15,6 +15,7 @@ afterEach(async () => { await Promise.all(directories.splice(0).map((d) => rm(d, { recursive: true, force: true }))); }); +/** A Box project with the runtime `agent`, with OTEL off: only the OTEL tests turn it on. */ const SPEC = { name: "demo", projectType: "box", @@ -24,6 +25,7 @@ const SPEC = { build: "CodeZip", entrypoint: "start.py", codeLocation: "app/agent", + instrumentation: { enableOtel: false }, box: { version: "1.0.0" }, }, ], @@ -403,6 +405,77 @@ describe("the telemetry of each box runtime", () => { }); }); +describe("the default X-Ray target of a runtime with OTEL on", () => { + const OTEL_SPEC = { + ...SPEC, + runtimes: [{ ...SPEC.runtimes[0]!, instrumentation: undefined }], + } as unknown as ProjectSpec; + const TRACES = { + name: "agentcore_traces", + kind: "cloudwatch_traces" as const, + region: "us-west-2", + include: ["deny", "permit", "trace"], + }; + const X_RAY = { + Sid: "BoxTelemetryToXRay", + Effect: "Allow", + Action: ["xray:PutSpans", "xray:PutTraceSegments"], + Resource: "*", + }; + + /** The telemetry policy file that build writes for the model `network`. */ + const policyOf = (network: NonNullable>[number]>) => + telemetryPolicyWriter.edits([network], { + spec: OTEL_SPEC, + projectRoot: "/project", + env: {}, + envLocal: {}, + }).files ?? []; + + test("gets the grant of the execution role for X-Ray, and with OTEL off gets none", async () => { + const root = await project([]); + const box = new FakeBoxProfileResolver(boxShowOf({ egress: API_ROUTE })); + + const [on] = await readBoxNetworks(OTEL_SPEC, root, box); + const [off] = await readBoxNetworks(SPEC, root, box); + + expect(on?.telemetry).toEqual([TRACES]); + const telemetry = on?.runtimeProfile?.telemetry as Record; + expect(telemetry.agentcore_traces).toEqual({ + kind: "cloudwatch_traces", + region: "${{ inputs.aws.region }}", + include: ["deny", "permit", "trace"], + secret: { ref: "aws://" }, + }); + const [file] = policyOf(on!); + expect(JSON.parse((file as { text: string }).text)).toEqual({ + Version: "2012-10-17", + Statement: [X_RAY], + }); + expect(off?.telemetry).toBeUndefined(); + expect(policyOf(off!)).toEqual([{ path: boxTelemetryPolicyFile("agent"), remove: true }]); + }); + + test("is not added, and gets no second grant, when the profile has its own cloudwatch_traces target", async () => { + const root = await project([]); + const shown = boxShowOf({ egress: API_ROUTE }); + const own = { kind: "cloudwatch_traces", region: "eu-west-1", secret: { ref: "aws://" } }; + const box = new FakeBoxProfileResolver({ + ...shown, + boxToml: { ...shown.boxToml, telemetry: { mine: own } }, + }); + + const [network] = await readBoxNetworks(OTEL_SPEC, root, box); + + expect(network?.telemetry).toEqual([ + { name: "mine", kind: "cloudwatch_traces", region: "eu-west-1" }, + ]); + expect(network?.runtimeProfile?.telemetry).not.toHaveProperty("agentcore_traces"); + const [file] = policyOf(network!); + expect(JSON.parse((file as { text: string }).text).Statement).toEqual([X_RAY]); + }); +}); + describe("two routes of one profile to one host", () => { /** The Box network model of `agent` for the local profile `text`, which box resolves to `egress`. */ async function networkOf(text: string, egress: Record) { diff --git a/src/core/project/boxNetworkWriters.ts b/src/core/project/boxNetworkWriters.ts index cd8b28324..df8d6da6a 100644 --- a/src/core/project/boxNetworkWriters.ts +++ b/src/core/project/boxNetworkWriters.ts @@ -402,9 +402,10 @@ export function runtimeProfileWriter(staged: readonly StagedRuntime[]): BoxNetwo edits: (networks, { spec }) => ({ files: staged.flatMap(({ runtime, dist, kind }) => { const network = networkOf(networks, runtime); + const entry = spec.runtimes.find(({ name }) => name === runtime) ?? { name: runtime }; return [ { path: `${dist}/profile.toml`, text: runtimeProfileText(runtime, network, spec) }, - { path: `${dist}/${AGENTCORE_PACK_PATH}`, text: renderAgentCorePack(spec) }, + { path: `${dist}/${AGENTCORE_PACK_PATH}`, text: renderAgentCorePack(spec, entry) }, ...(kind === "codezip" ? [ { diff --git a/src/core/project/manager.boxCredentials.test.ts b/src/core/project/manager.boxCredentials.test.ts index 48dbe4586..8cefab80b 100644 --- a/src/core/project/manager.boxCredentials.test.ts +++ b/src/core/project/manager.boxCredentials.test.ts @@ -105,10 +105,19 @@ describe("FsProjectManager.build: box env:// credentials", () => { ...((before.credentials as unknown[] | undefined) ?? []), { authorizerType: "ApiKeyCredentialProvider", name: "github" }, ]); - // Only `credentials` and the Box gateway change: every other field keeps its value and place. + // Only `credentials`, the Box gateway, and the X-Ray grant of the runtime's default traces + // target change: every other field keeps its value and place. const others = (spec: Record) => - Object.entries(spec).filter(([key]) => !["credentials", "agentCoreGateways"].includes(key)); + Object.entries({ + ...spec, + runtimes: (spec.runtimes as Record[]).map( + ({ additionalPolicies: _grant, ...runtime }) => runtime, + ), + }).filter(([key]) => !["credentials", "agentCoreGateways"].includes(key)); expect(others(after)).toEqual(others(before)); + expect((after.runtimes as Record[])[0]?.additionalPolicies).toEqual([ + `../../agentcore/box-telemetry-policy-${AGENT_PYTHON.runtimeName}.json`, + ]); const stored = await new EnvLocalFile(created.rootPath).read(); expect(stored[RESERVED]).toBe("ghp_shell"); expect(messages).toContain(`writing ${RESERVED} to ${ENV_LOCAL_RELATIVE_PATH}`); diff --git a/src/core/project/manager.boxNetwork.test.ts b/src/core/project/manager.boxNetwork.test.ts index 006c052d4..3e587ef94 100644 --- a/src/core/project/manager.boxNetwork.test.ts +++ b/src/core/project/manager.boxNetwork.test.ts @@ -197,6 +197,13 @@ async function editGateway(root: string, edit: (gateway: { targets: unknown[] }) await writeFile(path(root, SPEC), JSON.stringify(spec, undefined, 2)); } +/** Sets `instrumentation.enableOtel` of the runtime in agentcore.json. */ +async function setOtel(root: string, enableOtel: boolean) { + const spec = JSON.parse(await text(root, SPEC)); + spec.runtimes[0].instrumentation = { enableOtel }; + await writeFile(path(root, SPEC), JSON.stringify(spec, undefined, 2)); +} + async function build(manager: FsProjectManager, root: string) { const project = (await manager.resolve({ filePath: root })) as Project; return drain(manager.build(project)); @@ -657,6 +664,13 @@ describe("build writes the telemetry policy of the box profile", () => { `adding ${TELEMETRY_POLICY} to additionalPolicies of 'agent_python_strands', so its box can ` + "send telemetry to CloudWatch"; + /** {@link strandsProject} with OTEL off, so its Runtime profile gets no default X-Ray target. */ + async function otelOffProject(options?: Parameters[0]) { + const created = await strandsProject(options); + await setOtel(created.root, false); + return created; + } + /** What box resolves the created profile to, with `telemetry` as its [telemetry] tables. */ const showsTelemetry = (telemetry: Record) => { const answer = shows(); @@ -675,7 +689,7 @@ describe("build writes the telemetry policy of the box profile", () => { ])( "for %s, writes the grant and lists it on the box runtime", async (_label, telemetry, statements) => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); box.answer = showsTelemetry(telemetry); const { steps } = await build(manager, root); @@ -694,7 +708,7 @@ describe("build writes the telemetry policy of the box profile", () => { ); test("sets only additionalPolicies of the runtime, and keeps its other keys in their place", async () => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); const spec = JSON.parse(await text(root, SPEC)); spec.runtimes[0].myNote = "kept"; await writeFile(path(root, SPEC), JSON.stringify(spec, undefined, 2)); @@ -709,7 +723,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("copies both CloudWatch targets into the Runtime profile, signed with the execution role", async () => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); const records = { kind: "file", destination: "/tmp/records.jsonl" }; box.answer = showsTelemetry({ logs: LOGS, traces: TRACES, records }); @@ -724,7 +738,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("changes nothing on a second build", async () => { - const { manager, root, counter, box } = await strandsProject(); + const { manager, root, counter, box } = await otelOffProject(); box.answer = showsTelemetry({ logs: LOGS, traces: TRACES }); await build(manager, root); const policy = await text(root, TELEMETRY_POLICY); @@ -738,7 +752,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("writes no file and lists nothing when the profile sends no telemetry to CloudWatch", async () => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); box.answer = showsTelemetry({ langfuse: { kind: "langfuse", secret: { ref: "env://KEY" } } }); const { steps } = await build(manager, root); @@ -749,7 +763,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("removes the file and the entry when the profile no longer sends telemetry to CloudWatch", async () => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); box.answer = showsTelemetry({ logs: LOGS }); await build(manager, root); box.answer = shows(); @@ -774,7 +788,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("on build, sets the region build builds for, keeps a target's own region, and warns that dev needs a region", async () => { - const { manager, root, box } = await strandsProject({ env: { AWS_REGION: "eu-central-1" } }); + const { manager, root, box } = await otelOffProject({ env: { AWS_REGION: "eu-central-1" } }); box.answer = showsTelemetry({ logs: REGIONLESS_LOGS, traces: { ...TRACES, region: "eu-west-1" }, @@ -814,7 +828,7 @@ describe("build writes the telemetry policy of the box profile", () => { return []; }, }; - const { manager, root, box } = await strandsProject({ backends: { CDK: backend } }); + const { manager, root, box } = await otelOffProject({ backends: { CDK: backend } }); box.answer = showsTelemetry({ logs: REGIONLESS_LOGS }); await writeFile( path(root, "agentcore/aws-targets.json"), @@ -844,6 +858,78 @@ describe("build writes the telemetry policy of the box profile", () => { }); }); +describe("build with OTEL on, as create writes a box runtime", () => { + const TELEMETRY_POLICY = "agentcore/box-telemetry-policy-agent_python_strands.json"; + const X_RAY = { + Sid: "BoxTelemetryToXRay", + Effect: "Allow", + Action: ["xray:PutSpans", "xray:PutTraceSegments"], + Resource: "*", + }; + const DEFAULT_TRACES = { + kind: "cloudwatch_traces", + region: "${{ inputs.aws.region }}", + include: ["deny", "permit", "trace"], + secret: { ref: "aws://" }, + }; + const PACK = "app/agent_python_strands/.box-dist/resources/packs/agentcore/pack.toml"; + + test("starts the agent under opentelemetry-instrument, sets up its ADOT SDK, and sends its spans to X-Ray with the grant", async () => { + const { manager, root } = await strandsProject(); + expect(JSON.parse(await text(root, SPEC)).runtimes[0].instrumentation).toBeUndefined(); + + const { warnings } = await build(manager, root); + + const profile = await runtimeProfile(root); + expect((profile.agent as { command: string[] }).command.slice(0, 3)).toEqual([ + "${{ inputs.python.path }}", + "${{ inputs.code.dir }}/bin/opentelemetry-instrument", + "${{ inputs.python.path }}", + ]); + expect((profile.telemetry as Record).agentcore_traces).toEqual(DEFAULT_TRACES); + const pack = parse(await text(root, PACK)) as { agent: { env: Record } }; + expect(pack.agent.env).toMatchObject({ + OTEL_PYTHON_DISTRO: "aws_distro", + OTEL_TRACES_EXPORTER: "otlp", + OTEL_RESOURCE_ATTRIBUTES: "${{ inputs.runtime.resource_attributes }}", + }); + expect(JSON.parse(await text(root, TELEMETRY_POLICY)).Statement).toEqual([X_RAY]); + expect(JSON.parse(await text(root, SPEC)).runtimes[0].additionalPolicies).toEqual([ + "../../agentcore/box-telemetry-policy-agent_python_strands.json", + ]); + expect(warnings.filter((warning) => warning.includes("opentelemetry-instrument"))).toEqual([]); + }); + + test("adds nothing for OTEL when enableOtel is false", async () => { + const { manager, root } = await strandsProject(); + await setOtel(root, false); + + await build(manager, root); + + const profile = await runtimeProfile(root); + expect(JSON.stringify(profile.agent)).not.toContain("opentelemetry-instrument"); + expect(profile.telemetry).not.toHaveProperty("agentcore_traces"); + const pack = parse(await text(root, PACK)) as { agent: { env: Record } }; + expect(Object.keys(pack.agent.env)).toEqual(["MEMORY_AGENT_PYTHON_STRANDSMEMORY_ID"]); + expect(existsSync(path(root, TELEMETRY_POLICY))).toBe(false); + }); + + test("changes nothing on a second build", async () => { + const { manager, root, counter } = await strandsProject(); + await build(manager, root); + const policy = await text(root, TELEMETRY_POLICY); + const staged = await text(root, RUNTIME_PROFILE); + counter.specWrites = 0; + + const { steps } = await build(manager, root); + + expect(steps).toEqual(["synthesizing CloudFormation templates"]); + expect(counter.specWrites).toBe(0); + expect(await text(root, TELEMETRY_POLICY)).toBe(policy); + expect(await text(root, RUNTIME_PROFILE)).toBe(staged); + }); +}); + describe("build reads the box profile as box resolves it", () => { test("takes the route a pack adds, which the profile itself does not hold", async () => { const { manager, root, box } = await strandsProject({ env: { SEARCH_KEY: "key-1" } }); diff --git a/src/core/project/manager.test.ts b/src/core/project/manager.test.ts index 99f66dc18..f72d95162 100644 --- a/src/core/project/manager.test.ts +++ b/src/core/project/manager.test.ts @@ -2106,7 +2106,8 @@ describe("FsProjectManager.build", () => { }, ]); // Build adds the Bedrock and Memory targets, and grants the Box gateway's role each, since the - // gateway signs the agent's calls with that role. + // gateway signs the agent's calls with that role. With OTEL on, it grants the execution role + // X-Ray for the default traces target of the Runtime profile. expect(events).toEqual([ { type: "step", @@ -2128,8 +2129,17 @@ describe("FsProjectManager.build", () => { message: "adding 'agentcore/box-memory-policy.json' to gateway 'agent-python-strands-box-box-egress' additionalPolicies in agentcore.json, so its role can call Memory", }, + { + type: "step", + message: + "adding agentcore/box-telemetry-policy-agent_python_strands_box.json to additionalPolicies of 'agent_python_strands_box', so its box can send telemetry to CloudWatch", + }, { type: "step", message: "writing agentcore/box-bedrock-policy.json" }, { type: "step", message: "writing agentcore/box-memory-policy.json" }, + { + type: "step", + message: "writing agentcore/box-telemetry-policy-agent_python_strands_box.json", + }, { type: "step", message: "synthesizing CloudFormation templates" }, ]); // The zip carries box and the Runtime profile in the code folder's .box-dist. diff --git a/src/core/project/runtimeProfile/context.ts b/src/core/project/runtimeProfile/context.ts index 9b7a21cf7..16c3ab092 100644 --- a/src/core/project/runtimeProfile/context.ts +++ b/src/core/project/runtimeProfile/context.ts @@ -35,6 +35,7 @@ export function runtimePaths( runtimeCode: WORKSPACE, runtimePython: `${WORKSPACE}/.venv/bin/python`, runtimePythonPrefix: "/usr/local", + otelInstrument: [`${WORKSPACE}/.venv/bin/opentelemetry-instrument`], shownCode: "/app/", }; } @@ -50,6 +51,10 @@ export function runtimePaths( runtimeCode: RUNTIME_INPUTS.codeDir, runtimePython: RUNTIME_INPUTS.pythonPath, runtimePythonPrefix: RUNTIME_INPUTS.pythonPrefix, + otelInstrument: [ + RUNTIME_INPUTS.pythonPath, + `${RUNTIME_INPUTS.codeDir}/bin/opentelemetry-instrument`, + ], shownCode: root ? "./" : `${folder}/`, }; } diff --git a/src/core/project/runtimeProfile/derive.test.ts b/src/core/project/runtimeProfile/derive.test.ts index dff787616..eaec6ce02 100644 --- a/src/core/project/runtimeProfile/derive.test.ts +++ b/src/core/project/runtimeProfile/derive.test.ts @@ -82,6 +82,10 @@ describe("runtimePaths", () => { runtimeCode: "${{ inputs.code.dir }}", runtimePython: "${{ inputs.python.path }}", runtimePythonPrefix: "${{ inputs.python.prefix }}", + otelInstrument: [ + "${{ inputs.python.path }}", + "${{ inputs.code.dir }}/bin/opentelemetry-instrument", + ], shownCode: "app/agent/", }); expect(runtimePaths({ ...CODEZIP, codeLocation: "./app/agent/" } as ProjectRuntime)).toEqual( @@ -94,6 +98,7 @@ describe("runtimePaths", () => { expect(paths.runtimeCode).toBe(paths.localCode); expect(paths.runtimePython).toBe("${{ workspace }}/.venv/bin/python"); + expect(paths.otelInstrument).toEqual(["${{ workspace }}/.venv/bin/opentelemetry-instrument"]); }); }); diff --git a/src/core/project/runtimeProfile/projectPaths.ts b/src/core/project/runtimeProfile/projectPaths.ts index edbdd3b6f..a2fc4a80b 100644 --- a/src/core/project/runtimeProfile/projectPaths.ts +++ b/src/core/project/runtimeProfile/projectPaths.ts @@ -218,6 +218,12 @@ export function programOf(table: Readonly): string | undefined { return Array.isArray(command) && typeof command[0] === "string" ? command[0] : undefined; } +/** Whether `value` is the venv's Python as the local profile names it: `/bin/python[3[.N]]`. */ +export function isVenvPython(value: string, paths: RuntimePaths): boolean { + const normal = value.replace(/\$\{\{\s*workspace\s*\}\}/g, () => WORKSPACE); + return new RegExp(`^${literal(paths.localVenv)}/bin/python(?:3(?:\\.\\d+)?)?$`).test(normal); +} + /** Whether `program`, as a CodeZip Runtime profile names it, is on the Runtime. */ export function isRuntimeProgram(program: string, paths: RuntimePaths): boolean { return program === paths.runtimePython || program.startsWith(`${paths.runtimeCode}/`); diff --git a/src/core/project/runtimeProfile/rules/additions.test.ts b/src/core/project/runtimeProfile/rules/additions.test.ts index 1b3dc8202..3cc35bdc5 100644 --- a/src/core/project/runtimeProfile/rules/additions.test.ts +++ b/src/core/project/runtimeProfile/rules/additions.test.ts @@ -17,7 +17,14 @@ import { runtimeProfileContext } from "../context"; import { deriveRuntimeProfile } from "../derive"; import { serializeRuntimeProfile } from "../serialize"; import type { TomlTable } from "../types"; -import { apply, DEFAULT_TELEMETRY_TARGET, runtimeProfileHeader } from "./additions"; +import { + agentStartsUnderOtel, + apply, + DEFAULT_TELEMETRY_TARGET, + DEFAULT_TRACES_TARGET, + defaultTracesTelemetry, + runtimeProfileHeader, +} from "./additions"; const PROFILE_FILE = ".box/agent/local/profile.toml"; /** The region create resolved, which the local profile names: not the build region. */ @@ -27,7 +34,16 @@ const BUILD_REGION = "us-west-2"; type Build = "CodeZip" | "Container"; type Protocol = NonNullable; -function runtimeOf(build: Build, protocol: Protocol = "HTTP", modelProvider = "Bedrock") { +/** + * A box runtime, with OTEL off unless `otel`: only the OTEL tests depend on it. With OTEL on it has + * no `instrumentation`, as `agentcore create` writes it. + */ +function runtimeOf( + build: Build, + protocol: Protocol = "HTTP", + modelProvider = "Bedrock", + otel = false, +) { return { name: "agent", build, @@ -35,6 +51,7 @@ function runtimeOf(build: Build, protocol: Protocol = "HTTP", modelProvider = "B codeLocation: "app/agent", protocol, modelProvider, + ...(!otel && { instrumentation: { enableOtel: false } }), box: { version: "1.0.0" }, } as ProjectRuntime; } @@ -128,6 +145,36 @@ const DEFAULT_TARGET = { secret: { ref: "aws://" }, }; +/** The default X-Ray target of a Runtime profile with OTEL on. */ +const DEFAULT_TRACES = { + kind: "cloudwatch_traces", + region: "${{ inputs.aws.region }}", + include: ["deny", "permit", "trace"], + secret: { ref: "aws://" }, +}; + +/** The words that start the agent under opentelemetry-instrument on the Runtime. */ +const OTEL_START: Record = { + CodeZip: ["${{ inputs.python.path }}", "${{ inputs.code.dir }}/bin/opentelemetry-instrument"], + Container: ["${{ workspace }}/.venv/bin/opentelemetry-instrument"], +}; + +/** + * `profile`, a Runtime profile with OTEL off, as OTEL on gives it: the agent starts under + * opentelemetry-instrument, and the default X-Ray target comes last when no target is + * `cloudwatch_traces`. Nothing else changes. + */ +function withOtel(profile: TomlTable, build: Build): TomlTable { + const result = structuredClone(profile); + const agent = result.agent as TomlTable; + agent.command = [...OTEL_START[build], ...(agent.command as string[])]; + const telemetry = result.telemetry as Record; + if (!Object.values(telemetry).some(({ kind }) => kind === "cloudwatch_traces")) { + telemetry.agentcore_traces = DEFAULT_TRACES; + } + return result; +} + /** The CloudWatch targets that box takes from the local profile `local`, as build gives them. */ const acceptedTelemetry = (local: TomlTable) => withDeployRegion(cloudWatchTelemetry(local), BUILD_REGION, PROFILE_FILE).telemetry; @@ -185,21 +232,17 @@ describe("deriveRuntimeProfile on the created templates", () => { ).flatMap(([build, protocol]) => ["Bedrock", "OpenAI"].flatMap((provider) => [[], ["agentMemory"]].flatMap((memories) => - Object.keys(TELEMETRY_CASES).map((telemetry) => ({ - build, - protocol, - provider, - memories, - telemetry, - })), + Object.keys(TELEMETRY_CASES).flatMap((telemetry) => + [false, true].map((otel) => ({ build, protocol, provider, memories, telemetry, otel })), + ), ), ), ); test.each(cases)( - "gives the tables the old renderer wrote (recorded in __fixtures__), and the default target with no cloudwatch_logs target ($build $protocol $provider, memories $memories, telemetry $telemetry)", - ({ build, protocol, provider, memories, telemetry: telemetryCase }) => { - const runtime = runtimeOf(build, protocol, provider); + "gives the tables the old renderer wrote (recorded in __fixtures__), and the default target with no cloudwatch_logs target, and with OTEL on only the OTEL start and the default X-Ray target ($build $protocol $provider, memories $memories, telemetry $telemetry, OTEL $otel)", + ({ build, protocol, provider, memories, telemetry: telemetryCase, otel }) => { + const runtime = runtimeOf(build, protocol, provider, otel); const local = createdProfile(runtime); const tables = TELEMETRY_CASES[telemetryCase]; const extras = tables !== undefined; @@ -217,9 +260,10 @@ describe("deriveRuntimeProfile on the created templates", () => { contextOf(runtime, { memories, telemetry }), ); - const expected = OLD_RENDERER.templates[ + const recorded = OLD_RENDERER.templates[ `${build} ${protocol} ${provider} memories=${memories.length} telemetry=${telemetryCase}` ] as TomlTable; + const expected = otel ? withOtel(recorded, build) : recorded; expect(problems).toEqual([]); for (const key of OWNED) expect([key, profile[key]]).toEqual([key, expected[key] ?? undefined]); @@ -232,8 +276,11 @@ const BOX_TEMPLATES = (Object.keys(RUNTIME_TEMPLATE_SHORTCUTS) as RuntimeTemplat .filter((name) => (RUNTIME_TEMPLATE_SHORTCUTS[name] as { box?: boolean }).box === true) .sort(); -/** The runtime that `agentcore create` writes into agentcore.json for the template `input`. */ -function createdRuntime(input: ScaffoldRuntimeInput): ProjectRuntime { +/** + * The runtime that `agentcore create` writes into agentcore.json for the template `input`, with + * `instrumentation.enableOtel` false unless `otel`. + */ +function createdRuntime(input: ScaffoldRuntimeInput, otel: boolean): ProjectRuntime { return { name: input.runtimeName, build: input.build, @@ -241,6 +288,7 @@ function createdRuntime(input: ScaffoldRuntimeInput): ProjectRuntime { codeLocation: `app/${input.runtimeName}`, ...(input.protocol !== undefined && { protocol: input.protocol }), ...(input.modelProvider !== undefined && { modelProvider: input.modelProvider }), + ...(!otel && { instrumentation: { enableOtel: false } }), box: { version: "1.0.0" }, } as ProjectRuntime; } @@ -252,24 +300,21 @@ describe("deriveRuntimeProfile on each box template of create", () => { : ["of the template"]; return providers.flatMap((provider) => [[], ["agentMemory"]].flatMap((memories) => - Object.keys(TELEMETRY_CASES).map((telemetry) => ({ - template, - provider, - memories, - telemetry, - })), + Object.keys(TELEMETRY_CASES).flatMap((telemetry) => + [false, true].map((otel) => ({ template, provider, memories, telemetry, otel })), + ), ), ); }); test.each(cases)( - "gives the whole profile that the old renderer wrote (recorded in __fixtures__), with the default target when no target is cloudwatch_logs ($template, provider $provider, memories $memories, telemetry $telemetry)", - ({ template, provider, memories, telemetry: telemetryCase }) => { + "gives the whole profile that the old renderer wrote (recorded in __fixtures__), with the default target when no target is cloudwatch_logs, and with OTEL on only the OTEL start and the default X-Ray target ($template, provider $provider, memories $memories, telemetry $telemetry, OTEL $otel)", + ({ template, provider, memories, telemetry: telemetryCase, otel }) => { const input = resolveRuntimeTemplateShortcut( template as RuntimeTemplateShortcutName, provider === "OpenAI" ? { modelProvider: "OpenAI", apiKey: "sk-test" } : {}, ); - const runtime = createdRuntime(input); + const runtime = createdRuntime(input, otel); const local = parse(renderBoxProfile("demo", input, CREATE_REGION)!) as TomlTable; const tables = TELEMETRY_CASES[telemetryCase]; const extras = tables !== undefined; @@ -288,12 +333,12 @@ describe("deriveRuntimeProfile on each box template of create", () => { contextOf(runtime, { memories, telemetry }), ); - expect(problems).toEqual([]); - expect(profile).toEqual( + const recorded = OLD_RENDERER.boxTemplates[ `${template} provider=${provider} memories=${memories.length} telemetry=${telemetryCase}` - ]!, - ); + ]!; + expect(problems).toEqual([]); + expect(profile).toEqual(otel ? withOtel(recorded, input.build as Build) : recorded); }, ); }); @@ -815,7 +860,7 @@ describe("the additions rule", () => { }); test("reads each value through an input that the @box/agentcore pack declares", () => { - const { inputs } = parse(renderAgentCorePack(specOf(CODEZIP))) as { + const { inputs } = parse(renderAgentCorePack(specOf(CODEZIP), CODEZIP)) as { inputs: Record>; }; const declared = (placeholder: string) => { @@ -920,6 +965,190 @@ describe("the additions rule", () => { }); }); + describe("with OTEL on", () => { + const OTEL_CODEZIP = runtimeOf("CodeZip", "HTTP", "Bedrock", true); + const OTEL_CONTAINER = runtimeOf("Container", "HTTP", "Bedrock", true); + const otelRuntime = (build: Build) => (build === "CodeZip" ? OTEL_CODEZIP : OTEL_CONTAINER); + const BUILDS: Build[] = ["CodeZip", "Container"]; + + test.each(BUILDS)( + "starts the template's own command under opentelemetry-instrument, and with OTEL off does not (%s)", + (build) => { + const local = createdProfile(otelRuntime(build)); + const off = agentOf(deriveRuntimeProfile(local, contextOf(runtimeOf(build))).profile); + const on = agentOf(deriveRuntimeProfile(local, contextOf(otelRuntime(build))).profile); + + expect(on.command).toEqual([...OTEL_START[build], ...off.command]); + expect(off.command.join(" ")).not.toContain("opentelemetry-instrument"); + }, + ); + + test.each(["MCP", "A2A", "AGUI"] as const)( + "starts a CodeZip %s template with the Runtime's Python and the script of the zip", + (protocol) => { + const runtime = runtimeOf("CodeZip", protocol, "Bedrock", true); + + expect( + agentOf(deriveRuntimeProfile(createdProfile(runtime), contextOf(runtime)).profile) + .command, + ).toEqual([...OTEL_START.CodeZip, "${{ inputs.python.path }}", "-m", "main"]); + }, + ); + + test.each(BUILDS)( + "starts a command of the user that runs the venv's Python under opentelemetry-instrument (%s)", + (build) => { + const command = ["${{workspace}}/.venv/bin/python3.12", "-m", "main", "--debug"]; + + expect( + agentOf(apply({ agent: { command } }, contextOf(otelRuntime(build))).table).command, + ).toEqual([...OTEL_START[build], ...command]); + }, + ); + + test.each(BUILDS)( + "does not start a command under opentelemetry-instrument twice (%s)", + (build) => { + const python = "${{ workspace }}/.venv/bin/python"; + const script = "${{ workspace }}/.venv/bin/opentelemetry-instrument"; + // The script as the program, and as the script that the venv's Python runs. + for (const own of [ + [script, python, "-m", "main"], + [python, script, python, "-m", "main"], + ]) { + expect( + agentOf(apply({ agent: { command: own } }, contextOf(otelRuntime(build))).table) + .command, + ).toEqual(own); + } + }, + ); + + test("leaves a command of another program as it is, and an [agent] with no command", () => { + const command = ["/usr/local/bin/node", "${{ workspace }}/server.js"]; + + expect( + agentOf(apply({ agent: { command } }, contextOf(OTEL_CONTAINER)).table).command, + ).toEqual(command); + expect(agentOf(apply({ agent: {} }, contextOf(OTEL_CONTAINER)).table)).not.toHaveProperty( + "command", + ); + }); + + test("tells whether the agent of a Runtime profile starts under opentelemetry-instrument", () => { + const started = (command: unknown) => agentStartsUnderOtel({ agent: { command } }); + + expect(started([...OTEL_START.Container, "${{ workspace }}/.venv/bin/python"])).toBe(true); + expect(started([...OTEL_START.CodeZip, "${{ inputs.python.path }}"])).toBe(true); + expect(started(["${{ workspace }}/.venv/bin/python", "-m", "main"])).toBe(false); + expect(started(undefined)).toBe(false); + expect(agentStartsUnderOtel({})).toBe(false); + }); + + test.each(BUILDS)( + "adds the default X-Ray target when no target is cloudwatch_traces, and with OTEL off does not (%s)", + (build) => { + expect(tableOf(apply({}, contextOf(otelRuntime(build))).table)).toEqual({ + agentcore: DEFAULT_TARGET, + agentcore_traces: DEFAULT_TRACES, + }); + expect(tableOf(apply({}, contextOf(runtimeOf(build))).table)).toEqual({ + agentcore: DEFAULT_TARGET, + }); + expect({ ...DEFAULT_TRACES_TARGET } as TomlTable).toEqual(DEFAULT_TRACES); + }, + ); + + test("adds the default X-Ray target beside a cloudwatch_logs target of the user", () => { + const local = { telemetry: { logs: CLOUDWATCH.logs } }; + + expect( + Object.keys( + tableOf( + apply(local, contextOf(OTEL_CODEZIP, { telemetry: acceptedTelemetry(local) })).table, + ), + ), + ).toEqual(["logs", "agentcore_traces"]); + }); + + test("adds no default X-Ray target when the profile has its own cloudwatch_traces target", () => { + const local = { telemetry: { "x-ray": CLOUDWATCH["x-ray"] } }; + + expect( + tableOf( + apply(local, contextOf(OTEL_CODEZIP, { telemetry: acceptedTelemetry(local) })).table, + ), + ).toEqual({ + "x-ray": { kind: "cloudwatch_traces", region: BUILD_REGION, secret: { ref: "aws://" } }, + agentcore: DEFAULT_TARGET, + }); + }); + + test("adds no default X-Ray target when box resolves a cloudwatch_traces target that the profile does not declare", () => { + const telemetry: CloudWatchTelemetry[] = [ + { name: "pack-traces", kind: "cloudwatch_traces", region: "eu-west-1" }, + ]; + + expect(Object.keys(tableOf(apply({}, contextOf(OTEL_CODEZIP, { telemetry })).table))).toEqual( + ["pack-traces", "agentcore"], + ); + }); + + test("names the default X-Ray target agentcore_traces_default when a target is agentcore_traces", () => { + const local = { telemetry: { agentcore_traces: CLOUDWATCH.logs } }; + const telemetry = tableOf( + apply(local, contextOf(OTEL_CODEZIP, { telemetry: acceptedTelemetry(local) })).table, + ); + + expect(Object.keys(telemetry)).toEqual(["agentcore_traces", "agentcore_traces_default"]); + expect(telemetry.agentcore_traces_default).toEqual(DEFAULT_TRACES); + }); + + test("reads the region of the default X-Ray target through the input of @box/agentcore", () => { + const { inputs } = parse(renderAgentCorePack(specOf(OTEL_CODEZIP), OTEL_CODEZIP)) as { + inputs: { aws: { region: unknown } }; + }; + + expect(DEFAULT_TRACES.region).toBe("${{ inputs.aws.region }}"); + expect(inputs.aws.region).toMatchObject({ from: "env:AWS_REGION" }); + }); + + test("names the default X-Ray target in the header", () => { + const context = contextOf(OTEL_CODEZIP); + + expect(runtimeProfileHeader(context, apply({}, context).table).at(-1)).toBe( + "[telemetry.agentcore_traces] is the default X-Ray target: the agent's spans and box's " + + `own, in the Runtime's region. Add a cloudwatch_traces target to ${PROFILE_FILE} to ` + + "use another, or set instrumentation.enableOtel to false in agentcore.json to send none.", + ); + }); + + test("gives the default X-Ray target to the grant of the execution role, and no other target", () => { + const { table } = apply({ telemetry: { logs: CLOUDWATCH.logs } }, contextOf(OTEL_CODEZIP)); + + expect(defaultTracesTelemetry(table, BUILD_REGION)).toEqual([ + { + name: "agentcore_traces", + kind: "cloudwatch_traces", + region: BUILD_REGION, + include: ["deny", "permit", "trace"], + }, + ]); + expect(defaultTracesTelemetry(apply({}, contextOf(CODEZIP)).table, BUILD_REGION)).toEqual([]); + }); + + test.each(BUILDS)( + "keeps each OTEL variable that the user sets in [agent.env], so it wins over the pack (%s)", + (build) => { + const env = { OTEL_TRACES_EXPORTER: "none", OTEL_RESOURCE_ATTRIBUTES: "service.name=mine" }; + + expect( + agentOf(apply({ agent: { env } }, contextOf(otelRuntime(build))).table).env, + ).toMatchObject(env); + }, + ); + }); + test("keeps the policies list as the local profile names it", () => { const local = { policies: ["./policy.dw", "rules/extra.dw"] }; diff --git a/src/core/project/runtimeProfile/rules/additions.ts b/src/core/project/runtimeProfile/rules/additions.ts index 085f5042b..a8d75976e 100644 --- a/src/core/project/runtimeProfile/rules/additions.ts +++ b/src/core/project/runtimeProfile/rules/additions.ts @@ -7,31 +7,36 @@ * | `packs` | the user's packs in their order, then @box/agentcore, and @box/agentcore-codezip for CodeZip | * | `[profile] name` | the project's name | * | `[options] bedrock.region` | the build region, where the profile sets it or imports @box/bedrock | - * | `[agent] command` | the template's own command becomes its Runtime command; a command of the user loses `--reload` and `--reload-dir ` | + * | `[agent] command` | the template's own command becomes its Runtime command; a command of the user loses `--reload` and `--reload-dir `; with OTEL on, a command of the venv's Python starts under opentelemetry-instrument | * | `[agent.env]` | a template value becomes its Runtime value; `LOCAL_DEV` goes; the region of the Runtime replaces a region of the user, with a warning; `AWS_PROFILE` goes, with a warning; the AWS stand-ins and (CodeZip) a `PYTHONPATH` of the code folder come in; each other value of the user stays | * | `[agent.filesystem]` | the template's grants become its Runtime grants; each grant of the user stays | * | `[[agent.ingress]]` | the first listens on `0.0.0.0` on the port of the protocol, and forwards to the workload port of the template when the agent keeps its command and port, with a warning for each port that changes; each other listens on `0.0.0.0` on its own port | * | `[tool.]` | a table that the template writes, as it writes it, goes; a table of the user stays | * | `[telemetry.]` | each CloudWatch target that box takes is signed with a bare `aws://`, in the deploy region when it names none; one that box refuses goes; a `file` target goes, with a warning | * | `[telemetry.agentcore]` | added when no target is `cloudwatch_logs`: the runtime's own log group and log stream | + * | `[telemetry.agentcore_traces]` | added with OTEL on when no target is `cloudwatch_traces`: X-Ray in the Runtime's region | * * It moves no path: the paths rule runs after it and moves each path of the user. */ +import { posix } from "node:path"; import { AGENTCORE_CODEZIP_PACK_NAME, AGENTCORE_PACK_NAME, BOX_BEDROCK_PACK, boxIngressPort, boxWorkloadPort, + otelOn, RUNTIME_LOGS_HEADERS_ENV, } from "../../templates/boxProfile"; import type { ProjectRuntime } from "../../../../projectSchemas/runtime"; -import { isTable } from "../projectPaths"; +import type { CloudWatchTelemetry } from "../../boxNetwork"; +import { isTable, isVenvPython } from "../projectPaths"; import { FILE_TELEMETRY_KIND } from "./refusals"; import { RUNTIME_AWS_SIGNER, RUNTIME_INPUTS, type RuleResult, + type RuntimePaths, type RuntimeProfileContext, type TomlTable, } from "../types"; @@ -78,6 +83,30 @@ export const DEFAULT_TELEMETRY_TARGET = { secret: { ref: RUNTIME_AWS_SIGNER }, } as const; +/** + * The X-Ray target of a Runtime profile with OTEL on that has no `cloudwatch_traces` target: the + * agent's spans and box's own, in the Runtime's region, signed with the execution role. + */ +export const DEFAULT_TRACES_TARGET = { + kind: "cloudwatch_traces", + region: RUNTIME_INPUTS.awsRegion, + include: ["deny", "permit", "trace"], + secret: { ref: RUNTIME_AWS_SIGNER }, +} as const; + +/** The program that starts a Python command with the OpenTelemetry instrumentation. */ +const OTEL_INSTRUMENT = "opentelemetry-instrument"; + +/** Whether `command` starts under opentelemetry-instrument, as its program or as the script of it. */ +function startsUnderOtel(command: unknown): boolean { + return ( + Array.isArray(command) && + command + .slice(0, 2) + .some((word) => typeof word === "string" && posix.basename(word) === OTEL_INSTRUMENT) + ); +} + /** * The Runtime's region in the agent's environment, from the @box/agentcore `aws.region` input. * @box/bedrock sets `AWS_REGION` itself, so the profile sets it only without that pack. @@ -139,9 +168,43 @@ export function runtimeProfileHeader( `and log stream, which box reads from ${RUNTIME_LOGS_HEADERS_ENV} when the run starts. ` + `Add a cloudwatch_logs target to ${context.profileFile} to use another.`, ), + ...defaultTracesTargets(profile).map( + (name) => + `[telemetry.${name}] is the default X-Ray target: the agent's spans and box's own, in the ` + + `Runtime's region. Add a cloudwatch_traces target to ${context.profileFile} to use ` + + "another, or set instrumentation.enableOtel to false in agentcore.json to send none.", + ), ]; } +/** The name of each {@link DEFAULT_TRACES_TARGET} in the `[telemetry]` of `profile`. */ +function defaultTracesTargets(profile: Readonly): string[] { + return Object.entries(tableOf(profile.telemetry)) + .filter(([, target]) => sameValue(target, DEFAULT_TRACES_TARGET)) + .map(([name]) => name); +} + +/** + * Each {@link DEFAULT_TRACES_TARGET} of `profile`, a Runtime profile, as a CloudWatch target in + * `region`, the region build builds for: the execution role needs a grant for it. + */ +export function defaultTracesTelemetry( + profile: Readonly, + region: string, +): CloudWatchTelemetry[] { + return defaultTracesTargets(profile).map((name) => ({ + name, + kind: "cloudwatch_traces", + region, + include: [...DEFAULT_TRACES_TARGET.include], + })); +} + +/** Whether the `[agent] command` of `profile`, a Runtime profile, starts under opentelemetry-instrument. */ +export function agentStartsUnderOtel(profile: Readonly): boolean { + return startsUnderOtel(tableOf(profile.agent).command); +} + export function apply(table: Readonly, context: RuntimeProfileContext): RuleResult { const result = structuredClone(table) as TomlTable; const packs = runtimePacks(result.packs, context); @@ -202,9 +265,10 @@ function runtimeAgent( const templateRuntime = tableOf(context.template.runtime.agent); const result: TomlTable = { ...local }; if (local.command !== undefined) { - result.command = sameValue(local.command, templateLocal.command) + const command = sameValue(local.command, templateLocal.command) ? structuredClone(templateRuntime.command) : withoutReload(local.command); + result.command = otelOn(context.runtime) ? underOtel(command, context.paths) : command; } result.env = runtimeEnv( local.env, @@ -264,6 +328,18 @@ function withoutReload(command: unknown): unknown { return result; } +/** + * `command` under opentelemetry-instrument (`paths.otelInstrument`) when its program is the venv's + * Python, whose dependencies hold the script. A command that already starts under it, or that runs + * another program, stays. + */ +function underOtel(command: unknown, paths: RuntimePaths): unknown { + if (!Array.isArray(command) || startsUnderOtel(command)) return command; + const program: unknown = command[0]; + if (typeof program !== "string" || !isVenvPython(program, paths)) return command; + return [...paths.otelInstrument, ...command]; +} + /** * `[agent.env]` for the Runtime. A variable of the template that the local profile keeps with the * template's value gets its Runtime value, or goes when the Runtime has none. A variable that only @@ -440,9 +516,11 @@ function onAllAddresses(entry: unknown): unknown { * names of `context.telemetry`) with its keys, signed with a bare `aws://`, in the deploy region * when it names none; one that box refuses goes; then each CloudWatch target of the * context that the local profile does not declare, as box resolves it. A target of another kind - * stays as it is. With no `cloudwatch_logs` target, {@link DEFAULT_TELEMETRY_TARGET} comes last, as + * stays as it is. With no `cloudwatch_logs` target, {@link DEFAULT_TELEMETRY_TARGET} comes next, as * `agentcore`, or as `agentcore_default` when a target has that name, unless the envVars of the - * runtime turn observability off ({@link OBSERVABILITY_OFF_ENV}). + * runtime turn observability off ({@link OBSERVABILITY_OFF_ENV}). With OTEL on and no + * `cloudwatch_traces` target, {@link DEFAULT_TRACES_TARGET} comes last, as `agentcore_traces`, or + * as `agentcore_traces_default` when a target has that name. */ function runtimeTelemetry( telemetry: unknown, @@ -491,14 +569,20 @@ function runtimeTelemetry( secret: { ref: RUNTIME_AWS_SIGNER }, }; } - const logs = Object.entries(result).some( - ([name, entry]) => - isTable(entry) && (entry.kind ?? resolved.get(name)?.kind) === "cloudwatch_logs", - ); - if (!logs && !observabilityOff(context.runtime)) { + const has = (kind: string) => + Object.entries(result).some( + ([name, entry]) => isTable(entry) && (entry.kind ?? resolved.get(name)?.kind) === kind, + ); + if (!has("cloudwatch_logs") && !observabilityOff(context.runtime)) { const name = Object.hasOwn(result, "agentcore") ? "agentcore_default" : "agentcore"; result[name] = structuredClone(DEFAULT_TELEMETRY_TARGET); } + if (!has("cloudwatch_traces") && otelOn(context.runtime)) { + const name = Object.hasOwn(result, "agentcore_traces") + ? "agentcore_traces_default" + : "agentcore_traces"; + result[name] = structuredClone(DEFAULT_TRACES_TARGET); + } return result; } diff --git a/src/core/project/runtimeProfile/types.ts b/src/core/project/runtimeProfile/types.ts index 628d1520f..777c650b5 100644 --- a/src/core/project/runtimeProfile/types.ts +++ b/src/core/project/runtimeProfile/types.ts @@ -50,6 +50,12 @@ export interface RuntimePaths { readonly runtimePython: string; /** The prefix of that Python, for the uv Pythons: `${{ inputs.python.prefix }}` on CodeZip. */ readonly runtimePythonPrefix: string; + /** + * The words before a command that start it under opentelemetry-instrument on the Runtime. The + * CodeZip zip has the script with a `#!/usr/bin/env python3` line, which the box cannot run, so + * the Runtime's Python runs it. + */ + readonly otelInstrument: readonly string[]; /** The code folder as a message names it: `app/agent/`. */ readonly shownCode: string; } diff --git a/src/core/project/templates/boxProfile.test.ts b/src/core/project/templates/boxProfile.test.ts index b4f65b42a..5f2f2eef1 100644 --- a/src/core/project/templates/boxProfile.test.ts +++ b/src/core/project/templates/boxProfile.test.ts @@ -401,9 +401,13 @@ describe("startsInBoxOnCodeZip", () => { }); describe("renderAgentCorePack", () => { - function pack(memories: { name: string }[]) { + /** The pack of a runtime with OTEL off unless `otel`. */ + function pack(memories: { name: string }[], otel = false) { return parse( - renderAgentCorePack({ name: "V3Demo", memories } as unknown as ProjectSpec), + renderAgentCorePack({ name: "V3Demo", memories } as unknown as ProjectSpec, { + name: "agent", + ...(!otel && { instrumentation: { enableOtel: false } }), + }), ) as Record; } @@ -474,8 +478,48 @@ describe("renderAgentCorePack", () => { }); }); - test("has no agent.env when the project has no memory", () => { + test("has no agent.env when the project has no memory and OTEL is off", () => { expect(pack([]).agent).toBeUndefined(); + expect(pack([]).inputs.runtime).not.toHaveProperty("resource_attributes"); + }); + + test("with OTEL on, sets up the agent's ADOT SDK and sets no OTEL_EXPORTER_OTLP_ variable, which box owns", () => { + const { env } = pack([{ name: "agentMemory" }], true).agent; + + expect(env).toEqual({ + MEMORY_AGENTMEMORY_ID: "${{ inputs.memory.agentMemory.id }}", + OTEL_PYTHON_DISTRO: "aws_distro", + OTEL_PYTHON_CONFIGURATOR: "aws_configurator", + AGENT_OBSERVABILITY_ENABLED: "true", + OTEL_TRACES_EXPORTER: "otlp", + OTEL_LOGS_EXPORTER: "otlp", + OTEL_METRICS_EXPORTER: "none", + OTEL_PYTHON_EXCLUDED_URLS: "/ping", + OTEL_RESOURCE_ATTRIBUTES: "${{ inputs.runtime.resource_attributes }}", + }); + expect(Object.keys(env).filter((name) => name.startsWith("OTEL_EXPORTER_OTLP_"))).toEqual([]); + }); + + test("with OTEL on, reads the Runtime's OTEL_RESOURCE_ATTRIBUTES, with the runtime's name when the Runtime sets none", () => { + expect(pack([], true).inputs.runtime.resource_attributes).toEqual({ + description: "The OpenTelemetry resource attributes of the runtime.", + from: "env:OTEL_RESOURCE_ATTRIBUTES", + default: "service.name=agent", + }); + }); + + test("turns OTEL on when the runtime has no instrumentation, as create writes it", () => { + const text = renderAgentCorePack({ name: "V3Demo" } as unknown as ProjectSpec, { + name: "agent", + }); + + expect(text).toBe( + renderAgentCorePack({ name: "V3Demo" } as unknown as ProjectSpec, { + name: "agent", + instrumentation: { enableOtel: true }, + }), + ); + expect(text).toContain('OTEL_PYTHON_DISTRO = "aws_distro"'); }); }); diff --git a/src/core/project/templates/boxProfile.ts b/src/core/project/templates/boxProfile.ts index 231f59ed8..aa5f7f6d7 100644 --- a/src/core/project/templates/boxProfile.ts +++ b/src/core/project/templates/boxProfile.ts @@ -224,6 +224,11 @@ export function runsInBox(boxProject: boolean, runtime: Pick): boolean { + return runtime.instrumentation?.enableOtel ?? true; +} + /** {@link runsInBox} for a runtime of `spec`. */ export function runtimeRunsInBox(spec: ProjectSpec, runtime: ProjectRuntime): boolean { return runsInBox(isBoxProject(spec), runtime); @@ -605,6 +610,24 @@ export function boxProfileWarnings(runtime: ScaffoldRuntimeInput): string[] { ]; } +/** + * The variables that set up the ADOT SDK of an agent with OTEL on: the AWS distro and its + * configurator, spans and logs over OTLP to the box's own receiver (box sets each + * `OTEL_EXPORTER_OTLP_*`), no metrics, and no span for each health check of the Runtime. + */ +const AGENT_OTEL_ENV: Readonly> = { + OTEL_PYTHON_DISTRO: "aws_distro", + OTEL_PYTHON_CONFIGURATOR: "aws_configurator", + AGENT_OBSERVABILITY_ENABLED: "true", + OTEL_TRACES_EXPORTER: "otlp", + OTEL_LOGS_EXPORTER: "otlp", + OTEL_METRICS_EXPORTER: "none", + OTEL_PYTHON_EXCLUDED_URLS: "/ping", +}; + +/** The variable in which AgentCore Runtime names the resource of the runtime's spans. */ +const RUNTIME_RESOURCE_ATTRIBUTES_ENV = "OTEL_RESOURCE_ATTRIBUTES"; + /** * The `@box/agentcore` pack each Runtime profile imports, CodeZip and container: it declares the * inputs the profile reads from the environment the Runtime sets after deploy. The `aws.region` @@ -612,10 +635,16 @@ export function boxProfileWarnings(runtime: ScaffoldRuntimeInput): string[] { * egress backend `url`, bounded by a pattern so a value from the environment names only the host the * profile expects. The `runtime.log_group` and `runtime.log_stream` inputs each read one key of * {@link RUNTIME_LOGS_HEADERS_ENV}, for the default telemetry target of the Runtime profile. One - * input per project memory carries its ID into the agent's environment. box finds the pack beside - * its binary at {@link AGENTCORE_PACK_PATH}; build and deploy stage it there. + * input per project memory carries its ID into the agent's environment. With OTEL on for `runtime`, + * the agent gets {@link AGENT_OTEL_ENV} and the Runtime's resource attributes, so the console links + * its spans to the runtime; the profile merges after the pack, so a value of the user wins. box + * finds the pack beside its binary at {@link AGENTCORE_PACK_PATH}; build and deploy stage it there. */ -export function renderAgentCorePack(spec: ProjectSpec): string { +export function renderAgentCorePack( + spec: ProjectSpec, + runtime: Pick, +): string { + const otel = otelOn(runtime); const memories = spec.memories ?? []; const memoryInputs = memories .map( @@ -631,7 +660,23 @@ from = ${str(`env:${memoryEnvVarName(memory.name)}`)}`, `${memoryEnvVarName(memory.name)} = ${str(`\${{ inputs.memory.${tomlKey(memory.name)}.id }}`)}`, ) .join("\n"); - const agentEnv = memoryEnv ? `\n[agent.env]\n${memoryEnv}\n` : ""; + const otelEnv = otel + ? [ + ...Object.entries(AGENT_OTEL_ENV).map(([name, value]) => `${name} = ${str(value)}`), + `${RUNTIME_RESOURCE_ATTRIBUTES_ENV} = "\${{ inputs.runtime.resource_attributes }}"`, + ].join("\n") + : ""; + const otelInput = otel + ? ` + +# The resource of the agent's spans, which the Runtime names so the console links them to it. +[inputs.runtime.resource_attributes] +description = "The OpenTelemetry resource attributes of the runtime." +from = ${str(`env:${RUNTIME_RESOURCE_ATTRIBUTES_ENV}`)} +default = ${str(`service.name=${runtime.name}`)}` + : ""; + const env = [memoryEnv, otelEnv].filter((lines) => lines !== "").join("\n"); + const agentEnv = env ? `\n[agent.env]\n${env}\n` : ""; return `# The deployed AgentCore resources of '${spec.name}', read from the environment the Runtime sets. # Written by agentcore build and deploy. The runtime profile imports it as ${AGENTCORE_PACK_NAME}. [pack] @@ -669,7 +714,7 @@ key = "x-aws-log-stream" [inputs.runtime.url] description = "The URL an A2A agent card advertises on the Runtime." from = "env:AGENTCORE_RUNTIME_URL" -default = "http://localhost:9000/"${memoryInputs} +default = "http://localhost:9000/"${memoryInputs}${otelInput} ${agentEnv}`; } diff --git a/src/core/project/templates/runtime.test.ts b/src/core/project/templates/runtime.test.ts index e7632bf00..0eae85be2 100644 --- a/src/core/project/templates/runtime.test.ts +++ b/src/core/project/templates/runtime.test.ts @@ -96,20 +96,20 @@ describe("runtime templates", () => { expect(await Bun.file(join(appDir, ".dockerignore")).exists()).toBe(isContainer); }); - // The agent in a box does not run under opentelemetry-instrument, CodeZip or container, so a box - // template turns OTEL off; any other template keeps it on. - test.each<[RuntimeTemplateShortcutName, boolean]>([ - ["agent-python-strands-box", false], - ["agent-python-strands-box-container", false], - ["agent-python-minimal-box", false], - ["agent-python-langchain-box", false], - ["a2a-python-strands-box", false], - ["agui-python-strands-box", false], - ["mcp-python-fastmcp-box", false], - ["agent-python-strands", true], - ])("%s writes enableOtel %p", async (template, enableOtel) => { + // Build starts the agent in a box under opentelemetry-instrument, so a box template keeps OTEL on + // as every other template does, and writes no instrumentation. + test.each([ + "agent-python-strands-box", + "agent-python-strands-box-container", + "agent-python-minimal-box", + "agent-python-langchain-box", + "a2a-python-strands-box", + "agui-python-strands-box", + "mcp-python-fastmcp-box", + "agent-python-strands", + ])("%s keeps enableOtel on", async (template) => { const { runtime } = await addTemplate("my_agent", template); - expect(runtime.instrumentation?.enableOtel ?? true).toBe(enableOtel); + expect(runtime.instrumentation).toBeUndefined(); }); test("Bedrock Managed Agents keeps infrastructure overrides and its required policy", async () => { diff --git a/src/core/project/templates/runtime.ts b/src/core/project/templates/runtime.ts index 05722b94d..a90a29698 100644 --- a/src/core/project/templates/runtime.ts +++ b/src/core/project/templates/runtime.ts @@ -2,7 +2,6 @@ import { FsTreeNode } from "./fsTree"; import type { AssetSource } from "../source"; import type { RuntimeResourceConfig } from "../../../handlers/project/add/runtime/types"; import { DEFAULT_MODEL_IDS, type ProjectRuntime } from "../../../projectSchemas/runtime"; -import { hasBoxProfile } from "./boxProfile"; import { mergeSpecEntries } from "./spec"; import type { SpecEntries, TemplateRenderer, TemplateResolver } from "./types"; import type { @@ -132,10 +131,6 @@ function buildRuntimeSpec(input: RuntimeResourceConfig): ProjectRuntime { ...(scaffoldRuntimeInput.framework === "bma" && { bedrockManagedAgents: true, }), - // The agent in a box does not run under opentelemetry-instrument: a container's image does not - // run it, and on CodeZip it wraps start.py, which hands over to box run. So say so in - // agentcore.json until traces work in a box. - ...(hasBoxProfile(scaffoldRuntimeInput) && { instrumentation: { enableOtel: false } }), ...(infra.envVars && { envVars: infra.envVars }), ...(infra.networkMode && { networkMode: infra.networkMode }), ...(infra.networkConfig && { networkConfig: infra.networkConfig }), From b1634b5e29934725afa18b6964c1f32eca22f777 Mon Sep 17 00:00:00 2001 From: Sai Abhishek Date: Fri, 20 Nov 2026 14:53:53 -0800 Subject: [PATCH 2/3] fix(box): Add no default X-Ray target when DISABLE_ADOT_OBSERVABILITY is true --- docs/box-local-testing.md | 6 +++--- src/core/project/boxNetworkSource.test.ts | 20 +++++++++++++++++++ .../runtimeProfile/rules/additions.test.ts | 18 +++++++++++++++++ .../project/runtimeProfile/rules/additions.ts | 11 +++++----- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/docs/box-local-testing.md b/docs/box-local-testing.md index 481ece1d9..d20b4591a 100644 --- a/docs/box-local-testing.md +++ b/docs/box-local-testing.md @@ -169,9 +169,9 @@ the generated packs, and the policy files into `.box//aws/` for you to Runtime does not set it, box does not start. The Runtime makes that log group, so its retention applies, and it stays after you remove the runtime. CloudWatch Logs charges for the records it gets. To send none, set `DISABLE_ADOT_OBSERVABILITY` to `"true"` in the `envVars` of the runtime - in `agentcore.json`. To send them to another log group, add a `cloudwatch_logs` target to the - local profile. `agentcore create` and `agentcore add runtime` give this notice for each box - runtime. + in `agentcore.json`. Then the Runtime profile also gets no default X-Ray target. To send them to + another log group, add a `cloudwatch_logs` target to the local profile. `agentcore create` and + `agentcore add runtime` give this notice for each box runtime. **The default X-Ray target.** With `enableOtel` on and no `cloudwatch_traces` target, the Runtime profile gets `[telemetry.agentcore_traces]`: box sends the spans of the agent, and its diff --git a/src/core/project/boxNetworkSource.test.ts b/src/core/project/boxNetworkSource.test.ts index c82097db9..695fcb44b 100644 --- a/src/core/project/boxNetworkSource.test.ts +++ b/src/core/project/boxNetworkSource.test.ts @@ -456,6 +456,26 @@ describe("the default X-Ray target of a runtime with OTEL on", () => { expect(policyOf(off!)).toEqual([{ path: boxTelemetryPolicyFile("agent"), remove: true }]); }); + test("is not added, and gets no grant, when the envVars set DISABLE_ADOT_OBSERVABILITY to true", async () => { + const root = await project([]); + const box = new FakeBoxProfileResolver(boxShowOf({ egress: API_ROUTE })); + const spec = { + ...OTEL_SPEC, + runtimes: [ + { + ...OTEL_SPEC.runtimes[0]!, + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }], + }, + ], + } as unknown as ProjectSpec; + + const [network] = await readBoxNetworks(spec, root, box); + + expect(network?.runtimeProfile?.telemetry).toEqual({}); + expect(network?.telemetry).toBeUndefined(); + expect(policyOf(network!)).toEqual([{ path: boxTelemetryPolicyFile("agent"), remove: true }]); + }); + test("is not added, and gets no second grant, when the profile has its own cloudwatch_traces target", async () => { const root = await project([]); const shown = boxShowOf({ egress: API_ROUTE }); diff --git a/src/core/project/runtimeProfile/rules/additions.test.ts b/src/core/project/runtimeProfile/rules/additions.test.ts index 3cc35bdc5..56a01dbec 100644 --- a/src/core/project/runtimeProfile/rules/additions.test.ts +++ b/src/core/project/runtimeProfile/rules/additions.test.ts @@ -1094,6 +1094,24 @@ describe("the additions rule", () => { ); }); + test.each([ + ["true", false], + [" TRUE ", false], + ["false", true], + ])( + "adds the default X-Ray target only when the envVars do not set DISABLE_ADOT_OBSERVABILITY to true (%j)", + (value, added) => { + const runtime = { + ...OTEL_CODEZIP, + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value }], + } as ProjectRuntime; + const { table } = apply({}, contextOf(runtime)); + + expect(Object.hasOwn(tableOf(table), "agentcore_traces")).toBe(added); + expect(defaultTracesTelemetry(table, BUILD_REGION)).toHaveLength(added ? 1 : 0); + }, + ); + test("names the default X-Ray target agentcore_traces_default when a target is agentcore_traces", () => { const local = { telemetry: { agentcore_traces: CLOUDWATCH.logs } }; const telemetry = tableOf( diff --git a/src/core/project/runtimeProfile/rules/additions.ts b/src/core/project/runtimeProfile/rules/additions.ts index a8d75976e..f5fe63f8b 100644 --- a/src/core/project/runtimeProfile/rules/additions.ts +++ b/src/core/project/runtimeProfile/rules/additions.ts @@ -14,7 +14,7 @@ * | `[tool.]` | a table that the template writes, as it writes it, goes; a table of the user stays | * | `[telemetry.]` | each CloudWatch target that box takes is signed with a bare `aws://`, in the deploy region when it names none; one that box refuses goes; a `file` target goes, with a warning | * | `[telemetry.agentcore]` | added when no target is `cloudwatch_logs`: the runtime's own log group and log stream | - * | `[telemetry.agentcore_traces]` | added with OTEL on when no target is `cloudwatch_traces`: X-Ray in the Runtime's region | + * | `[telemetry.agentcore_traces]` | added with OTEL on when no target is `cloudwatch_traces`, unless observability is off: X-Ray in the Runtime's region | * * It moves no path: the paths rule runs after it and moves each path of the user. */ @@ -84,8 +84,9 @@ export const DEFAULT_TELEMETRY_TARGET = { } as const; /** - * The X-Ray target of a Runtime profile with OTEL on that has no `cloudwatch_traces` target: the - * agent's spans and box's own, in the Runtime's region, signed with the execution role. + * The X-Ray target of a Runtime profile with OTEL on that has no `cloudwatch_traces` target, unless + * the envVars of the runtime turn observability off: the agent's spans and box's own, in the + * Runtime's region, signed with the execution role. */ export const DEFAULT_TRACES_TARGET = { kind: "cloudwatch_traces", @@ -520,7 +521,7 @@ function onAllAddresses(entry: unknown): unknown { * `agentcore`, or as `agentcore_default` when a target has that name, unless the envVars of the * runtime turn observability off ({@link OBSERVABILITY_OFF_ENV}). With OTEL on and no * `cloudwatch_traces` target, {@link DEFAULT_TRACES_TARGET} comes last, as `agentcore_traces`, or - * as `agentcore_traces_default` when a target has that name. + * as `agentcore_traces_default` when a target has that name, unless observability is off. */ function runtimeTelemetry( telemetry: unknown, @@ -577,7 +578,7 @@ function runtimeTelemetry( const name = Object.hasOwn(result, "agentcore") ? "agentcore_default" : "agentcore"; result[name] = structuredClone(DEFAULT_TELEMETRY_TARGET); } - if (!has("cloudwatch_traces") && otelOn(context.runtime)) { + if (!has("cloudwatch_traces") && otelOn(context.runtime) && !observabilityOff(context.runtime)) { const name = Object.hasOwn(result, "agentcore_traces") ? "agentcore_traces_default" : "agentcore_traces"; From 365ec2bc971bd5bf851d03e09083e20314f54128 Mon Sep 17 00:00:00 2001 From: Sai Abhishek Date: Fri, 20 Nov 2026 16:16:13 -0800 Subject: [PATCH 3/3] fix(box): Give the agent no OTEL when DISABLE_ADOT_OBSERVABILITY is true (comment of Leo on additions.ts) --- docs/box-local-testing.md | 7 +++-- src/core/project/backends/cdk.test.ts | 21 ++++++++++++++ src/core/project/manager.boxNetwork.test.ts | 14 ++++++++-- src/core/project/manager.tsx | 3 +- .../runtimeProfile/rules/additions.test.ts | 15 ++++++++++ .../project/runtimeProfile/rules/additions.ts | 28 ++++++------------- src/core/project/templates/boxProfile.test.ts | 12 ++++++++ src/core/project/templates/boxProfile.ts | 26 ++++++++++++++--- 8 files changed, 96 insertions(+), 30 deletions(-) diff --git a/docs/box-local-testing.md b/docs/box-local-testing.md index 945cfbc1c..830aaae1f 100644 --- a/docs/box-local-testing.md +++ b/docs/box-local-testing.md @@ -256,9 +256,10 @@ A profile that names a pack `box/` fails: box refuses it and names the `@b Runtime does not set it, box does not start. The Runtime makes that log group, so its retention applies, and it stays after you remove the runtime. CloudWatch Logs charges for the records it gets. To send none, set `DISABLE_ADOT_OBSERVABILITY` to `"true"` in the `envVars` of the runtime - in `agentcore.json`. Then the Runtime profile also gets no default X-Ray target. To send them to - another log group, add a `cloudwatch_logs` target to the local profile. `agentcore create` and - `agentcore add runtime` give this notice for each box runtime. + in `agentcore.json`. Then the Runtime profile also gets no default X-Ray target. The agent does + not start with `opentelemetry-instrument`, and the `@box/agentcore` pack gives it no ADOT + variable. To send the records to another log group, add a `cloudwatch_logs` target to the local + profile. `agentcore create` and `agentcore add runtime` give this notice for each box runtime. **The default X-Ray target.** With `enableOtel` on and no `cloudwatch_traces` target, the Runtime profile gets `[telemetry.agentcore_traces]`: box sends the spans of the agent, and its diff --git a/src/core/project/backends/cdk.test.ts b/src/core/project/backends/cdk.test.ts index 05a8f6f90..9a28b88ca 100644 --- a/src/core/project/backends/cdk.test.ts +++ b/src/core/project/backends/cdk.test.ts @@ -410,6 +410,27 @@ describe("CdkBackend.build", () => { }, ); + test("gives no OTEL warning for a box runtime whose envVars set DISABLE_ADOT_OBSERVABILITY to true", () => { + const spec = ProjectSpecSchema.parse({ + name: "example", + version: 2, + projectType: "box", + runtimes: [ + { + name: "agent", + build: "Container", + entrypoint: "main.py", + codeLocation: "app/agent", + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }], + box: { version: BOX_RELEASE.version }, + }, + ], + agentCoreGateways: [boxGatewaySpec("agent")], + }); + + expect(boxOtelWarnings(spec, [boxNetwork({ agent: { command: PLAIN } })])).toEqual([]); + }); + test("gives no OTEL warning for a box runtime that has no Box network model", () => { const spec = ProjectSpecSchema.parse({ name: "example", diff --git a/src/core/project/manager.boxNetwork.test.ts b/src/core/project/manager.boxNetwork.test.ts index 3e587ef94..74800cd04 100644 --- a/src/core/project/manager.boxNetwork.test.ts +++ b/src/core/project/manager.boxNetwork.test.ts @@ -204,6 +204,13 @@ async function setOtel(root: string, enableOtel: boolean) { await writeFile(path(root, SPEC), JSON.stringify(spec, undefined, 2)); } +/** Sets `DISABLE_ADOT_OBSERVABILITY` to `"true"` in the envVars of the runtime in agentcore.json. */ +async function optOut(root: string) { + const spec = JSON.parse(await text(root, SPEC)); + spec.runtimes[0].envVars = [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }]; + await writeFile(path(root, SPEC), JSON.stringify(spec, undefined, 2)); +} + async function build(manager: FsProjectManager, root: string) { const project = (await manager.resolve({ filePath: root })) as Project; return drain(manager.build(project)); @@ -900,9 +907,12 @@ describe("build with OTEL on, as create writes a box runtime", () => { expect(warnings.filter((warning) => warning.includes("opentelemetry-instrument"))).toEqual([]); }); - test("adds nothing for OTEL when enableOtel is false", async () => { + test.each([ + ["enableOtel is false", (root: string) => setOtel(root, false)], + ["the envVars set DISABLE_ADOT_OBSERVABILITY to true", (root: string) => optOut(root)], + ])("adds nothing for OTEL when %s", async (_label, turnOff) => { const { manager, root } = await strandsProject(); - await setOtel(root, false); + await turnOff(root); await build(manager, root); diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index d7c3b9bbe..480709beb 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -69,11 +69,12 @@ import { boxProfileWarnings, boxUvSyncEnv, hasBoxProfile, + observabilityOff, renderBoxProfile, runsInBox, startsInBoxOnCodeZip, } from "./templates/boxProfile"; -import { defaultLogTargetNotice, observabilityOff } from "./runtimeProfile/rules/additions"; +import { defaultLogTargetNotice } from "./runtimeProfile/rules/additions"; import { validateBoxRuntimes } from "./boxValidation"; import { BOX_INSTALL_HINT, BOX_RELEASE } from "./backends/boxRelease"; import { getRuntimeTemplateResolver } from "./templates/runtime"; diff --git a/src/core/project/runtimeProfile/rules/additions.test.ts b/src/core/project/runtimeProfile/rules/additions.test.ts index 56a01dbec..b0327d932 100644 --- a/src/core/project/runtimeProfile/rules/additions.test.ts +++ b/src/core/project/runtimeProfile/rules/additions.test.ts @@ -1112,6 +1112,21 @@ describe("the additions rule", () => { }, ); + test.each(BUILDS)( + "does not start the agent under opentelemetry-instrument when the envVars set DISABLE_ADOT_OBSERVABILITY to true (%s)", + (build) => { + const optedOut = { + ...otelRuntime(build), + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }], + } as ProjectRuntime; + const local = createdProfile(optedOut); + + expect(agentOf(deriveRuntimeProfile(local, contextOf(optedOut)).profile)).toEqual( + agentOf(deriveRuntimeProfile(local, contextOf(runtimeOf(build))).profile), + ); + }, + ); + test("names the default X-Ray target agentcore_traces_default when a target is agentcore_traces", () => { const local = { telemetry: { agentcore_traces: CLOUDWATCH.logs } }; const telemetry = tableOf( diff --git a/src/core/project/runtimeProfile/rules/additions.ts b/src/core/project/runtimeProfile/rules/additions.ts index f5fe63f8b..2416a7662 100644 --- a/src/core/project/runtimeProfile/rules/additions.ts +++ b/src/core/project/runtimeProfile/rules/additions.ts @@ -14,7 +14,7 @@ * | `[tool.]` | a table that the template writes, as it writes it, goes; a table of the user stays | * | `[telemetry.]` | each CloudWatch target that box takes is signed with a bare `aws://`, in the deploy region when it names none; one that box refuses goes; a `file` target goes, with a warning | * | `[telemetry.agentcore]` | added when no target is `cloudwatch_logs`: the runtime's own log group and log stream | - * | `[telemetry.agentcore_traces]` | added with OTEL on when no target is `cloudwatch_traces`, unless observability is off: X-Ray in the Runtime's region | + * | `[telemetry.agentcore_traces]` | added with OTEL on when no target is `cloudwatch_traces`: X-Ray in the Runtime's region | * * It moves no path: the paths rule runs after it and moves each path of the user. */ @@ -25,10 +25,11 @@ import { BOX_BEDROCK_PACK, boxIngressPort, boxWorkloadPort, + OBSERVABILITY_OFF_ENV, + observabilityOff, otelOn, RUNTIME_LOGS_HEADERS_ENV, } from "../../templates/boxProfile"; -import type { ProjectRuntime } from "../../../../projectSchemas/runtime"; import type { CloudWatchTelemetry } from "../../boxNetwork"; import { isTable, isVenvPython } from "../projectPaths"; import { FILE_TELEMETRY_KIND } from "./refusals"; @@ -84,9 +85,9 @@ export const DEFAULT_TELEMETRY_TARGET = { } as const; /** - * The X-Ray target of a Runtime profile with OTEL on that has no `cloudwatch_traces` target, unless - * the envVars of the runtime turn observability off: the agent's spans and box's own, in the - * Runtime's region, signed with the execution role. + * The X-Ray target of a Runtime profile with OTEL on ({@link otelOn}) that has no + * `cloudwatch_traces` target: the agent's spans and box's own, in the Runtime's region, signed with + * the execution role. */ export const DEFAULT_TRACES_TARGET = { kind: "cloudwatch_traces", @@ -117,19 +118,6 @@ function runtimeRegionEnv(bedrock: boolean): Record { return { ...(!bedrock && { AWS_REGION: region }), AWS_DEFAULT_REGION: region }; } -/** - * The variable that turns off the observability of a runtime on AgentCore Runtime. When the envVars - * of the runtime set it to `true`, the Runtime profile gets no default log target. - */ -const OBSERVABILITY_OFF_ENV = "DISABLE_ADOT_OBSERVABILITY"; - -/** Whether the envVars of `runtime` set {@link OBSERVABILITY_OFF_ENV} to `true`. */ -export function observabilityOff(runtime: Pick): boolean { - return (runtime.envVars ?? []).some( - ({ name, value }) => name === OBSERVABILITY_OFF_ENV && value.trim().toLowerCase() === "true", - ); -} - /** * The notice that `agentcore create` and `agentcore add runtime` give for a box runtime: what the * default log target sends, where, what it costs, and how to turn it off. @@ -521,7 +509,7 @@ function onAllAddresses(entry: unknown): unknown { * `agentcore`, or as `agentcore_default` when a target has that name, unless the envVars of the * runtime turn observability off ({@link OBSERVABILITY_OFF_ENV}). With OTEL on and no * `cloudwatch_traces` target, {@link DEFAULT_TRACES_TARGET} comes last, as `agentcore_traces`, or - * as `agentcore_traces_default` when a target has that name, unless observability is off. + * as `agentcore_traces_default` when a target has that name ({@link otelOn}). */ function runtimeTelemetry( telemetry: unknown, @@ -578,7 +566,7 @@ function runtimeTelemetry( const name = Object.hasOwn(result, "agentcore") ? "agentcore_default" : "agentcore"; result[name] = structuredClone(DEFAULT_TELEMETRY_TARGET); } - if (!has("cloudwatch_traces") && otelOn(context.runtime) && !observabilityOff(context.runtime)) { + if (!has("cloudwatch_traces") && otelOn(context.runtime)) { const name = Object.hasOwn(result, "agentcore_traces") ? "agentcore_traces_default" : "agentcore_traces"; diff --git a/src/core/project/templates/boxProfile.test.ts b/src/core/project/templates/boxProfile.test.ts index 5f2f2eef1..77c4a37f9 100644 --- a/src/core/project/templates/boxProfile.test.ts +++ b/src/core/project/templates/boxProfile.test.ts @@ -508,6 +508,18 @@ describe("renderAgentCorePack", () => { }); }); + test("gives the agent no ADOT variable and declares no resource attributes input when the envVars set DISABLE_ADOT_OBSERVABILITY to true", () => { + const spec = { name: "V3Demo" } as unknown as ProjectSpec; + const optedOut = renderAgentCorePack(spec, { + name: "agent", + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }], + }); + + expect(optedOut).toBe( + renderAgentCorePack(spec, { name: "agent", instrumentation: { enableOtel: false } }), + ); + }); + test("turns OTEL on when the runtime has no instrumentation, as create writes it", () => { const text = renderAgentCorePack({ name: "V3Demo" } as unknown as ProjectSpec, { name: "agent", diff --git a/src/core/project/templates/boxProfile.ts b/src/core/project/templates/boxProfile.ts index aa5f7f6d7..3c78d3331 100644 --- a/src/core/project/templates/boxProfile.ts +++ b/src/core/project/templates/boxProfile.ts @@ -224,9 +224,27 @@ export function runsInBox(boxProject: boolean, runtime: Pick): boolean { - return runtime.instrumentation?.enableOtel ?? true; +/** + * The variable that turns off the observability of a runtime on AgentCore Runtime. When the envVars + * of the runtime set it to `true`, the Runtime profile gets no default log target, and the agent + * gets no OTEL ({@link otelOn}). + */ +export const OBSERVABILITY_OFF_ENV = "DISABLE_ADOT_OBSERVABILITY"; + +/** Whether the envVars of `runtime` set {@link OBSERVABILITY_OFF_ENV} to `true`. */ +export function observabilityOff(runtime: Pick): boolean { + return (runtime.envVars ?? []).some( + ({ name, value }) => name === OBSERVABILITY_OFF_ENV && value.trim().toLowerCase() === "true", + ); +} + +/** + * Whether the agent of `runtime` sends OTEL on AgentCore Runtime: `instrumentation.enableOtel`, + * which is `true` when absent, and the envVars do not turn observability off. It decides the + * opentelemetry-instrument start, the ADOT variables, and the default X-Ray target together. + */ +export function otelOn(runtime: Pick): boolean { + return (runtime.instrumentation?.enableOtel ?? true) && !observabilityOff(runtime); } /** {@link runsInBox} for a runtime of `spec`. */ @@ -642,7 +660,7 @@ const RUNTIME_RESOURCE_ATTRIBUTES_ENV = "OTEL_RESOURCE_ATTRIBUTES"; */ export function renderAgentCorePack( spec: ProjectSpec, - runtime: Pick, + runtime: Pick, ): string { const otel = otelOn(runtime); const memories = spec.memories ?? [];