diff --git a/docs/box-local-testing.md b/docs/box-local-testing.md index 9373542ed..5b147d5fb 100644 --- a/docs/box-local-testing.md +++ b/docs/box-local-testing.md @@ -242,6 +242,12 @@ A profile that names a pack `box/` fails: box refuses it and names the `@b `[tool.]`; - the `@box/agentcore` packs, the project name, and the build region come in, and each CloudWatch target signs with the execution role. + - with `instrumentation.enableOtel` on, which is the default for a box template too, a command + of the Python of the venv starts with `opentelemetry-instrument`. On CodeZip, the + Python of the Runtime runs the script in the zip, `/bin/opentelemetry-instrument`. The + `@box/agentcore` pack gives the agent the ADOT variables and the `OTEL_RESOURCE_ATTRIBUTES` + that the Runtime sets. When your `[agent.env]` sets one of these variables, box uses your value. + The pack sets no `OTEL_EXPORTER_OTLP_*` variable, because box sets each one. **The default log target.** With no `cloudwatch_logs` target, the Runtime profile gets `[telemetry.agentcore]`: box sends its deny, permit, and trace records to the log group of the @@ -250,9 +256,19 @@ A profile that names a pack `box/` fails: box refuses it and names the `@b Runtime does not set it, box does not start. The Runtime makes that log group, so its retention applies, and it stays after you remove the runtime. CloudWatch Logs charges for the records it gets. To send none, set `DISABLE_ADOT_OBSERVABILITY` to `"true"` in the `envVars` of the runtime - in `agentcore.json`. To send them to another log group, add a `cloudwatch_logs` target to the - local profile. `agentcore create` and `agentcore add runtime` give this notice for each box - runtime. + in `agentcore.json`. Then the Runtime profile also gets no default X-Ray target. The agent does + not start with `opentelemetry-instrument`, and the `@box/agentcore` pack gives it no ADOT + variable. To send the records to another log group, add a `cloudwatch_logs` target to the local + profile. `agentcore create` and `agentcore add runtime` give this notice for each box runtime. + + **The default X-Ray target.** With `enableOtel` on and no `cloudwatch_traces` target, the + Runtime profile gets `[telemetry.agentcore_traces]`: box sends the spans of the agent, and its + own deny, permit, and trace spans, to X-Ray in the region of the Runtime. Build gives the + execution role `xray:PutSpans` and `xray:PutTraceSegments` in + `agentcore/box-telemetry-policy-.json`. To send the spans to a different target, add a + `cloudwatch_traces` target to the local profile. To send no spans, set + `instrumentation.enableOtel` to `false` in `agentcore.json`. When `enableOtel` is on and the + agent does not start with `opentelemetry-instrument`, build gives a warning. Build stops, and names the key and its line, for a value that cannot work on the Runtime: a required grant or a command path in the project outside the code folder, in the venv, or in a @@ -466,6 +482,11 @@ to the Runtime profile. If the profile has a target with the name `agentcore`, t `AWS_REGION`, and its log group and stream from `OTEL_EXPORTER_OTLP_LOGS_HEADERS`. The target sends the deny, permit, and trace records, and box signs it with the execution role. +With `instrumentation.enableOtel` on, box also sends the spans of the agent and its own spans to +X-Ray, through the target `[telemetry.agentcore_traces]`. Build gives the execution role the X-Ray +grant for it. "The default X-Ray target" in step 4 gives the rules. CloudWatch Transaction Search +keeps the spans in the log group `aws/spans`. + The CLI and the L3 constructs make no log group for box. The role that the CDK makes for the runtime can write to the log group of the Runtime, thus build adds no grant for it. A runtime that names an `executionRoleArn` must have `logs:CreateLogStream` and `logs:PutLogEvents` on diff --git a/src/core/project/backends/boxCodeZip.test.ts b/src/core/project/backends/boxCodeZip.test.ts index 36f99bc4d..5ea9460a3 100644 --- a/src/core/project/backends/boxCodeZip.test.ts +++ b/src/core/project/backends/boxCodeZip.test.ts @@ -103,7 +103,7 @@ describe("stageBoxCodeZip", () => { // The profile imports these generated packs, which box finds beside its binary. They are // written after box's own resources are copied, so the copy does not remove them. expect(await readFile(join(dist, AGENTCORE_PACK_PATH), "utf8")).toBe( - renderAgentCorePack(demo.spec), + renderAgentCorePack(demo.spec, demo.spec.runtimes[0]!), ); expect(await readFile(join(dist, AGENTCORE_CODEZIP_PACK_PATH), "utf8")).toBe( renderAgentCoreCodeZipPack(), diff --git a/src/core/project/backends/boxContainer.test.ts b/src/core/project/backends/boxContainer.test.ts index 2bb9f1a03..834de51ad 100644 --- a/src/core/project/backends/boxContainer.test.ts +++ b/src/core/project/backends/boxContainer.test.ts @@ -90,7 +90,9 @@ describe("stageBoxContainers", () => { `BOX_SHA256=${BOX_RELEASE.linux.sha256}\n`, ); // The container also stages the @box/agentcore pack, beside box's own packs in the image. - expect(await readFile(join(dist, PACK_PATH), "utf8")).toBe(renderAgentCorePack(demo.spec)); + expect(await readFile(join(dist, PACK_PATH), "utf8")).toBe( + renderAgentCorePack(demo.spec, demo.spec.runtimes[0]!), + ); expect((await readdir(dist)).sort()).toEqual([...STAGED, "resources"].sort()); // The runtime's .box/agent/aws/ holds a copy of what the profile writer stages, and no box. const aws = join(demo.rootPath, ".box", "agent", "aws"); diff --git a/src/core/project/backends/cdk.test.ts b/src/core/project/backends/cdk.test.ts index 8466da265..9a28b88ca 100644 --- a/src/core/project/backends/cdk.test.ts +++ b/src/core/project/backends/cdk.test.ts @@ -69,8 +69,10 @@ function unusedIdentity(): CredentialProviderCalls { const NO_BOX_NETWORKS = { boxNetworks: [] }; const OTEL_WARNING = - "Runtime 'agent' runs its agent in a box, where the agent does not run under " + - "opentelemetry-instrument, so it sends no traces. Set instrumentation.enableOtel to false."; + "Runtime 'agent' runs its agent in a box, and [agent] command in " + + ".box/agent/local/profile.toml does not run the venv's Python, so on AgentCore Runtime the " + + "agent does not start under opentelemetry-instrument and sends no traces. Run the agent with " + + "the venv's Python, or set instrumentation.enableOtel to false."; function deployInput(overrides: Partial = {}): DeployBackendInput { return { target: TARGET, confirmTeardown: async () => false, boxNetworks: [], ...overrides }; @@ -332,32 +334,121 @@ describe("CdkBackend.build", () => { expect(subject.commands).toEqual([{ command: synthCommand(input), cwd: cdkDirectory(input) }]); }); + /** The Box network model of the runtime `agent`, with its derived Runtime profile. */ + const boxNetwork = (runtimeProfile: Record) => ({ + runtime: "agent", + gateway: "agent-box-egress", + region: "us-west-2", + hosts: [], + envRoutes: [], + policyFiles: [], + runtimeProfile, + grantedActions: [], + warnings: [], + failures: [], + }); + const PLAIN = ["${{ workspace }}/.venv/bin/python", "-m", "main"]; + const UNDER_OTEL: Record = { + Container: ["${{ workspace }}/.venv/bin/opentelemetry-instrument", ...PLAIN], + CodeZip: [ + "${{ inputs.python.path }}", + "${{ inputs.code.dir }}/bin/opentelemetry-instrument", + "${{ inputs.python.path }}", + "-m", + "main", + ], + }; + // A CodeZip box build downloads box before it warns, so the cases check the warning itself. test.each([ - ["a box container", "box", "Container", true, true], - ["a box CodeZip runtime", "box", "CodeZip", true, true], - ["a box container with enableOtel false", "box", "Container", false, false], - ["a container outside a Box project", undefined, "Container", true, false], - ] as const)("warns that %s sends no traces: %p", (_label, projectType, build, otel, warns) => { + [ + "a box container that starts under opentelemetry-instrument", + "box", + "Container", + true, + true, + false, + ], + [ + "a box CodeZip runtime that starts under opentelemetry-instrument", + "box", + "CodeZip", + true, + true, + false, + ], + ["a box container that does not", "box", "Container", true, false, true], + ["a box CodeZip runtime that does not", "box", "CodeZip", true, false, true], + ["a box container with enableOtel false", "box", "Container", false, false, false], + ["a container outside a Box project", undefined, "Container", true, false, false], + ] as const)( + "warns that %s sends no traces: %p", + (_label, projectType, build, otel, started, warns) => { + const spec = ProjectSpecSchema.parse({ + name: "example", + version: 2, + ...(projectType && { projectType }), + runtimes: [ + { + name: "agent", + build, + ...(build === "CodeZip" && { runtimeVersion: "PYTHON_3_14" }), + entrypoint: "main.py", + codeLocation: "app/agent", + ...(projectType && { box: { version: BOX_RELEASE.version } }), + ...(!otel && { instrumentation: { enableOtel: false } }), + }, + ], + agentCoreGateways: projectType ? [boxGatewaySpec("agent")] : [], + }); + + const command = started ? UNDER_OTEL[build]! : PLAIN; + + expect(boxOtelWarnings(spec, [boxNetwork({ agent: { command } })])).toEqual( + warns ? [OTEL_WARNING] : [], + ); + }, + ); + + test("gives no OTEL warning for a box runtime whose envVars set DISABLE_ADOT_OBSERVABILITY to true", () => { + const spec = ProjectSpecSchema.parse({ + name: "example", + version: 2, + projectType: "box", + runtimes: [ + { + name: "agent", + build: "Container", + entrypoint: "main.py", + codeLocation: "app/agent", + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }], + box: { version: BOX_RELEASE.version }, + }, + ], + agentCoreGateways: [boxGatewaySpec("agent")], + }); + + expect(boxOtelWarnings(spec, [boxNetwork({ agent: { command: PLAIN } })])).toEqual([]); + }); + + test("gives no OTEL warning for a box runtime that has no Box network model", () => { const spec = ProjectSpecSchema.parse({ name: "example", version: 2, - ...(projectType && { projectType }), + projectType: "box", runtimes: [ { name: "agent", - build, - ...(build === "CodeZip" && { runtimeVersion: "PYTHON_3_14" }), + build: "Container", entrypoint: "main.py", codeLocation: "app/agent", - ...(projectType && { box: { version: BOX_RELEASE.version } }), - ...(!otel && { instrumentation: { enableOtel: false } }), + box: { version: BOX_RELEASE.version }, }, ], - agentCoreGateways: projectType ? [boxGatewaySpec("agent")] : [], + agentCoreGateways: [boxGatewaySpec("agent")], }); - expect(boxOtelWarnings(spec)).toEqual(warns ? [OTEL_WARNING] : []); + expect(boxOtelWarnings(spec, [])).toEqual([]); }); test("build yields the OTEL warning of a box runtime", async () => { @@ -381,23 +472,17 @@ describe("CdkBackend.build", () => { await writeFile(join(input.rootPath, "app", "agent", "Dockerfile"), "FROM scratch\n"); // Build stages box from the runtime's Box network model, with its derived Runtime profile. - const boxNetworks = [ - { - runtime: "agent", - gateway: "agent-box-egress", - region: "us-west-2", - hosts: [], - envRoutes: [], - policyFiles: [], - runtimeProfile: { packs: ["@box/base", "@box/agentcore"] }, - grantedActions: [], - warnings: [], - failures: [], - }, - ]; - const events = await collect(harness().backend.build(input, { boxNetworks })); + const plain = { packs: ["@box/base", "@box/agentcore"], agent: { command: PLAIN } }; + const started = { ...plain, agent: { command: UNDER_OTEL.Container } }; + const warned = await collect( + harness().backend.build(input, { boxNetworks: [boxNetwork(plain)] }), + ); + const quiet = await collect( + harness().backend.build(input, { boxNetworks: [boxNetwork(started)] }), + ); - expect(events).toContainEqual({ type: "warning", message: OTEL_WARNING } as ProjectEvent); + expect(warned).toContainEqual({ type: "warning", message: OTEL_WARNING } as ProjectEvent); + expect(quiet).not.toContainEqual({ type: "warning", message: OTEL_WARNING } as ProjectEvent); }); test("warns but continues when the installed CDK is legacy", async () => { diff --git a/src/core/project/backends/cdk.ts b/src/core/project/backends/cdk.ts index ea65fd5ad..4bea42a52 100644 --- a/src/core/project/backends/cdk.ts +++ b/src/core/project/backends/cdk.ts @@ -12,7 +12,8 @@ import type { } from "../../../handlers/project/types"; import { stageBoxCodeZip } from "./boxCodeZip"; import { stageBoxContainers } from "./boxContainer"; -import { runtimeRunsInBox } from "../templates/boxProfile"; +import { boxLocalProfileFile, otelOn, runtimeRunsInBox } from "../templates/boxProfile"; +import { agentStartsUnderOtel } from "../runtimeProfile/rules/additions"; import { boxDownloadMessage, boxReleaseFor, resolveBoxLinuxBuild } from "./boxRelease"; import { cdkCompatibilityWarning } from "./cdk/compatibility"; import { @@ -160,18 +161,28 @@ export type CdkBackendConfig = { }; /** - * A warning for each runtime of `spec` that runs its agent in a box with OTEL on: the agent in the - * box does not run under opentelemetry-instrument, CodeZip or container, so it sends no traces. + * A warning for each runtime of `spec` that runs its agent in a box with OTEL on, and whose + * AgentCore Runtime profile in `boxNetworks` does not start the agent under opentelemetry-instrument: + * build starts only a command of the venv's Python under it, and the agent then sends no traces. */ -export function boxOtelWarnings(spec: Project["spec"]): string[] { +export function boxOtelWarnings( + spec: Project["spec"], + boxNetworks: BuildBackendInput["boxNetworks"], +): string[] { return spec.runtimes - .filter( - (runtime) => runtimeRunsInBox(spec, runtime) && (runtime.instrumentation?.enableOtel ?? true), - ) + .filter((runtime) => runtimeRunsInBox(spec, runtime) && otelOn(runtime)) + .filter((runtime) => { + const profile = boxNetworks.find( + (network) => network.runtime === runtime.name, + )?.runtimeProfile; + return profile !== undefined && !agentStartsUnderOtel(profile); + }) .map( (runtime) => - `Runtime '${runtime.name}' runs its agent in a box, where the agent does not run under ` + - "opentelemetry-instrument, so it sends no traces. Set instrumentation.enableOtel to false.", + `Runtime '${runtime.name}' runs its agent in a box, and [agent] command in ` + + `${boxLocalProfileFile(runtime.name)} does not run the venv's Python, so on AgentCore ` + + "Runtime the agent does not start under opentelemetry-instrument and sends no traces. Run " + + "the agent with the venv's Python, or set instrumentation.enableOtel to false.", ); } @@ -267,7 +278,9 @@ export class CdkBackend implements ProjectBackend { ): AsyncGenerator { yield* await this.ensureCdkDependencies(project, boxNetworks); - for (const message of boxOtelWarnings(project.spec)) yield { type: "warning", message }; + for (const message of boxOtelWarnings(project.spec, boxNetworks)) { + yield { type: "warning", message }; + } const compatibilityWarning = await cdkCompatibilityWarning(this.cdkDirectory(project)); if (compatibilityWarning) { diff --git a/src/core/project/boxNetwork/network.ts b/src/core/project/boxNetwork/network.ts index 49d72b050..b23cbc380 100644 --- a/src/core/project/boxNetwork/network.ts +++ b/src/core/project/boxNetwork/network.ts @@ -28,7 +28,7 @@ import { describeEntry, describeRule, ruleOrigin } from "./places"; import { placeOf, type ParsedPolicy } from "./policyParser"; import { runtimeProfileContext } from "../runtimeProfile/context"; import { deriveRuntimeProfile } from "../runtimeProfile/derive"; -import { runtimeProfileHeader } from "../runtimeProfile/rules/additions"; +import { defaultTracesTelemetry, runtimeProfileHeader } from "../runtimeProfile/rules/additions"; import type { TomlTable } from "../runtimeProfile/types"; import { withDeployRegion, type CloudWatchTelemetry } from "./telemetry"; import { outboundAuthWords, sameOutboundAuth, targetOutboundAuth } from "./projectSpec"; @@ -122,7 +122,10 @@ export interface BoxNetwork { * Runtime profile. */ readonly policyFiles: readonly string[]; - /** The CloudWatch targets of the profile's `[telemetry]`; absent means none. */ + /** + * The CloudWatch targets of the profile's `[telemetry]`, then the default X-Ray target of + * {@link runtimeProfile} (`DEFAULT_TRACES_TARGET`); absent means none. + */ readonly telemetry?: readonly CloudWatchTelemetry[]; /** * The AgentCore Runtime profile that build derives from the local profile (`deriveRuntimeProfile` @@ -217,6 +220,11 @@ export function buildBoxNetwork( failures.push( ...(derived?.problems ?? []).map(({ code, reason, place }) => ({ code, reason, place })), ); + const defaultTraces = derived ? defaultTracesTelemetry(derived.profile, project.region) : []; + const sent = + telemetry || defaultTraces.length > 0 + ? [...(telemetry?.telemetry ?? []), ...defaultTraces] + : undefined; return { runtime: project.runtime.name, gateway: project.gateway, @@ -224,7 +232,7 @@ export function buildBoxNetwork( hosts: targets.hosts, envRoutes: envRoutesOf(targets.hosts), policyFiles: parsed.profile.policyFiles, - ...(telemetry && { telemetry: telemetry.telemetry }), + ...(sent && { telemetry: sent }), ...(derived && context && { runtimeProfile: derived.profile, diff --git a/src/core/project/boxNetworkSource.test.ts b/src/core/project/boxNetworkSource.test.ts index 85151893e..695fcb44b 100644 --- a/src/core/project/boxNetworkSource.test.ts +++ b/src/core/project/boxNetworkSource.test.ts @@ -15,6 +15,7 @@ afterEach(async () => { await Promise.all(directories.splice(0).map((d) => rm(d, { recursive: true, force: true }))); }); +/** A Box project with the runtime `agent`, with OTEL off: only the OTEL tests turn it on. */ const SPEC = { name: "demo", projectType: "box", @@ -24,6 +25,7 @@ const SPEC = { build: "CodeZip", entrypoint: "start.py", codeLocation: "app/agent", + instrumentation: { enableOtel: false }, box: { version: "1.0.0" }, }, ], @@ -403,6 +405,97 @@ describe("the telemetry of each box runtime", () => { }); }); +describe("the default X-Ray target of a runtime with OTEL on", () => { + const OTEL_SPEC = { + ...SPEC, + runtimes: [{ ...SPEC.runtimes[0]!, instrumentation: undefined }], + } as unknown as ProjectSpec; + const TRACES = { + name: "agentcore_traces", + kind: "cloudwatch_traces" as const, + region: "us-west-2", + include: ["deny", "permit", "trace"], + }; + const X_RAY = { + Sid: "BoxTelemetryToXRay", + Effect: "Allow", + Action: ["xray:PutSpans", "xray:PutTraceSegments"], + Resource: "*", + }; + + /** The telemetry policy file that build writes for the model `network`. */ + const policyOf = (network: NonNullable>[number]>) => + telemetryPolicyWriter.edits([network], { + spec: OTEL_SPEC, + projectRoot: "/project", + env: {}, + envLocal: {}, + }).files ?? []; + + test("gets the grant of the execution role for X-Ray, and with OTEL off gets none", async () => { + const root = await project([]); + const box = new FakeBoxProfileResolver(boxShowOf({ egress: API_ROUTE })); + + const [on] = await readBoxNetworks(OTEL_SPEC, root, box); + const [off] = await readBoxNetworks(SPEC, root, box); + + expect(on?.telemetry).toEqual([TRACES]); + const telemetry = on?.runtimeProfile?.telemetry as Record; + expect(telemetry.agentcore_traces).toEqual({ + kind: "cloudwatch_traces", + region: "${{ inputs.aws.region }}", + include: ["deny", "permit", "trace"], + secret: { ref: "aws://" }, + }); + const [file] = policyOf(on!); + expect(JSON.parse((file as { text: string }).text)).toEqual({ + Version: "2012-10-17", + Statement: [X_RAY], + }); + expect(off?.telemetry).toBeUndefined(); + expect(policyOf(off!)).toEqual([{ path: boxTelemetryPolicyFile("agent"), remove: true }]); + }); + + test("is not added, and gets no grant, when the envVars set DISABLE_ADOT_OBSERVABILITY to true", async () => { + const root = await project([]); + const box = new FakeBoxProfileResolver(boxShowOf({ egress: API_ROUTE })); + const spec = { + ...OTEL_SPEC, + runtimes: [ + { + ...OTEL_SPEC.runtimes[0]!, + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }], + }, + ], + } as unknown as ProjectSpec; + + const [network] = await readBoxNetworks(spec, root, box); + + expect(network?.runtimeProfile?.telemetry).toEqual({}); + expect(network?.telemetry).toBeUndefined(); + expect(policyOf(network!)).toEqual([{ path: boxTelemetryPolicyFile("agent"), remove: true }]); + }); + + test("is not added, and gets no second grant, when the profile has its own cloudwatch_traces target", async () => { + const root = await project([]); + const shown = boxShowOf({ egress: API_ROUTE }); + const own = { kind: "cloudwatch_traces", region: "eu-west-1", secret: { ref: "aws://" } }; + const box = new FakeBoxProfileResolver({ + ...shown, + boxToml: { ...shown.boxToml, telemetry: { mine: own } }, + }); + + const [network] = await readBoxNetworks(OTEL_SPEC, root, box); + + expect(network?.telemetry).toEqual([ + { name: "mine", kind: "cloudwatch_traces", region: "eu-west-1" }, + ]); + expect(network?.runtimeProfile?.telemetry).not.toHaveProperty("agentcore_traces"); + const [file] = policyOf(network!); + expect(JSON.parse((file as { text: string }).text).Statement).toEqual([X_RAY]); + }); +}); + describe("two routes of one profile to one host", () => { /** The Box network model of `agent` for the local profile `text`, which box resolves to `egress`. */ async function networkOf(text: string, egress: Record) { diff --git a/src/core/project/boxNetworkWriters.ts b/src/core/project/boxNetworkWriters.ts index 88bdbbe73..9a2d48c88 100644 --- a/src/core/project/boxNetworkWriters.ts +++ b/src/core/project/boxNetworkWriters.ts @@ -405,9 +405,10 @@ export function runtimeProfileWriter(staged: readonly StagedRuntime[]): BoxNetwo edits: (networks, { spec }) => ({ files: staged.flatMap(({ runtime, dist, kind }) => { const network = networkOf(networks, runtime); + const entry = spec.runtimes.find(({ name }) => name === runtime) ?? { name: runtime }; return [ { path: `${dist}/profile.toml`, text: runtimeProfileText(runtime, network, spec) }, - { path: `${dist}/${AGENTCORE_PACK_PATH}`, text: renderAgentCorePack(spec) }, + { path: `${dist}/${AGENTCORE_PACK_PATH}`, text: renderAgentCorePack(spec, entry) }, ...(kind === "codezip" ? [ { diff --git a/src/core/project/manager.boxCredentials.test.ts b/src/core/project/manager.boxCredentials.test.ts index 48dbe4586..8cefab80b 100644 --- a/src/core/project/manager.boxCredentials.test.ts +++ b/src/core/project/manager.boxCredentials.test.ts @@ -105,10 +105,19 @@ describe("FsProjectManager.build: box env:// credentials", () => { ...((before.credentials as unknown[] | undefined) ?? []), { authorizerType: "ApiKeyCredentialProvider", name: "github" }, ]); - // Only `credentials` and the Box gateway change: every other field keeps its value and place. + // Only `credentials`, the Box gateway, and the X-Ray grant of the runtime's default traces + // target change: every other field keeps its value and place. const others = (spec: Record) => - Object.entries(spec).filter(([key]) => !["credentials", "agentCoreGateways"].includes(key)); + Object.entries({ + ...spec, + runtimes: (spec.runtimes as Record[]).map( + ({ additionalPolicies: _grant, ...runtime }) => runtime, + ), + }).filter(([key]) => !["credentials", "agentCoreGateways"].includes(key)); expect(others(after)).toEqual(others(before)); + expect((after.runtimes as Record[])[0]?.additionalPolicies).toEqual([ + `../../agentcore/box-telemetry-policy-${AGENT_PYTHON.runtimeName}.json`, + ]); const stored = await new EnvLocalFile(created.rootPath).read(); expect(stored[RESERVED]).toBe("ghp_shell"); expect(messages).toContain(`writing ${RESERVED} to ${ENV_LOCAL_RELATIVE_PATH}`); diff --git a/src/core/project/manager.boxNetwork.test.ts b/src/core/project/manager.boxNetwork.test.ts index 91fa06fb8..9dff64eb3 100644 --- a/src/core/project/manager.boxNetwork.test.ts +++ b/src/core/project/manager.boxNetwork.test.ts @@ -197,6 +197,20 @@ async function editGateway(root: string, edit: (gateway: { targets: unknown[] }) await writeFile(path(root, SPEC), JSON.stringify(spec, undefined, 2)); } +/** Sets `instrumentation.enableOtel` of the runtime in agentcore.json. */ +async function setOtel(root: string, enableOtel: boolean) { + const spec = JSON.parse(await text(root, SPEC)); + spec.runtimes[0].instrumentation = { enableOtel }; + await writeFile(path(root, SPEC), JSON.stringify(spec, undefined, 2)); +} + +/** Sets `DISABLE_ADOT_OBSERVABILITY` to `"true"` in the envVars of the runtime in agentcore.json. */ +async function optOut(root: string) { + const spec = JSON.parse(await text(root, SPEC)); + spec.runtimes[0].envVars = [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }]; + await writeFile(path(root, SPEC), JSON.stringify(spec, undefined, 2)); +} + async function build(manager: FsProjectManager, root: string) { const project = (await manager.resolve({ filePath: root })) as Project; return drain(manager.build(project)); @@ -657,6 +671,13 @@ describe("build writes the telemetry policy of the box profile", () => { `adding ${TELEMETRY_POLICY} to additionalPolicies of 'agent_python_strands', so its box can ` + "send telemetry to CloudWatch"; + /** {@link strandsProject} with OTEL off, so its Runtime profile gets no default X-Ray target. */ + async function otelOffProject(options?: Parameters[0]) { + const created = await strandsProject(options); + await setOtel(created.root, false); + return created; + } + /** What box resolves the created profile to, with `telemetry` as its [telemetry] tables. */ const showsTelemetry = (telemetry: Record) => { const answer = shows(); @@ -675,7 +696,7 @@ describe("build writes the telemetry policy of the box profile", () => { ])( "for %s, writes the grant and lists it on the box runtime", async (_label, telemetry, statements) => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); box.answer = showsTelemetry(telemetry); const { steps } = await build(manager, root); @@ -694,7 +715,7 @@ describe("build writes the telemetry policy of the box profile", () => { ); test("sets only additionalPolicies of the runtime, and keeps its other keys in their place", async () => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); const spec = JSON.parse(await text(root, SPEC)); spec.runtimes[0].myNote = "kept"; await writeFile(path(root, SPEC), JSON.stringify(spec, undefined, 2)); @@ -709,7 +730,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("copies both CloudWatch targets into the Runtime profile, signed with the execution role", async () => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); const records = { kind: "file", destination: "/tmp/records.jsonl" }; box.answer = showsTelemetry({ logs: LOGS, traces: TRACES, records }); @@ -724,7 +745,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("changes nothing on a second build", async () => { - const { manager, root, counter, box } = await strandsProject(); + const { manager, root, counter, box } = await otelOffProject(); box.answer = showsTelemetry({ logs: LOGS, traces: TRACES }); await build(manager, root); const policy = await text(root, TELEMETRY_POLICY); @@ -738,7 +759,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("writes no file and lists nothing when the profile sends no telemetry to CloudWatch", async () => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); box.answer = showsTelemetry({ langfuse: { kind: "langfuse", secret: { ref: "env://KEY" } } }); const { steps } = await build(manager, root); @@ -749,7 +770,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("removes the file and the entry when the profile no longer sends telemetry to CloudWatch", async () => { - const { manager, root, box } = await strandsProject(); + const { manager, root, box } = await otelOffProject(); box.answer = showsTelemetry({ logs: LOGS }); await build(manager, root); box.answer = shows(); @@ -774,7 +795,7 @@ describe("build writes the telemetry policy of the box profile", () => { }); test("on build, sets the region build builds for, keeps a target's own region, and warns that dev needs a region", async () => { - const { manager, root, box } = await strandsProject({ env: { AWS_REGION: "eu-central-1" } }); + const { manager, root, box } = await otelOffProject({ env: { AWS_REGION: "eu-central-1" } }); box.answer = showsTelemetry({ logs: REGIONLESS_LOGS, traces: { ...TRACES, region: "eu-west-1" }, @@ -814,7 +835,7 @@ describe("build writes the telemetry policy of the box profile", () => { return []; }, }; - const { manager, root, box } = await strandsProject({ backends: { CDK: backend } }); + const { manager, root, box } = await otelOffProject({ backends: { CDK: backend } }); box.answer = showsTelemetry({ logs: REGIONLESS_LOGS }); await writeFile( path(root, "agentcore/aws-targets.json"), @@ -844,6 +865,81 @@ describe("build writes the telemetry policy of the box profile", () => { }); }); +describe("build with OTEL on, as create writes a box runtime", () => { + const TELEMETRY_POLICY = "agentcore/box-telemetry-policy-agent_python_strands.json"; + const X_RAY = { + Sid: "BoxTelemetryToXRay", + Effect: "Allow", + Action: ["xray:PutSpans", "xray:PutTraceSegments"], + Resource: "*", + }; + const DEFAULT_TRACES = { + kind: "cloudwatch_traces", + region: "${{ inputs.aws.region }}", + include: ["deny", "permit", "trace"], + secret: { ref: "aws://" }, + }; + const PACK = "app/agent_python_strands/.box-dist/resources/packs/agentcore/pack.toml"; + + test("starts the agent under opentelemetry-instrument, sets up its ADOT SDK, and sends its spans to X-Ray with the grant", async () => { + const { manager, root } = await strandsProject(); + expect(JSON.parse(await text(root, SPEC)).runtimes[0].instrumentation).toBeUndefined(); + + const { warnings } = await build(manager, root); + + const profile = await runtimeProfile(root); + expect((profile.agent as { command: string[] }).command.slice(0, 3)).toEqual([ + "${{ inputs.python.path }}", + "${{ inputs.code.dir }}/bin/opentelemetry-instrument", + "${{ inputs.python.path }}", + ]); + expect((profile.telemetry as Record).agentcore_traces).toEqual(DEFAULT_TRACES); + const pack = parse(await text(root, PACK)) as { agent: { env: Record } }; + expect(pack.agent.env).toMatchObject({ + OTEL_PYTHON_DISTRO: "aws_distro", + OTEL_TRACES_EXPORTER: "otlp", + OTEL_RESOURCE_ATTRIBUTES: "${{ inputs.runtime.resource_attributes }}", + }); + expect(JSON.parse(await text(root, TELEMETRY_POLICY)).Statement).toEqual([X_RAY]); + expect(JSON.parse(await text(root, SPEC)).runtimes[0].additionalPolicies).toEqual([ + "../../agentcore/box-telemetry-policy-agent_python_strands.json", + ]); + expect(warnings.filter((warning) => warning.includes("opentelemetry-instrument"))).toEqual([]); + }); + + test.each([ + ["enableOtel is false", (root: string) => setOtel(root, false)], + ["the envVars set DISABLE_ADOT_OBSERVABILITY to true", (root: string) => optOut(root)], + ])("adds nothing for OTEL when %s", async (_label, turnOff) => { + const { manager, root } = await strandsProject(); + await turnOff(root); + + await build(manager, root); + + const profile = await runtimeProfile(root); + expect(JSON.stringify(profile.agent)).not.toContain("opentelemetry-instrument"); + expect(profile.telemetry).not.toHaveProperty("agentcore_traces"); + const pack = parse(await text(root, PACK)) as { agent: { env: Record } }; + expect(Object.keys(pack.agent.env)).toEqual(["MEMORY_AGENT_PYTHON_STRANDSMEMORY_ID"]); + expect(existsSync(path(root, TELEMETRY_POLICY))).toBe(false); + }); + + test("changes nothing on a second build", async () => { + const { manager, root, counter } = await strandsProject(); + await build(manager, root); + const policy = await text(root, TELEMETRY_POLICY); + const staged = await text(root, RUNTIME_PROFILE); + counter.specWrites = 0; + + const { steps } = await build(manager, root); + + expect(steps).toEqual(["synthesizing CloudFormation templates"]); + expect(counter.specWrites).toBe(0); + expect(await text(root, TELEMETRY_POLICY)).toBe(policy); + expect(await text(root, RUNTIME_PROFILE)).toBe(staged); + }); +}); + describe("build reads the box profile as box resolves it", () => { test("takes the route a pack adds, which the profile itself does not hold", async () => { const { manager, root, box } = await strandsProject({ env: { SEARCH_KEY: "key-1" } }); diff --git a/src/core/project/manager.test.ts b/src/core/project/manager.test.ts index 99f66dc18..f72d95162 100644 --- a/src/core/project/manager.test.ts +++ b/src/core/project/manager.test.ts @@ -2106,7 +2106,8 @@ describe("FsProjectManager.build", () => { }, ]); // Build adds the Bedrock and Memory targets, and grants the Box gateway's role each, since the - // gateway signs the agent's calls with that role. + // gateway signs the agent's calls with that role. With OTEL on, it grants the execution role + // X-Ray for the default traces target of the Runtime profile. expect(events).toEqual([ { type: "step", @@ -2128,8 +2129,17 @@ describe("FsProjectManager.build", () => { message: "adding 'agentcore/box-memory-policy.json' to gateway 'agent-python-strands-box-box-egress' additionalPolicies in agentcore.json, so its role can call Memory", }, + { + type: "step", + message: + "adding agentcore/box-telemetry-policy-agent_python_strands_box.json to additionalPolicies of 'agent_python_strands_box', so its box can send telemetry to CloudWatch", + }, { type: "step", message: "writing agentcore/box-bedrock-policy.json" }, { type: "step", message: "writing agentcore/box-memory-policy.json" }, + { + type: "step", + message: "writing agentcore/box-telemetry-policy-agent_python_strands_box.json", + }, { type: "step", message: "synthesizing CloudFormation templates" }, ]); // The zip carries box and the Runtime profile in the code folder's .box-dist. diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index d7c3b9bbe..480709beb 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -69,11 +69,12 @@ import { boxProfileWarnings, boxUvSyncEnv, hasBoxProfile, + observabilityOff, renderBoxProfile, runsInBox, startsInBoxOnCodeZip, } from "./templates/boxProfile"; -import { defaultLogTargetNotice, observabilityOff } from "./runtimeProfile/rules/additions"; +import { defaultLogTargetNotice } from "./runtimeProfile/rules/additions"; import { validateBoxRuntimes } from "./boxValidation"; import { BOX_INSTALL_HINT, BOX_RELEASE } from "./backends/boxRelease"; import { getRuntimeTemplateResolver } from "./templates/runtime"; diff --git a/src/core/project/runtimeProfile/context.ts b/src/core/project/runtimeProfile/context.ts index 9b7a21cf7..16c3ab092 100644 --- a/src/core/project/runtimeProfile/context.ts +++ b/src/core/project/runtimeProfile/context.ts @@ -35,6 +35,7 @@ export function runtimePaths( runtimeCode: WORKSPACE, runtimePython: `${WORKSPACE}/.venv/bin/python`, runtimePythonPrefix: "/usr/local", + otelInstrument: [`${WORKSPACE}/.venv/bin/opentelemetry-instrument`], shownCode: "/app/", }; } @@ -50,6 +51,10 @@ export function runtimePaths( runtimeCode: RUNTIME_INPUTS.codeDir, runtimePython: RUNTIME_INPUTS.pythonPath, runtimePythonPrefix: RUNTIME_INPUTS.pythonPrefix, + otelInstrument: [ + RUNTIME_INPUTS.pythonPath, + `${RUNTIME_INPUTS.codeDir}/bin/opentelemetry-instrument`, + ], shownCode: root ? "./" : `${folder}/`, }; } diff --git a/src/core/project/runtimeProfile/derive.test.ts b/src/core/project/runtimeProfile/derive.test.ts index dff787616..eaec6ce02 100644 --- a/src/core/project/runtimeProfile/derive.test.ts +++ b/src/core/project/runtimeProfile/derive.test.ts @@ -82,6 +82,10 @@ describe("runtimePaths", () => { runtimeCode: "${{ inputs.code.dir }}", runtimePython: "${{ inputs.python.path }}", runtimePythonPrefix: "${{ inputs.python.prefix }}", + otelInstrument: [ + "${{ inputs.python.path }}", + "${{ inputs.code.dir }}/bin/opentelemetry-instrument", + ], shownCode: "app/agent/", }); expect(runtimePaths({ ...CODEZIP, codeLocation: "./app/agent/" } as ProjectRuntime)).toEqual( @@ -94,6 +98,7 @@ describe("runtimePaths", () => { expect(paths.runtimeCode).toBe(paths.localCode); expect(paths.runtimePython).toBe("${{ workspace }}/.venv/bin/python"); + expect(paths.otelInstrument).toEqual(["${{ workspace }}/.venv/bin/opentelemetry-instrument"]); }); }); diff --git a/src/core/project/runtimeProfile/projectPaths.ts b/src/core/project/runtimeProfile/projectPaths.ts index edbdd3b6f..a2fc4a80b 100644 --- a/src/core/project/runtimeProfile/projectPaths.ts +++ b/src/core/project/runtimeProfile/projectPaths.ts @@ -218,6 +218,12 @@ export function programOf(table: Readonly): string | undefined { return Array.isArray(command) && typeof command[0] === "string" ? command[0] : undefined; } +/** Whether `value` is the venv's Python as the local profile names it: `/bin/python[3[.N]]`. */ +export function isVenvPython(value: string, paths: RuntimePaths): boolean { + const normal = value.replace(/\$\{\{\s*workspace\s*\}\}/g, () => WORKSPACE); + return new RegExp(`^${literal(paths.localVenv)}/bin/python(?:3(?:\\.\\d+)?)?$`).test(normal); +} + /** Whether `program`, as a CodeZip Runtime profile names it, is on the Runtime. */ export function isRuntimeProgram(program: string, paths: RuntimePaths): boolean { return program === paths.runtimePython || program.startsWith(`${paths.runtimeCode}/`); diff --git a/src/core/project/runtimeProfile/rules/additions.test.ts b/src/core/project/runtimeProfile/rules/additions.test.ts index 1b3dc8202..b0327d932 100644 --- a/src/core/project/runtimeProfile/rules/additions.test.ts +++ b/src/core/project/runtimeProfile/rules/additions.test.ts @@ -17,7 +17,14 @@ import { runtimeProfileContext } from "../context"; import { deriveRuntimeProfile } from "../derive"; import { serializeRuntimeProfile } from "../serialize"; import type { TomlTable } from "../types"; -import { apply, DEFAULT_TELEMETRY_TARGET, runtimeProfileHeader } from "./additions"; +import { + agentStartsUnderOtel, + apply, + DEFAULT_TELEMETRY_TARGET, + DEFAULT_TRACES_TARGET, + defaultTracesTelemetry, + runtimeProfileHeader, +} from "./additions"; const PROFILE_FILE = ".box/agent/local/profile.toml"; /** The region create resolved, which the local profile names: not the build region. */ @@ -27,7 +34,16 @@ const BUILD_REGION = "us-west-2"; type Build = "CodeZip" | "Container"; type Protocol = NonNullable; -function runtimeOf(build: Build, protocol: Protocol = "HTTP", modelProvider = "Bedrock") { +/** + * A box runtime, with OTEL off unless `otel`: only the OTEL tests depend on it. With OTEL on it has + * no `instrumentation`, as `agentcore create` writes it. + */ +function runtimeOf( + build: Build, + protocol: Protocol = "HTTP", + modelProvider = "Bedrock", + otel = false, +) { return { name: "agent", build, @@ -35,6 +51,7 @@ function runtimeOf(build: Build, protocol: Protocol = "HTTP", modelProvider = "B codeLocation: "app/agent", protocol, modelProvider, + ...(!otel && { instrumentation: { enableOtel: false } }), box: { version: "1.0.0" }, } as ProjectRuntime; } @@ -128,6 +145,36 @@ const DEFAULT_TARGET = { secret: { ref: "aws://" }, }; +/** The default X-Ray target of a Runtime profile with OTEL on. */ +const DEFAULT_TRACES = { + kind: "cloudwatch_traces", + region: "${{ inputs.aws.region }}", + include: ["deny", "permit", "trace"], + secret: { ref: "aws://" }, +}; + +/** The words that start the agent under opentelemetry-instrument on the Runtime. */ +const OTEL_START: Record = { + CodeZip: ["${{ inputs.python.path }}", "${{ inputs.code.dir }}/bin/opentelemetry-instrument"], + Container: ["${{ workspace }}/.venv/bin/opentelemetry-instrument"], +}; + +/** + * `profile`, a Runtime profile with OTEL off, as OTEL on gives it: the agent starts under + * opentelemetry-instrument, and the default X-Ray target comes last when no target is + * `cloudwatch_traces`. Nothing else changes. + */ +function withOtel(profile: TomlTable, build: Build): TomlTable { + const result = structuredClone(profile); + const agent = result.agent as TomlTable; + agent.command = [...OTEL_START[build], ...(agent.command as string[])]; + const telemetry = result.telemetry as Record; + if (!Object.values(telemetry).some(({ kind }) => kind === "cloudwatch_traces")) { + telemetry.agentcore_traces = DEFAULT_TRACES; + } + return result; +} + /** The CloudWatch targets that box takes from the local profile `local`, as build gives them. */ const acceptedTelemetry = (local: TomlTable) => withDeployRegion(cloudWatchTelemetry(local), BUILD_REGION, PROFILE_FILE).telemetry; @@ -185,21 +232,17 @@ describe("deriveRuntimeProfile on the created templates", () => { ).flatMap(([build, protocol]) => ["Bedrock", "OpenAI"].flatMap((provider) => [[], ["agentMemory"]].flatMap((memories) => - Object.keys(TELEMETRY_CASES).map((telemetry) => ({ - build, - protocol, - provider, - memories, - telemetry, - })), + Object.keys(TELEMETRY_CASES).flatMap((telemetry) => + [false, true].map((otel) => ({ build, protocol, provider, memories, telemetry, otel })), + ), ), ), ); test.each(cases)( - "gives the tables the old renderer wrote (recorded in __fixtures__), and the default target with no cloudwatch_logs target ($build $protocol $provider, memories $memories, telemetry $telemetry)", - ({ build, protocol, provider, memories, telemetry: telemetryCase }) => { - const runtime = runtimeOf(build, protocol, provider); + "gives the tables the old renderer wrote (recorded in __fixtures__), and the default target with no cloudwatch_logs target, and with OTEL on only the OTEL start and the default X-Ray target ($build $protocol $provider, memories $memories, telemetry $telemetry, OTEL $otel)", + ({ build, protocol, provider, memories, telemetry: telemetryCase, otel }) => { + const runtime = runtimeOf(build, protocol, provider, otel); const local = createdProfile(runtime); const tables = TELEMETRY_CASES[telemetryCase]; const extras = tables !== undefined; @@ -217,9 +260,10 @@ describe("deriveRuntimeProfile on the created templates", () => { contextOf(runtime, { memories, telemetry }), ); - const expected = OLD_RENDERER.templates[ + const recorded = OLD_RENDERER.templates[ `${build} ${protocol} ${provider} memories=${memories.length} telemetry=${telemetryCase}` ] as TomlTable; + const expected = otel ? withOtel(recorded, build) : recorded; expect(problems).toEqual([]); for (const key of OWNED) expect([key, profile[key]]).toEqual([key, expected[key] ?? undefined]); @@ -232,8 +276,11 @@ const BOX_TEMPLATES = (Object.keys(RUNTIME_TEMPLATE_SHORTCUTS) as RuntimeTemplat .filter((name) => (RUNTIME_TEMPLATE_SHORTCUTS[name] as { box?: boolean }).box === true) .sort(); -/** The runtime that `agentcore create` writes into agentcore.json for the template `input`. */ -function createdRuntime(input: ScaffoldRuntimeInput): ProjectRuntime { +/** + * The runtime that `agentcore create` writes into agentcore.json for the template `input`, with + * `instrumentation.enableOtel` false unless `otel`. + */ +function createdRuntime(input: ScaffoldRuntimeInput, otel: boolean): ProjectRuntime { return { name: input.runtimeName, build: input.build, @@ -241,6 +288,7 @@ function createdRuntime(input: ScaffoldRuntimeInput): ProjectRuntime { codeLocation: `app/${input.runtimeName}`, ...(input.protocol !== undefined && { protocol: input.protocol }), ...(input.modelProvider !== undefined && { modelProvider: input.modelProvider }), + ...(!otel && { instrumentation: { enableOtel: false } }), box: { version: "1.0.0" }, } as ProjectRuntime; } @@ -252,24 +300,21 @@ describe("deriveRuntimeProfile on each box template of create", () => { : ["of the template"]; return providers.flatMap((provider) => [[], ["agentMemory"]].flatMap((memories) => - Object.keys(TELEMETRY_CASES).map((telemetry) => ({ - template, - provider, - memories, - telemetry, - })), + Object.keys(TELEMETRY_CASES).flatMap((telemetry) => + [false, true].map((otel) => ({ template, provider, memories, telemetry, otel })), + ), ), ); }); test.each(cases)( - "gives the whole profile that the old renderer wrote (recorded in __fixtures__), with the default target when no target is cloudwatch_logs ($template, provider $provider, memories $memories, telemetry $telemetry)", - ({ template, provider, memories, telemetry: telemetryCase }) => { + "gives the whole profile that the old renderer wrote (recorded in __fixtures__), with the default target when no target is cloudwatch_logs, and with OTEL on only the OTEL start and the default X-Ray target ($template, provider $provider, memories $memories, telemetry $telemetry, OTEL $otel)", + ({ template, provider, memories, telemetry: telemetryCase, otel }) => { const input = resolveRuntimeTemplateShortcut( template as RuntimeTemplateShortcutName, provider === "OpenAI" ? { modelProvider: "OpenAI", apiKey: "sk-test" } : {}, ); - const runtime = createdRuntime(input); + const runtime = createdRuntime(input, otel); const local = parse(renderBoxProfile("demo", input, CREATE_REGION)!) as TomlTable; const tables = TELEMETRY_CASES[telemetryCase]; const extras = tables !== undefined; @@ -288,12 +333,12 @@ describe("deriveRuntimeProfile on each box template of create", () => { contextOf(runtime, { memories, telemetry }), ); - expect(problems).toEqual([]); - expect(profile).toEqual( + const recorded = OLD_RENDERER.boxTemplates[ `${template} provider=${provider} memories=${memories.length} telemetry=${telemetryCase}` - ]!, - ); + ]!; + expect(problems).toEqual([]); + expect(profile).toEqual(otel ? withOtel(recorded, input.build as Build) : recorded); }, ); }); @@ -815,7 +860,7 @@ describe("the additions rule", () => { }); test("reads each value through an input that the @box/agentcore pack declares", () => { - const { inputs } = parse(renderAgentCorePack(specOf(CODEZIP))) as { + const { inputs } = parse(renderAgentCorePack(specOf(CODEZIP), CODEZIP)) as { inputs: Record>; }; const declared = (placeholder: string) => { @@ -920,6 +965,223 @@ describe("the additions rule", () => { }); }); + describe("with OTEL on", () => { + const OTEL_CODEZIP = runtimeOf("CodeZip", "HTTP", "Bedrock", true); + const OTEL_CONTAINER = runtimeOf("Container", "HTTP", "Bedrock", true); + const otelRuntime = (build: Build) => (build === "CodeZip" ? OTEL_CODEZIP : OTEL_CONTAINER); + const BUILDS: Build[] = ["CodeZip", "Container"]; + + test.each(BUILDS)( + "starts the template's own command under opentelemetry-instrument, and with OTEL off does not (%s)", + (build) => { + const local = createdProfile(otelRuntime(build)); + const off = agentOf(deriveRuntimeProfile(local, contextOf(runtimeOf(build))).profile); + const on = agentOf(deriveRuntimeProfile(local, contextOf(otelRuntime(build))).profile); + + expect(on.command).toEqual([...OTEL_START[build], ...off.command]); + expect(off.command.join(" ")).not.toContain("opentelemetry-instrument"); + }, + ); + + test.each(["MCP", "A2A", "AGUI"] as const)( + "starts a CodeZip %s template with the Runtime's Python and the script of the zip", + (protocol) => { + const runtime = runtimeOf("CodeZip", protocol, "Bedrock", true); + + expect( + agentOf(deriveRuntimeProfile(createdProfile(runtime), contextOf(runtime)).profile) + .command, + ).toEqual([...OTEL_START.CodeZip, "${{ inputs.python.path }}", "-m", "main"]); + }, + ); + + test.each(BUILDS)( + "starts a command of the user that runs the venv's Python under opentelemetry-instrument (%s)", + (build) => { + const command = ["${{workspace}}/.venv/bin/python3.12", "-m", "main", "--debug"]; + + expect( + agentOf(apply({ agent: { command } }, contextOf(otelRuntime(build))).table).command, + ).toEqual([...OTEL_START[build], ...command]); + }, + ); + + test.each(BUILDS)( + "does not start a command under opentelemetry-instrument twice (%s)", + (build) => { + const python = "${{ workspace }}/.venv/bin/python"; + const script = "${{ workspace }}/.venv/bin/opentelemetry-instrument"; + // The script as the program, and as the script that the venv's Python runs. + for (const own of [ + [script, python, "-m", "main"], + [python, script, python, "-m", "main"], + ]) { + expect( + agentOf(apply({ agent: { command: own } }, contextOf(otelRuntime(build))).table) + .command, + ).toEqual(own); + } + }, + ); + + test("leaves a command of another program as it is, and an [agent] with no command", () => { + const command = ["/usr/local/bin/node", "${{ workspace }}/server.js"]; + + expect( + agentOf(apply({ agent: { command } }, contextOf(OTEL_CONTAINER)).table).command, + ).toEqual(command); + expect(agentOf(apply({ agent: {} }, contextOf(OTEL_CONTAINER)).table)).not.toHaveProperty( + "command", + ); + }); + + test("tells whether the agent of a Runtime profile starts under opentelemetry-instrument", () => { + const started = (command: unknown) => agentStartsUnderOtel({ agent: { command } }); + + expect(started([...OTEL_START.Container, "${{ workspace }}/.venv/bin/python"])).toBe(true); + expect(started([...OTEL_START.CodeZip, "${{ inputs.python.path }}"])).toBe(true); + expect(started(["${{ workspace }}/.venv/bin/python", "-m", "main"])).toBe(false); + expect(started(undefined)).toBe(false); + expect(agentStartsUnderOtel({})).toBe(false); + }); + + test.each(BUILDS)( + "adds the default X-Ray target when no target is cloudwatch_traces, and with OTEL off does not (%s)", + (build) => { + expect(tableOf(apply({}, contextOf(otelRuntime(build))).table)).toEqual({ + agentcore: DEFAULT_TARGET, + agentcore_traces: DEFAULT_TRACES, + }); + expect(tableOf(apply({}, contextOf(runtimeOf(build))).table)).toEqual({ + agentcore: DEFAULT_TARGET, + }); + expect({ ...DEFAULT_TRACES_TARGET } as TomlTable).toEqual(DEFAULT_TRACES); + }, + ); + + test("adds the default X-Ray target beside a cloudwatch_logs target of the user", () => { + const local = { telemetry: { logs: CLOUDWATCH.logs } }; + + expect( + Object.keys( + tableOf( + apply(local, contextOf(OTEL_CODEZIP, { telemetry: acceptedTelemetry(local) })).table, + ), + ), + ).toEqual(["logs", "agentcore_traces"]); + }); + + test("adds no default X-Ray target when the profile has its own cloudwatch_traces target", () => { + const local = { telemetry: { "x-ray": CLOUDWATCH["x-ray"] } }; + + expect( + tableOf( + apply(local, contextOf(OTEL_CODEZIP, { telemetry: acceptedTelemetry(local) })).table, + ), + ).toEqual({ + "x-ray": { kind: "cloudwatch_traces", region: BUILD_REGION, secret: { ref: "aws://" } }, + agentcore: DEFAULT_TARGET, + }); + }); + + test("adds no default X-Ray target when box resolves a cloudwatch_traces target that the profile does not declare", () => { + const telemetry: CloudWatchTelemetry[] = [ + { name: "pack-traces", kind: "cloudwatch_traces", region: "eu-west-1" }, + ]; + + expect(Object.keys(tableOf(apply({}, contextOf(OTEL_CODEZIP, { telemetry })).table))).toEqual( + ["pack-traces", "agentcore"], + ); + }); + + test.each([ + ["true", false], + [" TRUE ", false], + ["false", true], + ])( + "adds the default X-Ray target only when the envVars do not set DISABLE_ADOT_OBSERVABILITY to true (%j)", + (value, added) => { + const runtime = { + ...OTEL_CODEZIP, + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value }], + } as ProjectRuntime; + const { table } = apply({}, contextOf(runtime)); + + expect(Object.hasOwn(tableOf(table), "agentcore_traces")).toBe(added); + expect(defaultTracesTelemetry(table, BUILD_REGION)).toHaveLength(added ? 1 : 0); + }, + ); + + test.each(BUILDS)( + "does not start the agent under opentelemetry-instrument when the envVars set DISABLE_ADOT_OBSERVABILITY to true (%s)", + (build) => { + const optedOut = { + ...otelRuntime(build), + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }], + } as ProjectRuntime; + const local = createdProfile(optedOut); + + expect(agentOf(deriveRuntimeProfile(local, contextOf(optedOut)).profile)).toEqual( + agentOf(deriveRuntimeProfile(local, contextOf(runtimeOf(build))).profile), + ); + }, + ); + + test("names the default X-Ray target agentcore_traces_default when a target is agentcore_traces", () => { + const local = { telemetry: { agentcore_traces: CLOUDWATCH.logs } }; + const telemetry = tableOf( + apply(local, contextOf(OTEL_CODEZIP, { telemetry: acceptedTelemetry(local) })).table, + ); + + expect(Object.keys(telemetry)).toEqual(["agentcore_traces", "agentcore_traces_default"]); + expect(telemetry.agentcore_traces_default).toEqual(DEFAULT_TRACES); + }); + + test("reads the region of the default X-Ray target through the input of @box/agentcore", () => { + const { inputs } = parse(renderAgentCorePack(specOf(OTEL_CODEZIP), OTEL_CODEZIP)) as { + inputs: { aws: { region: unknown } }; + }; + + expect(DEFAULT_TRACES.region).toBe("${{ inputs.aws.region }}"); + expect(inputs.aws.region).toMatchObject({ from: "env:AWS_REGION" }); + }); + + test("names the default X-Ray target in the header", () => { + const context = contextOf(OTEL_CODEZIP); + + expect(runtimeProfileHeader(context, apply({}, context).table).at(-1)).toBe( + "[telemetry.agentcore_traces] is the default X-Ray target: the agent's spans and box's " + + `own, in the Runtime's region. Add a cloudwatch_traces target to ${PROFILE_FILE} to ` + + "use another, or set instrumentation.enableOtel to false in agentcore.json to send none.", + ); + }); + + test("gives the default X-Ray target to the grant of the execution role, and no other target", () => { + const { table } = apply({ telemetry: { logs: CLOUDWATCH.logs } }, contextOf(OTEL_CODEZIP)); + + expect(defaultTracesTelemetry(table, BUILD_REGION)).toEqual([ + { + name: "agentcore_traces", + kind: "cloudwatch_traces", + region: BUILD_REGION, + include: ["deny", "permit", "trace"], + }, + ]); + expect(defaultTracesTelemetry(apply({}, contextOf(CODEZIP)).table, BUILD_REGION)).toEqual([]); + }); + + test.each(BUILDS)( + "keeps each OTEL variable that the user sets in [agent.env], so it wins over the pack (%s)", + (build) => { + const env = { OTEL_TRACES_EXPORTER: "none", OTEL_RESOURCE_ATTRIBUTES: "service.name=mine" }; + + expect( + agentOf(apply({ agent: { env } }, contextOf(otelRuntime(build))).table).env, + ).toMatchObject(env); + }, + ); + }); + test("keeps the policies list as the local profile names it", () => { const local = { policies: ["./policy.dw", "rules/extra.dw"] }; diff --git a/src/core/project/runtimeProfile/rules/additions.ts b/src/core/project/runtimeProfile/rules/additions.ts index 085f5042b..2416a7662 100644 --- a/src/core/project/runtimeProfile/rules/additions.ts +++ b/src/core/project/runtimeProfile/rules/additions.ts @@ -7,31 +7,37 @@ * | `packs` | the user's packs in their order, then @box/agentcore, and @box/agentcore-codezip for CodeZip | * | `[profile] name` | the project's name | * | `[options] bedrock.region` | the build region, where the profile sets it or imports @box/bedrock | - * | `[agent] command` | the template's own command becomes its Runtime command; a command of the user loses `--reload` and `--reload-dir ` | + * | `[agent] command` | the template's own command becomes its Runtime command; a command of the user loses `--reload` and `--reload-dir `; with OTEL on, a command of the venv's Python starts under opentelemetry-instrument | * | `[agent.env]` | a template value becomes its Runtime value; `LOCAL_DEV` goes; the region of the Runtime replaces a region of the user, with a warning; `AWS_PROFILE` goes, with a warning; the AWS stand-ins and (CodeZip) a `PYTHONPATH` of the code folder come in; each other value of the user stays | * | `[agent.filesystem]` | the template's grants become its Runtime grants; each grant of the user stays | * | `[[agent.ingress]]` | the first listens on `0.0.0.0` on the port of the protocol, and forwards to the workload port of the template when the agent keeps its command and port, with a warning for each port that changes; each other listens on `0.0.0.0` on its own port | * | `[tool.]` | a table that the template writes, as it writes it, goes; a table of the user stays | * | `[telemetry.]` | each CloudWatch target that box takes is signed with a bare `aws://`, in the deploy region when it names none; one that box refuses goes; a `file` target goes, with a warning | * | `[telemetry.agentcore]` | added when no target is `cloudwatch_logs`: the runtime's own log group and log stream | + * | `[telemetry.agentcore_traces]` | added with OTEL on when no target is `cloudwatch_traces`: X-Ray in the Runtime's region | * * It moves no path: the paths rule runs after it and moves each path of the user. */ +import { posix } from "node:path"; import { AGENTCORE_CODEZIP_PACK_NAME, AGENTCORE_PACK_NAME, BOX_BEDROCK_PACK, boxIngressPort, boxWorkloadPort, + OBSERVABILITY_OFF_ENV, + observabilityOff, + otelOn, RUNTIME_LOGS_HEADERS_ENV, } from "../../templates/boxProfile"; -import type { ProjectRuntime } from "../../../../projectSchemas/runtime"; -import { isTable } from "../projectPaths"; +import type { CloudWatchTelemetry } from "../../boxNetwork"; +import { isTable, isVenvPython } from "../projectPaths"; import { FILE_TELEMETRY_KIND } from "./refusals"; import { RUNTIME_AWS_SIGNER, RUNTIME_INPUTS, type RuleResult, + type RuntimePaths, type RuntimeProfileContext, type TomlTable, } from "../types"; @@ -78,6 +84,31 @@ export const DEFAULT_TELEMETRY_TARGET = { secret: { ref: RUNTIME_AWS_SIGNER }, } as const; +/** + * The X-Ray target of a Runtime profile with OTEL on ({@link otelOn}) that has no + * `cloudwatch_traces` target: the agent's spans and box's own, in the Runtime's region, signed with + * the execution role. + */ +export const DEFAULT_TRACES_TARGET = { + kind: "cloudwatch_traces", + region: RUNTIME_INPUTS.awsRegion, + include: ["deny", "permit", "trace"], + secret: { ref: RUNTIME_AWS_SIGNER }, +} as const; + +/** The program that starts a Python command with the OpenTelemetry instrumentation. */ +const OTEL_INSTRUMENT = "opentelemetry-instrument"; + +/** Whether `command` starts under opentelemetry-instrument, as its program or as the script of it. */ +function startsUnderOtel(command: unknown): boolean { + return ( + Array.isArray(command) && + command + .slice(0, 2) + .some((word) => typeof word === "string" && posix.basename(word) === OTEL_INSTRUMENT) + ); +} + /** * The Runtime's region in the agent's environment, from the @box/agentcore `aws.region` input. * @box/bedrock sets `AWS_REGION` itself, so the profile sets it only without that pack. @@ -87,19 +118,6 @@ function runtimeRegionEnv(bedrock: boolean): Record { return { ...(!bedrock && { AWS_REGION: region }), AWS_DEFAULT_REGION: region }; } -/** - * The variable that turns off the observability of a runtime on AgentCore Runtime. When the envVars - * of the runtime set it to `true`, the Runtime profile gets no default log target. - */ -const OBSERVABILITY_OFF_ENV = "DISABLE_ADOT_OBSERVABILITY"; - -/** Whether the envVars of `runtime` set {@link OBSERVABILITY_OFF_ENV} to `true`. */ -export function observabilityOff(runtime: Pick): boolean { - return (runtime.envVars ?? []).some( - ({ name, value }) => name === OBSERVABILITY_OFF_ENV && value.trim().toLowerCase() === "true", - ); -} - /** * The notice that `agentcore create` and `agentcore add runtime` give for a box runtime: what the * default log target sends, where, what it costs, and how to turn it off. @@ -139,9 +157,43 @@ export function runtimeProfileHeader( `and log stream, which box reads from ${RUNTIME_LOGS_HEADERS_ENV} when the run starts. ` + `Add a cloudwatch_logs target to ${context.profileFile} to use another.`, ), + ...defaultTracesTargets(profile).map( + (name) => + `[telemetry.${name}] is the default X-Ray target: the agent's spans and box's own, in the ` + + `Runtime's region. Add a cloudwatch_traces target to ${context.profileFile} to use ` + + "another, or set instrumentation.enableOtel to false in agentcore.json to send none.", + ), ]; } +/** The name of each {@link DEFAULT_TRACES_TARGET} in the `[telemetry]` of `profile`. */ +function defaultTracesTargets(profile: Readonly): string[] { + return Object.entries(tableOf(profile.telemetry)) + .filter(([, target]) => sameValue(target, DEFAULT_TRACES_TARGET)) + .map(([name]) => name); +} + +/** + * Each {@link DEFAULT_TRACES_TARGET} of `profile`, a Runtime profile, as a CloudWatch target in + * `region`, the region build builds for: the execution role needs a grant for it. + */ +export function defaultTracesTelemetry( + profile: Readonly, + region: string, +): CloudWatchTelemetry[] { + return defaultTracesTargets(profile).map((name) => ({ + name, + kind: "cloudwatch_traces", + region, + include: [...DEFAULT_TRACES_TARGET.include], + })); +} + +/** Whether the `[agent] command` of `profile`, a Runtime profile, starts under opentelemetry-instrument. */ +export function agentStartsUnderOtel(profile: Readonly): boolean { + return startsUnderOtel(tableOf(profile.agent).command); +} + export function apply(table: Readonly, context: RuntimeProfileContext): RuleResult { const result = structuredClone(table) as TomlTable; const packs = runtimePacks(result.packs, context); @@ -202,9 +254,10 @@ function runtimeAgent( const templateRuntime = tableOf(context.template.runtime.agent); const result: TomlTable = { ...local }; if (local.command !== undefined) { - result.command = sameValue(local.command, templateLocal.command) + const command = sameValue(local.command, templateLocal.command) ? structuredClone(templateRuntime.command) : withoutReload(local.command); + result.command = otelOn(context.runtime) ? underOtel(command, context.paths) : command; } result.env = runtimeEnv( local.env, @@ -264,6 +317,18 @@ function withoutReload(command: unknown): unknown { return result; } +/** + * `command` under opentelemetry-instrument (`paths.otelInstrument`) when its program is the venv's + * Python, whose dependencies hold the script. A command that already starts under it, or that runs + * another program, stays. + */ +function underOtel(command: unknown, paths: RuntimePaths): unknown { + if (!Array.isArray(command) || startsUnderOtel(command)) return command; + const program: unknown = command[0]; + if (typeof program !== "string" || !isVenvPython(program, paths)) return command; + return [...paths.otelInstrument, ...command]; +} + /** * `[agent.env]` for the Runtime. A variable of the template that the local profile keeps with the * template's value gets its Runtime value, or goes when the Runtime has none. A variable that only @@ -440,9 +505,11 @@ function onAllAddresses(entry: unknown): unknown { * names of `context.telemetry`) with its keys, signed with a bare `aws://`, in the deploy region * when it names none; one that box refuses goes; then each CloudWatch target of the * context that the local profile does not declare, as box resolves it. A target of another kind - * stays as it is. With no `cloudwatch_logs` target, {@link DEFAULT_TELEMETRY_TARGET} comes last, as + * stays as it is. With no `cloudwatch_logs` target, {@link DEFAULT_TELEMETRY_TARGET} comes next, as * `agentcore`, or as `agentcore_default` when a target has that name, unless the envVars of the - * runtime turn observability off ({@link OBSERVABILITY_OFF_ENV}). + * runtime turn observability off ({@link OBSERVABILITY_OFF_ENV}). With OTEL on and no + * `cloudwatch_traces` target, {@link DEFAULT_TRACES_TARGET} comes last, as `agentcore_traces`, or + * as `agentcore_traces_default` when a target has that name ({@link otelOn}). */ function runtimeTelemetry( telemetry: unknown, @@ -491,14 +558,20 @@ function runtimeTelemetry( secret: { ref: RUNTIME_AWS_SIGNER }, }; } - const logs = Object.entries(result).some( - ([name, entry]) => - isTable(entry) && (entry.kind ?? resolved.get(name)?.kind) === "cloudwatch_logs", - ); - if (!logs && !observabilityOff(context.runtime)) { + const has = (kind: string) => + Object.entries(result).some( + ([name, entry]) => isTable(entry) && (entry.kind ?? resolved.get(name)?.kind) === kind, + ); + if (!has("cloudwatch_logs") && !observabilityOff(context.runtime)) { const name = Object.hasOwn(result, "agentcore") ? "agentcore_default" : "agentcore"; result[name] = structuredClone(DEFAULT_TELEMETRY_TARGET); } + if (!has("cloudwatch_traces") && otelOn(context.runtime)) { + const name = Object.hasOwn(result, "agentcore_traces") + ? "agentcore_traces_default" + : "agentcore_traces"; + result[name] = structuredClone(DEFAULT_TRACES_TARGET); + } return result; } diff --git a/src/core/project/runtimeProfile/types.ts b/src/core/project/runtimeProfile/types.ts index 628d1520f..777c650b5 100644 --- a/src/core/project/runtimeProfile/types.ts +++ b/src/core/project/runtimeProfile/types.ts @@ -50,6 +50,12 @@ export interface RuntimePaths { readonly runtimePython: string; /** The prefix of that Python, for the uv Pythons: `${{ inputs.python.prefix }}` on CodeZip. */ readonly runtimePythonPrefix: string; + /** + * The words before a command that start it under opentelemetry-instrument on the Runtime. The + * CodeZip zip has the script with a `#!/usr/bin/env python3` line, which the box cannot run, so + * the Runtime's Python runs it. + */ + readonly otelInstrument: readonly string[]; /** The code folder as a message names it: `app/agent/`. */ readonly shownCode: string; } diff --git a/src/core/project/templates/boxProfile.test.ts b/src/core/project/templates/boxProfile.test.ts index c6bf6105d..abebfc965 100644 --- a/src/core/project/templates/boxProfile.test.ts +++ b/src/core/project/templates/boxProfile.test.ts @@ -401,9 +401,13 @@ describe("startsInBoxOnCodeZip", () => { }); describe("renderAgentCorePack", () => { - function pack(memories: { name: string }[]) { + /** The pack of a runtime with OTEL off unless `otel`. */ + function pack(memories: { name: string }[], otel = false) { return parse( - renderAgentCorePack({ name: "V3Demo", memories } as unknown as ProjectSpec), + renderAgentCorePack({ name: "V3Demo", memories } as unknown as ProjectSpec, { + name: "agent", + ...(!otel && { instrumentation: { enableOtel: false } }), + }), ) as Record; } @@ -474,8 +478,60 @@ describe("renderAgentCorePack", () => { }); }); - test("has no agent.env when the project has no memory", () => { + test("has no agent.env when the project has no memory and OTEL is off", () => { expect(pack([]).agent).toBeUndefined(); + expect(pack([]).inputs.runtime).not.toHaveProperty("resource_attributes"); + }); + + test("with OTEL on, sets up the agent's ADOT SDK and sets no OTEL_EXPORTER_OTLP_ variable, which box owns", () => { + const { env } = pack([{ name: "agentMemory" }], true).agent; + + expect(env).toEqual({ + MEMORY_AGENTMEMORY_ID: "${{ inputs.memory.agentMemory.id }}", + OTEL_PYTHON_DISTRO: "aws_distro", + OTEL_PYTHON_CONFIGURATOR: "aws_configurator", + AGENT_OBSERVABILITY_ENABLED: "true", + OTEL_TRACES_EXPORTER: "otlp", + OTEL_LOGS_EXPORTER: "otlp", + OTEL_METRICS_EXPORTER: "none", + OTEL_PYTHON_EXCLUDED_URLS: "/ping", + OTEL_RESOURCE_ATTRIBUTES: "${{ inputs.runtime.resource_attributes }}", + }); + expect(Object.keys(env).filter((name) => name.startsWith("OTEL_EXPORTER_OTLP_"))).toEqual([]); + }); + + test("with OTEL on, reads the Runtime's OTEL_RESOURCE_ATTRIBUTES, with the runtime's name when the Runtime sets none", () => { + expect(pack([], true).inputs.runtime.resource_attributes).toEqual({ + description: "The OpenTelemetry resource attributes of the runtime.", + from: "env:OTEL_RESOURCE_ATTRIBUTES", + default: "service.name=agent", + }); + }); + + test("gives the agent no ADOT variable and declares no resource attributes input when the envVars set DISABLE_ADOT_OBSERVABILITY to true", () => { + const spec = { name: "V3Demo" } as unknown as ProjectSpec; + const optedOut = renderAgentCorePack(spec, { + name: "agent", + envVars: [{ name: "DISABLE_ADOT_OBSERVABILITY", value: "true" }], + }); + + expect(optedOut).toBe( + renderAgentCorePack(spec, { name: "agent", instrumentation: { enableOtel: false } }), + ); + }); + + test("turns OTEL on when the runtime has no instrumentation, as create writes it", () => { + const text = renderAgentCorePack({ name: "V3Demo" } as unknown as ProjectSpec, { + name: "agent", + }); + + expect(text).toBe( + renderAgentCorePack({ name: "V3Demo" } as unknown as ProjectSpec, { + name: "agent", + instrumentation: { enableOtel: true }, + }), + ); + expect(text).toContain('OTEL_PYTHON_DISTRO = "aws_distro"'); }); }); diff --git a/src/core/project/templates/boxProfile.ts b/src/core/project/templates/boxProfile.ts index cfc8aae6f..724cc560b 100644 --- a/src/core/project/templates/boxProfile.ts +++ b/src/core/project/templates/boxProfile.ts @@ -224,6 +224,29 @@ export function runsInBox(boxProject: boolean, runtime: Pick): boolean { + return (runtime.envVars ?? []).some( + ({ name, value }) => name === OBSERVABILITY_OFF_ENV && value.trim().toLowerCase() === "true", + ); +} + +/** + * Whether the agent of `runtime` sends OTEL on AgentCore Runtime: `instrumentation.enableOtel`, + * which is `true` when absent, and the envVars do not turn observability off. It decides the + * opentelemetry-instrument start, the ADOT variables, and the default X-Ray target together. + */ +export function otelOn(runtime: Pick): boolean { + return (runtime.instrumentation?.enableOtel ?? true) && !observabilityOff(runtime); +} + /** {@link runsInBox} for a runtime of `spec`. */ export function runtimeRunsInBox(spec: ProjectSpec, runtime: ProjectRuntime): boolean { return runsInBox(isBoxProject(spec), runtime); @@ -605,6 +628,24 @@ export function boxProfileWarnings(runtime: ScaffoldRuntimeInput): string[] { ]; } +/** + * The variables that set up the ADOT SDK of an agent with OTEL on: the AWS distro and its + * configurator, spans and logs over OTLP to the box's own receiver (box sets each + * `OTEL_EXPORTER_OTLP_*`), no metrics, and no span for each health check of the Runtime. + */ +const AGENT_OTEL_ENV: Readonly> = { + OTEL_PYTHON_DISTRO: "aws_distro", + OTEL_PYTHON_CONFIGURATOR: "aws_configurator", + AGENT_OBSERVABILITY_ENABLED: "true", + OTEL_TRACES_EXPORTER: "otlp", + OTEL_LOGS_EXPORTER: "otlp", + OTEL_METRICS_EXPORTER: "none", + OTEL_PYTHON_EXCLUDED_URLS: "/ping", +}; + +/** The variable in which AgentCore Runtime names the resource of the runtime's spans. */ +const RUNTIME_RESOURCE_ATTRIBUTES_ENV = "OTEL_RESOURCE_ATTRIBUTES"; + /** * The `@box/agentcore` pack each Runtime profile imports, CodeZip and container: it declares the * inputs the profile reads from the environment the Runtime sets after deploy. The `aws.region` @@ -612,10 +653,16 @@ export function boxProfileWarnings(runtime: ScaffoldRuntimeInput): string[] { * egress backend `url`, bounded by a pattern so a value from the environment names only the host the * profile expects. The `runtime.log_group` and `runtime.log_stream` inputs each read one key of * {@link RUNTIME_LOGS_HEADERS_ENV}, for the default telemetry target of the Runtime profile. One - * input per project memory carries its ID into the agent's environment. box finds the pack beside - * its binary at {@link AGENTCORE_PACK_PATH}; build and deploy stage it there. + * input per project memory carries its ID into the agent's environment. With OTEL on for `runtime`, + * the agent gets {@link AGENT_OTEL_ENV} and the Runtime's resource attributes, so the console links + * its spans to the runtime; the profile merges after the pack, so a value of the user wins. box + * finds the pack beside its binary at {@link AGENTCORE_PACK_PATH}; build and deploy stage it there. */ -export function renderAgentCorePack(spec: ProjectSpec): string { +export function renderAgentCorePack( + spec: ProjectSpec, + runtime: Pick, +): string { + const otel = otelOn(runtime); const memories = spec.memories ?? []; const memoryInputs = memories .map( @@ -631,7 +678,23 @@ from = ${str(`env:${memoryEnvVarName(memory.name)}`)}`, `${memoryEnvVarName(memory.name)} = ${str(`\${{ inputs.memory.${tomlKey(memory.name)}.id }}`)}`, ) .join("\n"); - const agentEnv = memoryEnv ? `\n[agent.env]\n${memoryEnv}\n` : ""; + const otelEnv = otel + ? [ + ...Object.entries(AGENT_OTEL_ENV).map(([name, value]) => `${name} = ${str(value)}`), + `${RUNTIME_RESOURCE_ATTRIBUTES_ENV} = "\${{ inputs.runtime.resource_attributes }}"`, + ].join("\n") + : ""; + const otelInput = otel + ? ` + +# The resource of the agent's spans, which the Runtime names so the console links them to it. +[inputs.runtime.resource_attributes] +description = "The OpenTelemetry resource attributes of the runtime." +from = ${str(`env:${RUNTIME_RESOURCE_ATTRIBUTES_ENV}`)} +default = ${str(`service.name=${runtime.name}`)}` + : ""; + const env = [memoryEnv, otelEnv].filter((lines) => lines !== "").join("\n"); + const agentEnv = env ? `\n[agent.env]\n${env}\n` : ""; return `# The deployed AgentCore resources of '${spec.name}', read from the environment the Runtime sets. # Written by agentcore build and deploy. The runtime profile imports it as ${AGENTCORE_PACK_NAME}. [pack] @@ -669,7 +732,7 @@ key = "x-aws-log-stream" [inputs.runtime.url] description = "The URL an A2A agent card advertises on the Runtime." from = "env:AGENTCORE_RUNTIME_URL" -default = "http://localhost:9000/"${memoryInputs} +default = "http://localhost:9000/"${memoryInputs}${otelInput} ${agentEnv}`; } diff --git a/src/core/project/templates/runtime.test.ts b/src/core/project/templates/runtime.test.ts index e7632bf00..0eae85be2 100644 --- a/src/core/project/templates/runtime.test.ts +++ b/src/core/project/templates/runtime.test.ts @@ -96,20 +96,20 @@ describe("runtime templates", () => { expect(await Bun.file(join(appDir, ".dockerignore")).exists()).toBe(isContainer); }); - // The agent in a box does not run under opentelemetry-instrument, CodeZip or container, so a box - // template turns OTEL off; any other template keeps it on. - test.each<[RuntimeTemplateShortcutName, boolean]>([ - ["agent-python-strands-box", false], - ["agent-python-strands-box-container", false], - ["agent-python-minimal-box", false], - ["agent-python-langchain-box", false], - ["a2a-python-strands-box", false], - ["agui-python-strands-box", false], - ["mcp-python-fastmcp-box", false], - ["agent-python-strands", true], - ])("%s writes enableOtel %p", async (template, enableOtel) => { + // Build starts the agent in a box under opentelemetry-instrument, so a box template keeps OTEL on + // as every other template does, and writes no instrumentation. + test.each([ + "agent-python-strands-box", + "agent-python-strands-box-container", + "agent-python-minimal-box", + "agent-python-langchain-box", + "a2a-python-strands-box", + "agui-python-strands-box", + "mcp-python-fastmcp-box", + "agent-python-strands", + ])("%s keeps enableOtel on", async (template) => { const { runtime } = await addTemplate("my_agent", template); - expect(runtime.instrumentation?.enableOtel ?? true).toBe(enableOtel); + expect(runtime.instrumentation).toBeUndefined(); }); test("Bedrock Managed Agents keeps infrastructure overrides and its required policy", async () => { diff --git a/src/core/project/templates/runtime.ts b/src/core/project/templates/runtime.ts index 05722b94d..a90a29698 100644 --- a/src/core/project/templates/runtime.ts +++ b/src/core/project/templates/runtime.ts @@ -2,7 +2,6 @@ import { FsTreeNode } from "./fsTree"; import type { AssetSource } from "../source"; import type { RuntimeResourceConfig } from "../../../handlers/project/add/runtime/types"; import { DEFAULT_MODEL_IDS, type ProjectRuntime } from "../../../projectSchemas/runtime"; -import { hasBoxProfile } from "./boxProfile"; import { mergeSpecEntries } from "./spec"; import type { SpecEntries, TemplateRenderer, TemplateResolver } from "./types"; import type { @@ -132,10 +131,6 @@ function buildRuntimeSpec(input: RuntimeResourceConfig): ProjectRuntime { ...(scaffoldRuntimeInput.framework === "bma" && { bedrockManagedAgents: true, }), - // The agent in a box does not run under opentelemetry-instrument: a container's image does not - // run it, and on CodeZip it wraps start.py, which hands over to box run. So say so in - // agentcore.json until traces work in a box. - ...(hasBoxProfile(scaffoldRuntimeInput) && { instrumentation: { enableOtel: false } }), ...(infra.envVars && { envVars: infra.envVars }), ...(infra.networkMode && { networkMode: infra.networkMode }), ...(infra.networkConfig && { networkConfig: infra.networkConfig }),