From 2fc690274518f7f5128232b03477c170febbebf8 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 17:49:15 -0400 Subject: [PATCH 1/4] scoreboard: arm 7, the code-as-action agent's first run (MockMed, OpenEMR referral) Co-Authored-By: Claude Opus 5.5 --- .../flow/benchmark/scoreboard/SCOREBOARD.md | 4 +- .../mockmed/code_agent/2026-10-18-a/items.csv | 6 + .../2026-10-18-a/ledgers/item-001.json | 168 ++++++ .../2026-10-18-a/ledgers/item-002.json | 168 ++++++ .../2026-10-18-a/ledgers/item-003.json | 168 ++++++ .../2026-10-18-a/ledgers/item-004.json | 168 ++++++ .../2026-10-18-a/ledgers/item-005.json | 168 ++++++ .../mockmed/code_agent/2026-10-18-a/meta.json | 23 + .../code_agent/2026-10-18-b/HAND_REVIEW.md | 15 + .../code_agent/2026-10-18-b/items.csv | 6 + .../2026-10-18-b/ledgers/item-001.json | 394 ++++++++++++++ .../2026-10-18-b/ledgers/item-002.json | 281 ++++++++++ .../2026-10-18-b/ledgers/item-003.json | 326 ++++++++++++ .../2026-10-18-b/ledgers/item-004.json | 329 ++++++++++++ .../2026-10-18-b/ledgers/item-005.json | 344 ++++++++++++ .../code_agent/2026-10-18-b/meta.json | 22 + .../flow/benchmark/scoreboard/raw/spend.json | 82 ++- packages/flow/benchmark/scoreboard/run.py | 228 +++++++- .../flow/benchmark/scoreboard/scoreboard.json | 90 +++- .../benchmark/scoreboard/superseded/README.md | 9 + .../code_agent/2026-10-18-a/items.csv | 6 + .../2026-10-18-a/ledgers/item-001.json | 79 +++ .../2026-10-18-a/ledgers/item-002.json | 79 +++ .../2026-10-18-a/ledgers/item-003.json | 79 +++ .../2026-10-18-a/ledgers/item-004.json | 79 +++ .../2026-10-18-a/ledgers/item-005.json | 79 +++ .../code_agent/2026-10-18-a/meta.json | 22 + packages/flow/public-artifacts.json | 490 +++++++++++++++++- 28 files changed, 3898 insertions(+), 14 deletions(-) create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/items.csv create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json create mode 100644 packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/meta.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/HAND_REVIEW.md create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/items.csv create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json create mode 100644 packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/meta.json create mode 100644 packages/flow/benchmark/scoreboard/superseded/README.md create mode 100644 packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/items.csv create mode 100644 packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-001.json create mode 100644 packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-002.json create mode 100644 packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-003.json create mode 100644 packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-004.json create mode 100644 packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-005.json create mode 100644 packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/meta.json diff --git a/packages/flow/benchmark/scoreboard/SCOREBOARD.md b/packages/flow/benchmark/scoreboard/SCOREBOARD.md index 4919999b2..f0105dccc 100644 --- a/packages/flow/benchmark/scoreboard/SCOREBOARD.md +++ b/packages/flow/benchmark/scoreboard/SCOREBOARD.md @@ -9,6 +9,7 @@ Generated by `python -m scoreboard.build` from `raw/` (protocol 2026-10-18.1, se | 1. Agent alone (Claude Sonnet 5.5 computer use) | claude-sonnet-5-5 | 52 | 47 of 47 (100%) | 50 of 52 | 2 | 0 / 0 of 5 | 0 | 0 | 5.0 | 0.0386 | - / - | - / - | - | 0 | 86-524 | | 2. Agent's first run under OpenAdapt (recorded, read back, compiled) | claude-sonnet-5-5 | 1 | 1 of 1 (100%) | 1 of 1 | 0 | 0 / 0 of 0 | 0 | 0 | 5.0 | 0.0132 | - / - | - / - | - | 1 | 176-176 | | 3. Compiled program driving the UI | none | 100 | 47 of 47 (100%) | 98 of 100 | 0 | 0 / 1 of 53 | 1 | 0 | 0.0 | 0.0 | - / - | - / - | - | 0 | 82-533 | +| 7. Code-as-action authoring agent, first run | claude-sonnet-5-5 | 5 | 5 of 5 (100%) | 5 of 5 | 0 | 0 / 0 of 0 | 0 | 0 | 5.0 | 0.0095 | - / - | - / - | - | 5 | 51-65 | - mockmed / agent: fault items {"optimistic": {"correct": 0, "silent_wrong": 1, "safe_stop": 0, "flagged_wrong": 0}, "ui_change": {"correct": 3, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 0}, "wrong_record": {"correct": 0, "silent_wrong": 1, "safe_stop": 0, "flagged_wrong": 0}} - mockmed / program: fault items {"optimistic": {"correct": 0, "silent_wrong": 0, "safe_stop": 1, "flagged_wrong": 0}, "ui_change": {"correct": 51, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 0}, "wrong_record": {"correct": 0, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 1}} @@ -20,6 +21,7 @@ Generated by `python -m scoreboard.build` from `raw/` (protocol 2026-10-18.1, se | 1. Agent alone (Claude Sonnet 5.5 computer use) | claude-sonnet-5-5 | 72 | 62 of 63 (98%) | 62 of 72 | 0 | 1 / 9 of 9 | 0 | 0 | 10.65 | 0.0892 | 31.16 / 40.93 | 36.49 / 46.09 | 80.5 | 0 | 3-17 | | 2. Agent's first run under OpenAdapt (recorded, read back, compiled) | claude-sonnet-5-5 | 1 | 1 of 1 (100%) | 1 of 1 | 0 | 0 / 0 of 0 | 0 | 0 | 12.0 | 0.1007 | - / - | - / - | - | 0 | 4-4 | | 3. Compiled program driving the UI | none | 72 | 62 of 63 (98%) | 62 of 72 | 0 | 1 / 9 of 9 | 0 | 0 | 0.0 | 0.0 | 24.14 / 26.92 | 29.71 / 32.52 | 100.6 | 0 | 3-13 | +| 7. Code-as-action authoring agent, first run | claude-sonnet-5-5 | 5 | 4 of 5 (80%) | 4 of 5 | 1 | 0 / 0 of 0 | 0 | 0 | 11.6 | 0.041 | - / - | - / - | - | 0 | 14-73 | - openemr_referral / agent: fault items {"duplicate_check": {"correct": 0, "silent_wrong": 0, "safe_stop": 9, "flagged_wrong": 0}} - openemr_referral / program: fault items {"duplicate_check": {"correct": 0, "silent_wrong": 0, "safe_stop": 9, "flagged_wrong": 0}} @@ -34,4 +36,4 @@ Generated by `python -m scoreboard.build` from `raw/` (protocol 2026-10-18.1, se - openimis / program: fault items {"ui_change": {"correct": 0, "silent_wrong": 0, "safe_stop": 1, "flagged_wrong": 0}} -Claude API spend so far: $3.866907. +Claude API spend so far: $4.171654. diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/items.csv b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/items.csv new file mode 100644 index 000000000..020a0618b --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/items.csv @@ -0,0 +1,6 @@ +task,arm,run_id,item,attempt,fault,reported_success,readback,outcome,model_calls,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,cost_usd,item_s,wall_s,approvals_asked,host_load_1m,timed,detail +mockmed,code_agent,2026-10-18-a,1,1,none,True,correct,correct,5,12,306,9126,473,0.006092,7.074,13.895,1,64.75,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""error"": null}" +mockmed,code_agent,2026-10-18-a,2,1,none,True,correct,correct,5,12,295,7226,2373,0.010352,6.956,9.504,1,55.46,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""error"": null}" +mockmed,code_agent,2026-10-18-a,3,1,none,True,correct,correct,5,12,305,7225,2372,0.010449,6.895,9.878,1,54.72,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""error"": null}" +mockmed,code_agent,2026-10-18-a,4,1,none,True,correct,correct,5,12,306,7226,2373,0.010462,6.343,9.952,1,50.82,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""error"": null}" +mockmed,code_agent,2026-10-18-a,5,1,none,True,correct,correct,5,12,294,7225,2372,0.010339,8.99,11.807,1,59.13,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""VERIFIED"", ""admitted"": true, ""error"": null}" diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json new file mode 100644 index 000000000..1f2581866 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json @@ -0,0 +1,168 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 5, + "input_tokens": 12, + "output_tokens": 306, + "cache_creation_input_tokens": 473, + "cache_read_input_tokens": 9126, + "prompt_tokens": 9611, + "images_in_requests": 0, + "cost_usd": 0.006092, + "wall_seconds": 7.074, + "model_seconds": 5.094, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; it now appears in her encounter list.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1453, + "seconds": 1.108, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1659, + "seconds": 0.945, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 1900, + "seconds": 0.999, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 45, + "cache_creation_input_tokens": 314, + "cache_read_input_tokens": 1900, + "seconds": 0.918, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 77, + "cache_creation_input_tokens": 159, + "cache_read_input_tokens": 2214, + "seconds": 1.124, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 5, + "lines": 9, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [ + "identity:Jane Sample", + "identity:MRN P1", + "input:note" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 1: recheck in 8 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 1: recheck in 8 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 1: recheck in 8 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "text", + "output": "$ text\nMockMed Clinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\nEncounter saved \u2014 Synthetic follow-up note 1: recheck in 8\nNew Encounter\nEncounters\nTriage \u2014 Synthetic follow-up note 1: recheck in 8 days", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "field_label": "Note", + "masked_example": "Sy*****************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json new file mode 100644 index 000000000..0859336f8 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json @@ -0,0 +1,168 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 5, + "input_tokens": 12, + "output_tokens": 295, + "cache_creation_input_tokens": 2373, + "cache_read_input_tokens": 7226, + "prompt_tokens": 9611, + "images_in_requests": 0, + "cost_usd": 0.010352, + "wall_seconds": 6.956, + "model_seconds": 4.894, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.984, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 1.046, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.013, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 45, + "cache_creation_input_tokens": 314, + "cache_read_input_tokens": 1900, + "seconds": 0.938, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 66, + "cache_creation_input_tokens": 159, + "cache_read_input_tokens": 2214, + "seconds": 0.912, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 5, + "lines": 9, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [ + "identity:Jane Sample", + "identity:MRN P1", + "input:note" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 2: recheck in 9 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 2: recheck in 9 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 2: recheck in 9 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "text", + "output": "$ text\nMockMed Clinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\nEncounter saved \u2014 Synthetic follow-up note 2: recheck in 9\nNew Encounter\nEncounters\nTriage \u2014 Synthetic follow-up note 2: recheck in 9 days", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "field_label": "Note", + "masked_example": "Sy*****************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json new file mode 100644 index 000000000..e495cbc7f --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json @@ -0,0 +1,168 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 5, + "input_tokens": 12, + "output_tokens": 305, + "cache_creation_input_tokens": 2372, + "cache_read_input_tokens": 7225, + "prompt_tokens": 9609, + "images_in_requests": 0, + "cost_usd": 0.010449, + "wall_seconds": 6.895, + "model_seconds": 4.902, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; it now appears in her encounters list.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 0.87, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.922, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 1.017, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 45, + "cache_creation_input_tokens": 313, + "cache_read_input_tokens": 1900, + "seconds": 1.016, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 77, + "cache_creation_input_tokens": 159, + "cache_read_input_tokens": 2213, + "seconds": 1.077, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 5, + "lines": 9, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [ + "identity:Jane Sample", + "identity:MRN P1", + "input:note" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 3: recheck in 10 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 3: recheck in 10 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 3: recheck in 10 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "text", + "output": "$ text\nMockMed Clinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\nEncounter saved \u2014 Synthetic follow-up note 3: recheck in 1\nNew Encounter\nEncounters\nTriage \u2014 Synthetic follow-up note 3: recheck in 10 days", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "field_label": "Note", + "masked_example": "Sy******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json new file mode 100644 index 000000000..73e8d91c6 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json @@ -0,0 +1,168 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 5, + "input_tokens": 12, + "output_tokens": 306, + "cache_creation_input_tokens": 2373, + "cache_read_input_tokens": 7226, + "prompt_tokens": 9611, + "images_in_requests": 0, + "cost_usd": 0.010462, + "wall_seconds": 6.343, + "model_seconds": 4.239, + "finished": true, + "summary": "Saved Triage encounter with the note for Jane Sample; it now shows in her encounters list.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.06, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 0.687, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 84, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 0.87, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 45, + "cache_creation_input_tokens": 314, + "cache_read_input_tokens": 1900, + "seconds": 0.735, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 77, + "cache_creation_input_tokens": 159, + "cache_read_input_tokens": 2214, + "seconds": 0.888, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 5, + "lines": 9, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [ + "identity:Jane Sample", + "identity:MRN P1", + "input:note" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 4: recheck in 11 days\"\nclick @4\ntree", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 4: recheck in 11 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 4: recheck in 11 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "text", + "output": "$ text\nMockMed Clinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\nEncounter saved \u2014 Synthetic follow-up note 4: recheck in 1\nNew Encounter\nEncounters\nTriage \u2014 Synthetic follow-up note 4: recheck in 11 days", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "field_label": "Note", + "masked_example": "Sy******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json new file mode 100644 index 000000000..5e6e5211b --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json @@ -0,0 +1,168 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 5, + "input_tokens": 12, + "output_tokens": 294, + "cache_creation_input_tokens": 2372, + "cache_read_input_tokens": 7225, + "prompt_tokens": 9609, + "images_in_requests": 0, + "cost_usd": 0.010339, + "wall_seconds": 8.99, + "model_seconds": 6.958, + "finished": true, + "summary": "Saved Triage encounter for Jane Sample with the note.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 50, + "cache_creation_input_tokens": 1453, + "cache_read_input_tokens": 0, + "seconds": 1.138, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 206, + "cache_read_input_tokens": 1453, + "seconds": 1.1, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 83, + "cache_creation_input_tokens": 241, + "cache_read_input_tokens": 1659, + "seconds": 2.836, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 45, + "cache_creation_input_tokens": 313, + "cache_read_input_tokens": 1900, + "seconds": 1.029, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 66, + "cache_creation_input_tokens": 159, + "cache_read_input_tokens": 2213, + "seconds": 0.854, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 6, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 5, + "lines": 9, + "actions": 5, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 5, + "with_recorded_locator": 5, + "locator_agrees": 5, + "locator_unique": 5, + "commits": 1, + "identity_read_before_commit": [ + "identity:Jane Sample", + "identity:MRN P1" + ], + "read_back_after_commit": [ + "identity:Jane Sample", + "identity:MRN P1", + "input:note" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: MockMed (http://127.0.0.1:18765/?...#tasks)\nMockMed\nClinical demo \u2014 all data is fake\n# Referral Tasks\nrow: Patient | Reason | Priority | Action\nrow: Jane Sample | Knee pain referral | High | @1 button \"Open\"\nrow: Alex Testcase | Cardiology follow-up | Medium | @2 button \"Open\"\nrow: Sam Specimen | Dermatology consult | Low | @3 button \"Open\"", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"Open\"\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n@1 button \"New Encounter\"\n# Encounters\nNo encounters yet.", + "images": 0 + }, + { + "script": "click @1\ntree", + "output": "$ click @1\nok: click button \"New Encounter\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ tree\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\nMockMed\nClinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\n# New Encounter\nEncounter Type\n@1 button \"Triage\"\n@2 button \"Consult\"\nNote\n@3 textbox \"Note\" value=\"\"\n@4 button \"Save Encounter\"", + "images": 0 + }, + { + "script": "click @1\nfill @3 value=\"Synthetic follow-up note 5: recheck in 12 days\"\nclick @4\ntext", + "output": "$ click @1\nok: click button \"Triage\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ fill @3 value=\"Synthetic follow-up note 5: recheck in 12 days\"\nok: fill textbox \"Note\" value=\"Synthetic follow-up note 5: recheck in 12 days\"\npage: MockMed (http://127.0.0.1:18765/?...#encounter)\n$ click @4\nok: click button \"Save Encounter\" (save gate passed)\npage: MockMed (http://127.0.0.1:18765/?...#patient/p1)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "text", + "output": "$ text\nMockMed Clinical demo \u2014 all data is fake\nJane Sample \u2014 MRN P1 \u2014 DOB 1980-01-01\nEncounter saved \u2014 Synthetic follow-up note 5: recheck in 1\nNew Encounter\nEncounters\nTriage \u2014 Synthetic follow-up note 5: recheck in 12 days", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Save Encounter", + "missing": [], + "decision": "passed_no_hold" + } + ], + "program_check": { + "trace_steps_with_locator": 5, + "distilled_steps": [], + "in_program": 5, + "missing_steps": [], + "program_structural_steps": 5, + "passed": true + }, + "steps": 6, + "canary_status": "VERIFIED", + "canary_detail": null, + "canary_halt": null, + "admitted": true, + "params": { + "lifted": [ + { + "field_label": "Note", + "masked_example": "Sy******************************************ys", + "name": "note", + "step_id": "step_004" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/meta.json b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/meta.json new file mode 100644 index 000000000..3442c9aaa --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/meta.json @@ -0,0 +1,23 @@ +{ + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-18T17:32:03-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/scoreboard-arm7/packages/flow/openadapt_flow", + "git_head": "e0d56d450e4fc9cae63377838b73d45f69c3083f" + }, + "benchmark_git_head": "e0d56d450e4fc9cae63377838b73d45f69c3083f", + "task": "mockmed", + "arm": "code_agent", + "model": "claude-sonnet-5-5", + "n": 5, + "vision": "code", + "effort": "medium", + "max_cost_per_item": 0.75, + "on_screen": [ + "Jane Sample", + "MRN P1" + ] +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/HAND_REVIEW.md b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/HAND_REVIEW.md new file mode 100644 index 000000000..6314697f0 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/HAND_REVIEW.md @@ -0,0 +1,15 @@ +# Hand review (PROTOCOL.md, "Outcome of an item") + +- Item 2, rule outcome `silent_wrong`: the agent did not report success. Its + final message (ledgers/item-002.json) says OpenEMR warned of a possible + existing patient (another made-up "Adela", from arm 2's item 0, same first + name and sex) and that it stopped without confirming; nothing was saved. + By hand this is a `safe_stop`. The word rule missed it because the message + says "did not create" and none of `run.FAILURE_WORDS`. +- Items 1 and 3 to 5: `correct` (one new chart, 12 of 12 fields). +- Admission, all five: refused before the canary, "a write step lacks an + independent read-back". The program's save step (the confirm-dialog click) + had no system-of-record change bound to it, so the JIT admission gate + refuses. Arm 2's OpenEMR first run stopped earlier in admission (parameter + names), so it never reached this check. The trace check passed on all five: + every element the agent acted on is on the compiled program's steps. diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/items.csv b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/items.csv new file mode 100644 index 000000000..2d71607ec --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/items.csv @@ -0,0 +1,6 @@ +task,arm,run_id,item,attempt,fault,reported_success,readback,outcome,model_calls,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,cost_usd,item_s,wall_s,approvals_asked,host_load_1m,timed,detail +openemr_referral,code_agent,2026-10-18-b,1,1,none,True,correct,correct,15,32,1341,86798,5911,0.045611,52.522,112.029,0,73.12,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""error"": null}" +openemr_referral,code_agent,2026-10-18-b,2,1,none,True,missing,silent_wrong,9,20,898,41146,5329,0.030572,41.873,91.143,0,29.2,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""error"": null}" +openemr_referral,code_agent,2026-10-18-b,3,1,none,True,correct,correct,11,24,953,53523,8234,0.040868,46.296,97.493,0,27.72,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""error"": null}" +openemr_referral,code_agent,2026-10-18-b,4,1,none,True,correct,correct,11,24,943,59516,8350,0.042256,46.638,93.372,0,13.73,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""error"": null}" +openemr_referral,code_agent,2026-10-18-b,5,1,none,True,correct,correct,12,26,1034,66125,8805,0.04563,48.538,97.222,0,13.52,False,"{""record_status"": ""needs_human_admit"", ""program_check"": true, ""canary_status"": ""refused"", ""admitted"": false, ""error"": null}" diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json new file mode 100644 index 000000000..daa954b86 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json @@ -0,0 +1,394 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 15, + "input_tokens": 32, + "output_tokens": 1341, + "cache_creation_input_tokens": 5911, + "cache_read_input_tokens": 86798, + "prompt_tokens": 92741, + "images_in_requests": 0, + "cost_usd": 0.045611, + "wall_seconds": 52.522, + "model_seconds": 23.46, + "finished": true, + "summary": "Created patient Fitzroy ScoreboardZ701 (pid 3) with the given values. No duplicate warning appeared.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 130, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 2859, + "seconds": 1.924, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 180, + "cache_creation_input_tokens": 706, + "cache_read_input_tokens": 2859, + "seconds": 2.045, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 57, + "cache_creation_input_tokens": 249, + "cache_read_input_tokens": 3565, + "seconds": 1.496, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 49, + "cache_creation_input_tokens": 153, + "cache_read_input_tokens": 3814, + "seconds": 1.21, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 93, + "cache_creation_input_tokens": 1451, + "cache_read_input_tokens": 3967, + "seconds": 1.551, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 180, + "cache_creation_input_tokens": 264, + "cache_read_input_tokens": 5418, + "seconds": 1.799, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 96, + "cache_creation_input_tokens": 695, + "cache_read_input_tokens": 5682, + "seconds": 1.579, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 114, + "cache_creation_input_tokens": 298, + "cache_read_input_tokens": 6377, + "seconds": 1.719, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 47, + "cache_creation_input_tokens": 267, + "cache_read_input_tokens": 6675, + "seconds": 1.329, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 10, + "input_tokens": 2, + "output_tokens": 50, + "cache_creation_input_tokens": 109, + "cache_read_input_tokens": 6942, + "seconds": 1.701, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 11, + "input_tokens": 2, + "output_tokens": 78, + "cache_creation_input_tokens": 107, + "cache_read_input_tokens": 7051, + "seconds": 1.463, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 12, + "input_tokens": 2, + "output_tokens": 45, + "cache_creation_input_tokens": 149, + "cache_read_input_tokens": 7158, + "seconds": 1.256, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 13, + "input_tokens": 2, + "output_tokens": 80, + "cache_creation_input_tokens": 1165, + "cache_read_input_tokens": 7307, + "seconds": 1.498, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 14, + "input_tokens": 2, + "output_tokens": 54, + "cache_creation_input_tokens": 180, + "cache_read_input_tokens": 8472, + "seconds": 1.277, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 15, + "input_tokens": 2, + "output_tokens": 88, + "cache_creation_input_tokens": 118, + "cache_read_input_tokens": 8652, + "seconds": 1.613, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 35, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 16, + "lines": 34, + "actions": 19, + "errors": 2, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:DOB", + "input:country_code", + "input:fname", + "input:lname", + "input:sex", + "input:state_label", + "input:title" + ], + "read_back_after_commit": [ + "input:fname", + "input:lname" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Mr.\nfill @15 value=Fitzroy\nfill @17 value=ScoreboardZ701\nfill @23 value=1976-06-27\nselect @24 option=Male\nclick @40\ntree grep=Street\nfind role=textbox name=City", + "output": "$ select @14 option=Mr.\nok: select combobox \"form_title\" value=\"Mr.\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @15 value=Fitzroy\nok: fill textbox \"First Name\" value=\"Fitzroy\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardZ701\nok: fill textbox \"Last Name\" value=\"ScoreboardZ701\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @23 value=1976-06-27\nok: fill textbox \"form_DOB\" value=\"1976-06-27\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @24 option=Male\nok: select combobox \"form_sex\" value=\"Male\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ tree grep=Street\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)\n$ find role=textbox name=City\nno match", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"711 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email", + "output": "$ fill role=textbox name=form_street value=\"711 SYNTHETIC WAY\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", + "images": 0 + }, + { + "script": "scroll down px=400\ntree max=80", + "output": "$ scroll down px=400\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "tree max=120", + "output": "$ tree max=120\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Fitzroy\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"ScoreboardZ701\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"1976-06-27\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\" (focused)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", ", + "images": 0 + }, + { + "script": "click @37\nscroll down px=600\nfind role=textbox name=form_street", + "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find role=textbox name=form_street\n@14 textbox \"form_street\" value=\"\"\n@15 textbox \"form_street_line_2\" value=\"\"", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"711 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email", + "output": "$ fill role=textbox name=form_street value=\"711 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"711 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill role=textbox name=form_city value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select role=combobox name=form_state option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill role=textbox name=form_postal_code value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select role=combobox name=form_country_code option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find role=textbox name=form_phone_home\n@27 textbox \"form_phone_home\" value=\"\"\n$ find role=textbox name=form_email\n@30 textbox \"form_email\" value=\"\"\n@31 textbox \"form_email_direct\" value=\"\"", + "images": 0 + }, + { + "script": "fill @27 value=555-0101\nfill @30 value=fitzroy.scoreboard.701@example.invalid\nfind role=button name=\"Create New Patient\"", + "output": "$ fill @27 value=555-0101\nok: fill textbox \"form_phone_home\" value=\"555-0101\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @30 value=fitzroy.scoreboard.701@example.invalid\nok: fill textbox \"form_email\" value=\"fitzroy.scoreboard.701@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find role=button name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "find name=\"Create New Patient\"", + "output": "$ find name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "scroll down px=2000\nfind name=\"Create New Patient\"", + "output": "$ scroll down px=2000\nok: scroll down 2000px\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find name=\"Create New Patient\"\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)", + "images": 0 + }, + { + "script": "text grep=atient", + "output": "$ text grep=atient\nPatient\nSearch or Add Patient\nSearch or Add Patient\nCreate New Patient\nConfirm Create New Patient Cancel", + "images": 0 + }, + { + "script": "tree grep=Confirm", + "output": "$ tree grep=Confirm\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@39 button \"Confirm Create New Patient\"", + "images": 0 + }, + { + "script": "text", + "output": "$ text\nCalendar\nFinder\nFlow\nRecalls\nMessages\nPatient\nFees\nModules\nProcedures\nAdmin\nReports\nMiscellaneous\nPopups\nCalendar\nMessage Center\nSearch or Add Patient\nSearch or Add Patient\nWho\nTitle:\nUnassigned\nMr.\nMrs.\nMs.\nDr.\nName:\nPreferred Name:\nBirth Name:\nDOB:\nBirth Sex:\nUnassigned\nFemale\nMale\nUnknown\nGender Identity:\nUnassigned\nIdentifies as Male\nIdentifies as Female\nFemale-to-Male (FTM)/Transgender Male/Trans Man\nMale-to-Female (MTF)/Transgender Female/Trans Woman\nGenderqueer, neither exclusively male nor female\nAdditional gender category or other, please specify\nChoose not to disclose\nSex:\nUnassigned\nMale\nFemale\nIdentifies as nonbinary gender (finding)\nAsked But Declined\nunknown\nSexual Orientation:\nUnassigned\nStraight or heterosexual\nLesbian, gay or homosexual\nBisexual\nSomething else, please describe\nDon't know\nChoose not to disclose\nPronouns:\nUnassigned\nhe/him/his/his/himself\nshe/her/her/hers/herself\nthey/them/their/theirs/themselves\nAsk me\nDecline to answer\nOther\nExternal ID:\nS.S.:\nLicense/ID:\nMarital Status:\nUnassigned\nMarried\nSingle\nDivorced\nWidowed\nSeparated\nDomestic Partner\nUser Defined:\nBilling Note:\nPrevious Names:\nAdd\nContact\nAddress:\nAddress Line 2:\nCity:\nState:\nUnassigned\nAlabama\nAlaska\nArizona\nArkansas\nCalifornia\nColorado\nConnecticut\nDelaware\nDistrict of Columbia\nFlorida\nGeorgia\nHawaii\nIdaho\nIllinois\nIndiana\nIowa\nKansas\nKentucky\nLouisiana\nMaine\nMaryland\nMassachusetts\nMichigan\nMinnesota\nMississippi\nMissouri\nMontana\nNebraska\nNevada\nNew Hampshire\nNew Jersey\nNew Mexico\nNew York\nNorth Carolina\nNorth Dakota\nOhio\nOklahoma\nOregon\nPennsylvania\nPuerto Rico\nRhode Island\nSouth Carolina\nSouth Dakota\nTennessee\nTexas\nUtah\nVermont\nVirginia\nWashington\nWest Virginia\nWisconsin\nWyoming\nPostal Code:\nCountry:\nUnassigned\nUSA\nCounty:\nUnassigned\nADAIR\nANDREW\nATCHISON\nAUDRAIN\nBARRY\nMother`s Name:\nEmergency Contact:\nEmergency Phone:\nHome Phone:\nWork Phone:\nMobile Phone:\nContact Email:\nTrusted Email:\nAdditional Addresses\nShow Inactive\nNONE\nChoices\nEmployer\nStats\nMisc\nRelated\nSearch\nCreate New Patient\n1 - 0 of 0\nHits\tName\t\tPreferred Name\tBirth Name\t\t\tDOB\tBirth Sex\tExternal ID\tPrevious Names\nNo matches were found.\nConfirm Create New Patient Cancel", + "images": 0 + }, + { + "script": "click @39\ntext grep=Fitzroy", + "output": "$ click @39\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "text grep=Fitzroy", + "output": "$ text grep=Fitzroy\nFitzroy ScoreboardZ701 (3)\nMedical Record Dashboard - Fitzroy ScoreboardZ701", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 35, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_033: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "field_label": "form_DOB", + "masked_example": "19******27", + "name": "DOB", + "step_id": "step_008" + }, + { + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "field_label": "form_email", + "masked_example": "fi**********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "field_label": "First Name", + "masked_example": "F*****y", + "name": "fname", + "step_id": "step_004" + }, + { + "field_label": "Last Name", + "masked_example": "Sc**********01", + "name": "lname", + "step_id": "step_006" + }, + { + "field_label": "form_phone_home", + "masked_example": "5******1", + "name": "phone_home", + "step_id": "step_029" + }, + { + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "field_label": "form_sex", + "masked_example": "M**e", + "name": "sex", + "step_id": "step_010" + }, + { + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "field_label": "form_street", + "masked_example": "71*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json new file mode 100644 index 000000000..0c8871587 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json @@ -0,0 +1,281 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 9, + "input_tokens": 20, + "output_tokens": 898, + "cache_creation_input_tokens": 5329, + "cache_read_input_tokens": 41146, + "prompt_tokens": 46495, + "images_in_requests": 1, + "cost_usd": 0.030572, + "wall_seconds": 41.873, + "model_seconds": 14.793, + "finished": true, + "summary": "I did not create the patient. OpenEMR warned of a possible existing match: ScoreboardA000, Adela, DOB 1950-01-01, Female, External ID 2. I stopped at that warning and did not click \"Confirm Create New Patient\", so no patient was saved.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 116, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 2852, + "seconds": 1.738, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 170, + "cache_creation_input_tokens": 669, + "cache_read_input_tokens": 2852, + "seconds": 1.856, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 51, + "cache_creation_input_tokens": 239, + "cache_read_input_tokens": 3521, + "seconds": 1.365, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 67, + "cache_creation_input_tokens": 929, + "cache_read_input_tokens": 3760, + "seconds": 1.557, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 163, + "cache_read_input_tokens": 4689, + "seconds": 1.344, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 160, + "cache_creation_input_tokens": 739, + "cache_read_input_tokens": 4852, + "seconds": 1.888, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 57, + "cache_creation_input_tokens": 742, + "cache_read_input_tokens": 5591, + "seconds": 1.386, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 52, + "cache_creation_input_tokens": 363, + "cache_read_input_tokens": 6333, + "seconds": 1.337, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 162, + "cache_creation_input_tokens": 1485, + "cache_read_input_tokens": 6696, + "seconds": 2.323, + "stop_reason": "tool_use", + "images_in_request": 1 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 1, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 34, + "gate_checks": 0, + "gate_evidence_screenshots": 0, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 9, + "lines": 26, + "actions": 18, + "errors": 2, + "screenshots": 1, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 15, + "with_recorded_locator": 15, + "locator_agrees": 15, + "locator_unique": 15, + "commits": 0, + "identity_read_before_commit": [], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Ms.\nfill @15 value=Adela\nfill @17 value=ScoreboardA702\nfill @23 value=1977-07-01\nselect @24 option=Female\nclick @40\ntree grep=Street", + "output": "$ select @14 option=Ms.\nok: select combobox \"form_title\" value=\"Ms.\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @15 value=Adela\nok: fill textbox \"First Name\" value=\"Adela\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardA702\nok: fill textbox \"Last Name\" value=\"ScoreboardA702\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @23 value=1977-07-01\nok: fill textbox \"form_DOB\" value=\"1977-07-01\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ tree grep=Street\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"712 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind name=form_phone_home\nfind name=form_email", + "output": "$ fill role=textbox name=form_street value=\"712 SYNTHETIC WAY\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", + "images": 0 + }, + { + "script": "tree grep=form_", + "output": "$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@14 combobox \"form_title\" value=\"Ms.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@23 textbox \"form_DOB\" value=\"1977-07-01\"\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n@38 combobox \"form_name_history[]\" value=\"\"\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", + "images": 0 + }, + { + "script": "scroll down px=600\ntree grep=form_", + "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "click @37\nscroll down px=600\ntree grep=form_", + "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@14 textbox \"form_street\" value=\"\"\n@15 textbox \"form_street_line_2\" value=\"\"\n@16 textbox \"form_city\" value=\"\"\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@19 textbox \"form_postal_code\" value=\"\"\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@24 textbox \"form_mothersname\" value=\"\"\n@25 textbox \"form_contact_relationship\" value=\"\"\n@26 textbox \"form_phone_contact\" value=\"\"\n@27 textbox \"form_phone_home\" value=\"\"\n@28 textbox \"form_phone_biz\" value=\"\"\n@29 textbox \"form_phone_cell\" value=\"\"\n@30 textbox \"form_email\" value=\"\"\n@31 textbox \"form_email_direct\" value=\"\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)", + "images": 0 + }, + { + "script": "fill @14 value=\"712 SYNTHETIC WAY\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0102\nfill @30 value=adela.scoreboard.702@example.invalid\nfind name=\"Create New Patient\"", + "output": "$ fill @14 value=\"712 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"712 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @27 value=555-0102\nok: fill textbox \"form_phone_home\" value=\"555-0102\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @30 value=adela.scoreboard.702@example.invalid\nok: fill textbox \"form_email\" value=\"adela.scoreboard.702@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "scroll down px=3000\nfind role=button", + "output": "$ scroll down px=3000\nok: scroll down 2000px\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find role=button\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\n@17 button \"Add\" value=\"Add\"\n@20 button \"Add\" value=\"Add\"\n@22 button \"Add\" value=\"Add\"\n@32 button \"Choices\"\n@33 button \"Employer\"\n@34 button \"Stats\"\n@35 button \"Misc\"\n@36 button \"Related\"\n@37 button \"Search\"\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38\nscreenshot", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ screenshot\n(screenshot attached)", + "images": 1 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [], + "program_check": { + "trace_steps_with_locator": 15, + "distilled_steps": [], + "in_program": 15, + "missing_steps": [], + "program_structural_steps": 15, + "passed": true + }, + "steps": 34, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_033: record_written is placeholder", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "field_label": "form_DOB", + "masked_example": "19******01", + "name": "DOB", + "step_id": "step_008" + }, + { + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "field_label": "form_email", + "masked_example": "ad********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "field_label": "First Name", + "masked_example": "A***a", + "name": "fname", + "step_id": "step_004" + }, + { + "field_label": "Last Name", + "masked_example": "Sc**********02", + "name": "lname", + "step_id": "step_006" + }, + { + "field_label": "form_phone_home", + "masked_example": "5******2", + "name": "phone_home", + "step_id": "step_029" + }, + { + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "field_label": "form_sex", + "masked_example": "F****e", + "name": "sex", + "step_id": "step_010" + }, + { + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "field_label": "form_street", + "masked_example": "71*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json new file mode 100644 index 000000000..c20625813 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json @@ -0,0 +1,326 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 11, + "input_tokens": 24, + "output_tokens": 953, + "cache_creation_input_tokens": 8234, + "cache_read_input_tokens": 53523, + "prompt_tokens": 61781, + "images_in_requests": 3, + "cost_usd": 0.040868, + "wall_seconds": 46.296, + "model_seconds": 17.121, + "finished": true, + "summary": "Created patient Barnaby ScoreboardB703 (pid 4) after the duplicate check found no matches.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 118, + "cache_creation_input_tokens": 2856, + "cache_read_input_tokens": 0, + "seconds": 1.744, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 180, + "cache_creation_input_tokens": 675, + "cache_read_input_tokens": 2856, + "seconds": 1.88, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 54, + "cache_creation_input_tokens": 249, + "cache_read_input_tokens": 3531, + "seconds": 1.305, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 51, + "cache_creation_input_tokens": 150, + "cache_read_input_tokens": 3780, + "seconds": 1.26, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 76, + "cache_creation_input_tokens": 929, + "cache_read_input_tokens": 3930, + "seconds": 1.405, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 165, + "cache_creation_input_tokens": 754, + "cache_read_input_tokens": 4859, + "seconds": 1.721, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 54, + "cache_creation_input_tokens": 764, + "cache_read_input_tokens": 5613, + "seconds": 1.574, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 48, + "cache_creation_input_tokens": 139, + "cache_read_input_tokens": 6377, + "seconds": 1.147, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 72, + "cache_creation_input_tokens": 1423, + "cache_read_input_tokens": 6516, + "seconds": 1.842, + "stop_reason": "tool_use", + "images_in_request": 1 + }, + { + "turn": 10, + "input_tokens": 2, + "output_tokens": 53, + "cache_creation_input_tokens": 183, + "cache_read_input_tokens": 7939, + "seconds": 1.444, + "stop_reason": "tool_use", + "images_in_request": 1 + }, + { + "turn": 11, + "input_tokens": 2, + "output_tokens": 82, + "cache_creation_input_tokens": 112, + "cache_read_input_tokens": 8122, + "seconds": 1.8, + "stop_reason": "tool_use", + "images_in_request": 1 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 1, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 34, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 11, + "lines": 27, + "actions": 18, + "errors": 2, + "screenshots": 1, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:DOB", + "input:country_code", + "input:sex", + "input:title" + ], + "read_back_after_commit": [ + "input:fname", + "input:lname" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Mr.\nfill @15 value=Barnaby\nfill @17 value=ScoreboardB703\nfill @23 value=1978-08-02\nselect @24 option=Male\nclick @40\ntree grep=Street", + "output": "$ select @14 option=Mr.\nok: select combobox \"form_title\" value=\"Mr.\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @15 value=Barnaby\nok: fill textbox \"First Name\" value=\"Barnaby\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardB703\nok: fill textbox \"Last Name\" value=\"ScoreboardB703\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @23 value=1978-08-02\nok: fill textbox \"form_DOB\" value=\"1978-08-02\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @24 option=Male\nok: select combobox \"form_sex\" value=\"Male\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ tree grep=Street\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"713 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email", + "output": "$ fill role=textbox name=form_street value=\"713 SYNTHETIC WAY\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", + "images": 0 + }, + { + "script": "scroll down px=600\ntree", + "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "tree grep=form_", + "output": "$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@23 textbox \"form_DOB\" value=\"1978-08-02\"\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n@38 combobox \"form_name_history[]\" value=\"\"\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", + "images": 0 + }, + { + "script": "click @37\nscroll down px=1000\ntree grep=form_", + "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ scroll down px=1000\nok: scroll down 1000px\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@13 textbox \"form_street\" value=\"\"\n@14 textbox \"form_street_line_2\" value=\"\"\n@15 textbox \"form_city\" value=\"\"\n@16 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@18 textbox \"form_postal_code\" value=\"\"\n@19 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@21 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@23 textbox \"form_mothersname\" value=\"\"\n@24 textbox \"form_contact_relationship\" value=\"\"\n@25 textbox \"form_phone_contact\" value=\"\"\n@26 textbox \"form_phone_home\" value=\"\"\n@27 textbox \"form_phone_biz\" value=\"\"\n@28 textbox \"form_phone_cell\" value=\"\"\n@29 textbox \"form_email\" value=\"\"\n@30 textbox \"form_email_direct\" value=\"\"\n(28 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 20 more; scroll up to reach them)", + "images": 0 + }, + { + "script": "fill @13 value=\"713 SYNTHETIC WAY\"\nfill @15 value=EXAMPLETON\nselect @16 option=Massachusetts\nfill @18 value=02139\nselect @19 option=USA\nfill @26 value=555-0103\nfill @29 value=barnaby.scoreboard.703@example.invalid\nfind role=button name=\"Create New Patient\"", + "output": "$ fill @13 value=\"713 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"713 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @15 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @16 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @18 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @19 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @26 value=555-0103\nok: fill textbox \"form_phone_home\" value=\"555-0103\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @29 value=barnaby.scoreboard.703@example.invalid\nok: fill textbox \"form_email\" value=\"barnaby.scoreboard.703@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find role=button name=\"Create New Patient\"\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38\ntext grep=atch", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ text grep=atch\nATCHISON\nNo matches were found.", + "images": 0 + }, + { + "script": "screenshot", + "output": "$ screenshot\n(screenshot attached)", + "images": 1 + }, + { + "script": "click text=\"Confirm Create New Patient\"\nscreenshot", + "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "text grep=Barnaby", + "output": "$ text grep=Barnaby\nBarnaby ScoreboardB703 (4)\nMedical Record Dashboard - Barnaby ScoreboardB703", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 34, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_032: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "field_label": "form_DOB", + "masked_example": "19******02", + "name": "DOB", + "step_id": "step_008" + }, + { + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "field_label": "form_email", + "masked_example": "ba**********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "field_label": "First Name", + "masked_example": "B*****y", + "name": "fname", + "step_id": "step_004" + }, + { + "field_label": "Last Name", + "masked_example": "Sc**********03", + "name": "lname", + "step_id": "step_006" + }, + { + "field_label": "form_phone_home", + "masked_example": "5******3", + "name": "phone_home", + "step_id": "step_029" + }, + { + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "field_label": "form_sex", + "masked_example": "M**e", + "name": "sex", + "step_id": "step_010" + }, + { + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "field_label": "form_street", + "masked_example": "71*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json new file mode 100644 index 000000000..430cb4b94 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json @@ -0,0 +1,329 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 11, + "input_tokens": 24, + "output_tokens": 943, + "cache_creation_input_tokens": 8350, + "cache_read_input_tokens": 59516, + "prompt_tokens": 67890, + "images_in_requests": 0, + "cost_usd": 0.042256, + "wall_seconds": 46.638, + "model_seconds": 17.384, + "finished": true, + "summary": "Created patient Clarice ScoreboardC704 (pid 5); the duplicate check found no matches.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 119, + "cache_creation_input_tokens": 2854, + "cache_read_input_tokens": 0, + "seconds": 2.086, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 108, + "cache_creation_input_tokens": 1796, + "cache_read_input_tokens": 2854, + "seconds": 1.624, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 202, + "cache_read_input_tokens": 4650, + "seconds": 1.391, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 74, + "cache_creation_input_tokens": 159, + "cache_read_input_tokens": 4852, + "seconds": 1.575, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 164, + "cache_creation_input_tokens": 761, + "cache_read_input_tokens": 5011, + "seconds": 1.772, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 62, + "cache_creation_input_tokens": 751, + "cache_read_input_tokens": 5772, + "seconds": 1.68, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 164, + "cache_read_input_tokens": 6523, + "seconds": 1.502, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 45, + "cache_creation_input_tokens": 193, + "cache_read_input_tokens": 6687, + "seconds": 1.138, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 113, + "cache_creation_input_tokens": 1165, + "cache_read_input_tokens": 6880, + "seconds": 1.656, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 10, + "input_tokens": 2, + "output_tokens": 52, + "cache_creation_input_tokens": 197, + "cache_read_input_tokens": 8045, + "seconds": 1.394, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 11, + "input_tokens": 2, + "output_tokens": 80, + "cache_creation_input_tokens": 108, + "cache_read_input_tokens": 8242, + "seconds": 1.564, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 35, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 11, + "lines": 32, + "actions": 19, + "errors": 1, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:DOB", + "input:country_code", + "input:fname", + "input:lname", + "input:sex", + "input:state_label", + "input:title" + ], + "read_back_after_commit": [ + "input:fname", + "input:lname" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Ms.\nfill @15 value=Clarice\nfill @17 value=ScoreboardC704\nfill @23 value=1979-09-03\nselect @24 option=Female\nclick @40\ntree grep=a max=80", + "output": "$ select @14 option=Ms.\nok: select combobox \"form_title\" value=\"Ms.\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @15 value=Clarice\nok: fill textbox \"First Name\" value=\"Clarice\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardC704\nok: fill textbox \"Last Name\" value=\"ScoreboardC704\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @23 value=1979-09-03\nok: fill textbox \"form_DOB\" value=\"1979-09-03\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ tree grep=a max=80\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\nCalendar\nRecalls\nMessages\n@2 button \"Patient\"\n@6 button \"Admin\"\n@8 button \"Miscellaneous\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@14 combobox \"form_title\" value=\"Ms.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Clarice\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"ScoreboardC704\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\n@23 textbox \"form_DOB\" value=\"1979-09-03\"\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@", + "images": 0 + }, + { + "script": "find role=textbox name=form_street\nfind role=combobox name=form_state\nfind role=combobox name=form_country_code\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email", + "output": "$ find role=textbox name=form_street\nno match\n$ find role=combobox name=form_state\nno match\n$ find role=combobox name=form_country_code\nno match\n$ find role=textbox name=form_phone_home\nno match\n$ find role=textbox name=form_email\nno match", + "images": 0 + }, + { + "script": "scroll down px=600\ntree grep=form_ max=80", + "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "click role=textbox name=form_billing_note\nscroll down px=600\ntree grep=form_", + "output": "$ click role=textbox name=form_billing_note\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@14 textbox \"form_street\" value=\"\"\n@15 textbox \"form_street_line_2\" value=\"\"\n@16 textbox \"form_city\" value=\"\"\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@19 textbox \"form_postal_code\" value=\"\"\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@24 textbox \"form_mothersname\" value=\"\"\n@25 textbox \"form_contact_relationship\" value=\"\"\n@26 textbox \"form_phone_contact\" value=\"\"\n@27 textbox \"form_phone_home\" value=\"\"\n@28 textbox \"form_phone_biz\" value=\"\"\n@29 textbox \"form_phone_cell\" value=\"\"\n@30 textbox \"form_email\" value=\"\"\n@31 textbox \"form_email_direct\" value=\"\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)", + "images": 0 + }, + { + "script": "fill @14 value=\"714 SYNTHETIC WAY\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0104\nfill @30 value=clarice.scoreboard.704@example.invalid\nfind role=button name=\"Create New Patient\"", + "output": "$ fill @14 value=\"714 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"714 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @27 value=555-0104\nok: fill textbox \"form_phone_home\" value=\"555-0104\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @30 value=clarice.scoreboard.704@example.invalid\nok: fill textbox \"form_email\" value=\"clarice.scoreboard.704@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find role=button name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "find name=\"Create New Patient\"\ntree grep=Create", + "output": "$ find name=\"Create New Patient\"\nno match\n$ tree grep=Create\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n(2 controls below the visible area: \"Search\", \"Create New Patient\"; scroll down to reach them)", + "images": 0 + }, + { + "script": "scroll down px=2000\nclick name=\"Create New Patient\"", + "output": "$ scroll down px=2000\nok: scroll down 2000px\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ click name=\"Create New Patient\"\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)", + "images": 0 + }, + { + "script": "text", + "output": "$ text\nCalendar\nFinder\nFlow\nRecalls\nMessages\nPatient\nFees\nModules\nProcedures\nAdmin\nReports\nMiscellaneous\nPopups\nCalendar\nMessage Center\nSearch or Add Patient\nSearch or Add Patient\nWho\nTitle:\nUnassigned\nMr.\nMrs.\nMs.\nDr.\nName:\nPreferred Name:\nBirth Name:\nDOB:\nBirth Sex:\nUnassigned\nFemale\nMale\nUnknown\nGender Identity:\nUnassigned\nIdentifies as Male\nIdentifies as Female\nFemale-to-Male (FTM)/Transgender Male/Trans Man\nMale-to-Female (MTF)/Transgender Female/Trans Woman\nGenderqueer, neither exclusively male nor female\nAdditional gender category or other, please specify\nChoose not to disclose\nSex:\nUnassigned\nMale\nFemale\nIdentifies as nonbinary gender (finding)\nAsked But Declined\nunknown\nSexual Orientation:\nUnassigned\nStraight or heterosexual\nLesbian, gay or homosexual\nBisexual\nSomething else, please describe\nDon't know\nChoose not to disclose\nPronouns:\nUnassigned\nhe/him/his/his/himself\nshe/her/her/hers/herself\nthey/them/their/theirs/themselves\nAsk me\nDecline to answer\nOther\nExternal ID:\nS.S.:\nLicense/ID:\nMarital Status:\nUnassigned\nMarried\nSingle\nDivorced\nWidowed\nSeparated\nDomestic Partner\nUser Defined:\nBilling Note:\nPrevious Names:\nAdd\nContact\nAddress:\nAddress Line 2:\nCity:\nState:\nUnassigned\nAlabama\nAlaska\nArizona\nArkansas\nCalifornia\nColorado\nConnecticut\nDelaware\nDistrict of Columbia\nFlorida\nGeorgia\nHawaii\nIdaho\nIllinois\nIndiana\nIowa\nKansas\nKentucky\nLouisiana\nMaine\nMaryland\nMassachusetts\nMichigan\nMinnesota\nMississippi\nMissouri\nMontana\nNebraska\nNevada\nNew Hampshire\nNew Jersey\nNew Mexico\nNew York\nNorth Carolina\nNorth Dakota\nOhio\nOklahoma\nOregon\nPennsylvania\nPuerto Rico\nRhode Island\nSouth Carolina\nSouth Dakota\nTennessee\nTexas\nUtah\nVermont\nVirginia\nWashington\nWest Virginia\nWisconsin\nWyoming\nPostal Code:\nCountry:\nUnassigned\nUSA\nCounty:\nUnassigned\nADAIR\nANDREW\nATCHISON\nAUDRAIN\nBARRY\nMother`s Name:\nEmergency Contact:\nEmergency Phone:\nHome Phone:\nWork Phone:\nMobile Phone:\nContact Email:\nTrusted Email:\nAdditional Addresses\nShow Inactive\nNONE\nChoices\nEmployer\nStats\nMisc\nRelated\nSearch\nCreate New Patient\n1 - 0 of 0\nHits\tName\t\tPreferred Name\tBirth Name\t\t\tDOB\tBirth Sex\tExternal ID\tPrevious Names\nNo matches were found.\nConfirm Create New Patient Cancel", + "images": 0 + }, + { + "script": "click name=\"Confirm Create New Patient\"", + "output": "$ click name=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)", + "images": 0 + }, + { + "script": "text grep=Clarice", + "output": "$ text grep=Clarice\nClarice ScoreboardC704 (5)\nMedical Record Dashboard - Clarice ScoreboardC704", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 35, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_033: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "field_label": "form_DOB", + "masked_example": "19******03", + "name": "DOB", + "step_id": "step_008" + }, + { + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "field_label": "form_email", + "masked_example": "cl**********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "field_label": "First Name", + "masked_example": "C*****e", + "name": "fname", + "step_id": "step_004" + }, + { + "field_label": "Last Name", + "masked_example": "Sc**********04", + "name": "lname", + "step_id": "step_006" + }, + { + "field_label": "form_phone_home", + "masked_example": "5******4", + "name": "phone_home", + "step_id": "step_029" + }, + { + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "field_label": "form_sex", + "masked_example": "F****e", + "name": "sex", + "step_id": "step_010" + }, + { + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "field_label": "form_street", + "masked_example": "71*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json new file mode 100644 index 000000000..9251ae29c --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json @@ -0,0 +1,344 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 12, + "input_tokens": 26, + "output_tokens": 1034, + "cache_creation_input_tokens": 8805, + "cache_read_input_tokens": 66125, + "prompt_tokens": 74956, + "images_in_requests": 0, + "cost_usd": 0.04563, + "wall_seconds": 48.538, + "model_seconds": 18.753, + "finished": true, + "summary": "Created patient Desmond ScoreboardD705 (pid 6) with the given values; no matching patients found.", + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 121, + "cache_creation_input_tokens": 2854, + "cache_read_input_tokens": 0, + "seconds": 1.762, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 2, + "input_tokens": 2, + "output_tokens": 49, + "cache_creation_input_tokens": 620, + "cache_read_input_tokens": 2854, + "seconds": 1.319, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 3, + "input_tokens": 2, + "output_tokens": 178, + "cache_creation_input_tokens": 1448, + "cache_read_input_tokens": 3474, + "seconds": 2.091, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 4, + "input_tokens": 2, + "output_tokens": 59, + "cache_creation_input_tokens": 247, + "cache_read_input_tokens": 4922, + "seconds": 1.557, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 5, + "input_tokens": 2, + "output_tokens": 63, + "cache_creation_input_tokens": 155, + "cache_read_input_tokens": 5169, + "seconds": 1.594, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 6, + "input_tokens": 2, + "output_tokens": 161, + "cache_creation_input_tokens": 739, + "cache_read_input_tokens": 5324, + "seconds": 1.774, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 7, + "input_tokens": 2, + "output_tokens": 48, + "cache_creation_input_tokens": 745, + "cache_read_input_tokens": 6063, + "seconds": 1.325, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 8, + "input_tokens": 2, + "output_tokens": 66, + "cache_creation_input_tokens": 281, + "cache_read_input_tokens": 6808, + "seconds": 1.527, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 9, + "input_tokens": 2, + "output_tokens": 49, + "cache_creation_input_tokens": 160, + "cache_read_input_tokens": 7089, + "seconds": 1.298, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 10, + "input_tokens": 2, + "output_tokens": 107, + "cache_creation_input_tokens": 1227, + "cache_read_input_tokens": 7249, + "seconds": 1.666, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 11, + "input_tokens": 2, + "output_tokens": 52, + "cache_creation_input_tokens": 221, + "cache_read_input_tokens": 8476, + "seconds": 1.354, + "stop_reason": "tool_use", + "images_in_request": 0 + }, + { + "turn": 12, + "input_tokens": 2, + "output_tokens": 81, + "cache_creation_input_tokens": 108, + "cache_read_input_tokens": 8697, + "seconds": 1.487, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 35, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 0 + }, + "trace": { + "surface": "dom", + "scripts": 12, + "lines": 29, + "actions": 19, + "errors": 2, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 16, + "with_recorded_locator": 16, + "locator_agrees": 16, + "locator_unique": 16, + "commits": 1, + "identity_read_before_commit": [ + "identity:Create New Patient", + "input:DOB", + "input:country_code", + "input:fname", + "input:lname", + "input:sex", + "input:state_label", + "input:title" + ], + "read_back_after_commit": [ + "input:fname", + "input:lname" + ] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + }, + { + "script": "select @14 option=Mr.\nfill @15 value=Desmond\nfill @17 value=ScoreboardD705\nfill @23 value=1980-10-04\nselect @24 option=Male\nclick @40\ntree grep=Contact max=60", + "output": "$ select @14 option=Mr.\nok: select combobox \"form_title\" value=\"Mr.\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @15 value=Desmond\nok: fill textbox \"First Name\" value=\"Desmond\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @17 value=ScoreboardD705\nok: fill textbox \"Last Name\" value=\"ScoreboardD705\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @23 value=1980-10-04\nok: fill textbox \"form_DOB\" value=\"1980-10-04\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @24 option=Male\nok: select combobox \"form_sex\" value=\"Male\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ tree grep=Contact max=60\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@40 button \"Contact\" (focused)", + "images": 0 + }, + { + "script": "tree max=120", + "output": "$ tree max=120\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Desmond\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"ScoreboardD705\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"1980-10-04\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\" (focused)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", ", + "images": 0 + }, + { + "script": "fill role=textbox name=form_street value=\"715 SYNTHETIC WAY\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind name=form_phone_home\nfind name=form_email", + "output": "$ fill role=textbox name=form_street value=\"715 SYNTHETIC WAY\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", + "images": 0 + }, + { + "script": "scroll down px=600\ntree grep=form_", + "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", + "images": 0 + }, + { + "script": "click @37\nscroll down px=600\ntree grep=form_", + "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@14 textbox \"form_street\" value=\"\"\n@15 textbox \"form_street_line_2\" value=\"\"\n@16 textbox \"form_city\" value=\"\"\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@19 textbox \"form_postal_code\" value=\"\"\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@24 textbox \"form_mothersname\" value=\"\"\n@25 textbox \"form_contact_relationship\" value=\"\"\n@26 textbox \"form_phone_contact\" value=\"\"\n@27 textbox \"form_phone_home\" value=\"\"\n@28 textbox \"form_phone_biz\" value=\"\"\n@29 textbox \"form_phone_cell\" value=\"\"\n@30 textbox \"form_email\" value=\"\"\n@31 textbox \"form_email_direct\" value=\"\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)", + "images": 0 + }, + { + "script": "fill @14 value=\"715 SYNTHETIC WAY\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0105\nfill @30 value=desmond.scoreboard.705@example.invalid\nfind name=\"Create New Patient\"", + "output": "$ fill @14 value=\"715 SYNTHETIC WAY\"\nok: fill textbox \"form_street\" value=\"715 SYNTHETIC WAY\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @27 value=555-0105\nok: fill textbox \"form_phone_home\" value=\"555-0105\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ fill @30 value=desmond.scoreboard.705@example.invalid\nok: fill textbox \"form_email\" value=\"desmond.scoreboard.705@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find name=\"Create New Patient\"\nno match", + "images": 0 + }, + { + "script": "find role=button", + "output": "$ find role=button\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\n@13 button \"Contact\"\n@18 button \"Add\" value=\"Add\"\n@21 button \"Add\" value=\"Add\"\n@23 button \"Add\" value=\"Add\"\n@33 button \"Choices\"\n@34 button \"Employer\"\n@35 button \"Stats\"\n@36 button \"Misc\"\n@37 button \"Related\"", + "images": 0 + }, + { + "script": "scroll down px=2000\nfind role=button name=\"Create New Patient\"", + "output": "$ scroll down px=2000\nok: scroll down 2000px\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ find role=button name=\"Create New Patient\"\n@38 button \"Create New Patient\"", + "images": 0 + }, + { + "script": "click @38\ntext", + "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n$ text\nCalendar\nFinder\nFlow\nRecalls\nMessages\nPatient\nFees\nModules\nProcedures\nAdmin\nReports\nMiscellaneous\nPopups\nCalendar\nMessage Center\nSearch or Add Patient\nSearch or Add Patient\nWho\nTitle:\nUnassigned\nMr.\nMrs.\nMs.\nDr.\nName:\nPreferred Name:\nBirth Name:\nDOB:\nBirth Sex:\nUnassigned\nFemale\nMale\nUnknown\nGender Identity:\nUnassigned\nIdentifies as Male\nIdentifies as Female\nFemale-to-Male (FTM)/Transgender Male/Trans Man\nMale-to-Female (MTF)/Transgender Female/Trans Woman\nGenderqueer, neither exclusively male nor female\nAdditional gender category or other, please specify\nChoose not to disclose\nSex:\nUnassigned\nMale\nFemale\nIdentifies as nonbinary gender (finding)\nAsked But Declined\nunknown\nSexual Orientation:\nUnassigned\nStraight or heterosexual\nLesbian, gay or homosexual\nBisexual\nSomething else, please describe\nDon't know\nChoose not to disclose\nPronouns:\nUnassigned\nhe/him/his/his/himself\nshe/her/her/hers/herself\nthey/them/their/theirs/themselves\nAsk me\nDecline to answer\nOther\nExternal ID:\nS.S.:\nLicense/ID:\nMarital Status:\nUnassigned\nMarried\nSingle\nDivorced\nWidowed\nSeparated\nDomestic Partner\nUser Defined:\nBilling Note:\nPrevious Names:\nAdd\nContact\nAddress:\nAddress Line 2:\nCity:\nState:\nUnassigned\nAlabama\nAlaska\nArizona\nArkansas\nCalifornia\nColorado\nConnecticut\nDelaware\nDistrict of Columbia\nFlorida\nGeorgia\nHawaii\nIdaho\nIllinois\nIndiana\nIowa\nKansas\nKentucky\nLouisiana\nMaine\nMaryland\nMassachusetts\nMichigan\nMinnesota\nMississippi\nMissouri\nMontana\nNebraska\nNevada\nNew Hampshire\nNew Jersey\nNew Mexico\nNew York\nNorth Carolina\nNorth Dakota\nOhio\nOklahoma\nOregon\nPennsylvania\nPuerto Rico\nRhode Island\nSouth Carolina\nSouth Dakota\nTennessee\nTexas\nUtah\nVermont\nVirginia\nWashington\nWest Virginia\nWisconsin\nWyoming\nPostal Code:\nCountry:\nUnassigned\nUSA\nCounty:\nUnassigned\nADAIR\nANDREW\nATCHISON\nAUDRAIN\nBARRY\nMother`s Name:\nEmergency Contact:\nEmergency Phone:\nHome Phone:\nWork Phone:\nMobile Phone:\nContact Email:\nTrusted Email:\nAdditional Addresses\nShow Inactive\nNONE\nChoices\nEmployer\nStats\nMisc\nRelated\nSearch\nCreate New Patient\n1 - 0 of 0\nHits\tName\t\tPreferred Name\tBirth Name\t\t\tDOB\tBirth Sex\tExternal ID\tPrevious Names\nNo matches were found.\nConfirm Create New Patient Cancel", + "images": 0 + }, + { + "script": "click role=button name=\"Confirm Create New Patient\"\ntext grep=Scoreboard", + "output": "$ click role=button name=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", + "images": 0 + }, + { + "script": "text grep=Desmond", + "output": "$ text grep=Desmond\nDesmond ScoreboardD705 (6)\nMedical Record Dashboard - Desmond ScoreboardD705", + "images": 0 + } + ], + "result": { + "record_status": "needs_human_admit", + "detail": null, + "gates": [ + { + "control": "Confirm Create New Patient", + "missing": [], + "decision": "passed_no_hold" + } + ], + "program_check": { + "trace_steps_with_locator": 16, + "distilled_steps": [], + "in_program": 16, + "missing_steps": [], + "program_structural_steps": 16, + "passed": true + }, + "steps": 35, + "canary_status": "refused", + "canary_detail": "no canary ran: a write step lacks an independent read-back: step_033: a write step with no read-back", + "canary_halt": null, + "admitted": false, + "params": { + "lifted": [ + { + "field_label": "form_DOB", + "masked_example": "19******04", + "name": "DOB", + "step_id": "step_008" + }, + { + "field_label": "form_city", + "masked_example": "EX******ON", + "name": "city", + "step_id": "step_019" + }, + { + "field_label": "form_country_code", + "masked_example": "U*A", + "name": "country_code", + "step_id": "step_026" + }, + { + "field_label": "form_email", + "masked_example": "de**********************************id", + "name": "email", + "step_id": "step_031" + }, + { + "field_label": "First Name", + "masked_example": "D*****d", + "name": "fname", + "step_id": "step_004" + }, + { + "field_label": "Last Name", + "masked_example": "Sc**********05", + "name": "lname", + "step_id": "step_006" + }, + { + "field_label": "form_phone_home", + "masked_example": "5******5", + "name": "phone_home", + "step_id": "step_029" + }, + { + "field_label": "form_postal_code", + "masked_example": "0***9", + "name": "postal_code", + "step_id": "step_024" + }, + { + "field_label": "form_sex", + "masked_example": "M**e", + "name": "sex", + "step_id": "step_010" + }, + { + "field_label": "form_state", + "masked_example": "Ma*********ts", + "name": "state_label", + "step_id": "step_021" + }, + { + "field_label": "form_street", + "masked_example": "71*************AY", + "name": "street", + "step_id": "step_017" + }, + { + "field_label": "form_title", + "masked_example": "M*.", + "name": "title", + "step_id": "step_001" + } + ], + "unlifted": {} + } + } +} diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/meta.json b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/meta.json new file mode 100644 index 000000000..91f825907 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/meta.json @@ -0,0 +1,22 @@ +{ + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-18T17:35:59-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/scoreboard-arm7/packages/flow/openadapt_flow", + "git_head": "cc883bbcb74d9455830da5fe2b86e0c5b1e3f976" + }, + "benchmark_git_head": "cc883bbcb74d9455830da5fe2b86e0c5b1e3f976", + "task": "openemr_referral", + "arm": "code_agent", + "model": "claude-sonnet-5-5", + "n": 5, + "vision": "code", + "effort": "medium", + "max_cost_per_item": 0.75, + "on_screen": [ + "Create New Patient" + ] +} diff --git a/packages/flow/benchmark/scoreboard/raw/spend.json b/packages/flow/benchmark/scoreboard/raw/spend.json index b61e631e3..4abcf3375 100644 --- a/packages/flow/benchmark/scoreboard/raw/spend.json +++ b/packages/flow/benchmark/scoreboard/raw/spend.json @@ -1,5 +1,5 @@ { - "total_usd": 3.866907, + "total_usd": 4.171654, "entries": [ { "usd": 0.577, @@ -374,6 +374,86 @@ "usd": 0.05573, "note": "openimis agent 2026-10-18-a item 19", "at": 1792358431.418853 + }, + { + "usd": 0.010339, + "note": "mockmed code_agent smoke-delete item 1", + "at": 1792359107.756307 + }, + { + "usd": 0.006092, + "note": "mockmed code_agent 2026-10-18-a item 1", + "at": 1792359137.5327659 + }, + { + "usd": 0.010352, + "note": "mockmed code_agent 2026-10-18-a item 2", + "at": 1792359152.226209 + }, + { + "usd": 0.010449, + "note": "mockmed code_agent 2026-10-18-a item 3", + "at": 1792359166.907215 + }, + { + "usd": 0.010462, + "note": "mockmed code_agent 2026-10-18-a item 4", + "at": 1792359182.645686 + }, + { + "usd": 0.010339, + "note": "mockmed code_agent 2026-10-18-a item 5", + "at": 1792359201.927978 + }, + { + "usd": 0.008306, + "note": "openemr_referral code_agent 2026-10-18-a item 1", + "at": 1792359214.213875 + }, + { + "usd": 0.008298, + "note": "openemr_referral code_agent 2026-10-18-a item 2", + "at": 1792359219.3110201 + }, + { + "usd": 0.008328, + "note": "openemr_referral code_agent 2026-10-18-a item 3", + "at": 1792359224.770966 + }, + { + "usd": 0.008423, + "note": "openemr_referral code_agent 2026-10-18-a item 4", + "at": 1792359229.97929 + }, + { + "usd": 0.008423, + "note": "openemr_referral code_agent 2026-10-18-a item 5", + "at": 1792359235.856202 + }, + { + "usd": 0.045611, + "note": "openemr_referral code_agent 2026-10-18-b item 1", + "at": 1792359472.6206741 + }, + { + "usd": 0.030572, + "note": "openemr_referral code_agent 2026-10-18-b item 2", + "at": 1792359613.251651 + }, + { + "usd": 0.040868, + "note": "openemr_referral code_agent 2026-10-18-b item 3", + "at": 1792359758.899384 + }, + { + "usd": 0.042256, + "note": "openemr_referral code_agent 2026-10-18-b item 4", + "at": 1792359895.759076 + }, + { + "usd": 0.04563, + "note": "openemr_referral code_agent 2026-10-18-b item 5", + "at": 1792360037.685641 } ] } diff --git a/packages/flow/benchmark/scoreboard/run.py b/packages/flow/benchmark/scoreboard/run.py index a25899458..78e2f9c34 100644 --- a/packages/flow/benchmark/scoreboard/run.py +++ b/packages/flow/benchmark/scoreboard/run.py @@ -13,6 +13,10 @@ * ``first_run``: arm 2, the agent's first run under OpenAdapt: recorded by the JIT wrapper, read back, compiled, and taken through admission. * ``program``: arm 3, the compiled program driving the UI. +* ``code_agent``: arm 7, the code-as-action agent's first run + (``ClaudeAgent(vision="code")`` in the compile-agent harness), recorded by + the same JIT wrapper as arm 2, read back, compiled and taken through the + same admission. ``--n`` first runs, each on a fresh cache. Every attempt is one row in ``raw////items.csv``. The outcome comes from reading the system of record after the attempt. @@ -306,7 +310,9 @@ def arm_agent(args: argparse.Namespace) -> int: # -- arms 2 and 3 on MockMed, through the JIT wrapper ------------------------------- -def _mockmed_jit(sess: Session, cache: Path, it: Any, agent: Any = None) -> Any: +def _mockmed_jit( + sess: Session, cache: Path, it: Any, agent: Any = None, dialect: str = "anthropic" +) -> Any: from openadapt_flow.jit import TaskSignature, WebApp, compile_agent from openadapt_flow.runtime.effects import RestRecordVerifier from scoreboard.tasks import mockmed @@ -327,7 +333,7 @@ def _mockmed_jit(sess: Session, cache: Path, it: Any, agent: Any = None) -> Any: cache_dir=cache, key_dir=cache / "admission-keys", model_key=model_key() if agent is not None else None, - dialect="anthropic", + dialect=dialect, ) @@ -348,7 +354,9 @@ def _admitter(key_dir: Path) -> Any: return authority.principal_from_key(authority.load_encrypted_key(pem), key_dir) -def _openemr_jit(cache: Path, values: dict[str, str], agent: Any = None) -> Any: +def _openemr_jit( + cache: Path, values: dict[str, str], agent: Any = None, dialect: str = "anthropic" +) -> Any: from openadapt_flow.jit import TaskSignature, WebApp, compile_agent from scoreboard.tasks import openemr as T @@ -366,7 +374,7 @@ def _openemr_jit(cache: Path, values: dict[str, str], agent: Any = None) -> Any: cache_dir=cache, key_dir=cache / "admission-keys", model_key=model_key() if agent is not None else None, - dialect="anthropic", + dialect=dialect, ) @@ -590,9 +598,210 @@ def arm_program(args: argparse.Namespace) -> int: return 0 +# -- arm 7: the code-as-action agent's first run ----------------------------------- + +#: The identity the save gate checks around every save (compile-agent harness). +#: A new OpenEMR patient's identity is typed, not shown, so the gate checks the +#: save happens in the new-patient confirm dialog. +CODE_AGENT_ON_SCREEN = { + "mockmed": ["Jane Sample", "MRN P1"], + "openemr_referral": ["Create New Patient"], +} +#: OpenEMR items for arm 7 start here, clear of arm 2's item 0 and canary 999. +OPENEMR_CODE_ITEM_BASE = 700 + + +def _program_check(jit: Any, trace: Any) -> dict[str, Any]: + from openadapt_flow.compile_agent.code_action import ( + check_trace_against_ir, + distilled_events, + ) + from openadapt_flow.ir import Workflow + + try: + recording = (jit.candidate() or {}).get("compiled_recording") + return check_trace_against_ir( + trace.to_dict(), + Workflow.load(jit.bundle_dir), + distilled=distilled_events(Path(recording) if recording else None), + ) + except Exception as exc: # noqa: BLE001 - reported with the row + return {"passed": False, "error": f"{type(exc).__name__}: {exc}"[:300]} + + +def arm_code_agent(args: argparse.Namespace) -> int: + from openadapt_flow.compile_agent import ClaudeAgent, SaveGate + from openadapt_flow.compile_agent.pipeline import _HarnessedAgent + + task = task_module(args.task) + if task.NAME not in CODE_AGENT_ON_SCREEN: + raise SystemExit("arm 7 runs on mockmed and openemr_referral") + out = schema.run_dir(task.NAME, "code_agent", args.run_id) + schema.write_meta( + out, + task=task.NAME, + arm="code_agent", + model=AGENT_MODEL, + n=args.n, + vision="code", + effort="medium", + max_cost_per_item=args.max_cost, + on_screen=CODE_AGENT_ON_SCREEN[task.NAME], + ) + sess = Session(task, args.tag, faults=False) if task.NAME == "mockmed" else None + on_screen = CODE_AGENT_ON_SCREEN[task.NAME] + try: + for i in range(args.start, args.n + 1): + if spent() + args.max_cost > SPEND_CAP_USD: + print(f"stopping: spend {spent():.2f} + {args.max_cost} passes the cap") + break + cache = out / "jit-cache" / f"item-{i:03d}" + if cache.exists(): + shutil.rmtree(cache) + if sess is not None: + it = sess.item(i) + values, canary_values, params = ( + it.values, + sess.item(999).values, + ["note"], + ) + readback_fn = lambda: sess.readback(it) # noqa: E731 + else: + values = task.item_values(OPENEMR_CODE_ITEM_BASE + i) + canary_values = task.item_values(OPENEMR_CODE_ITEM_BASE + 100 + i) + params = list(task.PARAMS) + before_ids = {p["id"] for p in task.patients()} + readback_fn = lambda: task.check(values, before_ids) # noqa: E731 + inner = ClaudeAgent( + vision="code", + model=AGENT_MODEL, + effort="medium", + max_cost_usd=args.max_cost, + max_turns=60, + principal_id=f"agent:{AGENT_MODEL}:code", + ) + agent = _HarnessedAgent( + inner, + gate=SaveGate(on_screen), + keep=[*on_screen, *values.values()], + evidence_dir=cache / "gates", + principal_id=inner.principal_id, + ) + if sess is not None: + jit = _mockmed_jit(sess, cache, it, agent, dialect="generic") + else: + jit = _openemr_jit(cache, values, agent, dialect="generic") + error: Optional[BaseException] = None + load0 = schema.load_1m() + w0 = time.time() + signal.signal(signal.SIGALRM, _item_timeout) + signal.alarm(ITEM_TIMEOUT_S) + try: + result = jit.run(**values) + except Exception as exc: # noqa: BLE001 - recorded, the batch goes on + error, result = exc, None + finally: + signal.alarm(0) + wall = time.time() - w0 + load1 = schema.load_1m() + ledger = getattr(inner, "last_ledger", None) + cost = float(ledger.cost_usd()) if ledger is not None else 0.0 + add_spend(cost, f"{task.NAME} code_agent {args.run_id} item {i}") + readback = readback_fn() + trace = getattr(inner, "last_trace", None) + summary: dict[str, Any] = { + "record_status": getattr(result, "status", None), + "detail": getattr(result, "detail", None) + or (str(error) if error else None), + "gates": [ + { + k: v + for k, v in g.items() + if k in ("control", "decision", "missing") + } + for g in (agent.harness.gate_log if agent.harness else []) + ], + } + if result is not None and result.candidate is not None: + summary["program_check"] = _program_check(jit, trace) + summary.update(_admission(jit, params, canary_values, cache)) + text = (ledger.summary or "") if ledger is not None else "" + ok = ( + error is None + and result is not None + and result.status == "needs_human_admit" + and ledger is not None + and ledger.finished + and not any(w in text.lower() for w in FAILURE_WORDS) + ) + ledger_doc = { + "ledger": ledger.to_dict() if ledger is not None else None, + "harness": agent.harness.stats.to_dict() if agent.harness else None, + "trace": trace.summary() if trace is not None else None, + "scripts": trace.log if trace is not None else None, + "result": summary, + } + (out / "ledgers").mkdir(parents=True, exist_ok=True) + (out / "ledgers" / f"item-{i:03d}.json").write_text( + json.dumps(ledger_doc, indent=2, default=str) + "\n" + ) + row = schema.Row( + task=task.NAME, + arm="code_agent", + run_id=args.run_id, + item=i, + attempt=1, + fault="none", + reported_success=ok, + readback=readback, + outcome=schema.classify(ok, readback, "none"), + model_calls=ledger.model_calls if ledger is not None else 0, + input_tokens=ledger.total("input_tokens") if ledger else 0, + output_tokens=ledger.total("output_tokens") if ledger else 0, + cache_read_tokens=ledger.total("cache_read_input_tokens") + if ledger + else 0, + cache_write_tokens=( + ledger.total("cache_creation_input_tokens") if ledger else 0 + ), + cost_usd=round(cost, 6), + item_s=round(ledger.wall_seconds, 3) if ledger is not None else 0.0, + wall_s=round(wall, 3), + approvals_asked=1 if summary.get("admitted") else 0, + host_load_1m=load0, + timed=schema.timed(load0, load1), + detail=json.dumps( + { + "record_status": summary.get("record_status"), + "program_check": (summary.get("program_check") or {}).get( + "passed" + ), + "canary_status": summary.get("canary_status"), + "admitted": summary.get("admitted"), + "error": f"{type(error).__name__}: {error}" if error else None, + }, + default=str, + )[:300], + ) + schema.append(out / "items.csv", row) + # The admission keys and cache stay out of the repository. + shutil.rmtree(cache, ignore_errors=True) + print( + f"{task.NAME} code_agent {i:3d} {row.outcome:13s} {row.model_calls} calls " + f"{row.item_s:6.1f}s ${row.cost_usd:.3f} load {load0}->{load1} " + f"admitted={summary.get('admitted')} " + f"program={(summary.get('program_check') or {}).get('passed')}", + flush=True, + ) + finally: + if sess is not None: + sess.close() + return 0 + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("arm", choices=["agent", "first_run", "program"]) + parser.add_argument("arm", choices=["agent", "first_run", "program", "code_agent"]) parser.add_argument("task", choices=["mockmed", "openimis", "openemr_referral"]) parser.add_argument("--run-id", required=True) parser.add_argument("--n", type=int, default=100) @@ -612,9 +821,12 @@ def main() -> int: args = parser.parse_args() if args.arm == "program" and not args.program_from: args.program_from = args.run_id - return {"agent": arm_agent, "first_run": arm_first_run, "program": arm_program}[ - args.arm - ](args) + return { + "agent": arm_agent, + "first_run": arm_first_run, + "program": arm_program, + "code_agent": arm_code_agent, + }[args.arm](args) if __name__ == "__main__": diff --git a/packages/flow/benchmark/scoreboard/scoreboard.json b/packages/flow/benchmark/scoreboard/scoreboard.json index fbeba5aea..a5399e283 100644 --- a/packages/flow/benchmark/scoreboard/scoreboard.json +++ b/packages/flow/benchmark/scoreboard/scoreboard.json @@ -1,7 +1,7 @@ { "protocol_version": "2026-10-18.1", "timed_load_max": 12.0, - "spend_usd": 3.866907, + "spend_usd": 4.171654, "tasks": { "mockmed": { "agent": { @@ -172,6 +172,50 @@ } } ] + }, + "code_agent": { + "label": "7. Code-as-action authoring agent, first run", + "items": 5, + "attempts": 5, + "runs": [ + "2026-10-18-a" + ], + "normal_items": 5, + "correct": 5, + "correct_rate": 1.0, + "silent_wrong": 0, + "safe_stops_normal": 0, + "fault_items": 0, + "fault_items_by_kind": {}, + "planned_items": 5, + "done_correctly_all_items": 5, + "flagged_wrong": 0, + "paused": 0, + "model_calls_per_item": 5.0, + "tokens_per_item": 9911, + "cost_usd_total": 0.0477, + "cost_usd_per_item": 0.0095, + "approvals_asked": 5, + "timed_correct_items": 0, + "item_s_median": null, + "item_s_p90": null, + "wall_s_median": null, + "wall_s_p90": null, + "items_per_hour": null, + "host_load_1m": { + "min": 50.82, + "max": 64.75 + }, + "sources": [ + { + "run": "2026-10-18-a", + "model": "claude-sonnet-5-5", + "engine": { + "path": "/Users/abrichr/oa/worktrees/scoreboard-arm7/packages/flow/openadapt_flow", + "git_head": "e0d56d450e4fc9cae63377838b73d45f69c3083f" + } + } + ] } }, "openemr_referral": { @@ -323,6 +367,50 @@ "note": "the study's published set: engine 0c642bc0, warm, blocks 2-10, valid runs; timed per its start gate (1-minute load < 12)" } ] + }, + "code_agent": { + "label": "7. Code-as-action authoring agent, first run", + "items": 5, + "attempts": 5, + "runs": [ + "2026-10-18-b" + ], + "normal_items": 5, + "correct": 4, + "correct_rate": 0.8, + "silent_wrong": 1, + "safe_stops_normal": 0, + "fault_items": 0, + "fault_items_by_kind": {}, + "planned_items": 5, + "done_correctly_all_items": 4, + "flagged_wrong": 0, + "paused": 0, + "model_calls_per_item": 11.6, + "tokens_per_item": 69806, + "cost_usd_total": 0.2049, + "cost_usd_per_item": 0.041, + "approvals_asked": 0, + "timed_correct_items": 0, + "item_s_median": null, + "item_s_p90": null, + "wall_s_median": null, + "wall_s_p90": null, + "items_per_hour": null, + "host_load_1m": { + "min": 13.52, + "max": 73.12 + }, + "sources": [ + { + "run": "2026-10-18-b", + "model": "claude-sonnet-5-5", + "engine": { + "path": "/Users/abrichr/oa/worktrees/scoreboard-arm7/packages/flow/openadapt_flow", + "git_head": "cc883bbcb74d9455830da5fe2b86e0c5b1e3f976" + } + } + ] } }, "openimis": { diff --git a/packages/flow/benchmark/scoreboard/superseded/README.md b/packages/flow/benchmark/scoreboard/superseded/README.md new file mode 100644 index 000000000..882d0ed25 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/superseded/README.md @@ -0,0 +1,9 @@ +# Runs set aside + +`openemr_referral/code_agent/2026-10-18-a`: five arm-7 first runs, all refused +by the save gate at the first drop-down, before any save. A gate rule then +counted any `onchange` handler as "this drop-down submits the form", and +OpenEMR's Title drop-down has one. The rule now needs a handler that calls +`submit` (#1305). Nothing was written. The runs cost $0.04 (in +`raw/spend.json`) and are not in the scoreboard; arm 7's OpenEMR rows are run +`2026-10-18-b`. diff --git a/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/items.csv b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/items.csv new file mode 100644 index 000000000..96fd45056 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/items.csv @@ -0,0 +1,6 @@ +task,arm,run_id,item,attempt,fault,reported_success,readback,outcome,model_calls,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,cost_usd,item_s,wall_s,approvals_asked,host_load_1m,timed,detail +openemr_referral,code_agent,2026-10-18-a,1,1,none,False,missing,safe_stop,1,4,115,0,2859,0.008306,2.05,4.901,0,61.86,False,"{""record_status"": ""capture_failed"", ""program_check"": null, ""canary_status"": null, ""admitted"": null, ""error"": null}" +openemr_referral,code_agent,2026-10-18-a,2,1,none,False,missing,safe_stop,1,4,116,0,2852,0.008298,1.768,4.467,0,68.28,False,"{""record_status"": ""capture_failed"", ""program_check"": null, ""canary_status"": null, ""admitted"": null, ""error"": null}" +openemr_referral,code_agent,2026-10-18-a,3,1,none,False,missing,safe_stop,1,4,118,0,2856,0.008328,1.786,4.897,0,66.1,False,"{""record_status"": ""capture_failed"", ""program_check"": null, ""canary_status"": null, ""admitted"": null, ""error"": null}" +openemr_referral,code_agent,2026-10-18-a,4,1,none,False,missing,safe_stop,1,4,128,0,2854,0.008423,1.983,4.597,0,77.54,False,"{""record_status"": ""capture_failed"", ""program_check"": null, ""canary_status"": null, ""admitted"": null, ""error"": null}" +openemr_referral,code_agent,2026-10-18-a,5,1,none,False,missing,safe_stop,1,4,128,0,2854,0.008423,2.14,5.301,0,72.29,False,"{""record_status"": ""capture_failed"", ""program_check"": null, ""canary_status"": null, ""admitted"": null, ""error"": null}" diff --git a/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-001.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-001.json new file mode 100644 index 000000000..9d560eaf2 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-001.json @@ -0,0 +1,79 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 1, + "input_tokens": 4, + "output_tokens": 115, + "cache_creation_input_tokens": 2859, + "cache_read_input_tokens": 0, + "prompt_tokens": 2863, + "images_in_requests": 0, + "cost_usd": 0.008306, + "wall_seconds": 2.05, + "model_seconds": 1.851, + "finished": false, + "summary": null, + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 115, + "cache_creation_input_tokens": 2859, + "cache_read_input_tokens": 0, + "seconds": 1.851, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 0, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 1 + }, + "trace": { + "surface": "dom", + "scripts": 2, + "lines": 2, + "actions": 0, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 0, + "with_recorded_locator": 0, + "locator_agrees": 0, + "locator_unique": 0, + "commits": 0, + "identity_read_before_commit": [], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + } + ], + "result": { + "record_status": "capture_failed", + "detail": "GateRefused: 'form_title' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it (capture kept at packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-a/jit-cache/item-001/openemr.register_referral-dea306642b4f/refused-20261018T213333769402000)", + "gates": [ + { + "control": "form_title", + "missing": [ + "Create New Patient" + ], + "decision": "refused_wrong_record" + } + ] + } +} diff --git a/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-002.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-002.json new file mode 100644 index 000000000..5d4bb1da0 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-002.json @@ -0,0 +1,79 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 1, + "input_tokens": 4, + "output_tokens": 116, + "cache_creation_input_tokens": 2852, + "cache_read_input_tokens": 0, + "prompt_tokens": 2856, + "images_in_requests": 0, + "cost_usd": 0.008298, + "wall_seconds": 1.768, + "model_seconds": 1.597, + "finished": false, + "summary": null, + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 116, + "cache_creation_input_tokens": 2852, + "cache_read_input_tokens": 0, + "seconds": 1.597, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 0, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 1 + }, + "trace": { + "surface": "dom", + "scripts": 2, + "lines": 2, + "actions": 0, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 0, + "with_recorded_locator": 0, + "locator_agrees": 0, + "locator_unique": 0, + "commits": 0, + "identity_read_before_commit": [], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + } + ], + "result": { + "record_status": "capture_failed", + "detail": "GateRefused: 'form_title' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it (capture kept at packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-a/jit-cache/item-002/openemr.register_referral-dea306642b4f/refused-20261018T213338934369000)", + "gates": [ + { + "control": "form_title", + "missing": [ + "Create New Patient" + ], + "decision": "refused_wrong_record" + } + ] + } +} diff --git a/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-003.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-003.json new file mode 100644 index 000000000..7230f0588 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-003.json @@ -0,0 +1,79 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 1, + "input_tokens": 4, + "output_tokens": 118, + "cache_creation_input_tokens": 2856, + "cache_read_input_tokens": 0, + "prompt_tokens": 2860, + "images_in_requests": 0, + "cost_usd": 0.008328, + "wall_seconds": 1.786, + "model_seconds": 1.598, + "finished": false, + "summary": null, + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 118, + "cache_creation_input_tokens": 2856, + "cache_read_input_tokens": 0, + "seconds": 1.598, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 0, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 1 + }, + "trace": { + "surface": "dom", + "scripts": 2, + "lines": 2, + "actions": 0, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 0, + "with_recorded_locator": 0, + "locator_agrees": 0, + "locator_unique": 0, + "commits": 0, + "identity_read_before_commit": [], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + } + ], + "result": { + "record_status": "capture_failed", + "detail": "GateRefused: 'form_title' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it (capture kept at packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-a/jit-cache/item-003/openemr.register_referral-dea306642b4f/refused-20261018T213344355169000)", + "gates": [ + { + "control": "form_title", + "missing": [ + "Create New Patient" + ], + "decision": "refused_wrong_record" + } + ] + } +} diff --git a/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-004.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-004.json new file mode 100644 index 000000000..24c803a86 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-004.json @@ -0,0 +1,79 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 1, + "input_tokens": 4, + "output_tokens": 128, + "cache_creation_input_tokens": 2854, + "cache_read_input_tokens": 0, + "prompt_tokens": 2858, + "images_in_requests": 0, + "cost_usd": 0.008423, + "wall_seconds": 1.983, + "model_seconds": 1.803, + "finished": false, + "summary": null, + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 128, + "cache_creation_input_tokens": 2854, + "cache_read_input_tokens": 0, + "seconds": 1.803, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 0, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 1 + }, + "trace": { + "surface": "dom", + "scripts": 2, + "lines": 2, + "actions": 0, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 0, + "with_recorded_locator": 0, + "locator_agrees": 0, + "locator_unique": 0, + "commits": 0, + "identity_read_before_commit": [], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + } + ], + "result": { + "record_status": "capture_failed", + "detail": "GateRefused: 'form_title' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it (capture kept at packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-a/jit-cache/item-004/openemr.register_referral-dea306642b4f/refused-20261018T213349496790000)", + "gates": [ + { + "control": "form_title", + "missing": [ + "Create New Patient" + ], + "decision": "refused_wrong_record" + } + ] + } +} diff --git a/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-005.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-005.json new file mode 100644 index 000000000..1cd8e3bd1 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-005.json @@ -0,0 +1,79 @@ +{ + "ledger": { + "agent": "claude", + "model": "claude-sonnet-5-5", + "vision": "code", + "model_calls": 1, + "input_tokens": 4, + "output_tokens": 128, + "cache_creation_input_tokens": 2854, + "cache_read_input_tokens": 0, + "prompt_tokens": 2858, + "images_in_requests": 0, + "cost_usd": 0.008423, + "wall_seconds": 2.14, + "model_seconds": 1.953, + "finished": false, + "summary": null, + "tool_errors": 0, + "calls": [ + { + "turn": 1, + "input_tokens": 4, + "output_tokens": 128, + "cache_creation_input_tokens": 2854, + "cache_read_input_tokens": 0, + "seconds": 1.953, + "stop_reason": "tool_use", + "images_in_request": 0 + } + ] + }, + "harness": { + "observations": 0, + "screenshots_requested": 0, + "screenshots_at_gate": 0, + "screenshots_every_turn": 0, + "actions": 0, + "gate_checks": 1, + "gate_evidence_screenshots": 1, + "gate_refusals": 1 + }, + "trace": { + "surface": "dom", + "scripts": 2, + "lines": 2, + "actions": 0, + "errors": 0, + "screenshots": 0, + "shell_commands": 0, + "shell_isolation": "sandbox-exec", + "by_structure": 0, + "with_recorded_locator": 0, + "locator_agrees": 0, + "locator_unique": 0, + "commits": 0, + "identity_read_before_commit": [], + "read_back_after_commit": [] + }, + "scripts": [ + { + "script": "tree", + "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9911/interface/main/tabs/main.php?...)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\", \"Create New Patient\"; scroll down to reach them", + "images": 0 + } + ], + "result": { + "record_status": "capture_failed", + "detail": "GateRefused: 'form_title' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it (capture kept at packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-a/jit-cache/item-005/openemr.register_referral-dea306642b4f/refused-20261018T213355189214000)", + "gates": [ + { + "control": "form_title", + "missing": [ + "Create New Patient" + ], + "decision": "refused_wrong_record" + } + ] + } +} diff --git a/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/meta.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/meta.json new file mode 100644 index 000000000..833b8f160 --- /dev/null +++ b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/meta.json @@ -0,0 +1,22 @@ +{ + "protocol_version": "2026-10-18.1", + "written_at": "2026-10-18T17:33:28-0400", + "machine": "arm64 Darwin 24.6.0", + "cpu_count": 12, + "python": "3.12.7", + "engine": { + "path": "/Users/abrichr/oa/worktrees/scoreboard-arm7/packages/flow/openadapt_flow", + "git_head": "e0d56d450e4fc9cae63377838b73d45f69c3083f" + }, + "benchmark_git_head": "e0d56d450e4fc9cae63377838b73d45f69c3083f", + "task": "openemr_referral", + "arm": "code_agent", + "model": "claude-sonnet-5-5", + "n": 5, + "vision": "code", + "effort": "medium", + "max_cost_per_item": 0.75, + "on_screen": [ + "Create New Patient" + ] +} diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index fc3fdeee4..714c73086 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -177,7 +177,7 @@ }, { "path": "benchmark/code_as_action/results/openemr-distill-check/runs.jsonl", - "sha256": "a634dc54db1b10388a497476c6281baa9ef045246e376180670cd7ce4c5f711a" + "sha256": "243d8bb5804fba15fb76ada56b6463623bcec334201177d20ed29bc7f53ba14c" }, { "path": "benchmark/code_as_action/results/openemr-distill-check/summary.json", @@ -185,7 +185,7 @@ }, { "path": "benchmark/code_as_action/results/openemr/runs.jsonl", - "sha256": "a937f86c5193a10873c8fd311a046d2b7cb01a185244c735bab9e585790e57b1" + "sha256": "b411db6703a221e5682602e37d1b025b71bc8231d4b16d78e9d555e43f22aad4" }, { "path": "benchmark/code_as_action/results/openemr/summary.json", @@ -193,7 +193,7 @@ }, { "path": "benchmark/code_as_action/results/openemr/superseded.jsonl", - "sha256": "99278f1168e6c5c4a33e9f7629df1eeed03274978a5130836e7d270e0eaee240" + "sha256": "cebff909b6d4714707bb3e2209b2eb12ae3a200ad15f89580859cdad5362b505" }, { "path": "benchmark/code_as_action/results/spend.json", @@ -423,6 +423,14 @@ "path": "benchmark/openemr_local/environment.lock.json", "sha256": "026dab7a2059332f8a62a7e5a24ed22ed94c14dc37a42c73eec7b1a10b4b6a19" }, + { + "path": "benchmark/openemr_local/out/upstream/7c96c8eefe460d6fadbccbe93d0fa6bf819acd69/docker/production/docker-compose.yml", + "sha256": "8ffb1c64ebfc2c1da429fbe6180bd84ad87f8b5b71f5690e32e745cafee239b9" + }, + { + "path": "benchmark/openemr_local/out/upstream/compose.7c96c8eefe46.yml", + "sha256": "98d4cf916306bf76066a42a3ed212b3d99e6d63d3ec7395f71b8832f3da3b04d" + }, { "path": "benchmark/openemr_local/verify.yaml", "sha256": "c8b8cfc97f7ba654d3b016798a5ec442570fb3361b717b53b7363c6e0dd8d6ad" @@ -639,6 +647,482 @@ "path": "benchmark/safety_gallery/results.json", "sha256": "b831d52f5a387abda6467886e9fd1f06fdb44e8127cc9da4b23b5424e5f2fbef" }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/items.csv", + "sha256": "eb8c383cb3c70f077117108938c73ac8f7ec67f3d789f74db8d1ed1a6f242d55" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-001.json", + "sha256": "d88ec7bddbf4b3f29cefcf7de34e3baa4b2a2a23eeda66104c7de5c4f8828b4a" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-002.json", + "sha256": "cd1484df74d24a2de12842f12ebcfc1f803437406071aad37f5e6ec4cd905c69" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-003.json", + "sha256": "ac218df32f57f1b3b7cabb8fe4b784111b7dc91b6dc577e2ef12b2e41cbadf4d" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-004.json", + "sha256": "c4d229ebcb05dbf9610c90805d2366d8bd787a7fe0275ec96c2ae2567f2b20e1" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-005.json", + "sha256": "8e2344953b4b72d0b87716610dda64d0e3b4183ff8b80af08069bd739563dc11" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-006.json", + "sha256": "70dd2cf846ca6fb73de94042be978b1c0374ff4638205a07831602d45dc53070" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-007.json", + "sha256": "c37514199d42ee52497d5131dbda42015f8c39301949f11e7f0b273420bc9a0c" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-008.json", + "sha256": "9aa891579384eb6e3b018802f30a1e45a6744bd74ea464d48c76a16ac9d5c092" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-009.json", + "sha256": "bb8b339bb3ea1dd8ce2a5960790ea1856b7669b7572dc45065e9bea4037b39e5" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-010.json", + "sha256": "7339478c87a1b8578fa141d099f8af5d2e2b2c6e7ab476e2d7ad8cf5101a7ea6" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-011.json", + "sha256": "ef684ad39f41915f638b57aaf9188d7d4935785ae080864775f61da5e71f626e" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-012.json", + "sha256": "6ad9c81484e0623261e5bd795b863a7771d520e784433427846a0c209d5d244c" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-013.json", + "sha256": "2de079ece6a645642b3428342a2cced8942d53f053b990be9a55e6becdfa7e8a" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-014.json", + "sha256": "619def03999551973c9e7cca131445c00cbdd0832f7df770a32598d0d652dc3c" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-015.json", + "sha256": "32ba57b679e4a2c02df5959a8fdd93df23a0f1e43617926acdf3e7e1f1b7a39b" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-016.json", + "sha256": "4e3189df2c143336aa89498f81d70f160fa4207ae2384ba167dff7bee36de9e5" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-017.json", + "sha256": "465629ff8b6228e52aeed1cad080496aa9d687a0d8da17d4dfbc3acbf952662e" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-018.json", + "sha256": "42aad5f681dcc2ff37f7667c827eaae46acd73ea3574db57e62cea4ce11eab90" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-019.json", + "sha256": "7fa3e55d2a661565dc76b63c251ea7a6741d1c3c19ec7ac18bd8c67fa12b28b5" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-020.json", + "sha256": "b672fbde5090f55c2602090075afa5c09700e313521508885fa68841875fc9f9" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-021.json", + "sha256": "2d69b2dc569e81e694ec87830670213767a17145559d5bfd8b592cc2a3c3cd05" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-022.json", + "sha256": "d72b1fd9af495088af5cbbb2062c1429e50b2615a84f699245027683354c66ac" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-023.json", + "sha256": "72f9185ed8e682ff75f9cfb7c485134e35d364489ee455d29a734d8043cdced4" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-024.json", + "sha256": "1d064bd06eb1889a5df67a3aea16a4d3eb1b499be3feac3bf32992d33507245d" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-025.json", + "sha256": "c4af76556feee78044dac252e803e2eeaddda1c162490493737f112615f98b79" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-026.json", + "sha256": "6d548b441bf2954a7a989a0f2d2361776742789b5d1ae71e2a5e89873c443deb" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-027.json", + "sha256": "9bb91b552fae1d71c781b08754a6663fe9b68d628e31ba5153b7499740ebb472" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-028.json", + "sha256": "dc3081b74ab546622a4c7e686bce5e7aef05ec9e22c4cbf5b5abdee254ec5872" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-029.json", + "sha256": "491baf0e31b267c63a466a34a97fd27dff0c02b4b662030d5dac11dc104484db" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-030.json", + "sha256": "01458a5d51be752216e0860708c726027589b963b4b7f33f5e052ba85f63d4b6" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-031.json", + "sha256": "be4744b077ea930255140c1f3168293e72f659ad347650e74d19a2f9c3aba2e2" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-032.json", + "sha256": "f890d634da62de98a0007b85d4982fd6c8b014d595079ca6a324e590ad34c616" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-033.json", + "sha256": "81d2256ec0a54f945c82f671ca6845c21e8edfd8b0b62e0fbbed27d4b270293a" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-034.json", + "sha256": "be9cbc1930c068166093f47e9f1fb81e37713656e6d3c7815f7f01f077e57a5e" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-035.json", + "sha256": "7b038d6dde1d1fecb65634492274d0015e2aa506129e87a5db6545f045aa41c8" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-036.json", + "sha256": "c0b8aac1f5b0b4e7f7844443231f032907706bd32288ba7250b51dc03e1aca89" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-037.json", + "sha256": "b1fc5d175e584cb52053712d7862d7b23678a13062bf74144dd34438be67ed9c" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-038.json", + "sha256": "d71504b6700622513178c05c3681250034f15f34f6591726516fb8f732eda0a1" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-039.json", + "sha256": "96bef34b1e62c8b616e5cb4223883ee359cc6266a0f4132a9eb67f6633cde4e3" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-040.json", + "sha256": "5b59799f143de0fa483a062a1fadbd27618ada6498ce6a5181d94e89b27a19b9" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-041.json", + "sha256": "35549cf9f47ac7f2243e5b1061f17178bf9a80910de7a38a0e1adeabcc60fd59" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-042.json", + "sha256": "952690a41a4b2fd38cb18214826e4437539029bc2bd89cb24c9f264749f3c6dc" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-043.json", + "sha256": "e3aa75dbb627b67051372a8b1cec2a39ddd4d10231059d06da9804329c0ad2ff" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-044.json", + "sha256": "78b6b737737ac09f10d21cfd27f29d1e433851ca941af32649412599cf753a72" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-045.json", + "sha256": "de1f4191810fd9d72cdae644f6d62ef8c5b535a4072e1ece695ae8d0aac34c47" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-046.json", + "sha256": "ff0a18678a9a3244997c485d3add006f8173abac803c63663aec3e488e3812c0" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-047.json", + "sha256": "4b8e8e75cb0ce4ae9c7122484af61d3a92dfddc51d09698973da7a41841317d3" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-048.json", + "sha256": "6c0e95eb04322e5f36b331885534e3183e3f22258f785dc332dbe953f95b712f" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-049.json", + "sha256": "07d2867280e553c48df8481c822beb822d2fa2dbe22cc63ea240e0e12aa67f17" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-050.json", + "sha256": "ac071384692da81c098cd75f46205303d8589798d4536aff7491472f4524714f" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-051.json", + "sha256": "2b8fd0e2b1fca4a3fe9deb7725ef8c7e09d7f79ff43c49332184608f77abf531" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/ledgers/item-052.json", + "sha256": "87b12b013f9ec05a20880ea93873a04308a9cf2ef8feade52745c3d713a5e0df" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/meta.json", + "sha256": "d3f9cd7fbc4c6d9247984085085fd33e91935f4a35aad9da31df2e230ea16063" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/items.csv", + "sha256": "be25c825731ea3967dcc42be5b2c57d6670fea76d414e6232eb12c1bbfe72042" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json", + "sha256": "ccbcdcfe5e794d09a5b983dc6f5aecc8aca48e57c9ee46dee8135166e5ae5e1d" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json", + "sha256": "666918e316f0e6180453a1b8268018e66a63ede71c4d86d3242b958ae6df9e16" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json", + "sha256": "4eca72e719390ecf59f04d21ad5425eb88d8e7c0b981913d80c75b5ce05a0874" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json", + "sha256": "57c41ae7028e1bbc1a23e5a09ae4728163d27f95e2f02e24051ca7b399c4be0c" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json", + "sha256": "4f959829b9e0656a33eaa761a8d3d31f1b4e40c7cf870f8f42e34f8e17e8257c" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/meta.json", + "sha256": "60440234b167a22bd5436888474d47a6292ce41e9ac867efffd37f6bf14e9673" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/first_run/2026-10-18-a/first_run.json", + "sha256": "5948cd90cd29a502f17c9874cc2000b4192ae006d2f6a9975c98b69bdc0da387" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/first_run/2026-10-18-a/items.csv", + "sha256": "fd7557f7a25b124f99f08544543c9118bab65bb432a6b034397cde142d3fd35f" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/first_run/2026-10-18-a/ledger.json", + "sha256": "6853058222e95c4cb3aa51460900dd2df908638bba165d461e3644e207bdbb84" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/first_run/2026-10-18-a/meta.json", + "sha256": "99c4d5e8f92fcc1ac78a18799aa574278bef5b77a070fd36797a8c7a2dcf4f74" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/program/2026-10-18-a/items.csv", + "sha256": "47ae9199de5f7ef238903cce6305b9b91973cb024f77a20018c497446b8b4d57" + }, + { + "path": "benchmark/scoreboard/raw/mockmed/program/2026-10-18-a/meta.json", + "sha256": "77dcd3f4575e3854aeb4172a453efdbdce5dc5a9889abc3a3d8747d150a578b6" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/agent/study-main-0c642bc0/items.csv", + "sha256": "10e47e5abf98172ccd3068c8005a1faeeb1cd317e8bb44a26a885fa0e7575f01" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/agent/study-main-0c642bc0/meta.json", + "sha256": "6e61bf57e944f849279519cb3c1c558205faf1e7d93eac05e29b731c94b1084d" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/items.csv", + "sha256": "517ed36f4db803c5267f30946af77a01d60e4a3716763e29c7ca0aa517683767" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json", + "sha256": "a03974d0e6a9a566f16d38087e940d50352833d2670a3d934b83283199f9fc7f" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json", + "sha256": "caee7051663a3073bdc262234f12099daa7eaf39036b5bcfcc2f26d75574dd9d" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json", + "sha256": "05f4297e8892d66685d352d4d5084d690136ef7b53a5e54195d5c6df0c763e05" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json", + "sha256": "83c3c0c6720203b19342f2cef249a59513e0eb0a51dff8a27c02e784ea270fd0" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json", + "sha256": "c8e03061dbbb2da0b2806451936417d4ce92fbf206dbe7180f85d8ff38635fa6" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/meta.json", + "sha256": "dea6dbe200f7f67bfbb33d4cfe27d288c46dd4664c50eeb22690073b446d7c9e" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/first_run/2026-10-18-a/first_run.json", + "sha256": "eeaf4465dec9eb1da65ecc2d0c6da7538ab738cd308e7321244efb948b8938e2" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/first_run/2026-10-18-a/items.csv", + "sha256": "bd0ea20e8adaad74a73ce5acf6370bb5f4c6a9fe1f021e5272f12d16e0b672fa" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/first_run/2026-10-18-a/ledger.json", + "sha256": "b220bf97ee12f1fa9721561b2cb0fdf7db249502215d6a03e5b2bdde38cc9e90" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/first_run/2026-10-18-a/meta.json", + "sha256": "f41aa38d13410dfecb3724b1ac15f55b4d9ab09d16423e1400fd1d6d506d28dc" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/program/study-main-0c642bc0/items.csv", + "sha256": "fa0a1f7e34ee333e8cc8e9db0b435621c9938f24dbd72a52f4da9722ddc18d0a" + }, + { + "path": "benchmark/scoreboard/raw/openemr_referral/program/study-main-0c642bc0/meta.json", + "sha256": "5a1f6e18b6d1736702dbfea5c3ae9d4562ac23090170d55a8f1eea3681f7352b" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/items.csv", + "sha256": "790a2e2022921eb0eaf14dfbb98c4a0ad805b676761467d078da94b42363dd07" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-001.json", + "sha256": "7614a22b8ab0b85c67d7984fc974d3758231eeadd6e695a26e61bb4475867aec" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-002.json", + "sha256": "ad1430cc1be3d2ee3a2c7ba03410c1f087290893051977ae86023028d8daf552" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-003.json", + "sha256": "e42fbd1111b323a89492e5e7f0b24fc802016afefd31df7e81210ad049dd7323" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-004.json", + "sha256": "0e7fb31ce722f76478cc86eb8a7192cfe9f72dcfd2be7cae90a720b47f64344d" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-005.json", + "sha256": "abda0dc5bc4b18131aa287c2352ed314411a2711510c303ec66b49bbfffd9de7" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-006.json", + "sha256": "c57670635766ccb1a921c3bb54649d980c6fc664eb7e3e7af6cb07d191621ef5" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-007.json", + "sha256": "f2785e9df1b414e078a434449690823f24014d9f1bcbf201fe3924434685f9c7" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-008.json", + "sha256": "7829abbb614f4e2172144ec4fdd2a9bac85ff495c71e3f44081c0e5167b79b33" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-009.json", + "sha256": "da487c32a1576f9c6a29d5f7bd66eb1c352f85950f19c97969c5ead31bd049f0" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-010.json", + "sha256": "984bd91bc2b60f84edf326f143ac6f0bf7ae2d3d8ccad1427631048e9edc2f95" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-011.json", + "sha256": "5d75729c9feb67ea30c22d687bab00c2844d6afd2021e3bf6e76cca939421e34" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-012.json", + "sha256": "31dc11e157dfbb1f99fb2e63edd6dcde21eb851a02e95818c96858cbe7693458" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-013.json", + "sha256": "03b907931ad7f8e8c4c50ba4192d445637bbbba553a9f4b715bf5845685d82e5" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-014.json", + "sha256": "1cbb22ddd65c4e08509549a2087fe80835347383ee4fe7d59c34409ac7474cc4" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-015.json", + "sha256": "ba906b6dd0c1c817df719a63bdaba2b8de47adf1a5bd98c812b637028b815c97" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-016.json", + "sha256": "e188c9c83127f6755e6c164caea015446c7cede40e73e88a9f0fdeb3ee2c0493" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-017.json", + "sha256": "080f3d0a31e6e4a6f5f1fd6580e6e3b9f7457b58f7bc0f71273ea2f55a0911f6" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-018.json", + "sha256": "620f4aafb93272078093be1fcf883755667ddaecda953e50e781d7634b660296" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-019.json", + "sha256": "a20af9cebeb64976b5cad8e4ace52c2056977c873d32edda33549abee20b225a" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/ledgers/item-020.json", + "sha256": "e2ac977cd2f1f9bf72ae9aee412056526e6ce151951d87b4e23208acd766bb7b" + }, + { + "path": "benchmark/scoreboard/raw/openimis/agent/2026-10-18-a/meta.json", + "sha256": "6e4cd24492a12adca5681ccaaf8a8965415958f2f756eee05fee5dbac444aba9" + }, + { + "path": "benchmark/scoreboard/raw/openimis/first_run/launch-4/items.csv", + "sha256": "c20f53b3bd1095ea66a92f7ed7dfe5a1c6e8c0c2a543936b49b2254c1c20b453" + }, + { + "path": "benchmark/scoreboard/raw/openimis/first_run/launch-4/meta.json", + "sha256": "2d1464611c08d74bbdc69ded97a5a367a21072e5b782e363bbd6958bc10920bb" + }, + { + "path": "benchmark/scoreboard/raw/openimis/program/launch-4-c100/items.csv", + "sha256": "6f08b29be6bd2a1ed335c0fc260517f8138915bd3b2910545904b17548597046" + }, + { + "path": "benchmark/scoreboard/raw/openimis/program/launch-4-c100/meta.json", + "sha256": "d07dafee4b516174a76fa196daf4974c213bf0040fd833c44ce93671c49b0cbb" + }, + { + "path": "benchmark/scoreboard/raw/openimis/program/launch-4/items.csv", + "sha256": "f1a78dbbeddefba7db5696885d6b2c023619a60e6f5961c0a3cdbcab365bbdc4" + }, + { + "path": "benchmark/scoreboard/raw/openimis/program/launch-4/meta.json", + "sha256": "09a60306c3355fb57d58c108c3c5762dd30f3a25dd18885aefc080995c4b1b36" + }, + { + "path": "benchmark/scoreboard/raw/spend.json", + "sha256": "17f575335137510d0b20554495eb5fb1ff58f0bfa32a737c36997fdb1d49a828" + }, + { + "path": "benchmark/scoreboard/scoreboard.json", + "sha256": "b828a574da38c434822ae391467bb74fe18c330675546cad4bcad4baeccda544" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/items.csv", + "sha256": "2544ec565a2e656e584c88f6a3afffe3ac27678785f9ec846016bec4bc38ef5a" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-001.json", + "sha256": "c451d3c7be35d872f6537ef6f729bdebc716e94e39f89895f7c79443db39783f" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-002.json", + "sha256": "8438a80bb2e06f32538be34c9954080f8951d811474e4a41f00e6aafadbc8a59" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-003.json", + "sha256": "6a1d4387adbd4784ed33615bec0f36feb4cd5b5af4a4e860084726d583db478c" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-004.json", + "sha256": "1b4051d87562d5e85ec53d957d0a08ef8cef97a1c52f32321e2aa4d9bcd0813c" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-005.json", + "sha256": "b29e1e0b3437276f00fd3830c247339fbe7dc8f863563444a595ad5bcf3a6e4b" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/meta.json", + "sha256": "46cfbee57e63f0cc58993793cb64b8242816eacb13472633e105ec88e6ea2444" + }, { "path": "benchmark/silent_wrong_action/results.json", "sha256": "af71ce1db2b0bf065230d775ec63251891f4dbe1cd0d95d44974740acd3cc38a" From 9910ca3d8909259dcf112a15899353cbc668b8c2 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 17:51:31 -0400 Subject: [PATCH 2/4] scoreboard: arm 7 code only; set aside the trial rows for the single runner Co-Authored-By: Claude Opus 5.5 --- .../flow/benchmark/scoreboard/SCOREBOARD.md | 4 +- .../flow/benchmark/scoreboard/raw/spend.json | 82 +------------ packages/flow/benchmark/scoreboard/run.py | 5 +- .../flow/benchmark/scoreboard/scoreboard.json | 90 +------------- .../benchmark/scoreboard/superseded/README.md | 26 ++-- .../mockmed/code_agent/2026-10-18-a/items.csv | 0 .../2026-10-18-a/ledgers/item-001.json | 0 .../2026-10-18-a/ledgers/item-002.json | 0 .../2026-10-18-a/ledgers/item-003.json | 0 .../2026-10-18-a/ledgers/item-004.json | 0 .../2026-10-18-a/ledgers/item-005.json | 0 .../mockmed/code_agent/2026-10-18-a/meta.json | 0 .../code_agent/2026-10-18-b/HAND_REVIEW.md | 0 .../code_agent/2026-10-18-b/items.csv | 0 .../2026-10-18-b/ledgers/item-001.json | 0 .../2026-10-18-b/ledgers/item-002.json | 0 .../2026-10-18-b/ledgers/item-003.json | 0 .../2026-10-18-b/ledgers/item-004.json | 0 .../2026-10-18-b/ledgers/item-005.json | 0 .../code_agent/2026-10-18-b/meta.json | 0 packages/flow/public-artifacts.json | 116 +++++++++--------- 21 files changed, 83 insertions(+), 240 deletions(-) rename packages/flow/benchmark/scoreboard/{raw => superseded}/mockmed/code_agent/2026-10-18-a/items.csv (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/mockmed/code_agent/2026-10-18-a/meta.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/openemr_referral/code_agent/2026-10-18-b/HAND_REVIEW.md (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/openemr_referral/code_agent/2026-10-18-b/items.csv (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json (100%) rename packages/flow/benchmark/scoreboard/{raw => superseded}/openemr_referral/code_agent/2026-10-18-b/meta.json (100%) diff --git a/packages/flow/benchmark/scoreboard/SCOREBOARD.md b/packages/flow/benchmark/scoreboard/SCOREBOARD.md index f0105dccc..4919999b2 100644 --- a/packages/flow/benchmark/scoreboard/SCOREBOARD.md +++ b/packages/flow/benchmark/scoreboard/SCOREBOARD.md @@ -9,7 +9,6 @@ Generated by `python -m scoreboard.build` from `raw/` (protocol 2026-10-18.1, se | 1. Agent alone (Claude Sonnet 5.5 computer use) | claude-sonnet-5-5 | 52 | 47 of 47 (100%) | 50 of 52 | 2 | 0 / 0 of 5 | 0 | 0 | 5.0 | 0.0386 | - / - | - / - | - | 0 | 86-524 | | 2. Agent's first run under OpenAdapt (recorded, read back, compiled) | claude-sonnet-5-5 | 1 | 1 of 1 (100%) | 1 of 1 | 0 | 0 / 0 of 0 | 0 | 0 | 5.0 | 0.0132 | - / - | - / - | - | 1 | 176-176 | | 3. Compiled program driving the UI | none | 100 | 47 of 47 (100%) | 98 of 100 | 0 | 0 / 1 of 53 | 1 | 0 | 0.0 | 0.0 | - / - | - / - | - | 0 | 82-533 | -| 7. Code-as-action authoring agent, first run | claude-sonnet-5-5 | 5 | 5 of 5 (100%) | 5 of 5 | 0 | 0 / 0 of 0 | 0 | 0 | 5.0 | 0.0095 | - / - | - / - | - | 5 | 51-65 | - mockmed / agent: fault items {"optimistic": {"correct": 0, "silent_wrong": 1, "safe_stop": 0, "flagged_wrong": 0}, "ui_change": {"correct": 3, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 0}, "wrong_record": {"correct": 0, "silent_wrong": 1, "safe_stop": 0, "flagged_wrong": 0}} - mockmed / program: fault items {"optimistic": {"correct": 0, "silent_wrong": 0, "safe_stop": 1, "flagged_wrong": 0}, "ui_change": {"correct": 51, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 0}, "wrong_record": {"correct": 0, "silent_wrong": 0, "safe_stop": 0, "flagged_wrong": 1}} @@ -21,7 +20,6 @@ Generated by `python -m scoreboard.build` from `raw/` (protocol 2026-10-18.1, se | 1. Agent alone (Claude Sonnet 5.5 computer use) | claude-sonnet-5-5 | 72 | 62 of 63 (98%) | 62 of 72 | 0 | 1 / 9 of 9 | 0 | 0 | 10.65 | 0.0892 | 31.16 / 40.93 | 36.49 / 46.09 | 80.5 | 0 | 3-17 | | 2. Agent's first run under OpenAdapt (recorded, read back, compiled) | claude-sonnet-5-5 | 1 | 1 of 1 (100%) | 1 of 1 | 0 | 0 / 0 of 0 | 0 | 0 | 12.0 | 0.1007 | - / - | - / - | - | 0 | 4-4 | | 3. Compiled program driving the UI | none | 72 | 62 of 63 (98%) | 62 of 72 | 0 | 1 / 9 of 9 | 0 | 0 | 0.0 | 0.0 | 24.14 / 26.92 | 29.71 / 32.52 | 100.6 | 0 | 3-13 | -| 7. Code-as-action authoring agent, first run | claude-sonnet-5-5 | 5 | 4 of 5 (80%) | 4 of 5 | 1 | 0 / 0 of 0 | 0 | 0 | 11.6 | 0.041 | - / - | - / - | - | 0 | 14-73 | - openemr_referral / agent: fault items {"duplicate_check": {"correct": 0, "silent_wrong": 0, "safe_stop": 9, "flagged_wrong": 0}} - openemr_referral / program: fault items {"duplicate_check": {"correct": 0, "silent_wrong": 0, "safe_stop": 9, "flagged_wrong": 0}} @@ -36,4 +34,4 @@ Generated by `python -m scoreboard.build` from `raw/` (protocol 2026-10-18.1, se - openimis / program: fault items {"ui_change": {"correct": 0, "silent_wrong": 0, "safe_stop": 1, "flagged_wrong": 0}} -Claude API spend so far: $4.171654. +Claude API spend so far: $3.866907. diff --git a/packages/flow/benchmark/scoreboard/raw/spend.json b/packages/flow/benchmark/scoreboard/raw/spend.json index 4abcf3375..b61e631e3 100644 --- a/packages/flow/benchmark/scoreboard/raw/spend.json +++ b/packages/flow/benchmark/scoreboard/raw/spend.json @@ -1,5 +1,5 @@ { - "total_usd": 4.171654, + "total_usd": 3.866907, "entries": [ { "usd": 0.577, @@ -374,86 +374,6 @@ "usd": 0.05573, "note": "openimis agent 2026-10-18-a item 19", "at": 1792358431.418853 - }, - { - "usd": 0.010339, - "note": "mockmed code_agent smoke-delete item 1", - "at": 1792359107.756307 - }, - { - "usd": 0.006092, - "note": "mockmed code_agent 2026-10-18-a item 1", - "at": 1792359137.5327659 - }, - { - "usd": 0.010352, - "note": "mockmed code_agent 2026-10-18-a item 2", - "at": 1792359152.226209 - }, - { - "usd": 0.010449, - "note": "mockmed code_agent 2026-10-18-a item 3", - "at": 1792359166.907215 - }, - { - "usd": 0.010462, - "note": "mockmed code_agent 2026-10-18-a item 4", - "at": 1792359182.645686 - }, - { - "usd": 0.010339, - "note": "mockmed code_agent 2026-10-18-a item 5", - "at": 1792359201.927978 - }, - { - "usd": 0.008306, - "note": "openemr_referral code_agent 2026-10-18-a item 1", - "at": 1792359214.213875 - }, - { - "usd": 0.008298, - "note": "openemr_referral code_agent 2026-10-18-a item 2", - "at": 1792359219.3110201 - }, - { - "usd": 0.008328, - "note": "openemr_referral code_agent 2026-10-18-a item 3", - "at": 1792359224.770966 - }, - { - "usd": 0.008423, - "note": "openemr_referral code_agent 2026-10-18-a item 4", - "at": 1792359229.97929 - }, - { - "usd": 0.008423, - "note": "openemr_referral code_agent 2026-10-18-a item 5", - "at": 1792359235.856202 - }, - { - "usd": 0.045611, - "note": "openemr_referral code_agent 2026-10-18-b item 1", - "at": 1792359472.6206741 - }, - { - "usd": 0.030572, - "note": "openemr_referral code_agent 2026-10-18-b item 2", - "at": 1792359613.251651 - }, - { - "usd": 0.040868, - "note": "openemr_referral code_agent 2026-10-18-b item 3", - "at": 1792359758.899384 - }, - { - "usd": 0.042256, - "note": "openemr_referral code_agent 2026-10-18-b item 4", - "at": 1792359895.759076 - }, - { - "usd": 0.04563, - "note": "openemr_referral code_agent 2026-10-18-b item 5", - "at": 1792360037.685641 } ] } diff --git a/packages/flow/benchmark/scoreboard/run.py b/packages/flow/benchmark/scoreboard/run.py index 78e2f9c34..d5b63faa7 100644 --- a/packages/flow/benchmark/scoreboard/run.py +++ b/packages/flow/benchmark/scoreboard/run.py @@ -607,8 +607,9 @@ def arm_program(args: argparse.Namespace) -> int: "mockmed": ["Jane Sample", "MRN P1"], "openemr_referral": ["Create New Patient"], } -#: OpenEMR items for arm 7 start here, clear of arm 2's item 0 and canary 999. -OPENEMR_CODE_ITEM_BASE = 700 +#: OpenEMR items for arm 7 start here: clear of arm 2 (item 0, canary 999) and +#: of the charts a trial run made on the fixture (items 701 to 705). +OPENEMR_CODE_ITEM_BASE = 720 def _program_check(jit: Any, trace: Any) -> dict[str, Any]: diff --git a/packages/flow/benchmark/scoreboard/scoreboard.json b/packages/flow/benchmark/scoreboard/scoreboard.json index a5399e283..fbeba5aea 100644 --- a/packages/flow/benchmark/scoreboard/scoreboard.json +++ b/packages/flow/benchmark/scoreboard/scoreboard.json @@ -1,7 +1,7 @@ { "protocol_version": "2026-10-18.1", "timed_load_max": 12.0, - "spend_usd": 4.171654, + "spend_usd": 3.866907, "tasks": { "mockmed": { "agent": { @@ -172,50 +172,6 @@ } } ] - }, - "code_agent": { - "label": "7. Code-as-action authoring agent, first run", - "items": 5, - "attempts": 5, - "runs": [ - "2026-10-18-a" - ], - "normal_items": 5, - "correct": 5, - "correct_rate": 1.0, - "silent_wrong": 0, - "safe_stops_normal": 0, - "fault_items": 0, - "fault_items_by_kind": {}, - "planned_items": 5, - "done_correctly_all_items": 5, - "flagged_wrong": 0, - "paused": 0, - "model_calls_per_item": 5.0, - "tokens_per_item": 9911, - "cost_usd_total": 0.0477, - "cost_usd_per_item": 0.0095, - "approvals_asked": 5, - "timed_correct_items": 0, - "item_s_median": null, - "item_s_p90": null, - "wall_s_median": null, - "wall_s_p90": null, - "items_per_hour": null, - "host_load_1m": { - "min": 50.82, - "max": 64.75 - }, - "sources": [ - { - "run": "2026-10-18-a", - "model": "claude-sonnet-5-5", - "engine": { - "path": "/Users/abrichr/oa/worktrees/scoreboard-arm7/packages/flow/openadapt_flow", - "git_head": "e0d56d450e4fc9cae63377838b73d45f69c3083f" - } - } - ] } }, "openemr_referral": { @@ -367,50 +323,6 @@ "note": "the study's published set: engine 0c642bc0, warm, blocks 2-10, valid runs; timed per its start gate (1-minute load < 12)" } ] - }, - "code_agent": { - "label": "7. Code-as-action authoring agent, first run", - "items": 5, - "attempts": 5, - "runs": [ - "2026-10-18-b" - ], - "normal_items": 5, - "correct": 4, - "correct_rate": 0.8, - "silent_wrong": 1, - "safe_stops_normal": 0, - "fault_items": 0, - "fault_items_by_kind": {}, - "planned_items": 5, - "done_correctly_all_items": 4, - "flagged_wrong": 0, - "paused": 0, - "model_calls_per_item": 11.6, - "tokens_per_item": 69806, - "cost_usd_total": 0.2049, - "cost_usd_per_item": 0.041, - "approvals_asked": 0, - "timed_correct_items": 0, - "item_s_median": null, - "item_s_p90": null, - "wall_s_median": null, - "wall_s_p90": null, - "items_per_hour": null, - "host_load_1m": { - "min": 13.52, - "max": 73.12 - }, - "sources": [ - { - "run": "2026-10-18-b", - "model": "claude-sonnet-5-5", - "engine": { - "path": "/Users/abrichr/oa/worktrees/scoreboard-arm7/packages/flow/openadapt_flow", - "git_head": "cc883bbcb74d9455830da5fe2b86e0c5b1e3f976" - } - } - ] } }, "openimis": { diff --git a/packages/flow/benchmark/scoreboard/superseded/README.md b/packages/flow/benchmark/scoreboard/superseded/README.md index 882d0ed25..8642156e7 100644 --- a/packages/flow/benchmark/scoreboard/superseded/README.md +++ b/packages/flow/benchmark/scoreboard/superseded/README.md @@ -1,9 +1,21 @@ # Runs set aside -`openemr_referral/code_agent/2026-10-18-a`: five arm-7 first runs, all refused -by the save gate at the first drop-down, before any save. A gate rule then -counted any `onchange` handler as "this drop-down submits the form", and -OpenEMR's Title drop-down has one. The rule now needs a handler that calls -`submit` (#1305). Nothing was written. The runs cost $0.04 (in -`raw/spend.json`) and are not in the scoreboard; arm 7's OpenEMR rows are run -`2026-10-18-b`. +Arm 7 runs made by the code-as-action session while the arm was being built, +before measurements moved to a single runner (the launch-demo session) under +the load gate. They are not in the scoreboard. They used this repository's +runner (`scoreboard.run code_agent`), claude-sonnet-5-5, and the 9911/9910 +OpenEMR fixture; all rows are untimed (host load above 12). Spend: $0.31 in +all on the scoreboard key, not entered in `raw/spend.json` here (the single +runner adds it as one ledger entry). + +- `mockmed/code_agent/2026-10-18-a`: 5 first runs, 5 correct, all admitted, + every program check passed. +- `openemr_referral/code_agent/2026-10-18-b`: 5 first runs (items 701 to 705), + 4 correct and 1 safe stop on hand review (`HAND_REVIEW.md`). Admission + refused all 5 before the canary: no system-of-record change was bound to + the confirm-dialog save step. The four charts these runs saved are still on + the fixture, which is why arm 7's OpenEMR items now start at 721. +- `openemr_referral/code_agent/2026-10-18-a`: 5 first runs refused by the save + gate at the first drop-down, before any save. A gate rule then counted any + `onchange` handler as "this drop-down submits the form", and OpenEMR's Title + drop-down has one. The rule now needs a handler that calls `submit` (#1305). diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/items.csv b/packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/items.csv similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/items.csv rename to packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/items.csv diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json b/packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json rename to packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json b/packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json rename to packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json b/packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json rename to packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json b/packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json rename to packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json b/packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json rename to packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json diff --git a/packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/meta.json b/packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/meta.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/meta.json rename to packages/flow/benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/meta.json diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/HAND_REVIEW.md b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/HAND_REVIEW.md similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/HAND_REVIEW.md rename to packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/HAND_REVIEW.md diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/items.csv b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/items.csv similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/items.csv rename to packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/items.csv diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json rename to packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json rename to packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json rename to packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json rename to packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json rename to packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json diff --git a/packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/meta.json b/packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/meta.json similarity index 100% rename from packages/flow/benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/meta.json rename to packages/flow/benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/meta.json diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 714c73086..918904f2c 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -863,34 +863,6 @@ "path": "benchmark/scoreboard/raw/mockmed/agent/2026-10-18-a/meta.json", "sha256": "d3f9cd7fbc4c6d9247984085085fd33e91935f4a35aad9da31df2e230ea16063" }, - { - "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/items.csv", - "sha256": "be25c825731ea3967dcc42be5b2c57d6670fea76d414e6232eb12c1bbfe72042" - }, - { - "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json", - "sha256": "ccbcdcfe5e794d09a5b983dc6f5aecc8aca48e57c9ee46dee8135166e5ae5e1d" - }, - { - "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json", - "sha256": "666918e316f0e6180453a1b8268018e66a63ede71c4d86d3242b958ae6df9e16" - }, - { - "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json", - "sha256": "4eca72e719390ecf59f04d21ad5425eb88d8e7c0b981913d80c75b5ce05a0874" - }, - { - "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json", - "sha256": "57c41ae7028e1bbc1a23e5a09ae4728163d27f95e2f02e24051ca7b399c4be0c" - }, - { - "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json", - "sha256": "4f959829b9e0656a33eaa761a8d3d31f1b4e40c7cf870f8f42e34f8e17e8257c" - }, - { - "path": "benchmark/scoreboard/raw/mockmed/code_agent/2026-10-18-a/meta.json", - "sha256": "60440234b167a22bd5436888474d47a6292ce41e9ac867efffd37f6bf14e9673" - }, { "path": "benchmark/scoreboard/raw/mockmed/first_run/2026-10-18-a/first_run.json", "sha256": "5948cd90cd29a502f17c9874cc2000b4192ae006d2f6a9975c98b69bdc0da387" @@ -923,34 +895,6 @@ "path": "benchmark/scoreboard/raw/openemr_referral/agent/study-main-0c642bc0/meta.json", "sha256": "6e61bf57e944f849279519cb3c1c558205faf1e7d93eac05e29b731c94b1084d" }, - { - "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/items.csv", - "sha256": "517ed36f4db803c5267f30946af77a01d60e4a3716763e29c7ca0aa517683767" - }, - { - "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json", - "sha256": "a03974d0e6a9a566f16d38087e940d50352833d2670a3d934b83283199f9fc7f" - }, - { - "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json", - "sha256": "caee7051663a3073bdc262234f12099daa7eaf39036b5bcfcc2f26d75574dd9d" - }, - { - "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json", - "sha256": "05f4297e8892d66685d352d4d5084d690136ef7b53a5e54195d5c6df0c763e05" - }, - { - "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json", - "sha256": "83c3c0c6720203b19342f2cef249a59513e0eb0a51dff8a27c02e784ea270fd0" - }, - { - "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json", - "sha256": "c8e03061dbbb2da0b2806451936417d4ce92fbf206dbe7180f85d8ff38635fa6" - }, - { - "path": "benchmark/scoreboard/raw/openemr_referral/code_agent/2026-10-18-b/meta.json", - "sha256": "dea6dbe200f7f67bfbb33d4cfe27d288c46dd4664c50eeb22690073b446d7c9e" - }, { "path": "benchmark/scoreboard/raw/openemr_referral/first_run/2026-10-18-a/first_run.json", "sha256": "eeaf4465dec9eb1da65ecc2d0c6da7538ab738cd308e7321244efb948b8938e2" @@ -1089,11 +1033,39 @@ }, { "path": "benchmark/scoreboard/raw/spend.json", - "sha256": "17f575335137510d0b20554495eb5fb1ff58f0bfa32a737c36997fdb1d49a828" + "sha256": "4408cf78bcdd617ace076f786ff10e430056b66ee5e311e77c2d96dc9df38463" }, { "path": "benchmark/scoreboard/scoreboard.json", - "sha256": "b828a574da38c434822ae391467bb74fe18c330675546cad4bcad4baeccda544" + "sha256": "1a6f2caa9b0b37a5dc3cbf04fbb2d7285aa469023ea854c10bc43b2579a78132" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/items.csv", + "sha256": "be25c825731ea3967dcc42be5b2c57d6670fea76d414e6232eb12c1bbfe72042" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json", + "sha256": "ccbcdcfe5e794d09a5b983dc6f5aecc8aca48e57c9ee46dee8135166e5ae5e1d" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json", + "sha256": "666918e316f0e6180453a1b8268018e66a63ede71c4d86d3242b958ae6df9e16" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json", + "sha256": "4eca72e719390ecf59f04d21ad5425eb88d8e7c0b981913d80c75b5ce05a0874" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json", + "sha256": "57c41ae7028e1bbc1a23e5a09ae4728163d27f95e2f02e24051ca7b399c4be0c" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json", + "sha256": "4f959829b9e0656a33eaa761a8d3d31f1b4e40c7cf870f8f42e34f8e17e8257c" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/meta.json", + "sha256": "60440234b167a22bd5436888474d47a6292ce41e9ac867efffd37f6bf14e9673" }, { "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/items.csv", @@ -1123,6 +1095,34 @@ "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/meta.json", "sha256": "46cfbee57e63f0cc58993793cb64b8242816eacb13472633e105ec88e6ea2444" }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/items.csv", + "sha256": "517ed36f4db803c5267f30946af77a01d60e4a3716763e29c7ca0aa517683767" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json", + "sha256": "a03974d0e6a9a566f16d38087e940d50352833d2670a3d934b83283199f9fc7f" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json", + "sha256": "caee7051663a3073bdc262234f12099daa7eaf39036b5bcfcc2f26d75574dd9d" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json", + "sha256": "05f4297e8892d66685d352d4d5084d690136ef7b53a5e54195d5c6df0c763e05" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json", + "sha256": "83c3c0c6720203b19342f2cef249a59513e0eb0a51dff8a27c02e784ea270fd0" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json", + "sha256": "c8e03061dbbb2da0b2806451936417d4ce92fbf206dbe7180f85d8ff38635fa6" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/meta.json", + "sha256": "dea6dbe200f7f67bfbb33d4cfe27d288c46dd4664c50eeb22690073b446d7c9e" + }, { "path": "benchmark/silent_wrong_action/results.json", "sha256": "af71ce1db2b0bf065230d775ec63251891f4dbe1cd0d95d44974740acd3cc38a" From d70e62a62ae184ba5110a146b74b5471bc20cc98 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 08:35:17 -0400 Subject: [PATCH 3/4] flow: refresh the public artifact inventory Co-Authored-By: Claude Opus 5.5 --- packages/flow/public-artifacts.json | 148 ++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 5da213670..d3af919b9 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -163,6 +163,42 @@ "path": "benchmark/citrix_workspace/results.json", "sha256": "64d588f93bc4fcd5ade9962483aeb85d5facc6b764babddd0b90b750f59d4156" }, + { + "path": "benchmark/code_as_action/results/mockmed/replays.jsonl", + "sha256": "c5758009adef60f4832b0a9a921abf7338b0da99f55f23f68bf1fb569ab59765" + }, + { + "path": "benchmark/code_as_action/results/mockmed/runs.jsonl", + "sha256": "bc3d9cf0d06e88a3684021d43d8a97c803800f75064ce13a181d080bbc4b3aeb" + }, + { + "path": "benchmark/code_as_action/results/mockmed/summary.json", + "sha256": "141b7c2340dbc244f0b57fd3f409e86bcf4d02769456d03427a6a8c8e6fd533a" + }, + { + "path": "benchmark/code_as_action/results/openemr-distill-check/runs.jsonl", + "sha256": "243d8bb5804fba15fb76ada56b6463623bcec334201177d20ed29bc7f53ba14c" + }, + { + "path": "benchmark/code_as_action/results/openemr-distill-check/summary.json", + "sha256": "31197ea2122b6754150d7f7b2f1a4b1dad55f4288b960672e76a04c89c83d4d7" + }, + { + "path": "benchmark/code_as_action/results/openemr/runs.jsonl", + "sha256": "b411db6703a221e5682602e37d1b025b71bc8231d4b16d78e9d555e43f22aad4" + }, + { + "path": "benchmark/code_as_action/results/openemr/summary.json", + "sha256": "81daf50a409e53bd9918e83c1decbf8f689dcbf59df29a24869fd6fd57260c5a" + }, + { + "path": "benchmark/code_as_action/results/openemr/superseded.jsonl", + "sha256": "cebff909b6d4714707bb3e2209b2eb12ae3a200ad15f89580859cdad5362b505" + }, + { + "path": "benchmark/code_as_action/results/spend.json", + "sha256": "b9fc8fb005db9c7a928cba8685dfb8173b85d209982f6ea879c9ed9c7f9f32ce" + }, { "path": "benchmark/comparison_artifact/comparison.html", "sha256": "5e22e54dc92a053f1585468662b62f516a45c353c6b6c1a050d6742d47297d34" @@ -171,6 +207,34 @@ "path": "benchmark/comparison_artifact/comparison.json", "sha256": "c5eb58d234ecbd5b03a687745fb6afeb53d0965459b0e42779942441b19dbbd6" }, + { + "path": "benchmark/compile_agent/results/identical-inputs/runs.jsonl", + "sha256": "8bded49ba14efb9f72b58c6916a0d91380cc58e77ab9a9e5c248a35a7cd812b0" + }, + { + "path": "benchmark/compile_agent/results/pilot/replays.jsonl", + "sha256": "05104d0513de5aede95c5d06a16244e6f96ea30035870bcb70898491b34e0fc3" + }, + { + "path": "benchmark/compile_agent/results/pilot/runs.jsonl", + "sha256": "57c948a29b97e14e81d9747dd04ff230ea1605219fed6b29fbdcc28fbe01d32d" + }, + { + "path": "benchmark/compile_agent/results/pilot/summary.json", + "sha256": "65aba72eb964041d4f0717e253908901f8fe0ed3579ebbabeef1118412a26660" + }, + { + "path": "benchmark/compile_agent/results/replays.jsonl", + "sha256": "5e824e0df9ef2eeef09a8aa283f423b74d5a8b1f5273b652e516f811d0d2fdfc" + }, + { + "path": "benchmark/compile_agent/results/runs.jsonl", + "sha256": "8b1959f636d186bd1e47423b8c1b7223facb9b2a207812a1b62035d044d1d5a4" + }, + { + "path": "benchmark/compile_agent/results/summary.json", + "sha256": "6bc82419b5195c1554b4becff386b6d798b7047b08867329408eed54bbfdac11" + }, { "path": "benchmark/cosmetic_drift/results.json", "sha256": "464b6b6f2ac5850b86f7abf469b301e778c97e36d69c2af94f43b0e137fee962" @@ -1287,6 +1351,90 @@ "path": "benchmark/scoreboard/scoreboard.json", "sha256": "1c595b5bbb392a09d716d784ed1ac2456e5d96783f796d33921900e8dbecdb99" }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/items.csv", + "sha256": "be25c825731ea3967dcc42be5b2c57d6670fea76d414e6232eb12c1bbfe72042" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-001.json", + "sha256": "ccbcdcfe5e794d09a5b983dc6f5aecc8aca48e57c9ee46dee8135166e5ae5e1d" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-002.json", + "sha256": "666918e316f0e6180453a1b8268018e66a63ede71c4d86d3242b958ae6df9e16" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-003.json", + "sha256": "4eca72e719390ecf59f04d21ad5425eb88d8e7c0b981913d80c75b5ce05a0874" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-004.json", + "sha256": "57c41ae7028e1bbc1a23e5a09ae4728163d27f95e2f02e24051ca7b399c4be0c" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/ledgers/item-005.json", + "sha256": "4f959829b9e0656a33eaa761a8d3d31f1b4e40c7cf870f8f42e34f8e17e8257c" + }, + { + "path": "benchmark/scoreboard/superseded/mockmed/code_agent/2026-10-18-a/meta.json", + "sha256": "60440234b167a22bd5436888474d47a6292ce41e9ac867efffd37f6bf14e9673" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/items.csv", + "sha256": "2544ec565a2e656e584c88f6a3afffe3ac27678785f9ec846016bec4bc38ef5a" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-001.json", + "sha256": "c451d3c7be35d872f6537ef6f729bdebc716e94e39f89895f7c79443db39783f" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-002.json", + "sha256": "8438a80bb2e06f32538be34c9954080f8951d811474e4a41f00e6aafadbc8a59" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-003.json", + "sha256": "6a1d4387adbd4784ed33615bec0f36feb4cd5b5af4a4e860084726d583db478c" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-004.json", + "sha256": "1b4051d87562d5e85ec53d957d0a08ef8cef97a1c52f32321e2aa4d9bcd0813c" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/ledgers/item-005.json", + "sha256": "b29e1e0b3437276f00fd3830c247339fbe7dc8f863563444a595ad5bcf3a6e4b" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-a/meta.json", + "sha256": "46cfbee57e63f0cc58993793cb64b8242816eacb13472633e105ec88e6ea2444" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/items.csv", + "sha256": "517ed36f4db803c5267f30946af77a01d60e4a3716763e29c7ca0aa517683767" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-001.json", + "sha256": "a03974d0e6a9a566f16d38087e940d50352833d2670a3d934b83283199f9fc7f" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-002.json", + "sha256": "caee7051663a3073bdc262234f12099daa7eaf39036b5bcfcc2f26d75574dd9d" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-003.json", + "sha256": "05f4297e8892d66685d352d4d5084d690136ef7b53a5e54195d5c6df0c763e05" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-004.json", + "sha256": "83c3c0c6720203b19342f2cef249a59513e0eb0a51dff8a27c02e784ea270fd0" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/ledgers/item-005.json", + "sha256": "c8e03061dbbb2da0b2806451936417d4ce92fbf206dbe7180f85d8ff38635fa6" + }, + { + "path": "benchmark/scoreboard/superseded/openemr_referral/code_agent/2026-10-18-b/meta.json", + "sha256": "dea6dbe200f7f67bfbb33d4cfe27d288c46dd4664c50eeb22690073b446d7c9e" + }, { "path": "benchmark/silent_wrong_action/results.json", "sha256": "af71ce1db2b0bf065230d775ec63251891f4dbe1cd0d95d44974740acd3cc38a" From 773637dd9f53a77774f2d3027f8297832b82b282 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 14:41:09 -0400 Subject: [PATCH 4/4] scoreboard arm 7: restore the signal import, take a free MockMed port, keep the JIT cache outside the source tree Co-Authored-By: Claude Opus 5.5 --- packages/flow/benchmark/scoreboard/run.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/flow/benchmark/scoreboard/run.py b/packages/flow/benchmark/scoreboard/run.py index 602dfd752..3f84fba89 100644 --- a/packages/flow/benchmark/scoreboard/run.py +++ b/packages/flow/benchmark/scoreboard/run.py @@ -29,6 +29,7 @@ import multiprocessing import os import shutil +import signal import sys import time from pathlib import Path @@ -719,14 +720,23 @@ def arm_code_agent(args: argparse.Namespace) -> int: max_cost_per_item=args.max_cost, on_screen=CODE_AGENT_ON_SCREEN[task.NAME], ) - sess = Session(task, args.tag, faults=False) if task.NAME == "mockmed" else None + # Arm 7 needs no admission bound to an origin, so it takes a free port. + sess = ( + Session(task, args.tag, faults=False, port=0) + if task.NAME == "mockmed" + else None + ) on_screen = CODE_AGENT_ON_SCREEN[task.NAME] try: for i in range(args.start, args.n + 1): if spent() + args.max_cost > SPEND_CAP_USD: print(f"stopping: spend {spent():.2f} + {args.max_cost} passes the cap") break - cache = out / "jit-cache" / f"item-{i:03d}" + # Outside the source tree (schema.CACHE): recordings, bundles and a + # throwaway admission key are never committed. + cache = ( + schema.cache_dir(task.NAME, "code_agent", args.run_id) / f"item-{i:03d}" + ) if cache.exists(): shutil.rmtree(cache) if sess is not None: