From a2147d1a97c0adab46f372c8bbfc240da28acdf2 Mon Sep 17 00:00:00 2001 From: Chris Bonilla Date: Mon, 26 Oct 2026 16:04:32 +0100 Subject: [PATCH] feat(gooddata-eval): send the document format on agentic document runs gen-ai picks the document content format per turn from `options.documentFormat` (`aac-v1` or `aac-v2`) and otherwise from the `enablePublisherV2` flag, which is off for the evaluation org, so the `agent_document_skill` items always ran on the v1 Report names. `ChatClient` now takes `document_format`, validated by `normalize_document_format` when the client is built, and sends it beside `reasoningEffort` in one `options` object that is omitted when both are unset. `run_agentic_document_skill` and `evaluate_agentic_document_skill` take the same argument and validate it before any request. Every turn carries it, since gen-ai keeps a draft in the format of the turn that wrote it. `evaluate_agentic_document_skill` also records it as `document_format` in the dataset-run metadata. It is not part of the run name, so it separates runs in Langfuse only when their names already differ. jira: LX-3326 risk: low --- .../core/agentic/document_skill.py | 30 ++++++-- .../src/gooddata_eval/core/chat/sse_client.py | 23 ++++++- .../src/gooddata_eval/core/config.py | 19 ++++++ .../tests/test_agentic_document_skill.py | 68 +++++++++++++++++++ .../gooddata-eval/tests/test_sse_client.py | 55 ++++++++++++++- 5 files changed, 185 insertions(+), 10 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py index ba6332a4a..533b0d40b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py @@ -32,7 +32,7 @@ from gooddata_eval.core.agentic.dashboard_skill import _extract_tool_result, _skill_activated from gooddata_eval.core.chat.render import render_answer_text from gooddata_eval.core.chat.sse_client import ChatClient -from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.config import DocumentFormat, ReasoningEffort, normalize_document_format from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run from gooddata_eval.core.langfuse.item_scope import item_scope from gooddata_eval.core.models import ( @@ -801,18 +801,24 @@ def run_agentic_document_skill( agent_id: str | None = None, judge: LLMJudge | None = None, user_context: dict | None = None, + document_format: DocumentFormat | None = None, ) -> AgenticDocumentSummary: """Run the document-skill agentic evaluation K times and return a summary. The narrative judge is built only for a fixture that states a ``narrative``, so an item without one needs neither the llm-judge extra nor ``OPENAI_API_KEY``. + ``document_format`` is sent on every turn of every run, see ``ChatClient``; None leaves + the format to the server's ``enablePublisherV2`` feature flag. + Raises: ValueError: the fixture is unusable — see ``_validate_expectation`` and - ``_validate_user_context``. + ``_validate_user_context``, or ``document_format`` is not a format gen-ai + accepts. Raised before any chat request. """ _validate_expectation(expected_output) _validate_user_context(user_context) + document_format = normalize_document_format(document_format) if judge is None and _has_narrative(expected_output): judge = LLMJudge(_NARRATIVE_EVALUATION_STEPS) run_results: list[DocumentRunResult] = [] @@ -823,6 +829,7 @@ def run_agentic_document_skill( reasoning_effort=reasoning_effort, agent_id=agent_id, user_context=user_context, + document_format=document_format, ) try: @@ -882,20 +889,32 @@ def evaluate_agentic_document_skill( gate: EvalGate = DEFAULT_GATE, judge: LLMJudge | None = None, user_context: dict | None = None, + document_format: DocumentFormat | None = None, ) -> AgenticEvalOutcome: """Run document-skill evaluation, log to Langfuse, and raise on failure. Returns the best run's outcome on success; on failure the same values are attached to the raised ``DocumentSkillAssertionError`` so callers can retrieve them either way. + ``document_format`` (``aac-v1``/``aac-v2``) asks gen-ai for that document content format + on every turn and is recorded as ``document_format`` in the dataset-run metadata; None + leaves it to the server's ``enablePublisherV2`` feature flag. + Raises: DocumentSkillAssertionError: the gate did not pass. - ValueError: the fixture is unusable — see ``_validate_expectation`` and - ``_validate_user_context``. Raised before any - request, so it means a fixture to fix rather than a result to read. + ValueError: the fixture is unusable, see ``_validate_expectation`` and + ``_validate_user_context``. Raised before any chat request, so it means a + fixture to fix rather than a result to read. Also raised, equally before any + request, when the caller passes a ``document_format`` gen-ai does not accept. JudgeResponseError: the fixture states a narrative and the judge returned no readable verdict for any run -- an item without a result, not K failures. """ + document_format = normalize_document_format(document_format) + if document_format is not None: + # Metadata only, not part of the run name: it tells runs apart only when their names + # already differ, so two runs that share a run_timestamp, dataset and model but not the + # format still collide on one dataset-run name. + run_metadata_extra = {**(run_metadata_extra or {}), "document_format": document_format} identity = RunIdentity( host, token, @@ -921,6 +940,7 @@ def evaluate_agentic_document_skill( agent_id=agent_id, judge=judge, user_context=user_context, + document_format=document_format, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index dca29444d..0753f52b2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -26,7 +26,12 @@ import httpx from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, float_env, int_env, retry_transient -from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort +from gooddata_eval.core.config import ( + DocumentFormat, + ReasoningEffort, + normalize_document_format, + normalize_reasoning_effort, +) from gooddata_eval.core.langfuse.item_scope import JoinedRun, baggage_entries, current_scope, mark_joined, trace_ids_for from gooddata_eval.core.models import ChatResult, DatasetItem from gooddata_eval.core.timing import note_trace_latency @@ -460,6 +465,7 @@ def __init__( reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, user_context: dict[str, Any] | None = None, + document_format: DocumentFormat | None = None, ): """Create a chat client bound to one workspace. @@ -476,6 +482,13 @@ def __init__( back to it: a caller that changes or clears the context per message builds the client without one. The server grounds answers in it only while the ``enableAiContextSetup`` feature flag is on for the organization. + + ``document_format`` (``aac-v1``/``aac-v2``) is sent as ``options.documentFormat`` + on every message and selects the document content format gen-ai drafts in for + that turn. gen-ai keeps a draft in the format of the turn that wrote it and a + turn in the other format cannot continue it, so follow-up and clarification + turns carry it too. When None the key is omitted and the server picks the format + from the ``enablePublisherV2`` feature flag. Any other value raises ``ValueError``. """ self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations" self._auth = {"Authorization": f"Bearer {token}", **_trace_baggage()} @@ -494,6 +507,7 @@ def __init__( self._client = httpx.Client(timeout=http_timeout) self._preserve_failed = preserve_failed self._reasoning_effort = normalize_reasoning_effort(reasoning_effort) + self._document_format = normalize_document_format(document_format) self._agent_id = agent_id self._user_context = user_context # Run index of each conversation this client has sent to, in first-send order: every @@ -531,8 +545,13 @@ def send_message( join_headers, joined = self._join_item_trace(conversation_id) headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json", **join_headers} body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}} + options: dict[str, str] = {} if self._reasoning_effort is not None: - body["options"] = {"reasoningEffort": self._reasoning_effort} + options["reasoningEffort"] = self._reasoning_effort + if self._document_format is not None: + options["documentFormat"] = self._document_format + if options: + body["options"] = options # A per-call context overrides the client's. Only sent when there is one: gen-ai # accepts an explicit null, so assigning unconditionally would quietly change every # request that has no attachment. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 4d9032831..bb5f838ae 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -33,6 +33,9 @@ def judge_model() -> str: ReasoningEffort = Literal["LOW", "MEDIUM", "HIGH"] """Effort values the AI chat endpoint accepts, uppercase as the server enum requires.""" +DocumentFormat = Literal["aac-v1", "aac-v2"] +"""Document content formats the AI chat endpoint accepts as ``options.documentFormat``.""" + EvalGate = Literal["any", "power"] """`any` = pass@K, `power` = pass^K. Behaviour is in core/agentic/_gate.py; the type lives here because putting it there would make config import the agentic package, whose __init__ imports @@ -73,6 +76,22 @@ def normalize_reasoning_effort(value: str | None) -> ReasoningEffort | None: return cast("ReasoningEffort", candidate) +def normalize_document_format(value: str | None) -> DocumentFormat | None: + """Canonical document format, or None when unset. + + gen-ai rejects any value outside `DocumentFormat`, so an unknown value raises here, + before a run sends its first request. + """ + if value is None: + return None + candidate = value.strip().lower() + if not candidate: + return None + if candidate not in get_args(DocumentFormat): + raise ValueError(f"Invalid document format {value!r}; expected one of {', '.join(get_args(DocumentFormat))}.") + return cast("DocumentFormat", candidate) + + @dataclass class RunConfig: host: str diff --git a/packages/gooddata-eval/tests/test_agentic_document_skill.py b/packages/gooddata-eval/tests/test_agentic_document_skill.py index 9defdcb64..58c2d5f1f 100644 --- a/packages/gooddata-eval/tests/test_agentic_document_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_document_skill.py @@ -536,6 +536,74 @@ def test_a_user_context_with_view_report_fails_before_any_request(monkeypatch: p assert client.sent == [] +def test_the_document_format_reaches_the_chat_client(monkeypatch: pytest.MonkeyPatch) -> None: + client = _ScriptedChatClient([_drafting_turn()]) + built: list[dict[str, Any]] = [] + + def _build(**kwargs: Any) -> _ScriptedChatClient: + built.append(kwargs) + return client + + monkeypatch.setattr(document_skill, "ChatClient", _build) + evaluate_agentic_document_skill("https://h", "tok", "ws", "Create a document", {}, document_format="aac-v2") + assert [kwargs["document_format"] for kwargs in built] == ["aac-v2"] + + +def test_an_unknown_document_format_fails_before_any_request(monkeypatch: pytest.MonkeyPatch) -> None: + client = _ScriptedChatClient([_drafting_turn()]) + _install_client(monkeypatch, client) + with pytest.raises(ValueError, match="Invalid document format 'v2'"): + evaluate_agentic_document_skill("https://h", "tok", "ws", "Create a document", {}, document_format="v2") + assert client.created == client.sent == [] + + +def test_an_unknown_document_format_fails_before_the_narrative_judge_is_built( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + client = _ScriptedChatClient([_drafting_turn()]) + _install_client(monkeypatch, client) + with pytest.raises(ValueError, match="Invalid document format 'v2'"): + run_agentic_document_skill("https://h", "tok", "ws", "q", {"narrative": _NARRATIVE}, document_format="v2") + assert client.created == client.sent == [] + + +@pytest.mark.parametrize( + ("document_format", "extra", "expected"), + [ + ("AAC-V2", {"tags": ["ci"]}, {"tags": ["ci"], "document_format": "aac-v2"}), + (None, {"tags": ["ci"]}, {"tags": ["ci"]}), + (None, None, None), + ], +) +def test_the_document_format_is_recorded_in_the_run_metadata( + monkeypatch: pytest.MonkeyPatch, + document_format: str | None, + extra: dict[str, Any] | None, + expected: dict[str, Any] | None, +) -> None: + _install_client(monkeypatch, _ScriptedChatClient([_drafting_turn()])) + identities: list[Any] = [] + monkeypatch.setattr( + document_skill, "submit_trace_scoring", lambda _link, identity, **_kw: identities.append(identity) + ) + try: + evaluate_agentic_document_skill( + "https://h", + "tok", + "ws", + "Create a document", + {}, + langfuse=object(), + dataset_item_id="item-1", + run_metadata_extra=extra, + document_format=document_format, + ) + except DocumentSkillAssertionError: + pass # the identity is handed to scoring before the gate raises + assert [identity.run_metadata_extra for identity in identities] == [expected] + + # ── fixture validation ────────────────────────────────────────────────────── diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 8880ee283..510f2b0ca 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -2,6 +2,7 @@ import json import logging from collections.abc import Callable +from typing import Any from urllib.parse import unquote import httpx @@ -951,12 +952,23 @@ def handler(request): assert ei.value.conversation_id == "conv-exc" -def _capture_body_client(captured, *, reasoning_effort=None): - def handler(request): +def _capture_body_client( + captured: list[dict[str, Any]], + *, + reasoning_effort: str | None = None, + document_format: str | None = None, +) -> ChatClient: + def handler(request: httpx.Request) -> httpx.Response: captured.append(json.loads(request.content)) return httpx.Response(200, content=_OK_SSE) - client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort=reasoning_effort) + client = ChatClient( + host="https://example.invalid", + token="t", + workspace_id="w", + reasoning_effort=reasoning_effort, + document_format=document_format, + ) client._client = httpx.Client(transport=httpx.MockTransport(handler)) return client @@ -1026,6 +1038,43 @@ def test_invalid_reasoning_effort_fails_at_construction(): ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort="maximum") +@pytest.mark.parametrize("document_format", ["aac-v1", "aac-v2"]) +def test_send_message_sends_document_format_on_every_message(document_format: str) -> None: + """gen-ai keeps a draft in the format of the turn that wrote it, so follow-ups need it too.""" + captured: list[dict[str, Any]] = [] + client = _capture_body_client(captured, document_format=document_format) + client.send_message("conv", "first") + client.send_message("conv", "follow-up") + assert [c["options"] for c in captured] == [{"documentFormat": document_format}] * 2 + + +def test_send_message_sends_document_format_with_reasoning_effort() -> None: + captured: list[dict[str, Any]] = [] + _capture_body_client(captured, reasoning_effort="LOW", document_format="aac-v2").send_message("conv", "q") + assert captured[0]["options"] == {"reasoningEffort": "LOW", "documentFormat": "aac-v2"} + + +@pytest.mark.parametrize(("given", "sent"), [("AAC-V2", "aac-v2"), (" aac-v1 ", "aac-v1")]) +def test_send_message_normalizes_document_format(given: str, sent: str) -> None: + captured: list[dict[str, Any]] = [] + _capture_body_client(captured, document_format=given).send_message("conv", "q") + assert captured[0]["options"] == {"documentFormat": sent} + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_blank_document_format_is_treated_as_unset(blank: str) -> None: + captured: list[dict[str, Any]] = [] + _capture_body_client(captured, document_format=blank).send_message("conv", "q") + assert "options" not in captured[0] + + +@pytest.mark.parametrize("bad", ["v2", "aac-v3", "aac_v2"]) +def test_invalid_document_format_fails_at_construction(bad: str) -> None: + """gen-ai rejects any other value; fail before the first request instead.""" + with pytest.raises(ValueError, match="Invalid document format"): + ChatClient(host="https://example.invalid", token="t", workspace_id="w", document_format=bad) + + def test_turn_timeout_aborts_a_streaming_turn_and_is_not_retried(monkeypatch): """A chatty-but-slow agent must be cut off: httpx's per-read timeout never fires for one, so only the wall-clock budget bounds the item."""