Coverage for gco_mcp/mission/criteria_scaffold.py: 94.46%
227 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-30 21:22 +0000
1"""Helpers for ``gco mission scaffold-criteria``.
3The CLI subcommand turns a natural-language directive into a JSON
4array of Criterion objects that ``mission.validation.validate_criteria``
5accepts. Two paths are exposed:
7* :func:`generate_deterministic_criteria` — pure, no I/O. Keyword-matches
8 the directive against a small template table to pick a kind and shape
9 the criterion. The default fallback is a single ``predicate`` with
10 ``expression: "True"`` so the operator notices and edits before use.
11 Always emits at most ``max_criteria`` entries.
12* :func:`generate_sampled_criteria` — async, drives a resolved
13 :class:`SamplingBackend` to produce JSON. The response is parsed,
14 validated through ``validate_criteria``, and on rejection is retried
15 up to ``retries`` times with a feedback prompt mentioning the
16 rejection ``reason``. After the retry budget is exhausted, the helper
17 raises :class:`ScaffoldSamplingError` so the caller can fall back to
18 the deterministic path.
19* :func:`build_scaffold_prompt` — render the prompt the sampling
20 backend sees. Pure; lives here so tests can pin the exact text.
22The module is import-light: no FastMCP, no boto3, no MCP server. It
23imports the validators (and through them the predicate AST validator)
24and the sampling Protocol type, but nothing that touches a transport.
25The CLI wires the two paths together; this module keeps them
26decoupled so each can be tested in isolation.
27"""
29from __future__ import annotations
31import ast
32import json
33import re
34from dataclasses import dataclass
35from typing import TYPE_CHECKING, Any
37from gco.bedrock import BedrockFTUFormNotAcceptedError
39from . import validation as _validation
40from .predicate import PredicateRejected, parse_predicate
41from .validation import MissionValidationError
43# <pyflowchart-code-diagram> BEGIN - auto-inserted, do not edit
44# Generated at (UTC): 2026-07-18T01:03:40Z
45# Flowchart(s) generated from this file:
46# * ``generate_sampled_criteria`` -> ``diagrams/code_diagrams/gco_mcp/mission/criteria_scaffold.generate_sampled_criteria.html``
47# (PNG: ``diagrams/code_diagrams/gco_mcp/mission/criteria_scaffold.generate_sampled_criteria.png``)
48# Regenerate with ``python diagrams/code_diagrams/generate.py``.
49# <pyflowchart-code-diagram> END
52if TYPE_CHECKING: # pragma: no cover - type-checker only
53 from .sampling import SamplingBackend
56__all__ = [
57 "DEFAULT_MAX_CRITERIA",
58 "DEFAULT_RETRIES",
59 "ScaffoldSamplingError",
60 "build_scaffold_prompt",
61 "generate_deterministic_criteria",
62 "generate_sampled_criteria",
63]
66# ---------------------------------------------------------------------------
67# Tunables
68# ---------------------------------------------------------------------------
70#: Default cap on the number of criteria scaffolded per call.
71DEFAULT_MAX_CRITERIA: int = 5
73#: Default retry count for the sampling path. Each retry re-prompts
74#: the model with a feedback message containing the rejection reason.
75DEFAULT_RETRIES: int = 3
77# ---------------------------------------------------------------------------
78# Keyword templates for the deterministic fallback
79# ---------------------------------------------------------------------------
81# Each entry is (regex, builder). The first match wins; builders
82# return a single Criterion dict that ``validate_criteria`` accepts.
83# The regex is matched case-insensitively against the directive.
84# Order matters: more specific patterns appear first.
86# "Lower is better" metrics (loss, error rate, latency, cost).
87_LOWER_IS_BETTER_RE = re.compile(r"\b(loss|error|latency|cost)\b", re.IGNORECASE)
89# "Higher is better" metrics (accuracy, throughput, recall, F1).
90_HIGHER_IS_BETTER_RE = re.compile(r"\b(accuracy|throughput|f1|recall|precision)\b", re.IGNORECASE)
92# Search-flavoured directives.
93_SEARCH_RE = re.compile(r"\b(find|search|discover|locate|lookup)\b", re.IGNORECASE)
95# Event-style directives.
96_EVENT_RE = re.compile(
97 r"\b(succeed|succeeded|complete|completed|finish|finished|emit)\b",
98 re.IGNORECASE,
99)
102def _slugify(value: str, fallback: str = "criterion") -> str:
103 """Turn a directive snippet into a stable criterion_id-friendly slug.
105 Lowercase, ASCII letters / digits / underscores only. Empty input
106 falls back to ``fallback``. Non-empty results are capped at 32
107 chars so the audit log entries don't get unwieldy.
108 """
109 cleaned = re.sub(r"[^A-Za-z0-9]+", "_", value.strip().lower()).strip("_")
110 if not cleaned: 110 ↛ 111line 110 didn't jump to line 111 because the condition on line 110 was never true
111 return fallback
112 return cleaned[:32]
115@dataclass(frozen=True)
116class _DirectiveMatch:
117 """Internal: a directive's matched template plus the captured token."""
119 kind: str
120 captured: str # the matched keyword; informs slug + metric name
123def _classify_directive(directive: str) -> _DirectiveMatch | None:
124 """Pick the matching template for ``directive``, or ``None`` for default.
126 The first match wins so more specific patterns can take precedence
127 over the generic "search" template by listing first. Returns
128 ``None`` when nothing matches; the caller then emits the
129 placeholder predicate fallback.
130 """
131 if (m := _LOWER_IS_BETTER_RE.search(directive)) is not None:
132 return _DirectiveMatch(kind="metric_threshold_lower", captured=m.group(1).lower())
133 if (m := _HIGHER_IS_BETTER_RE.search(directive)) is not None:
134 return _DirectiveMatch(kind="metric_threshold_higher", captured=m.group(1).lower())
135 if _SEARCH_RE.search(directive) is not None:
136 return _DirectiveMatch(kind="predicate_search", captured="search")
137 if _EVENT_RE.search(directive) is not None:
138 return _DirectiveMatch(kind="event", captured="job_succeeded")
139 return None
142def _build_metric_threshold(directive: str, captured: str, op: str) -> dict[str, Any]:
143 """Build a ``metric_threshold`` criterion for the given keyword.
145 The metric name uses ``val_<keyword>`` so it lines up with the
146 common validation-loss / val-accuracy convention; the target is a
147 placeholder the operator should override (0.1 for lower-is-better
148 metrics, 0.9 for higher-is-better metrics).
150 The dot-path is prefixed with ``metrics.`` because the engine's
151 Observe_Phase merges the dispatcher's top-level ``metrics`` dict
152 into the Observation under the ``metrics`` key, and the
153 ``_evaluate_metric_threshold`` resolver walks the path against the
154 Observation root. A bare ``val_loss`` (no prefix) would land on
155 every iteration as ``inconclusive: metric_path_missing`` because
156 the Observation's top level carries ``tool_results``, ``metrics``,
157 ``events`` — not loose metric values. See
158 :data:`tests.test_mission_e2e_train_to_loss` for the canonical
159 end-to-end shape this prefix lines up with.
160 """
161 slug = _slugify(captured, fallback="metric")
162 target = 0.1 if op in ("<", "<=") else 0.9
163 metric_name = f"val_{captured}" if captured in ("loss", "accuracy") else captured
164 return {
165 "criterion_id": f"{slug}_target",
166 "kind": "metric_threshold",
167 "required": True,
168 "metric": f"metrics.{metric_name}",
169 "op": op,
170 "target": target,
171 }
174def _build_predicate_search() -> dict[str, Any]:
175 """The canonical search predicate: the iteration produced any results.
177 Uses subscript form (``obs["tool_results"]``) rather than
178 ``obs.get(...)`` because the predicate AST validator rejects
179 method calls on ``obs`` — only the eight pure stdlib callables
180 are allowed. Subscript notation is the documented surface for
181 reading from the Observation.
182 """
183 return {
184 "criterion_id": "results_present",
185 "kind": "predicate",
186 "required": True,
187 "expression": "len(obs['tool_results']) > 0",
188 }
191def _build_tool_call_succeeded(tool_name: str) -> dict[str, Any]:
192 """Build a ``tool_call_succeeded`` criterion targeting ``tool_name``.
194 The slug is derived from the tool name so two ``tool_call_succeeded``
195 entries in the same list don't collide on ``criterion_id``. The
196 default ``min_count`` of 1 is left implicit on the criterion shape
197 so the operator can edit it after scaffolding without first
198 deleting an explicit value.
199 """
200 slug = _slugify(tool_name, fallback="tool")
201 return {
202 "criterion_id": f"{slug}_called",
203 "kind": "tool_call_succeeded",
204 "required": True,
205 "tool_name": tool_name,
206 }
209def _build_event(captured: str) -> dict[str, Any]:
210 """Build an ``event`` criterion using the captured keyword as the name."""
211 return {
212 "criterion_id": "expected_event",
213 "kind": "event",
214 "required": True,
215 "event_name": captured,
216 }
219def _build_default_placeholder() -> dict[str, Any]:
220 """Return the deterministic placeholder predicate.
222 The expression is the literal ``True`` so the criterion is always
223 met — this is intentional. The TODO note in the description is the
224 cue for the operator to edit the file before running. Mission's
225 validators accept the criterion as-is so the scaffolded output is
226 always usable, but a session run with this criterion unmodified
227 completes on iteration 0.
228 """
229 return {
230 "criterion_id": "todo_placeholder",
231 "kind": "predicate",
232 "required": True,
233 "expression": "True",
234 # Non-required pass-through key (not on the validator's
235 # required-keys list) so we don't trip schema validation.
236 # It surfaces in the JSON for the operator to read.
237 "description": "TODO: replace this placeholder with a real success condition.",
238 }
241def generate_deterministic_criteria(
242 directive: str,
243 *,
244 allowlist: list[str] | None = None,
245 max_criteria: int = DEFAULT_MAX_CRITERIA,
246) -> list[dict[str, Any]]:
247 """Build a criteria list deterministically from a directive.
249 Always returns a list that ``validate_criteria`` accepts. The
250 keyword-template lookup is naive on purpose — the fallback is
251 *guidance for the operator*, not a substitute for thinking about
252 the goal. The placeholder predicate is the explicit signal that
253 no template matched.
255 Args:
256 directive: The natural-language goal.
257 allowlist: Optional list of tool names. When the directive is
258 a search-flavoured goal *and* an allowlist is supplied,
259 the generator emits one ``tool_call_succeeded`` criterion
260 per allowlisted tool (capped at ``max_criteria``) instead
261 of the loose ``len(obs['tool_results']) > 0`` predicate.
262 That gives the operator concrete per-tool success
263 signals out of the box and keeps the criterion server-
264 evaluated rather than going through the predicate AST
265 sandbox. Falls back to the predicate when no allowlist
266 is supplied so existing callers keep their shape.
267 max_criteria: Cap on the number of entries returned. Always
268 at least 1; values less than 1 are clamped.
270 Returns:
271 A list of one or more Criterion dicts. The list always
272 validates through :func:`mission.validation.validate_criteria`.
273 """
274 if max_criteria < 1: 274 ↛ 275line 274 didn't jump to line 275 because the condition on line 274 was never true
275 max_criteria = 1
276 match = _classify_directive(directive)
277 if match is None:
278 return [_build_default_placeholder()]
279 if match.kind == "metric_threshold_lower":
280 return [_build_metric_threshold(directive, match.captured, "<=")]
281 if match.kind == "metric_threshold_higher":
282 return [_build_metric_threshold(directive, match.captured, ">=")]
283 if match.kind == "predicate_search":
284 # Prefer per-tool ``tool_call_succeeded`` criteria when the
285 # operator told us what tools they intend to allowlist —
286 # those are server-evaluated and require zero predicate
287 # syntax. Fall back to the loose predicate when no
288 # allowlist is available so the no-allowlist call shape
289 # stays exactly as it was.
290 if allowlist:
291 tool_names = list(allowlist)[:max_criteria]
292 return [_build_tool_call_succeeded(name) for name in tool_names]
293 return [_build_predicate_search()]
294 if match.kind == "event":
295 return [_build_event(match.captured)]
296 # Defensive fallback — keeps mypy happy with the exhaustive return.
297 return [_build_default_placeholder()] # pragma: no cover
300# ---------------------------------------------------------------------------
301# Sampling path
302# ---------------------------------------------------------------------------
305class ScaffoldSamplingError(Exception):
306 """Raised when every sampling attempt was rejected.
308 The caller (the CLI) catches this and falls back to the
309 deterministic path. The ``last_reason`` attribute carries the
310 rejection token from the final retry so the CLI can surface it
311 in a one-line warning.
312 """
314 def __init__(self, last_reason: str, message: str | None = None) -> None:
315 self.last_reason: str = last_reason
316 super().__init__(message or last_reason)
319def build_scaffold_prompt(
320 directive: str,
321 *,
322 allowlist: list[str] | None = None,
323 max_criteria: int = DEFAULT_MAX_CRITERIA,
324 feedback: str | None = None,
325) -> str:
326 """Render the prompt the sampling backend sees.
328 The prompt asks for a strict JSON-array response, one entry per
329 Criterion. The shape is described inline so the model doesn't
330 need to fetch a schema document. The ``feedback`` argument carries
331 the rejection reason from a prior attempt — when present, it is
332 appended as a "feedback" block telling the model why the previous
333 response was rejected.
334 """
335 allowlist_block = "(none specified)" if not allowlist else ", ".join(allowlist)
336 sections: list[str] = []
337 sections.append(
338 "You are drafting Success_Criteria for a Mission goal-directed "
339 "iteration loop. The operator's directive and the tool "
340 "allowlist follow. Produce a JSON array of criterion objects "
341 "the operator can hand to `gco mission start --criteria-file`."
342 )
343 sections.append("")
344 sections.append("=== Directive ===")
345 sections.append(directive)
346 sections.append("")
347 sections.append("=== Tool allowlist ===")
348 sections.append(allowlist_block)
349 sections.append("")
350 sections.append(f"=== Cap: at most {max_criteria} criterion entries ===")
351 sections.append("")
352 sections.append("=== Observation shape (read by predicates and metric paths) ===")
353 sections.append(
354 "Each iteration's Observation is a dict with these fields:\n"
355 ' - "tool_results": list[dict] — every tool the iteration\n'
356 " called returns one entry. Each entry is whatever the\n"
357 " tool itself returned, plus a top-level ``_status`` flag.\n"
358 ' - "metrics": dict[str, Any] — numeric / scalar values\n'
359 " surfaced by tools that emit them. The dot-path for a\n"
360 " metric_threshold criterion against ``val_loss`` is\n"
361 ' ``"metrics.val_loss"`` (NOT ``"val_loss"``); the engine\n'
362 " walks the path against the Observation root and a bare\n"
363 " name will land as ``inconclusive: metric_path_missing``\n"
364 " on every iteration.\n"
365 ' - "events": list[dict] — emitted events, each with an\n'
366 " ``event_name`` key.\n"
367 ' - "errors" (optional): list[dict] — errors any tool raised.\n'
368 ' - "phase_started_at" / "phase_ended_at": ISO-8601 strings.'
369 )
370 sections.append("")
371 sections.append("=== Output schema ===")
372 sections.append(
373 "Return a single JSON array. Each entry is an object with "
374 "these required keys:\n"
375 ' - "criterion_id": unique non-empty string\n'
376 ' - "kind": one of "metric_threshold" / "event" / '
377 '"predicate" / "tool_call_succeeded"\n'
378 ' - "required": JSON boolean\n'
379 "Plus the kind-specific keys:\n"
380 ' metric_threshold -> "metric" (DOT-PATH into the\n'
381 " Observation, e.g.\n"
382 " ``metrics.val_loss``,\n"
383 " ``tool_results.0.score``), "
384 '"op" (one of <, <=, >, >=, ==, !=), "target" (number)\n'
385 ' event -> "event_name" (non-empty string;\n'
386 ' matched against entries in obs["events"])\n'
387 ' tool_call_succeeded -> "tool_name" (non-empty string;\n'
388 " matched against entries in\n"
389 ' ``obs["tool_results"]`` whose\n'
390 ' ``_status`` equals ``"ok"``).\n'
391 ' Optional: "min_count" (positive\n'
392 " int, default 1).\n"
393 " PREFER this kind over a predicate\n"
394 ' when the goal is "this tool ran\n'
395 ' and succeeded" — it is server-\n'
396 " evaluated and never goes through\n"
397 " the predicate AST sandbox.\n"
398 ' predicate -> "expression" (a Python expression\n'
399 " evaluated against `obs` — see\n"
400 " the predicate vocabulary section\n"
401 " below for the exact surface)"
402 )
403 sections.append("")
404 sections.append("=== Predicate vocabulary ===")
405 sections.append(
406 "Predicate expressions run inside a tight AST sandbox. The\n"
407 "allowed surface:\n"
408 "\n"
409 "Names: ``obs`` (the Observation dict).\n"
410 "Top-level callables (twelve, all pure stdlib):\n"
411 " ``len``, ``min``, ``max``, ``sum``, ``abs``,\n"
412 " ``any``, ``all``, ``sorted``,\n"
413 " ``str``, ``int``, ``float``, ``bool`` (type coercions).\n"
414 "Read-only method calls on any value (eight, all pure):\n"
415 " ``.get(key[, default])``, ``.keys()``, ``.values()``,\n"
416 " ``.items()``, ``.lower()``, ``.upper()``, ``.strip()``,\n"
417 " ``.startswith(prefix[, start[, end]])``\n"
418 "Operators: arithmetic, comparisons (<, <=, >, >=, ==, !=,\n"
419 " is, is not, in, not in), boolean (and, or, not), ternary\n"
420 " (a if b else c).\n"
421 "Containers: list/tuple/dict/set literals, list / set / dict\n"
422 " / generator comprehensions (the comprehension target may\n"
423 " not shadow ``obs`` or any callable name).\n"
424 "Subscripts: ``obs['key']``, ``obs['k']['nested']``,\n"
425 " ``obs['list'][0]``, etc.\n"
426 "Attribute access: ONLY single-level on ``obs`` (e.g. ``obs.events``\n"
427 " for read-only access; subscript form is preferred). Nested\n"
428 " walks like ``obs.a.b`` are rejected — use ``obs['a']['b']``.\n"
429 "\n"
430 "Method calls outside the eight pure-accessor names are\n"
431 "rejected (no ``.append``, ``.update``, ``.pop``, ``.count``,\n"
432 "``.split``, etc.). Calls to non-allowlisted names\n"
433 "(``list``, ``dict``, ``getattr``, ``isinstance``, ...)\n"
434 "are rejected."
435 )
436 sections.append("")
437 sections.append("=== Predicate examples (do NOT use rejected forms) ===")
438 sections.append(
439 "ACCEPTED predicate expressions:\n"
440 " len(obs['tool_results']) > 0\n"
441 " obs['metrics']['val_loss'] < 0.1\n"
442 " any(e['event_name'] == 'goal_reached' for e in obs['events'])\n"
443 " any(r.get('_status') == 'ok' for r in obs['tool_results'])\n"
444 " all(r.get('_status') == 'ok' for r in obs['tool_results'])\n"
445 " any(r.get('_status') == 'ok' and r.get('tool_name') == 'find_docs'\n"
446 " for r in obs['tool_results'])\n"
447 " any('inference' in str(r).lower() for r in obs['tool_results'])\n"
448 " len(obs.get('errors', [])) == 0\n"
449 " any(k == 'val_loss' for k in obs['metrics'].keys())\n"
450 " any(k.startswith('val_') for k in obs['metrics'].keys())\n"
451 "\n"
452 "REJECTED predicate expressions (will fail validation):\n"
453 " obs.metrics.val_loss < 0.1 # nested attribute walk; use obs['metrics']['val_loss']\n" # noqa: E501
454 " obs['tool_results'].count('ok') # ``.count`` is not on the method allowlist\n"
455 " obs['tool_results'].append(1) # ``.append`` mutates and is not allowed\n"
456 " any(r.split(',') for r in obs['tool_results']) # ``.split`` not on method allowlist\n"
457 " getattr(obs, 'tool_results') # ``getattr`` not on callable allowlist\n"
458 " obs['x'].y.z # attribute walk after subscript"
459 )
460 sections.append("")
461 sections.append("Output only the JSON array. No prose, no markdown fences.")
462 if feedback:
463 sections.append("")
464 sections.append("=== Feedback on previous attempt ===")
465 sections.append(feedback)
466 return "\n".join(sections)
469def _parse_response(text: str) -> list[dict[str, Any]]:
470 """Extract a JSON array from a model response.
472 Models occasionally wrap JSON in markdown fences; tolerate that.
473 Raises ``ValueError`` when no JSON array is recoverable.
474 """
475 stripped = text.strip()
476 # Strip markdown fences if present.
477 if stripped.startswith("```"):
478 # remove first fence line
479 first_newline = stripped.find("\n")
480 if first_newline != -1: 480 ↛ 482line 480 didn't jump to line 482 because the condition on line 480 was always true
481 stripped = stripped[first_newline + 1 :]
482 if stripped.endswith("```"): 482 ↛ 485line 482 didn't jump to line 485 because the condition on line 482 was always true
483 stripped = stripped[:-3].rstrip()
484 # Find the first '[' and last ']' so we tolerate trailing prose.
485 start = stripped.find("[")
486 end = stripped.rfind("]")
487 if start == -1 or end == -1 or end < start:
488 raise ValueError("no JSON array found in response")
489 parsed = json.loads(stripped[start : end + 1])
490 if not isinstance(parsed, list): 490 ↛ 491line 490 didn't jump to line 491 because the condition on line 490 was never true
491 raise ValueError("JSON payload is not a list")
492 out: list[dict[str, Any]] = []
493 for entry in parsed:
494 if not isinstance(entry, dict): 494 ↛ 495line 494 didn't jump to line 495 because the condition on line 494 was never true
495 raise ValueError("array entry is not an object")
496 out.append(entry)
497 return out
500def _normalize_kind_name(criterion: dict[str, Any]) -> dict[str, Any]:
501 """Rewrite obvious ``kind`` typos to the canonical names.
503 Models occasionally emit a near-miss for the criterion ``kind``
504 field — pluralising (``tool_calls_succeeded`` instead of the
505 canonical ``tool_call_succeeded``), abbreviating
506 (``threshold`` instead of ``metric_threshold``), or hyphenating
507 (``tool-call-succeeded`` instead of underscore form). The
508 structural validator rejects these with ``kind_invalid`` and the
509 retry-with-feedback path can recover, but the typos are
510 mechanical: a closed alias map covers every captured emission
511 we have seen across Bedrock models.
513 The map is intentionally narrow — we only canonicalise a name
514 when it is unambiguously a typo for one of the four valid
515 kinds, never a name a future kind extension might claim. Returns
516 the input unchanged when the kind is already canonical, missing,
517 or not a string. Returns a shallow copy when a rewrite fires so
518 the input dict is never mutated.
519 """
520 kind = criterion.get("kind")
521 if not isinstance(kind, str):
522 return criterion
523 canonical = _KIND_ALIASES.get(kind)
524 if canonical is None:
525 return criterion
526 if canonical == kind: 526 ↛ 527line 526 didn't jump to line 527 because the condition on line 526 was never true
527 return criterion
528 out = dict(criterion)
529 out["kind"] = canonical
530 return out
533# Closed alias map for ``_normalize_kind_name``. Every entry here was
534# observed in the captured fixture corpus under
535# ``tests/fixtures/scaffold_responses/`` — adding a new entry is the
536# right move only when a captured model emits a near-miss the
537# rejection-feedback retry doesn't recover on the next attempt.
538_KIND_ALIASES: dict[str, str] = {
539 # Llama 4 Scout pluralises the kind name in its first emission.
540 "tool_calls_succeeded": "tool_call_succeeded",
541 # Hyphenated forms occasionally surface from JSON-schema-trained
542 # smaller models that map ``snake_case`` onto ``kebab-case``.
543 "tool-call-succeeded": "tool_call_succeeded",
544 "metric-threshold": "metric_threshold",
545}
548def _normalize_metric_path(criterion: dict[str, Any]) -> dict[str, Any]:
549 """Auto-prefix bare metric names with ``metrics.`` for ``metric_threshold``.
551 The engine's metric path resolver walks the dot-path against the
552 Observation root, where canonical metric values live under the
553 ``metrics`` sub-dict. A bare ``"val_loss"`` lands as
554 ``inconclusive: metric_path_missing`` on every iteration.
556 Models trained on generic metric semantics tend to emit bare
557 names anyway. Rather than reject the response and burn a retry,
558 this normaliser injects the ``metrics.`` prefix when:
560 1. ``kind == "metric_threshold"``,
561 2. ``metric`` is a non-empty string,
562 3. The string contains no ``.`` separator (so already-qualified
563 paths like ``tool_results.0.score`` or
564 ``metrics.something.nested`` pass through verbatim).
566 Returns a shallow copy so the input is never mutated. The strip is
567 idempotent on already-prefixed values: ``"metrics.foo"`` has a
568 ``.`` so it falls through unchanged.
569 """
570 if criterion.get("kind") != "metric_threshold":
571 return criterion
572 metric = criterion.get("metric")
573 if not isinstance(metric, str) or not metric:
574 return criterion
575 if "." in metric:
576 return criterion
577 out = dict(criterion)
578 out["metric"] = f"metrics.{metric}"
579 return out
582class _AttributeToSubscriptRewriter(ast.NodeTransformer):
583 """Rewrite ``obs.<attr>`` chains as ``obs['<attr>']`` chains.
585 The predicate validator accepts a single-level attribute read on
586 ``obs`` (``obs.tool_results``) but rejects nested attribute walks
587 (``obs.metrics.val_loss``) and method-style calls
588 (``obs.x.any()``, ``obs.get('x')``). Models routinely emit those
589 shapes because they are the obvious Pythonic idioms. This
590 transformer rewrites the *attribute-walk* shapes mechanically;
591 method-call shapes that need creative rewriting are left alone so
592 the standard retry-with-feedback loop can teach the model.
594 The walk only rewrites attribute reads whose innermost base is the
595 ``Name('obs')`` — every other attribute access (e.g. on a list
596 element returned from a comprehension, on a number) is left
597 untouched so the validator's other guards still apply.
598 """
600 def visit_Attribute(self, node: ast.Attribute) -> ast.AST: # noqa: N802 - ast hook name
601 # Recurse into the value first so a nested attribute walk gets
602 # rewritten bottom-up: ``obs.metrics.val_loss`` -> visit
603 # ``obs.metrics`` first (which becomes ``obs['metrics']``)
604 # then wrap the result in ``[...]['val_loss']``.
605 self.generic_visit(node)
606 # Only rewrite when the rewritten base is one of:
607 # * Name('obs') — the simple ``obs.x`` case
608 # * Subscript whose ultimate base is Name('obs') — the
609 # already-rewritten ``obs['metrics']`` case
610 # Anything else (attribute on a Call, on a list literal, on a
611 # comprehension target) is left as-is so the validator's
612 # rejections still fire on shapes the autofix shouldn't try to
613 # silently rescue.
614 base = node.value
615 innermost = base
616 while isinstance(innermost, ast.Subscript):
617 innermost = innermost.value
618 if not (isinstance(innermost, ast.Name) and innermost.id == "obs"): 618 ↛ 619line 618 didn't jump to line 619 because the condition on line 618 was never true
619 return node
620 return ast.Subscript(
621 value=base,
622 slice=ast.Constant(value=node.attr),
623 ctx=node.ctx,
624 )
627def _autofix_predicate(criterion: dict[str, Any]) -> dict[str, Any]:
628 """Best-effort rewrite of attribute-walk predicates into subscript form.
630 Keeps the crit dict unchanged when:
632 * ``kind != "predicate"``
633 * the expression is missing or non-string
634 * the expression already parses cleanly through
635 :func:`mission.predicate.parse_predicate`
636 * source has a syntax error (the validator will reject it with the
637 original code anyway)
638 * the rewritten expression *still* fails validation (so the
639 retry-with-feedback path runs against the original source the
640 model emitted, not a partially-rewritten one)
642 Returns a shallow copy with the rewritten ``expression`` only when
643 the rewrite produced a predicate that clears the validator. This
644 mirrors :func:`_normalize_metric_path` — never mutates input,
645 always returns a JSON-safe dict.
646 """
647 if criterion.get("kind") != "predicate":
648 return criterion
649 expression = criterion.get("expression")
650 if not isinstance(expression, str) or not expression:
651 return criterion
652 # Cheap fast path: if the source is already valid, don't pay the
653 # cost of an AST round-trip on the happy case.
654 try:
655 parse_predicate(expression)
656 return criterion
657 except PredicateRejected:
658 pass
660 try:
661 tree = ast.parse(expression, mode="eval")
662 except SyntaxError:
663 return criterion
665 rewritten_tree = _AttributeToSubscriptRewriter().visit(tree)
666 ast.fix_missing_locations(rewritten_tree)
667 try:
668 rewritten_src = ast.unparse(rewritten_tree)
669 except Exception: # noqa: BLE001 - unparse failure leaves us no better off
670 return criterion
672 # Re-validate the rewrite. If the rewrite still doesn't validate
673 # (e.g. a method call like ``obs.x.any()`` produced
674 # ``obs['x'].any()`` which is still a method-call-on-subscript),
675 # fall back to the original so the retry-with-feedback loop sees
676 # the model's actual emission.
677 try:
678 parse_predicate(rewritten_src)
679 except PredicateRejected:
680 return criterion
682 out = dict(criterion)
683 out["expression"] = rewritten_src
684 return out
687async def generate_sampled_criteria(
688 backend: SamplingBackend,
689 directive: str,
690 *,
691 allowlist: list[str] | None = None,
692 max_criteria: int = DEFAULT_MAX_CRITERIA,
693 retries: int = DEFAULT_RETRIES,
694) -> list[dict[str, Any]]:
695 """Drive a sampling backend to produce a validated criteria list.
697 Builds the prompt, calls ``backend.sample(prompt_str)``, parses
698 the JSON, validates through :func:`validate_criteria`, and on
699 rejection retries up to ``retries`` times with feedback. Returns
700 the validated list (with private ``_parsed_ast`` keys stripped so
701 the result is JSON-safe). Raises :class:`ScaffoldSamplingError`
702 when every attempt was rejected, and propagates
703 :class:`gco.bedrock.BedrockFTUFormNotAcceptedError` unwrapped so a
704 missing Anthropic first-time-use form is reported rather than
705 silently downgraded to deterministic criteria.
707 The backend is duck-typed against the ``SamplingBackend`` protocol
708 on purpose — tests can substitute a stub object whose ``sample``
709 method returns canned strings without bringing in a transport.
710 """
711 feedback: str | None = None
712 last_reason = "no_attempts"
713 # We do retries + 1 total attempts — the first attempt is "free",
714 # then each retry is one extra try.
715 for attempt in range(retries + 1):
716 prompt_str = build_scaffold_prompt(
717 directive,
718 allowlist=allowlist,
719 max_criteria=max_criteria,
720 feedback=feedback,
721 )
722 try:
723 raw = await _call_backend(backend, prompt_str)
724 except BedrockFTUFormNotAcceptedError:
725 # A missing Anthropic FTU form is a permanent misconfiguration, not
726 # a transport fault. Let it escape the transport-agnostic catch
727 # below so the caller reports it instead of quietly scaffolding
728 # deterministic criteria.
729 raise
730 except Exception as exc: # noqa: BLE001 - transport-agnostic catch
731 # Transport-layer failures are not retriable from the
732 # scaffolder's point of view — the backend itself decides
733 # whether to recover. Surface as a sampling error so the
734 # CLI falls back deterministically.
735 raise ScaffoldSamplingError(
736 "transport_error",
737 message=f"sampling backend raised {type(exc).__name__}: {exc}",
738 ) from exc
739 try:
740 parsed = _parse_response(raw)
741 except (ValueError, json.JSONDecodeError) as exc:
742 last_reason = "json_parse"
743 feedback = (
744 "Your previous response could not be parsed as a JSON "
745 "array. Return a single JSON array, no prose, no "
746 f"markdown fences. ({exc})"
747 )
748 continue
749 # Cap to max_criteria — if the model returned more, truncate
750 # rather than rejecting outright. The structural validator
751 # below catches everything else.
752 if len(parsed) > max_criteria:
753 parsed = parsed[:max_criteria]
754 # Best-effort kind-name normalisation runs first so the
755 # metric-path / predicate-autofix passes branch correctly on
756 # the canonical ``kind``. Models occasionally pluralise
757 # (``tool_calls_succeeded``) or hyphenate
758 # (``tool-call-succeeded``) the kind name; rewriting to the
759 # canonical form here saves a retry round-trip. The map is
760 # closed and explicit — see ``_KIND_ALIASES``.
761 parsed = [_normalize_kind_name(c) for c in parsed]
762 # Best-effort normalisation: a model that emits a bare metric
763 # name (``"val_loss"``) instead of the dot-path
764 # (``"metrics.val_loss"``) the engine actually walks would
765 # otherwise produce a session whose metric_threshold criterion
766 # silently evaluates ``inconclusive: metric_path_missing`` on
767 # every iteration. The prompt now teaches this convention but
768 # we still post-process for robustness against older prompts
769 # and models that ignore the schema.
770 parsed = [_normalize_metric_path(c) for c in parsed]
771 # Best-effort autofix for predicate expressions: the predicate
772 # AST validator rejects attribute-walk patterns
773 # (``obs.metrics.val_loss``) and method-call shapes
774 # (``obs.get('x')``, ``obs.x.any()``) — both are common Python
775 # idioms the model defaults to. The rewriter rescues the
776 # attribute-walk shape into subscript notation; method-call
777 # shapes that need creative rewriting fall through to the
778 # standard retry-with-feedback path so the model gets the
779 # rejection token and tries again.
780 parsed = [_autofix_predicate(c) for c in parsed]
781 try:
782 validated = _validation.validate_criteria(parsed)
783 except MissionValidationError as exc:
784 details = exc.details or {}
785 last_reason = str(details.get("reason") or exc.code)
786 feedback = (
787 "Your previous response was rejected by the validator. "
788 f"Rejection reason: {last_reason}. Details: {details!r}. "
789 "Re-emit a corrected JSON array."
790 )
791 continue
792 # Strip private cached AST keys so the JSON written to disk is
793 # round-trippable. ``_parsed_ast`` is attached to predicate
794 # entries by ``validate_criteria``.
795 del attempt
796 return [
797 {k: v for k, v in entry.items() if not str(k).startswith("_")} for entry in validated
798 ]
799 raise ScaffoldSamplingError(last_reason)
802async def _call_backend(backend: SamplingBackend, prompt_str: str) -> str:
803 """Adapt the protocol's ``sample(SamplingPrompt)`` call to a string prompt.
805 The :class:`SamplingBackend` protocol takes a structured
806 :class:`SamplingPrompt`. The criteria-scaffold use case is a
807 one-off prompt rather than a full Mission round-trip, so we
808 construct a minimal ``SamplingPrompt`` whose render produces
809 exactly ``prompt_str``. Backends that need extra context
810 (Bedrock's region, MCP's model preferences) read from their bound
811 state and ignore the prompt's surrounding fields.
813 Tests can substitute a stub backend whose ``sample`` returns a
814 canned string; those tests pass the stub directly to
815 :func:`generate_sampled_criteria` and bypass the protocol entirely.
816 """
817 # Lazy import to avoid the import cycle: sampling imports validation
818 # which would otherwise import this module.
819 from .sampling import SamplingPrompt # noqa: PLC0415
821 # Wrap the prompt string in a dataclass that renders to itself.
822 # The full SamplingPrompt has many required fields; the scaffolder
823 # uses a thin adapter that overrides ``assemble`` so the existing
824 # backend implementations call ``assemble()`` and get the prompt.
825 prompt_obj = _PromptAdapter(prompt_str)
826 # Backends accept any object with an ``assemble`` method. Both
827 # MCPSamplingBackend and BedrockSamplingBackend call
828 # ``prompt.assemble()`` to get the rendered string.
829 del SamplingPrompt # imported only for documentation linkage
830 return await backend.sample(prompt_obj) # type: ignore[arg-type]
833class _PromptAdapter:
834 """Minimal duck-typed stand-in for :class:`SamplingPrompt`.
836 Both backends call ``prompt.assemble()`` to render the prompt
837 string. This adapter satisfies that single contract so the
838 scaffolder can route a free-form prompt through the same backend
839 surface the engine uses, without constructing a full
840 SamplingPrompt with iteration history that does not exist for a
841 one-off scaffolding call.
842 """
844 def __init__(self, text: str) -> None:
845 self._text: str = text
847 def assemble(self) -> str:
848 return self._text