3"""Mutation tests for the HIL convergence safety checker."""
5from __future__
import annotations
8from collections.abc
import Callable
11import hil_convergence_safety_fixtures
as fixtures
12import hil_convergence_safety_policy
as policy
13import hil_convergence_safety_selftest_environment
as environment
14import hil_convergence_safety_semantic_mutations
as semantic_mutations
15import hil_convergence_safety_v9
as v9
18Scan = Callable[[dict[str, str]], list[str]]
21class SelftestFixtureError(ValueError):
22 """A structural selftest no longer has one exact mutation target."""
25def _mutate(inputs: dict[str, str], key: str, old: str, new: str) -> dict[str, str]:
26 """Apply one unique must-fire mutation."""
27 if inputs[key].count(old) != 1:
28 message = f
"non-unique selftest fixture in {key}: {old!r}"
29 raise SelftestFixtureError(message)
30 changed = dict(inputs)
31 changed[key] = inputs[key].replace(old, new)
35def _replace_first(inputs: dict[str, str], key: str, old: str, new: str) -> dict[str, str]:
36 """Replace one selected occurrence where repetition is the safety policy."""
37 if old
not in inputs[key]:
38 message = f
"missing selftest fixture in {key}: {old!r}"
39 raise SelftestFixtureError(message)
40 changed = dict(inputs)
41 changed[key] = inputs[key].replace(old, new, 1)
45def _move_dev_task_before(
46 inputs: dict[str, str], moving_name: str, before_name: str
48 """Move one uniquely named dev-box transaction task before another."""
49 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"dev_main"]))
50 moving_matches = [index
for index, task
in enumerate(tasks)
if task.get(
"name") == moving_name]
51 before_matches = [index
for index, task
in enumerate(tasks)
if task.get(
"name") == before_name]
52 if len(moving_matches) != 1
or len(before_matches) != 1:
53 message = f
"non-unique task reorder fixture: {moving_name!r} before {before_name!r}"
54 raise SelftestFixtureError(message)
55 moving = tasks.pop(moving_matches[0])
56 before_at = next(index
for index, task
in enumerate(tasks)
if task.get(
"name") == before_name)
57 tasks.insert(before_at, moving)
58 changed = dict(inputs)
59 changed[
"dev_main"] = yaml.safe_dump(tasks, sort_keys=
False)
63def _move_apt_before_idle_proof(inputs: dict[str, str]) -> dict[str, str]:
64 """Return a role with a representative job-affecting mutator too early."""
65 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"dev_main"]))
66 name =
"Install the substrate every later step needs"
67 apt_at = next(index
for index, task
in enumerate(tasks)
if task.get(
"name") == name)
68 apt = tasks.pop(apt_at)
70 changed = dict(inputs)
71 changed[
"dev_main"] = yaml.safe_dump(tasks, sort_keys=
False)
75def _weaken_listener_state(inputs: dict[str, str]) -> dict[str, str]:
76 """Return a role that accepts transitional and unreadable service states."""
80 "dev_box_hil_runner_initial_activity.stdout | trim in ['inactive', 'failed']",
81 "dev_box_hil_runner_initial_activity.stdout | trim != 'active'",
85def _weaken_service_installer(inputs: dict[str, str]) -> dict[str, str]:
86 """Return a transaction that runs one installer through caller PATH."""
87 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"dev_main"]))
88 task = next(item
for item
in tasks
if item.get(
"name") ==
"Install the workspace reaper")
89 command = cast(dict[str, object], task[
"ansible.builtin.command"])
90 argv = cast(list[str], command[
"argv"])
92 changed = dict(inputs)
93 changed[
"dev_main"] = yaml.safe_dump(tasks, sort_keys=
False)
97def _weaken_hil_recipe_shell(inputs: dict[str, str]) -> dict[str, str]:
98 """Return one HIL recipe that re-enables caller startup processing."""
99 return _replace_first(inputs,
"hil_just",
"#!/bin/bash -p",
"#!/usr/bin/env bash")
102def _weaken_hil_script_shell(inputs: dict[str, str]) -> dict[str, str]:
103 """Return one HIL script that resolves its interpreter through PATH."""
104 sources = cast(dict[str, str], json.loads(inputs[
"hil_shells"]))
105 path = sorted(sources)[0]
106 sources[path] = sources[path].replace(
"#!/bin/bash -p",
"#!/usr/bin/env bash", 1)
107 changed = dict(inputs)
108 changed[
"hil_shells"] = json.dumps(sources, sort_keys=
True)
112def _weaken_monitor_service_shell(inputs: dict[str, str]) -> dict[str, str]:
113 """Return a monitor generator that resolves service Bash through env."""
114 sources = cast(dict[str, str], json.loads(inputs[
"hil_shells"]))
115 path =
"scripts/ci/monitor.sh"
116 sources[path] = sources[path].replace(
117 "ExecStart=/bin/bash -p $self daemon",
118 "ExecStart=/usr/bin/env bash $self daemon",
121 changed = dict(inputs)
122 changed[
"hil_shells"] = json.dumps(sources, sort_keys=
True)
126def _bypass_infra_boundary_recipe(inputs: dict[str, str]) -> dict[str, str]:
127 """Replace the public sanitation probe with an inert success command."""
131 "{{ infra }} --selftest-boundary",
136def _bypass_infra_boundary_endpoint(inputs: dict[str, str]) -> dict[str, str]:
137 """Make the dependency-free infra endpoint unconditional."""
141 'if [[ "${1:-}" == --selftest-boundary ]]; then',
146def _move_boundary_after_consumer(
147 inputs: dict[str, str], boundary_name: str, consumer_name: str
149 """Move one check-mode boundary after the bytes it must protect."""
150 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"dev_role"]))
152 index
for index, task
in enumerate(tasks)
if task.get(
"name") == boundary_name
154 boundary = tasks.pop(boundary_at)
156 index
for index, task
in enumerate(tasks)
if task.get(
"name") == consumer_name
158 tasks.insert(consumer_at + 1, boundary)
159 changed = dict(inputs)
160 changed[
"dev_role"] = yaml.safe_dump(tasks, sort_keys=
False)
165 inputs: dict[str, str], task_name: str, key: str, *, value: object
167 """Set one top-level control on one governed listener task."""
168 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"dev_role"]))
169 task = next(item
for item
in tasks
if item.get(
"name") == task_name)
171 changed = dict(inputs)
172 changed[
"dev_role"] = yaml.safe_dump(tasks, sort_keys=
False)
177 inputs: dict[str, str], task_name: str, key: str, *, value: object
179 """Replace one field in a governed no-follow stat."""
180 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"dev_role"]))
181 task = next(item
for item
in tasks
if item.get(
"name") == task_name)
182 stat = cast(dict[str, object], task[
"ansible.builtin.stat"])
184 changed = dict(inputs)
185 changed[
"dev_role"] = yaml.safe_dump(tasks, sort_keys=
False)
189def _task_module(inputs: dict[str, str], task_name: str, old: str, new: str) -> dict[str, str]:
190 """Replace one governed task module without changing its arguments."""
191 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"dev_role"]))
192 task = next(item
for item
in tasks
if item.get(
"name") == task_name)
193 task[new] = task.pop(old)
194 changed = dict(inputs)
195 changed[
"dev_role"] = yaml.safe_dump(tasks, sort_keys=
False)
199def _remove_loop_member(
200 inputs: dict[str, str], key: str, task_name: str, member: str
202 """Remove one exact string or destination-mapped task-loop member."""
203 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[key]))
204 matches = [task
for task
in tasks
if task.get(
"name") == task_name]
205 if len(matches) != 1
or not isinstance(matches[0].get(
"loop"), list):
206 message = f
"non-unique loop task in {key}: {task_name!r}"
207 raise SelftestFixtureError(message)
208 loop = cast(list[object], matches[0][
"loop"])
212 if item == member
or (isinstance(item, dict)
and item.get(
"dest") == member)
214 if len(selected) != 1:
215 message = f
"non-unique loop member in {key}: {member!r}"
216 raise SelftestFixtureError(message)
217 loop.remove(selected[0])
218 changed = dict(inputs)
219 changed[key] = yaml.safe_dump(tasks, sort_keys=
False)
223def _remove_manifest_member(inputs: dict[str, str], member: str) -> dict[str, str]:
224 """Remove one destination from the shared HIL Python authority manifest."""
225 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"bench_role"]))
229 if item.get(
"name") ==
"Define the one HIL Python execution-authority manifest"
231 facts = cast(dict[str, object], task[
"ansible.builtin.set_fact"])
232 authorities = cast(list[object], facts[
"hil_bench_python_authorities"])
234 item
for item
in authorities
if isinstance(item, dict)
and item.get(
"dest") == member
236 if len(selected) != 1:
237 message = f
"non-unique HIL authority: {member!r}"
238 raise SelftestFixtureError(message)
239 authorities.remove(selected[0])
240 changed = dict(inputs)
241 changed[
"bench_role"] = yaml.safe_dump(tasks, sort_keys=
False)
245def _remove_bench_task(inputs: dict[str, str], task_name: str) -> dict[str, str]:
246 """Remove one uniquely named HIL transaction task."""
247 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"bench_role"]))
248 selected = [task
for task
in tasks
if task.get(
"name") == task_name]
249 if len(selected) != 1:
250 message = f
"non-unique HIL task: {task_name!r}"
251 raise SelftestFixtureError(message)
252 tasks.remove(selected[0])
253 changed = dict(inputs)
254 changed[
"bench_role"] = yaml.safe_dump(tasks, sort_keys=
False)
258def _weaken_hil_authority_mode(inputs: dict[str, str]) -> dict[str, str]:
259 """Drift one executable-authority mode in the shared manifest."""
260 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"bench_role"]))
264 if item.get(
"name") ==
"Define the one HIL Python execution-authority manifest"
266 facts = cast(dict[str, object], task[
"ansible.builtin.set_fact"])
267 authorities = cast(list[dict[str, object]], facts[
"hil_bench_python_authorities"])
268 helper = next(item
for item
in authorities
if item.get(
"dest") ==
"bootstrap_uv_exec.py")
269 helper[
"mode"] =
"0755"
270 changed = dict(inputs)
271 changed[
"bench_role"] = yaml.safe_dump(tasks, sort_keys=
False)
275def _assert_condition(
276 inputs: dict[str, str], task_name: str, index: int, condition: str
278 """Replace one independently enforced identity predicate."""
279 tasks = cast(list[dict[str, object]], yaml.safe_load(inputs[
"dev_role"]))
280 task = next(item
for item
in tasks
if item.get(
"name") == task_name)
281 assertion = cast(dict[str, object], task[
"ansible.builtin.assert"])
282 conditions = cast(list[str], assertion[
"that"])
283 conditions[index] = condition
284 changed = dict(inputs)
285 changed[
"dev_role"] = yaml.safe_dump(tasks, sort_keys=
False)
289def _reports(inputs: dict[str, str], scan: Scan, expected: str) -> bool:
290 """Require the targeted defect class, not an unrelated scan failure."""
291 return expected
in scan(inputs)
294def _registration_preflight_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
295 """Return registration preflight identity mutations."""
296 registration =
"Check whether this runner is already registered"
297 registration_id =
"Refuse a linked or non-regular runner registration identity"
298 registration_error =
"hil_runner.yml: first-registration identity/token preflight is not exact"
301 "registration preflight path drift fires its own class",
307 value=
"{{ dev_box_hil_runner_root }}/.credentials",
314 "registration preflight module drift fires its own class",
316 _task_module(inputs, registration,
"ansible.builtin.stat",
"ansible.builtin.file"),
322 "registration link-following fires its own class",
324 _stat_field(inputs, registration,
"follow", value=
True),
330 "registration non-regular acceptance fires its own class",
332 _assert_condition(inputs, registration_id, 0,
"true"),
338 "registration link acceptance fires its own class",
340 _assert_condition(inputs, registration_id, 1,
"true"),
348def _runner_python_authority_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
349 """Return CI and HIL runner Python-authority mutations."""
353 f
"CI runner {member} {task_name} removal fires",
354 bool(scan(_remove_loop_member(inputs,
"ci_runner", task_name, member))),
357 "scripts/dev/managed_python_env.py",
358 "scripts/dev/managed_python_env_checks.py",
361 "Stage the root-context Python lock and bootstrap inputs",
362 "Read back every staged root-context authority byte-for-byte",
363 "Assert both Dockerfiles and every locked Python input arrived",
368 f
"HIL authority proof removal fires: {task_name}",
369 bool(scan(_remove_bench_task(inputs, task_name))),
372 "Inspect every deployed HIL Python execution authority without following links",
373 "Refuse check mode when any HIL Python authority needs apply",
374 "Prove every deployed HIL Python authority is regular and mode-exact",
375 "Read back every deployed HIL Python execution authority",
376 "Prove every deployed HIL Python authority is byte-exact",
381 "HIL shared authority mode drift fires",
382 bool(scan(_weaken_hil_authority_mode(inputs))),
388def _wsl_python_authority_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
389 """Return WSL Python-authority mutations."""
392 "WSL explicit uv directory removal fires",
398 'f"--no-config --directory {shlex.quote(stage)} sync --locked '
399 '--only-group infra "',
400 'f"--no-config sync --locked --only-group infra "',
406 "WSL authenticated uv status masking fires",
413 'f"{sync_flags} || true",',
419 "WSL helper mode proof removal fires",
425 'f"644 {helper_digest}",',
426 'f"755 {helper_digest}",',
434def _registration_identity_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
435 """Return registration and Python-authority identity mutations."""
436 preflight_cases, runner_cases, wsl_cases = (
437 _registration_preflight_cases(inputs, scan),
438 _runner_python_authority_cases(inputs, scan),
439 _wsl_python_authority_cases(inputs, scan),
441 if not all((preflight_cases, runner_cases, wsl_cases)):
442 message =
"registration case helper returned no mutation cases"
443 raise SelftestFixtureError(message)
444 return preflight_cases + runner_cases + wsl_cases
447def _public_key_identity_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
448 """Return public-key path, follow, type, and link mutations."""
449 public_key =
"Inspect the dedicated HIL public key before account planning"
450 identity =
"Refuse a linked or non-regular dedicated HIL public key"
451 expected =
"hil_runner.yml: fresh-account public-key identity preflight is not exact"
459 value=
"{{ dev_box_hil_runner_home }}/.ssh/id_rsa.pub",
462 (
"link-following", _stat_field(inputs, public_key,
"follow", value=
True)),
463 (
"non-regular acceptance", _assert_condition(inputs, identity, 0,
"true")),
464 (
"link acceptance", _assert_condition(inputs, identity, 1,
"true")),
467 (f
"public-key preflight {label} fires its own class", _reports(case, scan, expected))
468 for label, case
in mutations
472def _assert_failure_control_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
473 """Return waiver and token-bypass mutations for fail-closed assertions."""
474 registration =
"Refuse a linked or non-regular runner registration identity"
475 requirement_task =
"Require a short-lived token only for first registration"
476 public_key =
"Refuse a linked or non-regular dedicated HIL public key"
477 registration_error =
"hil_runner.yml: first-registration identity/token preflight is not exact"
478 public_key_error =
"hil_runner.yml: fresh-account public-key identity preflight is not exact"
480 for task, error, label
in (
481 (registration, registration_error,
"registration identity"),
482 (requirement_task, registration_error,
"first-registration token"),
483 (public_key, public_key_error,
"public-key identity"),
485 for control, value
in ((
"ignore_errors",
True), (
"failed_when",
False)):
486 changed = _task_control(inputs, task, control, value=value)
488 (f
"{label} {control} waiver fires its own class", _reports(changed, scan, error))
490 bypass = _task_control(inputs, requirement_task,
"when", value=
False)
492 (
"token decision bypass fires its own class", _reports(bypass, scan, registration_error))
494 removed = _assert_condition(inputs, requirement_task, 0,
"true")
497 "token requirement removal fires its own class",
498 _reports(removed, scan, registration_error),
504def _check_mode_control_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
505 """Return account, service-plan, and liveness failure-control mutations."""
506 user =
"Create the isolated HIL runner account and its dedicated SSH key"
507 home =
"Keep the isolated runner home private"
508 start =
"Enable and start the dedicated HIL listener"
509 verify =
"Verify the dedicated HIL listener is active"
510 account_error =
"hil_runner.yml: fresh-account user/home planning is not exact"
511 start_error =
"hil_runner.yml: listener start check-mode planning is not exact"
512 verify_error =
"hil_runner.yml: post-apply listener liveness proof is not exact"
515 "forced check-mode key generation fires its own class",
516 _reports(_task_control(inputs, user,
"check_mode", value=
False), scan, account_error),
519 "forced check-mode home mutation fires its own class",
520 _reports(_task_control(inputs, home,
"check_mode", value=
False), scan, account_error),
523 "forced check-mode listener start fires its own class",
524 _reports(_task_control(inputs, start,
"check_mode", value=
False), scan, start_error),
527 "hidden check-mode listener start fires its own class",
529 _task_control(inputs, start,
"when", value=
"not ansible_check_mode"),
535 "ignored liveness failure fires its own class",
537 _task_control(inputs, verify,
"ignore_errors", value=
True), scan, verify_error
541 "disabled liveness failure fires its own class",
542 _reports(_task_control(inputs, verify,
"failed_when", value=
False), scan, verify_error),
545 "check-mode liveness probe fires its own class",
547 _task_control(inputs, verify,
"when", value=
"ansible_check_mode"),
555def _fresh_boundary_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
556 """Return position-sensitive fresh-output boundary mutations."""
559 "fresh-account boundary position fires its own class",
561 _move_boundary_after_consumer(
563 "End a fresh-account check before consuming the planned public key",
564 "Read the dedicated HIL public key",
567 "hil_runner.yml: fresh-account boundary follows its key consumer",
571 "fresh-runner boundary position fires its own class",
573 _move_boundary_after_consumer(
575 "End a fresh-runner check before consuming planned package bytes",
576 "Install the official service launcher beside the runner",
579 "hil_runner.yml: fresh-runner package boundary follows a byte consumer",
585 + _registration_identity_cases(inputs, scan)
586 + _public_key_identity_cases(inputs, scan)
587 + _assert_failure_control_cases(inputs, scan)
588 + _check_mode_control_cases(inputs, scan)
592def _fleet_selftest_removed_case(
593 inputs: dict[str, str], scan: Scan, target: str, label: str
594) -> tuple[str, bool]:
595 """Return one exact required fleet selftest call-removal case."""
596 changed = _mutate(inputs,
"fleet", f
" + {target}\n",
"")
597 finding =
"fleet.py: executable HIL transaction selftests are not exact"
598 return label, _reports(changed, scan, finding)
601def _fleet_selftest_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
602 """Return live fleet selftest and independent required-call mutations."""
604 (
"fml.run_selftest()",
"fleet mutation-lock selftest removal fires"),
605 (
"fcc.run_selftest(data)",
"capacity-client selftest removal fires"),
606 (
"_bench_guard_inheritance_selftest()",
"guard selftest removal fires"),
607 (
"_inventory_publication_selftest()",
"inventory selftest removal fires"),
609 cases = [_fleet_selftest_removed_case(inputs, scan, *case)
for case
in targets]
610 return [(
"complete convergence boundary stays quiet",
not scan(inputs)), *cases]
613def _live_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
614 """Return live semantic and whole-role ordering cases."""
616 *_fleet_selftest_cases(inputs, scan),
618 "indented canonical startup authority stays quiet",
619 not v9.startup_authority_selftest(),
621 (
"v9 live capability attacks stay quiet",
not v9.semantic_errors()),
623 "public hostile startup stays inert",
624 not v9.public_boundary_selftest(policy.REPO_ROOT),
627 "infra public-boundary recipe bypass fires",
628 bool(scan(_bypass_infra_boundary_recipe(inputs))),
631 "infra boundary endpoint bypass fires",
632 bool(scan(_bypass_infra_boundary_endpoint(inputs))),
635 "recursive workflow closure selftest",
636 not policy.workflow_dependency_selftest(),
639 "package mutation before idle proof fires",
640 bool(scan(_move_apt_before_idle_proof(inputs))),
643 "transitional listener state fires",
644 bool(scan(_weaken_listener_state(inputs))),
647 "caller-PATH workspace installer fires",
648 bool(scan(_weaken_service_installer(inputs))),
651 "HIL recipe startup poisoning fires",
652 bool(scan(_weaken_hil_recipe_shell(inputs))),
655 "HIL script startup poisoning fires",
656 bool(scan(_weaken_hil_script_shell(inputs))),
659 "generated monitor startup poisoning fires",
660 bool(scan(_weaken_monitor_service_shell(inputs))),
662 *_fresh_boundary_cases(inputs, scan),
666def _base_cases(inputs: dict[str, str], scan: Scan) -> list[tuple[str, bool]]:
667 """Return every hand-authored boundary case."""
669 _live_cases(inputs, scan)
670 + environment.cases(inputs, scan, _mutate, _remove_manifest_member, _remove_loop_member)
671 + semantic_mutations.aggregator_cases(inputs, scan)
672 + semantic_mutations.fleet_split_cases(inputs, scan)
673 + semantic_mutations.fleet_activation_cases(inputs, scan)
674 + semantic_mutations.fleet_guard_dispatch_cases(inputs, scan)
675 + semantic_mutations.digest_cases(inputs, scan)
676 + semantic_mutations.wsl_clock_cases(inputs, scan)
680def run(scan: Scan, runner_scan: Scan) -> int:
681 """Prove the complete boundary stays quiet and independent removals fire."""
682 inputs = policy.load_inputs(policy.REPO_ROOT)
683 cases = _base_cases(inputs, scan)
684 cases.extend(environment.runner_runtime_directory_cases(inputs, runner_scan, _mutate))
685 for label, key, old, new
in fixtures.mutations():
686 changed = _mutate(inputs, key, old, new)
687 expected = semantic_mutations.semantic_image_findings(label, key)
688 if expected
is not None:
689 changed = semantic_mutations.rebind_helper_mutation(changed, key)
690 findings = scan(changed)
691 passed = len(findings) == len(expected)
and set(findings) == set(expected)
693 passed = bool(scan(changed))
694 cases.append((f
"{label} fires", passed))
695 for label, moving_name, before_name
in fixtures.reorders():
696 changed = _move_dev_task_before(inputs, moving_name, before_name)
697 cases.append((f
"{label} fires", bool(scan(changed))))
698 for event
in (
"push",
"pull_request"):
699 for path
in policy.workflow_paths(policy.REPO_ROOT):
700 changed = policy.remove_workflow_path(inputs, event, path)
701 cases.append((f
"{event} trigger removal fires: {path}", bool(scan(changed))))
702 for label, passed
in cases:
703 print(f
" [{'PASS' if passed else 'FAIL'}] {label}")
704 ok = all(passed
for _, passed
in cases)
705 print(f
"check_hil_convergence_safety.py --selftest: {'PASS' if ok else 'FAIL'}")
706 return 0
if ok
else 1