Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
bump: minor
---

### Added

- `/session-flow:audit-sessions` drift check: the census now counts each `toolDenialKind` value as a `denial_kind:<value>` key, so a denial kind that first appears in the newest Claude Code version is reported as a `new` change in `collect.py drift`. A new expect-absent canary, `invariant:subagent_files_missing`, counts sessions that have no subagent file even though an Agent or Task call returned without an error. A rejected, denied or failed call writes no file, so it does not count. It feeds `subagents.count`, which no longer shows as unchecked for lack of a canary.
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"validated_against": "2.1.312",
"verified": "2026-10-28",
"recheck": "a Claude Code release that changes the transcript format, or a drift run that reports new or vanished keys",
"basis": "collect.py census over 104 sessions on one reference machine (2.1.275-2.1.287), a different population from the spikes that set the thresholds in sweep-rules.json: each key below is on every record of its population in 2.1.284, 2.1.285 and 2.1.287, except effort (99.8%) and turn_duration (30-47% of system records); invariant:usage_split counted 16 groups, all in subagents, none in 2.1.287. key_path:user:entrypoint was added 2026-10-04: it decides which sessions sweep sets aside as automated, and its feeds are the four metrics whose pooled and interactive-only medians the 2026-10-03 audit compared (764 sessions), all of which moved; no other metric was compared, so widen the feeds if a later comparison shows one moving; it was on every user record of headless 2.1.288 sessions, not yet checked over the reference machine's corpus. 2026-10-18: the turn_duration canaries were dropped because time.active_s no longer reads that record (it sums main-session record gaps capped at 300 s, so it feeds from record timestamps); invariant:active_over_wall and invariant:stop_wait_over_sum count a session whose gap-based active time exceeds its wall clock and a Stop summary whose slowest hook exceeds the sum of its hooks, neither of which the definitions allow, so any count flags a broken reading. 2026-10-25: invariant:usage_split was replaced by invariant:usage_regress after a 2.1.296 sweep reported usage_split lost (#6785): records of one message may now differ on input and cache usage, as the API's message_delta usage is cumulative (https://platform.claude.com/docs/en/build-with-claude/streaming#event-types, read 2026-10-25), and last-record-wins dedup stays right as long as no earlier record of a message exceeds its last on input_tokens, cache_read_input_tokens or cache_creation_input_tokens, which usage_regress counts. 2026-10-28: collect and sweep over 2,387 sessions (665 interactive; Claude Code 2.1.280-2.1.312, 1,123 of them on 2.1.29x) on one reference machine: the stored census holds no invariant:usage_regress, invariant:active_over_wall or invariant:stop_wait_over_sum count on any version, the sweep reported no canary lost over its newest three versions (2.1.310-2.1.312), and tokens.main and tokens.sub had values (290 of 300 2.1.296 sessions with main usage, the other 10 with no assistant message); the census counts only regressions, not how many messages had several records, so that usage_regress held where records differed is inferred from the 2.1.296 sweep that reported usage_split lost",
"basis": "collect.py census over 104 sessions on one reference machine (2.1.275-2.1.287), a different population from the spikes that set the thresholds in sweep-rules.json: each key below is on every record of its population in 2.1.284, 2.1.285 and 2.1.287, except effort (99.8%) and turn_duration (30-47% of system records); invariant:usage_split counted 16 groups, all in subagents, none in 2.1.287. key_path:user:entrypoint was added 2026-10-04: it decides which sessions sweep sets aside as automated, and its feeds are the four metrics whose pooled and interactive-only medians the 2026-10-03 audit compared (764 sessions), all of which moved; no other metric was compared, so widen the feeds if a later comparison shows one moving; it was on every user record of headless 2.1.288 sessions, not yet checked over the reference machine's corpus. 2026-10-18: the turn_duration canaries were dropped because time.active_s no longer reads that record (it sums main-session record gaps capped at 300 s, so it feeds from record timestamps); invariant:active_over_wall and invariant:stop_wait_over_sum count a session whose gap-based active time exceeds its wall clock and a Stop summary whose slowest hook exceeds the sum of its hooks, neither of which the definitions allow, so any count flags a broken reading. 2026-10-25: invariant:usage_split was replaced by invariant:usage_regress after a 2.1.296 sweep reported usage_split lost (#6785): records of one message may now differ on input and cache usage, as the API's message_delta usage is cumulative (https://platform.claude.com/docs/en/build-with-claude/streaming#event-types, read 2026-10-25), and last-record-wins dedup stays right as long as no earlier record of a message exceeds its last on input_tokens, cache_read_input_tokens or cache_creation_input_tokens, which usage_regress counts. 2026-10-28: collect and sweep over 2,387 sessions (665 interactive; Claude Code 2.1.280-2.1.312, 1,123 of them on 2.1.29x) on one reference machine: the stored census holds no invariant:usage_regress, invariant:active_over_wall or invariant:stop_wait_over_sum count on any version, the sweep reported no canary lost over its newest three versions (2.1.310-2.1.312), and tokens.main and tokens.sub had values (290 of 300 2.1.296 sessions with main usage, the other 10 with no assistant message); the census counts only regressions, not how many messages had several records, so that usage_regress held where records differed is inferred from the 2.1.296 sweep that reported usage_split lost. 2026-10-29: invariant:subagent_files_missing counts a session with no subagent file whose Agent or Task call got a tool_result without is_error, since subagents.count counts files and no key canary can feed it; a rejected, denied or failed call writes no file, so it does not count. The 2026-10-03 audit found 16 sessions with Agent or Task calls and no subagent file over its whole store (#6078), counting every call whatever its result; neither count has been re-measured or checked against any one version's min_count. A toolDenialKind value is counted as denial_kind:<value>, so a new value is a new drift change; the values are undocumented and no canary watches them, since a presence canary would misfire in a version with no denial",
"canaries": [
{"key": "key_path:assistant:message.id", "record_type": "assistant", "feeds": ["tokens.main", "tokens.sub"]},
{"key": "usage_key:input_tokens", "record_type": "assistant", "feeds": ["tokens.main", "tokens.sub"]},
Expand All @@ -18,6 +18,7 @@
{"key": "invariant:active_over_wall", "record_type": "assistant", "expect": "absent", "feeds": ["time.active_s"]},
{"key": "key_path:system:hookInfos[].durationMs", "record_type": "system", "population": "system_subtype:stop_hook_summary", "feeds": ["hooks.stop_ms"]},
{"key": "invariant:stop_wait_over_sum", "record_type": "system", "population": "system_subtype:stop_hook_summary", "expect": "absent", "feeds": ["hooks.stop_ms"]},
{"key": "invariant:subagent_files_missing", "record_type": "assistant", "expect": "absent", "feeds": ["subagents.count"]},
{"key": "key_path:user:message.content", "record_type": "user", "feeds": ["human.turns", "tools.interrupts", "context.clear_commands"]},
{"key": "key_path:user:entrypoint", "record_type": "user", "feeds": ["tokens.main", "tokens.sub", "time.active_s", "human.turns"]}
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
"usage_key",
"effort_value",
"assistant_error",
"denial_kind",
"invariant",
}
)
Expand Down
24 changes: 24 additions & 0 deletions plugins/session-flow/skills/audit-sessions/scripts/collect.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@
# As of: 2026-10-13
# Recheck trigger: that section stops naming MultiEdit, or moves.
EDIT_TOOLS = frozenset({"Edit", "Write", "MultiEdit", "NotebookEdit"})
# Tool names whose calls `invariant:subagent_files_missing` expects a subagent transcript for.
AGENT_TOOLS = frozenset({"Agent", "Task"})
# System subtypes seen on current versions; any other is counted in `unknown.system_subtypes`.
SYSTEM_SUBTYPES = frozenset(
{
Expand Down Expand Up @@ -265,6 +267,8 @@ def census_keys(record: dict) -> tuple[str, list[str]]:
keys.append(f"assistant_error:{_label(record.get('error'))}")
if "effort" in record:
keys.append(f"effort_value:{_label(record.get('effort'))}")
elif kind == "user" and "toolDenialKind" in record:
keys.append(f"denial_kind:{_label(record.get('toolDenialKind'))}")
return f"{version}|{model}", keys


Expand Down Expand Up @@ -321,6 +325,8 @@ def __init__(self, excerpt_words: int) -> None:
self.bucket = ""
self.usage_peak: dict[tuple[str, str], tuple] = {}
self.usage_last: dict[tuple[str, str], tuple[str, tuple]] = {}
self.agent_calls: dict[str, str] = {}
self.agent_call_bucket: str | None = None
self.friction = friction_scan.FrictionScan()

def add(self, record: dict, side: str) -> None:
Expand Down Expand Up @@ -398,6 +404,16 @@ def usage_regressions(self) -> None:
if any(value < peak for value, peak in zip(values, self.usage_peak[group])):
self.census[bucket]["invariant:usage_regress"] += 1

def subagent_gap(self, subagent_files: int) -> None:
"""Count the session once when an Agent or Task call returned without error but no subagent file exists.

A rejected, denied or failed call writes no file, so only a call whose tool_result is not
an error counts. `subagents.count` counts files; the drift guard's invariant canary reads
this count.
"""
if self.agent_call_bucket is not None and not subagent_files:
self.census[self.agent_call_bucket]["invariant:subagent_files_missing"] += 1

def _tool_use(self, block: dict) -> None:
# Streaming can repeat a block in a later record; the id counts it once.
tool_id = block.get("id")
Expand All @@ -413,8 +429,15 @@ def _tool_use(self, block: dict) -> None:
self.edits[path] += 1
if name == "Skill":
self.skills[str(tool_input.get("skill"))] += 1
if name in AGENT_TOOLS and isinstance(tool_id, str):
self.agent_calls[tool_id] = self.bucket

def _user(self, record: dict, side: str, ts: float | None) -> None:
content = _obj(record.get("message")).get("content")
for block in content if isinstance(content, list) else ():
if isinstance(block, dict) and block.get("type") == "tool_result" and not block.get("is_error"):
bucket = self.agent_calls.pop(str(block.get("tool_use_id")), None)
self.agent_call_bucket = bucket or self.agent_call_bucket
result = record.get("toolUseResult")
if isinstance(result, str) and result.startswith("User rejected tool use"):
self.user_rejected += 1
Expand Down Expand Up @@ -706,6 +729,7 @@ def build_record(
scrub = Scrubber(redactor, excerpt_chars)
time_block = scan.time_block() # counts the invariant, so call it once per session
scan.usage_regressions() # likewise
scan.subagent_gap(len(subagents)) # likewise
# Repo identity above used the raw cwd; every string below is stored, so each goes through scrub.
# The dict is built in order, so the closing redaction block sees every scrub call's count.
return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
and unknown blocks only), and the collector case ingests the hand-written
`drift/usage/proj-u/usage-0001.jsonl`: msg_M's second record carries usage merged from a cumulative
message_delta (input and cache counts up), msg_D dips on input and recovers on its last record, and
msg_R's last record is below its first on input tokens. The bundled
msg_R's last record is below its first on input tokens. The denial-kind and subagent-file cases
collect transcripts they write under `tmp_path`. The bundled
`reference/canaries.json` is checked for shape.
"""

Expand Down Expand Up @@ -288,6 +289,75 @@ def test_collector_counts_only_a_last_record_below_an_earlier_one(data_dir, copy
assert (tokens["input"], tokens["cache_read"], tokens["cache_creation"]) == (8 + 5 + 4, 1500 * 3, 350)


def write_transcript(path: Path, version: str, records: list[dict]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
lines = (
json.dumps({"sessionId": path.stem, "version": version, "timestamp": f"2026-09-30T10:00:{i:02d}Z", **r})
for i, r in enumerate(records)
)
path.write_text("\n".join(lines) + "\n", encoding="utf-8")


def denied(n: int, kind: str) -> dict:
return {"type": "user", "uuid": f"u-{n}", "toolDenialKind": kind, "message": {"role": "user", "content": "denied"}}


def agent_call(tool: str) -> dict:
block = {"type": "tool_use", "id": "toolu_A1", "name": tool, "input": {"prompt": "look"}}
return {"type": "assistant", "uuid": "a-1", "message": {"id": "msg_A", "role": "assistant", "model": "model-s", "content": [block]}}


def agent_result(is_error: bool) -> dict:
block = {"type": "tool_result", "tool_use_id": "toolu_A1", "content": "done", "is_error": is_error}
return {"type": "user", "uuid": "u-r", "message": {"role": "user", "content": [block]}}


def collect(data_dir: Path, root: Path) -> dict[str, dict]:
collected = run_cli("collect.py", "collect", "--data-dir", str(data_dir), "--projects-root", str(root))
assert collected.returncode == 0, collected.stderr
stored = census.store_dir(data_dir).glob("p-*/*.json")
return {path.stem: json.loads(path.read_text(encoding="utf-8")) for path in stored}


def test_drift_cli_reports_a_denial_kind_first_seen_in_the_newest_version(tmp_path, data_dir, canaries_file):
root = tmp_path / "projects"
write_transcript(root / "proj-d" / "deny-0001.jsonl", "1.0.1", [denied(1, "rule")])
write_transcript(root / "proj-d" / "deny-0002.jsonl", "1.0.2", [denied(1, "rule"), denied(2, "fresh")])
collect(data_dir, root)
result = run_cli(
"collect.py", "drift", "--data-dir", str(data_dir), "--min-count", "1", "--versions", "1",
"--session-floor", "1", "--canaries", str(canaries_file),
)
assert result.returncode == 1, result.stdout + result.stderr
new = [c for c in envelope(result)["data"]["changes"] if c["class"] == "new" and c["key"].startswith("denial_kind:")]
assert [(c["key"], c["first_seen_version"]) for c in new] == [("denial_kind:fresh", "1.0.2")]


@pytest.mark.parametrize("tool", ["Agent", "Task"])
def test_collector_counts_a_returned_agent_call_without_a_subagent_file(tmp_path, data_dir, tool):
root = tmp_path / "projects"
write_transcript(root / "proj-g" / "gap-0001.jsonl", "2.0.9", [agent_call(tool), agent_result(False)])
write_transcript(root / "proj-g" / "kept-0001.jsonl", "2.0.9", [agent_call(tool), agent_result(False)])
write_transcript(root / "proj-g" / "kept-0001" / "subagents" / "agent-a1.jsonl", "2.0.9", [denied(1, "rule")])
# A rejected or failed call and one with no result yet leave no subagent file by design.
write_transcript(root / "proj-g" / "failed-0001.jsonl", "2.0.9", [agent_call(tool), agent_result(True)])
write_transcript(root / "proj-g" / "open-0001.jsonl", "2.0.9", [agent_call(tool)])
write_transcript(root / "proj-g" / "solo-0001.jsonl", "2.0.9", [denied(1, "rule")])
records = collect(data_dir, root)
missing = {s: sum(row.get("invariant:subagent_files_missing", 0) for row in r["census"].values()) for s, r in records.items()}
assert missing == {"gap-0001": 1, "kept-0001": 0, "failed-0001": 0, "open-0001": 0, "solo-0001": 0}
assert records["gap-0001"]["census"]["2.0.9|model-s"]["invariant:subagent_files_missing"] == 1


def test_bundled_subagent_gap_canary_degrades_the_subagent_count():
canaries = [c for c in census.load_canaries(census.BUNDLED_CANARIES)["canaries"] if c["key"] == "invariant:subagent_files_missing"]
assert [c["expect"] for c in canaries] == ["absent"]
held = [rec("a", {"1.0.2|m": {"record_type:assistant": 30, "invariant:subagent_files_missing": 2}})]
gaps = [rec("a", {"1.0.2|m": {"record_type:assistant": 30, "invariant:subagent_files_missing": 20}})]
assert drift(held, canaries)["degraded_metrics"] == []
assert drift(gaps, canaries)["degraded_metrics"] == ["subagents.count"]


@pytest.fixture
def canaries_file(tmp_path: Path) -> Path:
path = tmp_path / "canaries.json"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -304,8 +304,8 @@ def test_report_only_reasons_attribute_basis_numbers_to_the_reference_machine(da
def test_metrics_without_a_canary_are_listed_unchecked(data_dir):
write_store(data_dir, record("s1"))
unchecked = {row["what"]: row["reason"] for row in envelope(sweep(data_dir))["data"]["unchecked"]}
assert "canary" in unchecked["tools.denials"] and "canary" in unchecked["subagents.count"]
assert "tokens.main" not in unchecked
assert "canary" in unchecked["tools.denials"]
assert "tokens.main" not in unchecked and "subagents.count" not in unchecked


def test_suggested_skill_absent_from_catalog_renders_not_installed(data_dir):
Expand Down
Loading