From bba59ef9d668a051647fcdf9ba0f1267d311ef15 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 28 Sep 2026 04:10:57 +0000 Subject: [PATCH 1/7] feat(skill-quality): description-invocation probe harness Add measure-invocation.sh: validate a ~20-query train/validation probe set, score a lexical listing-overlap floor, compare two reports, and emit claude plugin eval cases. Seed probes cover skill-quality:check and mcp-tools:audit. The committed baseline records that current descriptions already saturate positives on the floor; false triggers are the gap a model-graded run still has to measure. Part of #3526. Co-authored-by: Kyle Sexton --- .../probes/baselines/listing-overlap.json | 462 +++++++++++++++ .../skill-quality/probes/mcp-tools.audit.json | 32 ++ .../probes/skill-quality.check.json | 32 ++ .../reference/invocation-probes.md | 98 ++++ .../scripts/measure-invocation.sh | 541 ++++++++++++++++++ .../scripts/measure-invocation.test.sh | 200 +++++++ 6 files changed, 1365 insertions(+) create mode 100644 plugins/skill-quality/probes/baselines/listing-overlap.json create mode 100644 plugins/skill-quality/probes/mcp-tools.audit.json create mode 100644 plugins/skill-quality/probes/skill-quality.check.json create mode 100644 plugins/skill-quality/reference/invocation-probes.md create mode 100755 plugins/skill-quality/scripts/measure-invocation.sh create mode 100755 plugins/skill-quality/scripts/measure-invocation.test.sh diff --git a/plugins/skill-quality/probes/baselines/listing-overlap.json b/plugins/skill-quality/probes/baselines/listing-overlap.json new file mode 100644 index 00000000000..7599e35a532 --- /dev/null +++ b/plugins/skill-quality/probes/baselines/listing-overlap.json @@ -0,0 +1,462 @@ +{ + "method": "listing-overlap", + "generated_at": "2026-09-28T04:09:46Z", + "skills": [ + { + "skill": "mcp-tools:audit", + "plugin": "mcp-tools", + "method": "listing-overlap", + "listing_file": "/tmp/wt-3526/plugins/mcp-tools/skills/audit/SKILL.md", + "splits": { + "train": { + "n": 12, + "n_positive": 6, + "n_negative": 6, + "trigger_hits": 6, + "false_triggers": 0, + "trigger_rate": 1, + "false_trigger_rate": 0 + }, + "validation": { + "n": 8, + "n_positive": 4, + "n_negative": 4, + "trigger_hits": 4, + "false_triggers": 1, + "trigger_rate": 1, + "false_trigger_rate": 0.25 + } + }, + "cases": [ + { + "id": "pos-train-01", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 34, + "winner": "mcp-tools:audit", + "request": "audit MCP tools" + }, + { + "id": "pos-train-02", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 38, + "winner": "mcp-tools:audit", + "request": "check MCP tool descriptions before I ship the server" + }, + { + "id": "pos-train-03", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 37, + "winner": "mcp-tools:audit", + "request": "review MCP server quality of our tool definitions" + }, + { + "id": "pos-train-04", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 19, + "winner": "mcp-tools:audit", + "request": "readOnlyHint missing on a read-only tool" + }, + { + "id": "pos-train-05", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 34, + "winner": "mcp-tools:audit", + "request": "parameter descriptions missing in the MCP tools" + }, + { + "id": "pos-train-06", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 26, + "winner": "mcp-tools:audit", + "request": "check the _meta annotations on these tools" + }, + { + "id": "pos-val-01", + "split": "validation", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 22, + "winner": "mcp-tools:audit", + "request": "are my server instructions too long" + }, + { + "id": "pos-val-02", + "split": "validation", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 33, + "winner": "mcp-tools:audit", + "request": "mcp audit of tool definitions in this repo" + }, + { + "id": "pos-val-03", + "split": "validation", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 37, + "winner": "mcp-tools:audit", + "request": "tool annotations on the MCP server I am authoring" + }, + { + "id": "pos-val-04", + "split": "validation", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 31, + "winner": "mcp-tools:audit", + "request": "maxResultSizeChars on our MCP tools" + }, + { + "id": "neg-train-01", + "split": "train", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 20, + "winner": "mcp-tools:audit-posture", + "request": "is it safe to run my MCP servers" + }, + { + "id": "neg-train-02", + "split": "train", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 12, + "winner": "mcp-tools:audit-posture", + "request": "floating MCP versions in my config" + }, + { + "id": "neg-train-03", + "split": "train", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 17, + "winner": "mcp-tools:audit-posture", + "request": "npx @latest MCP server on this machine" + }, + { + "id": "neg-train-04", + "split": "train", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 20, + "winner": "mcp-tools:audit-posture", + "request": "MCP server inventory of what Claude Code will launch" + }, + { + "id": "neg-train-05", + "split": "train", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 17, + "winner": "mcp-tools:audit-posture", + "request": "unpinned MCP server in user settings" + }, + { + "id": "neg-train-06", + "split": "train", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 28, + "winner": "mcp-tools:audit-posture", + "request": "mcp posture of configured servers, not tool design" + }, + { + "id": "neg-val-01", + "split": "validation", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 19, + "winner": "mcp-tools:audit-posture", + "request": "which MCP servers run on my host" + }, + { + "id": "neg-val-02", + "split": "validation", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 21, + "winner": "mcp-tools:audit-posture", + "request": "mcp supply chain of the servers I have configured" + }, + { + "id": "neg-val-03", + "split": "validation", + "expect_trigger": false, + "predicted": true, + "correct": false, + "target_score": 2, + "winner": "mcp-tools:audit", + "request": "check this SKILL.md before publishing" + }, + { + "id": "neg-val-04", + "split": "validation", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 0, + "winner": "mcp-tools:audit-posture", + "request": "what permissions are actually in effect" + } + ] + }, + { + "skill": "skill-quality:check", + "plugin": "skill-quality", + "method": "listing-overlap", + "listing_file": "/tmp/wt-3526/plugins/skill-quality/skills/check/SKILL.md", + "splits": { + "train": { + "n": 12, + "n_positive": 6, + "n_negative": 6, + "trigger_hits": 6, + "false_triggers": 4, + "trigger_rate": 1, + "false_trigger_rate": 0.6666666666666666 + }, + "validation": { + "n": 8, + "n_positive": 4, + "n_negative": 4, + "trigger_hits": 4, + "false_triggers": 2, + "trigger_rate": 1, + "false_trigger_rate": 0.5 + } + }, + "cases": [ + { + "id": "pos-train-01", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 30, + "winner": "skill-quality:check", + "request": "check this skill before I publish it" + }, + { + "id": "pos-train-02", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 27, + "winner": "skill-quality:check", + "request": "is this SKILL.md valid" + }, + { + "id": "pos-train-03", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 29, + "winner": "skill-quality:check", + "request": "lint my skill frontmatter" + }, + { + "id": "pos-train-04", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 35, + "winner": "skill-quality:check", + "request": "validate skill quality on plugins/foo/skills/bar" + }, + { + "id": "pos-train-05", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 31, + "winner": "skill-quality:check", + "request": "check skill before publishing" + }, + { + "id": "pos-train-06", + "split": "train", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 38, + "winner": "skill-quality:check", + "request": "validate evals.json for my new skill" + }, + { + "id": "pos-val-01", + "split": "validation", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 32, + "winner": "skill-quality:check", + "request": "is the skill listing overflowing" + }, + { + "id": "pos-val-02", + "split": "validation", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 25, + "winner": "skill-quality:check", + "request": "shared listing budget across every plugin" + }, + { + "id": "pos-val-03", + "split": "validation", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 31, + "winner": "skill-quality:check", + "request": "validate skill frontmatter before I ship" + }, + { + "id": "pos-val-04", + "split": "validation", + "expect_trigger": true, + "predicted": true, + "correct": true, + "target_score": 19, + "winner": "skill-quality:check", + "request": "run the skill-quality contract gate on this skill" + }, + { + "id": "neg-train-01", + "split": "train", + "expect_trigger": false, + "predicted": true, + "correct": false, + "target_score": 16, + "winner": "skill-quality:check", + "request": "write me a brand-new skill for parsing CSV files" + }, + { + "id": "neg-train-02", + "split": "train", + "expect_trigger": false, + "predicted": true, + "correct": false, + "target_score": 15, + "winner": "skill-quality:check", + "request": "how should I structure a SKILL.md with several reference topics" + }, + { + "id": "neg-train-03", + "split": "train", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 16, + "winner": "playbooks:skill-authoring", + "request": "create a skill from Anthropic's authoring playbook" + }, + { + "id": "neg-train-04", + "split": "train", + "expect_trigger": false, + "predicted": true, + "correct": false, + "target_score": 1, + "winner": "skill-quality:check", + "request": "what permissions are actually in effect on this machine" + }, + { + "id": "neg-train-05", + "split": "train", + "expect_trigger": false, + "predicted": true, + "correct": false, + "target_score": 3, + "winner": "skill-quality:check", + "request": "audit MCP tool descriptions against the spec" + }, + { + "id": "neg-train-06", + "split": "train", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 0, + "winner": "none", + "request": "babysit this pull request until it merges" + }, + { + "id": "neg-val-01", + "split": "validation", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 18, + "winner": "playbooks:skill-authoring", + "request": "skill best practices for writing a description" + }, + { + "id": "neg-val-02", + "split": "validation", + "expect_trigger": false, + "predicted": true, + "correct": false, + "target_score": 19, + "winner": "skill-quality:check", + "request": "configure where this repo's skills live for skill-quality" + }, + { + "id": "neg-val-03", + "split": "validation", + "expect_trigger": false, + "predicted": true, + "correct": false, + "target_score": 17, + "winner": "skill-quality:check", + "request": "show unused skills in this session and why they starve" + }, + { + "id": "neg-val-04", + "split": "validation", + "expect_trigger": false, + "predicted": false, + "correct": true, + "target_score": 15, + "winner": "playbooks:skill-authoring", + "request": "how to write SKILL.md progressive disclosure" + } + ] + } + ] +} diff --git a/plugins/skill-quality/probes/mcp-tools.audit.json b/plugins/skill-quality/probes/mcp-tools.audit.json new file mode 100644 index 00000000000..b773a9f6532 --- /dev/null +++ b/plugins/skill-quality/probes/mcp-tools.audit.json @@ -0,0 +1,32 @@ +{ + "skill": "mcp-tools:audit", + "plugin": "mcp-tools", + "skill_dir": "plugins/mcp-tools/skills/audit", + "ranking_note": "Seed skill: shares 'audit MCP' vocabulary with mcp-tools:audit-posture, so false triggers are the interesting failure. Starvation ranking is live-session input; this pair is the in-plugin competitor the issue's probe design needs.", + "competitors": ["mcp-tools:audit-posture"], + "competitor_dirs": { + "mcp-tools:audit-posture": "plugins/mcp-tools/skills/audit-posture" + }, + "queries": [ + {"id": "pos-train-01", "split": "train", "expect_trigger": true, "request": "audit MCP tools"}, + {"id": "pos-train-02", "split": "train", "expect_trigger": true, "request": "check MCP tool descriptions before I ship the server"}, + {"id": "pos-train-03", "split": "train", "expect_trigger": true, "request": "review MCP server quality of our tool definitions"}, + {"id": "pos-train-04", "split": "train", "expect_trigger": true, "request": "readOnlyHint missing on a read-only tool"}, + {"id": "pos-train-05", "split": "train", "expect_trigger": true, "request": "parameter descriptions missing in the MCP tools"}, + {"id": "pos-train-06", "split": "train", "expect_trigger": true, "request": "check the _meta annotations on these tools"}, + {"id": "pos-val-01", "split": "validation", "expect_trigger": true, "request": "are my server instructions too long"}, + {"id": "pos-val-02", "split": "validation", "expect_trigger": true, "request": "mcp audit of tool definitions in this repo"}, + {"id": "pos-val-03", "split": "validation", "expect_trigger": true, "request": "tool annotations on the MCP server I am authoring"}, + {"id": "pos-val-04", "split": "validation", "expect_trigger": true, "request": "maxResultSizeChars on our MCP tools"}, + {"id": "neg-train-01", "split": "train", "expect_trigger": false, "request": "is it safe to run my MCP servers"}, + {"id": "neg-train-02", "split": "train", "expect_trigger": false, "request": "floating MCP versions in my config"}, + {"id": "neg-train-03", "split": "train", "expect_trigger": false, "request": "npx @latest MCP server on this machine"}, + {"id": "neg-train-04", "split": "train", "expect_trigger": false, "request": "MCP server inventory of what Claude Code will launch"}, + {"id": "neg-train-05", "split": "train", "expect_trigger": false, "request": "unpinned MCP server in user settings"}, + {"id": "neg-train-06", "split": "train", "expect_trigger": false, "request": "mcp posture of configured servers, not tool design"}, + {"id": "neg-val-01", "split": "validation", "expect_trigger": false, "request": "which MCP servers run on my host"}, + {"id": "neg-val-02", "split": "validation", "expect_trigger": false, "request": "mcp supply chain of the servers I have configured"}, + {"id": "neg-val-03", "split": "validation", "expect_trigger": false, "request": "check this SKILL.md before publishing"}, + {"id": "neg-val-04", "split": "validation", "expect_trigger": false, "request": "what permissions are actually in effect"} + ] +} diff --git a/plugins/skill-quality/probes/skill-quality.check.json b/plugins/skill-quality/probes/skill-quality.check.json new file mode 100644 index 00000000000..c3fe3b9dd49 --- /dev/null +++ b/plugins/skill-quality/probes/skill-quality.check.json @@ -0,0 +1,32 @@ +{ + "skill": "skill-quality:check", + "plugin": "skill-quality", + "skill_dir": "plugins/skill-quality/skills/check", + "ranking_note": "Seed skill: high sibling-intent density (setup, skill-authoring, listing-budget). Fleet starvation ranking is a live-session input this CI harness does not have; replace the seed when an audit-skill-visibility report is in hand.", + "competitors": ["playbooks:skill-authoring"], + "competitor_dirs": { + "playbooks:skill-authoring": "plugins/playbooks/skills/skill-authoring" + }, + "queries": [ + {"id": "pos-train-01", "split": "train", "expect_trigger": true, "request": "check this skill before I publish it"}, + {"id": "pos-train-02", "split": "train", "expect_trigger": true, "request": "is this SKILL.md valid"}, + {"id": "pos-train-03", "split": "train", "expect_trigger": true, "request": "lint my skill frontmatter"}, + {"id": "pos-train-04", "split": "train", "expect_trigger": true, "request": "validate skill quality on plugins/foo/skills/bar"}, + {"id": "pos-train-05", "split": "train", "expect_trigger": true, "request": "check skill before publishing"}, + {"id": "pos-train-06", "split": "train", "expect_trigger": true, "request": "validate evals.json for my new skill"}, + {"id": "pos-val-01", "split": "validation", "expect_trigger": true, "request": "is the skill listing overflowing"}, + {"id": "pos-val-02", "split": "validation", "expect_trigger": true, "request": "shared listing budget across every plugin"}, + {"id": "pos-val-03", "split": "validation", "expect_trigger": true, "request": "validate skill frontmatter before I ship"}, + {"id": "pos-val-04", "split": "validation", "expect_trigger": true, "request": "run the skill-quality contract gate on this skill"}, + {"id": "neg-train-01", "split": "train", "expect_trigger": false, "request": "write me a brand-new skill for parsing CSV files"}, + {"id": "neg-train-02", "split": "train", "expect_trigger": false, "request": "how should I structure a SKILL.md with several reference topics"}, + {"id": "neg-train-03", "split": "train", "expect_trigger": false, "request": "create a skill from Anthropic's authoring playbook"}, + {"id": "neg-train-04", "split": "train", "expect_trigger": false, "request": "what permissions are actually in effect on this machine"}, + {"id": "neg-train-05", "split": "train", "expect_trigger": false, "request": "audit MCP tool descriptions against the spec"}, + {"id": "neg-train-06", "split": "train", "expect_trigger": false, "request": "babysit this pull request until it merges"}, + {"id": "neg-val-01", "split": "validation", "expect_trigger": false, "request": "skill best practices for writing a description"}, + {"id": "neg-val-02", "split": "validation", "expect_trigger": false, "request": "configure where this repo's skills live for skill-quality"}, + {"id": "neg-val-03", "split": "validation", "expect_trigger": false, "request": "show unused skills in this session and why they starve"}, + {"id": "neg-val-04", "split": "validation", "expect_trigger": false, "request": "how to write SKILL.md progressive disclosure"} + ] +} diff --git a/plugins/skill-quality/reference/invocation-probes.md b/plugins/skill-quality/reference/invocation-probes.md new file mode 100644 index 00000000000..e6c4668fe4f --- /dev/null +++ b/plugins/skill-quality/reference/invocation-probes.md @@ -0,0 +1,98 @@ +# Invocation probes + +On-demand harness for #3526: whether a skill's `description` would win the +requests it should, and stay quiet on the ones it should not. + +## What this measures + +Auto-invocation is the model matching a user request against listing text. Static +gates (`check-skill.sh` length cap, trigger-phrase preservation) do not measure +that match. This harness does, in two layers: + +| Method | What it is | When to run | +|---|---|---| +| `listing-overlap` | Deterministic floor: quoted-trigger hits plus content-token overlap against the target listing and named competitors | CI and every rewrite. The committed baseline uses this method. | +| `emit-plugin-eval` | Writes `claude plugin eval` cases (`tool_used: Skill`) per probe | On demand, within one plugin. The CLI loads a single plugin, so cross-plugin competitors are out of scope for this method. | +| Headless `claude -p` | Live cross-plugin selection | On demand, documented, not wrapped here. Replay its fire/quiet bits through `compare`. | + +`listing-overlap` is **not** a model-graded auto-invocation rate. A high floor +means the listing text already contains the request's nouns; a low floor means +a rewrite still has room. The exit criterion for a rewrite is a validation-split +gain on a model-graded run, with trigger-phrase preservation still passing. + +Packed `description` vs `when_to_use` is out of scope: the harness joins them +the same way the listing does (`check-skill.sh` combined-length check). + +## Command + +```shell +# Schema, polarity, train/validation split, roughly-20 count +bash plugins/skill-quality/scripts/measure-invocation.sh validate plugins/skill-quality/probes + +# Lexical floor, JSON on stdout, per-split rates on stderr +bash plugins/skill-quality/scripts/measure-invocation.sh score plugins/skill-quality/probes \ + > /tmp/invocation-score.json + +# Delta versus the committed baseline (train and validation separately) +bash plugins/skill-quality/scripts/measure-invocation.sh compare \ + plugins/skill-quality/probes/baselines/listing-overlap.json \ + /tmp/invocation-score.json + +# Emit plugin-eval cases for a live run +bash plugins/skill-quality/scripts/measure-invocation.sh emit-plugin-eval \ + plugins/skill-quality/probes /tmp/invocation-eval-cases +``` + +From a session, `/skill-quality:check measure-invocation` is the same script. + +## Probe shape + +One JSON file per skill under `probes/` (not `probes/baselines/`): + +- `skill`, `plugin`, `skill_dir` (repo-relative directory holding `SKILL.md`) +- `competitors[]` and `competitor_dirs` (how the floor finds rival listings) +- `queries[]`: `id`, `split` (`train` or `validation`), `expect_trigger`, `request` + +Target: roughly 20 labeled queries, both polarities, both splits. `validate` +FAILs under 8, WARNs outside 16–24. + +## Seed set + +CI has no live `claude-ops:audit-skill-visibility` starvation report. The seed +is two skills chosen for competitor density, not for a claimed starvation rank: + +- `skill-quality:check` against `playbooks:skill-authoring` (check vs write) +- `mcp-tools:audit` against `mcp-tools:audit-posture` (tool design vs supply chain) + +Replace the seed when a starvation report is in hand. Fleet-wide description +rewrites stay attended and are not filed from this harness. + +## Baseline (2026-09-28, listing-overlap) + +Committed at `probes/baselines/listing-overlap.json`. Positive trigger rate is 1.0 +on both skills and both splits: current descriptions already contain the request +nouns the seed positives use. False-trigger rates are the gap the floor can see: + +| Skill | Train false-trigger | Validation false-trigger | +|---|---|---| +| `mcp-tools:audit` | 0.00 | 0.25 | +| `skill-quality:check` | 0.67 | 0.50 | + +False positives on this floor are token-greedy: `check this SKILL.md before publishing` +scores as `mcp-tools:audit` because that listing contains `check`; `write me a brand-new +skill` scores as `skill-quality:check` because that listing contains `skill`. That is why +the floor is not a model-graded auto-invocation rate, and why a rewrite's exit criterion +is a validation-split gain on a live plugin-eval or `claude -p` run. + +## Report fields + +Per skill, per split: `n`, `n_positive`, `n_negative`, `trigger_rate` +(positives that the method selected), `false_trigger_rate` (negatives it +selected). `compare` prints treatment minus baseline for both rates on both +splits so a rewrite cannot hide a validation drop behind a train gain. + +## Trigger-phrase preservation + +A description rewrite still has to pass `check-skill.sh` check 3 (advisory drop +warning vs the base ref). A dropped quoted trigger that this harness needed is +a failed rewrite, not a measurement win. diff --git a/plugins/skill-quality/scripts/measure-invocation.sh b/plugins/skill-quality/scripts/measure-invocation.sh new file mode 100755 index 00000000000..6281bcffa21 --- /dev/null +++ b/plugins/skill-quality/scripts/measure-invocation.sh @@ -0,0 +1,541 @@ +#!/usr/bin/env bash +# Description-driven auto-invocation probe harness (#3526). +# +# Measures whether a skill's listing text would win the requests it should +# (and stay quiet on the ones it should not). Default method is a +# deterministic lexical listing-overlap floor that CI can re-run without a +# model. Model-graded `claude plugin eval` cases are emitted on demand; a +# replayed model run uses the same report shape so `compare` can show a +# rewrite delta. +# +# Exit 0 = success (WARN lines allowed); 1 = FAIL findings; 2 = usage / +# environment error (no jq, missing file). +# +# Usage: +# bash measure-invocation.sh validate +# bash measure-invocation.sh score [--method listing-overlap] +# bash measure-invocation.sh compare +# bash measure-invocation.sh emit-plugin-eval +# bash measure-invocation.sh --help +# +# Probe files: /*.json (not baselines/). Each file is one skill: +# skill, plugin, skill_dir (repo-relative), competitors[], queries[] +# query: id, split (train|validation), expect_trigger (bool), request +# Roughly 20 labeled queries per skill, both polarities, both splits. +# +# listing-overlap is a FLOOR, not a model-graded auto-invocation rate. It +# ranks the request against the target description and named competitors by +# quoted-trigger hits plus content-token overlap. The live method is +# emit-plugin-eval + `claude plugin eval` (within one plugin) or a headless +# `claude -p` session (cross-plugin); replay those results through compare. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./skill-frontmatter.sh +source "$SCRIPT_DIR/skill-frontmatter.sh" + +usage() { + awk 'NR > 1 && !/^#/ { exit } NR > 1 { sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}" +} + +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + usage + exit 0 +fi + +if ! command -v jq >/dev/null 2>&1; then + printf 'Error: jq is required\n' >&2 + exit 2 +fi + +if [[ $# -lt 1 ]]; then + printf 'Usage: measure-invocation.sh ...\n' >&2 + exit 2 +fi + +fails=0 +warns=0 +err() { + printf 'FAIL: %s\n' "$1" >&2 + fails=$((fails + 1)) +} +warn() { + printf 'WARN: %s\n' "$1" >&2 + warns=$((warns + 1)) +} +note() { printf 'INFO: %s\n' "$1" >&2; } + +repo_root() { + if [[ -n "${MEASURE_INVOCATION_REPO_ROOT:-}" ]]; then + printf '%s\n' "$MEASURE_INVOCATION_REPO_ROOT" + return + fi + git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null && return + printf '%s\n' "$PWD" +} + +probe_files() { + local dir="$1" + find "$dir" -maxdepth 1 -type f -name '*.json' ! -name '.*' | sort +} + +load_listing() { + local md="$1" + local fm desc wtu + fm="$(skill_frontmatter::extract <"$md")" + desc="$(skill_frontmatter::strip_quotes "$(skill_frontmatter::field description <<<"$fm")")" + wtu="$(skill_frontmatter::strip_quotes "$(skill_frontmatter::field when_to_use <<<"$fm")")" + printf '%s\n%s\n' "$desc" "$wtu" +} + +# Quoted 'trigger phrases' plus remaining content tokens (length >= 3). +listing_tokens() { + local text="$1" + printf '%s\n' "$text" | skill_frontmatter::extract_triggers | sed "s/^'//;s/'$//" | tr '[:upper:]' '[:lower:]' + printf '%s\n' "$text" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '\n' | + awk 'length($0) >= 3 && $0 !~ /^(the|and|for|with|when|use|this|that|from|into|not|your|our|are|was|were|has|have|had|its|but|or|nor|any|all|can|may|per|via|vs|than|then|also|just|only|each|both|same|such|into|over|under|after|before|about|into)$/' +} + +score_request() { + local request="$1" + local listing="$2" + local req_lc listing_lc + req_lc="$(printf '%s' "$request" | tr '[:upper:]' '[:lower:]')" + listing_lc="$(printf '%s' "$listing" | tr '[:upper:]' '[:lower:]')" + local quoted_hits=0 + local phrase + while IFS= read -r phrase; do + [[ -z "$phrase" ]] && continue + if [[ "$req_lc" == *"$phrase"* ]]; then + quoted_hits=$((quoted_hits + 1)) + fi + done < <(printf '%s\n' "$listing" | skill_frontmatter::extract_triggers | sed "s/^'//;s/'$//" | tr '[:upper:]' '[:lower:]') + local overlap=0 + local tok + while IFS= read -r tok; do + [[ -z "$tok" ]] && continue + if [[ "$req_lc" == *"$tok"* ]]; then + overlap=$((overlap + 1)) + fi + done < <(listing_tokens "$listing_lc") + printf '%s\n' $((quoted_hits * 10 + overlap)) +} + +resolve_skill_md() { + local root="$1" rel="$2" + if [[ -f "$rel" ]]; then + printf '%s\n' "$rel" + return + fi + if [[ -f "$root/$rel/SKILL.md" ]]; then + printf '%s\n' "$root/$rel/SKILL.md" + return + fi + if [[ -f "$root/$rel" ]]; then + printf '%s\n' "$root/$rel" + return + fi + return 1 +} + +cmd_validate() { + local dir="${1:-}" + if [[ -z "$dir" || ! -d "$dir" ]]; then + printf 'Error: validate needs a probes directory\n' >&2 + exit 2 + fi + local root + root="$(repo_root)" + local f nfiles=0 + local -a files + mapfile -t files < <(probe_files "$dir") + if [[ ${#files[@]} -eq 0 ]]; then + err "no probe JSON files in $dir" + return + fi + for f in "${files[@]}"; do + nfiles=$((nfiles + 1)) + if ! jq -e . "$f" >/dev/null 2>&1; then + err "$f is not valid JSON" + continue + fi + local skill + skill="$(jq -r '.skill // empty' "$f")" + if [[ -z "$skill" ]]; then + err "$f: missing skill" + fi + if ! jq -e '.queries | type == "array" and length > 0' "$f" >/dev/null; then + err "$f: queries must be a non-empty array" + continue + fi + local n pos neg train val + n="$(jq '.queries | length' "$f")" + pos="$(jq '[.queries[] | select(.expect_trigger == true)] | length' "$f")" + neg="$(jq '[.queries[] | select(.expect_trigger == false)] | length' "$f")" + train="$(jq '[.queries[] | select(.split == "train")] | length' "$f")" + val="$(jq '[.queries[] | select(.split == "validation")] | length' "$f")" + if ((n < 8)); then + err "$f ($skill): $n queries (need at least 8; roughly 20 is the target)" + elif ((n < 16 || n > 24)); then + warn "$f ($skill): $n queries (target is roughly 20)" + fi + if ((pos < 1)); then + err "$f ($skill): no positive (expect_trigger: true) queries" + fi + if ((neg < 1)); then + err "$f ($skill): no negative (expect_trigger: false) queries" + fi + if ((train < 1)); then + err "$f ($skill): no train split" + fi + if ((val < 1)); then + err "$f ($skill): no validation split" + fi + local dup + dup="$(jq -r '.queries | group_by(.id) | map(select(length > 1)[0].id) | .[]?' "$f")" + if [[ -n "$dup" ]]; then + err "$f ($skill): duplicate query id(s): $(printf '%s' "$dup" | tr '\n' ' ')" + fi + local bad_split + bad_split="$(jq -r '.queries[] | select(.split != "train" and .split != "validation") | .id' "$f")" + if [[ -n "$bad_split" ]]; then + err "$f ($skill): split must be train or validation; bad id(s): $(printf '%s' "$bad_split" | tr '\n' ' ')" + fi + local empty_req + empty_req="$(jq -r '.queries[] | select((.request | type != "string") or (.request | length < 1)) | .id' "$f")" + if [[ -n "$empty_req" ]]; then + err "$f ($skill): empty request; id(s): $(printf '%s' "$empty_req" | tr '\n' ' ')" + fi + local rel md + rel="$(jq -r '.skill_dir // empty' "$f")" + if [[ -n "$rel" ]]; then + if md="$(resolve_skill_md "$root" "$rel")"; then + note "$skill: listing $md" + else + warn "$f ($skill): skill_dir '$rel' does not resolve under $root" + fi + else + warn "$f ($skill): no skill_dir; score cannot load a listing" + fi + note "$skill: $n queries (pos=$pos neg=$neg train=$train validation=$val)" + done + note "validated $nfiles probe file(s)" +} + +rate() { + local hits="$1" n="$2" + if ((n == 0)); then + printf 'null' + return + fi + awk -v h="$hits" -v n="$n" 'BEGIN { printf "%.4f", h / n }' +} + +cmd_score() { + local method="listing-overlap" + if [[ "${1:-}" == "--method" ]]; then + method="${2:-}" + shift 2 + fi + if [[ "$method" != "listing-overlap" ]]; then + printf 'Error: unknown method %s (listing-overlap is the CI floor; emit-plugin-eval for the live path)\n' "$method" >&2 + exit 2 + fi + local dir="${1:-}" + if [[ -z "$dir" || ! -d "$dir" ]]; then + printf 'Error: score needs a probes directory\n' >&2 + exit 2 + fi + local root + root="$(repo_root)" + local tmp + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' RETURN + local f + local -a files + mapfile -t files < <(probe_files "$dir") + if [[ ${#files[@]} -eq 0 ]]; then + printf 'Error: no probe JSON files in %s\n' "$dir" >&2 + exit 2 + fi + local skill_idx=0 + local -A comp_listing + for f in "${files[@]}"; do + local skill plugin rel + skill="$(jq -r '.skill' "$f")" + plugin="$(jq -r '.plugin // empty' "$f")" + rel="$(jq -r '.skill_dir // empty' "$f")" + local md + if ! md="$(resolve_skill_md "$root" "$rel")"; then + err "$skill: cannot resolve skill_dir '$rel'" + continue + fi + local listing + listing="$(load_listing "$md")" + for k in "${!comp_listing[@]}"; do + unset "comp_listing[$k]" + done + local comp + while IFS= read -r comp; do + [[ -z "$comp" ]] && continue + local comp_dir + comp_dir="$(jq -r --arg c "$comp" '.competitor_dirs[$c] // empty' "$f")" + if [[ -z "$comp_dir" ]]; then + warn "$skill: competitor '$comp' has no competitor_dirs entry; skipped" + continue + fi + local cmd + if cmd="$(resolve_skill_md "$root" "$comp_dir")"; then + comp_listing["$comp"]="$(load_listing "$cmd")" + else + warn "$skill: competitor_dirs '$comp' -> '$comp_dir' does not resolve" + fi + done < <(jq -r '.competitors[]?' "$f") + + local cases_json='[]' + local nq + nq="$(jq '.queries | length' "$f")" + local i + for ((i = 0; i < nq; i++)); do + local id split expect request + id="$(jq -r ".queries[$i].id" "$f")" + split="$(jq -r ".queries[$i].split" "$f")" + expect="$(jq -r ".queries[$i].expect_trigger" "$f")" + request="$(jq -r ".queries[$i].request" "$f")" + local tscore + tscore="$(score_request "$request" "$listing")" + local winner="$skill" + local best="$tscore" + local predicted=true + if ((tscore == 0)); then + predicted=false + winner="none" + best=0 + fi + local cname + for cname in "${!comp_listing[@]}"; do + local cscore + cscore="$(score_request "$request" "${comp_listing[$cname]}")" + if ((cscore > best)); then + best="$cscore" + winner="$cname" + predicted=false + elif ((cscore == best)) && [[ "$winner" == "$skill" ]] && ((cscore > 0)); then + # Tie with a competitor: target did not uniquely win. + predicted=false + winner="$skill+$cname" + fi + done + if [[ "$expect" == "true" && "$predicted" == "true" ]]; then + : + elif [[ "$expect" == "false" && "$predicted" == "false" ]]; then + : + fi + local correct=false + if [[ "$expect" == "true" && "$predicted" == "true" ]]; then + correct=true + elif [[ "$expect" == "false" && "$predicted" == "false" ]]; then + correct=true + fi + cases_json="$(jq -c --arg id "$id" --arg split "$split" --argjson expect "$expect" \ + --argjson predicted "$predicted" --argjson correct "$correct" \ + --argjson tscore "$tscore" --arg winner "$winner" --arg request "$request" \ + '. + [{id:$id, split:$split, expect_trigger:$expect, predicted:$predicted, correct:$correct, target_score:$tscore, winner:$winner, request:$request}]' <<<"$cases_json")" + done + + local split_json + split_json="$(jq -c ' + def band($s; $pos): + (.[] | select(.split == $s and .expect_trigger == $pos)); + def rate(hits; n): if n == 0 then null else (hits / n) end; + { + train: { + n: [.[] | select(.split == "train")] | length, + n_positive: [band("train"; true)] | length, + n_negative: [band("train"; false)] | length, + trigger_hits: [band("train"; true) | select(.predicted == true)] | length, + false_triggers: [band("train"; false) | select(.predicted == true)] | length + }, + validation: { + n: [.[] | select(.split == "validation")] | length, + n_positive: [band("validation"; true)] | length, + n_negative: [band("validation"; false)] | length, + trigger_hits: [band("validation"; true) | select(.predicted == true)] | length, + false_triggers: [band("validation"; false) | select(.predicted == true)] | length + } + } + | .train.trigger_rate = (if .train.n_positive == 0 then null else (.train.trigger_hits / .train.n_positive) end) + | .train.false_trigger_rate = (if .train.n_negative == 0 then null else (.train.false_triggers / .train.n_negative) end) + | .validation.trigger_rate = (if .validation.n_positive == 0 then null else (.validation.trigger_hits / .validation.n_positive) end) + | .validation.false_trigger_rate = (if .validation.n_negative == 0 then null else (.validation.false_triggers / .validation.n_negative) end) + ' <<<"$cases_json")" + + jq -n --arg skill "$skill" --arg plugin "$plugin" --arg method "$method" \ + --arg listing_file "$md" --argjson splits "$split_json" --argjson cases "$cases_json" \ + '{skill:$skill, plugin:$plugin, method:$method, listing_file:$listing_file, splits:$splits, cases:$cases}' \ + >"$tmp/skill-$skill_idx.json" + skill_idx=$((skill_idx + 1)) + + local tr vr ft_t ft_v + tr="$(jq -r '.train.trigger_rate' <<<"$split_json")" + vr="$(jq -r '.validation.trigger_rate' <<<"$split_json")" + ft_t="$(jq -r '.train.false_trigger_rate' <<<"$split_json")" + ft_v="$(jq -r '.validation.false_trigger_rate' <<<"$split_json")" + note "$skill listing-overlap train trigger_rate=$tr false_trigger_rate=$ft_t validation trigger_rate=$vr false_trigger_rate=$ft_v" + done + + local generated + generated="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + if [[ $skill_idx -eq 0 ]]; then + printf 'Error: no skills scored\n' >&2 + exit 1 + fi + jq -s --arg generated "$generated" --arg method "$method" \ + '{method:$method, generated_at:$generated, skills:.}' "$tmp"/skill-*.json +} + +cmd_compare() { + local base="${1:-}" treat="${2:-}" + if [[ -z "$base" || -z "$treat" || ! -f "$base" || ! -f "$treat" ]]; then + printf 'Error: compare needs two report JSON files\n' >&2 + exit 2 + fi + jq -n --slurpfile b "$base" --slurpfile t "$treat" ' + ($b[0].skills) as $bs | ($t[0].skills) as $ts | + { + method_baseline: $b[0].method, + method_treatment: $t[0].method, + skills: [ + $ts[] as $t | + ($bs[] | select(.skill == $t.skill)) as $bb | + { + skill: $t.skill, + train: { + trigger_rate_baseline: $bb.splits.train.trigger_rate, + trigger_rate_treatment: $t.splits.train.trigger_rate, + trigger_rate_delta: (($t.splits.train.trigger_rate // 0) - ($bb.splits.train.trigger_rate // 0)), + false_trigger_rate_baseline: $bb.splits.train.false_trigger_rate, + false_trigger_rate_treatment: $t.splits.train.false_trigger_rate, + false_trigger_rate_delta: (($t.splits.train.false_trigger_rate // 0) - ($bb.splits.train.false_trigger_rate // 0)) + }, + validation: { + trigger_rate_baseline: $bb.splits.validation.trigger_rate, + trigger_rate_treatment: $t.splits.validation.trigger_rate, + trigger_rate_delta: (($t.splits.validation.trigger_rate // 0) - ($bb.splits.validation.trigger_rate // 0)), + false_trigger_rate_baseline: $bb.splits.validation.false_trigger_rate, + false_trigger_rate_treatment: $t.splits.validation.false_trigger_rate, + false_trigger_rate_delta: (($t.splits.validation.false_trigger_rate // 0) - ($bb.splits.validation.false_trigger_rate // 0)) + } + } + ] + } + ' +} + +cmd_emit() { + local dir="${1:-}" out="${2:-}" + if [[ -z "$dir" || ! -d "$dir" || -z "$out" ]]; then + printf 'Error: emit-plugin-eval needs \n' >&2 + exit 2 + fi + mkdir -p "$out" + local f + local -a files + mapfile -t files < <(probe_files "$dir") + for f in "${files[@]}"; do + local skill plugin leaf + skill="$(jq -r '.skill' "$f")" + plugin="$(jq -r '.plugin // empty' "$f")" + leaf="${skill#*:}" + local nq i + nq="$(jq '.queries | length' "$f")" + for ((i = 0; i < nq; i++)); do + local id split expect request + id="$(jq -r ".queries[$i].id" "$f")" + split="$(jq -r ".queries[$i].split" "$f")" + expect="$(jq -r ".queries[$i].expect_trigger" "$f")" + request="$(jq -r ".queries[$i].request" "$f")" + local case_dir="$out/${plugin}-${leaf}-${id}" + mkdir -p "$case_dir/graders" + printf '%s\n' "--- +description: invocation probe ${skill} ${id} (${split}, expect_trigger=${expect}) +tags: [invocation-probe, ${split}] +runs: 1 +max_turns: 8 +allowed_tools: [Skill] +expected_outcome: Skill ${skill} $(if [[ "$expect" == "true" ]]; then echo fires; else echo does not fire; fi) +--- + +${request} +" >"$case_dir/prompt.md" + if [[ "$expect" == "true" ]]; then + printf '%s\n' "--- +type: tool_used +tool: Skill +input_match: '\"skill\"\\\\s*:\\\\s*\"(?:${plugin}:)?${leaf}\"' +min: 1 +--- +" >"$case_dir/graders/skill-fired.md" + else + printf '%s\n' "--- +type: tool_used +tool: Skill +input_match: '\"skill\"\\\\s*:\\\\s*\"(?:${plugin}:)?${leaf}\"' +min: 0 +max: 0 +arm: both +--- +" >"$case_dir/graders/skill-quiet.md" + fi + done + note "emitted $nq plugin-eval cases for $skill under $out" + done + cat >"$out/README.md" <<'EOF' +Plugin-eval cases generated by `measure-invocation.sh emit-plugin-eval`. + +Run within one plugin at a time (the CLI loads a single plugin): + +``` +claude plugin eval plugins/ --eval-dir --trust-plugin +``` + +A with-only `tool_used: Skill` grader is a plugin-fired indicator, not a score. +Copy the per-case predicted fire/quiet bits into a report JSON matching +`measure-invocation.sh score` output, then `compare` against a baseline. +Cross-plugin competitors need a headless `claude -p` session with those +plugins enabled; plugin eval cannot load them. +EOF +} + +ACTION="$1" +shift +case "$ACTION" in + validate) + cmd_validate "$@" + if ((fails > 0)); then + printf 'measure-invocation validate: FAIL (%s error(s), %s warning(s))\n' "$fails" "$warns" >&2 + exit 1 + fi + printf 'measure-invocation validate: PASS (%s warning(s))\n' "$warns" >&2 + exit 0 + ;; + score) + cmd_score "$@" + if ((fails > 0)); then + exit 1 + fi + exit 0 + ;; + compare) + cmd_compare "$@" + exit 0 + ;; + emit-plugin-eval) + cmd_emit "$@" + exit 0 + ;; + *) + printf 'Error: unknown action %s (validate|score|compare|emit-plugin-eval)\n' "$ACTION" >&2 + exit 2 + ;; +esac diff --git a/plugins/skill-quality/scripts/measure-invocation.test.sh b/plugins/skill-quality/scripts/measure-invocation.test.sh new file mode 100755 index 00000000000..8b5a286b739 --- /dev/null +++ b/plugins/skill-quality/scripts/measure-invocation.test.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# Black-box contract test for measure-invocation.sh. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SUT="$SCRIPT_DIR/measure-invocation.sh" + +fails=0 +pass() { printf 'ok - %s\n' "$1"; } +fail() { + printf 'FAIL - %s\n' "$1" >&2 + fails=$((fails + 1)) +} + +if ! command -v jq >/dev/null 2>&1; then + printf 'FAIL - jq is required\n' >&2 + exit 1 +fi + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +export MEASURE_INVOCATION_REPO_ROOT="$TMP" + +run() { bash "$SUT" "$@"; } + +make_skill() { + local dir="$1" desc="$2" + mkdir -p "$dir" + printf '%s\n' "--- +description: \"$desc\" +disable-model-invocation: false +--- + +# Fixture +" >"$dir/SKILL.md" +} + +# A probe set of 20 queries, both splits, both polarities. +write_probes() { + local skill_dir="$1" + local competitor_dir="$2" + mkdir -p "$TMP/probes" + cat >"$TMP/probes/target.json" <&1)" +rc=$? +if [[ $rc -eq 0 ]] && grep -q 'validate: PASS' <<<"$out"; then + pass "validate accepts a 20-query both-split set" +else + fail "validate should pass the fixture (rc=$rc): $out" +fi + +mkdir -p "$TMP/thin/probes" +jq '.queries = .queries[0:3]' "$TMP/probes/target.json" >"$TMP/thin/probes/target.json" +# Fix skill_dir in the thin copy to still resolve. +out="$(run validate "$TMP/thin/probes" 2>&1)" +rc=$? +if [[ $rc -eq 1 ]] && grep -q 'FAIL:' <<<"$out"; then + pass "validate FAILs a set under 8 queries" +else + fail "thin set should FAIL (rc=$rc): $out" +fi + +mkdir -p "$TMP/nosplit/probes" +jq '.queries[].split = "holdout"' "$TMP/probes/target.json" >"$TMP/nosplit/probes/target.json" +out="$(run validate "$TMP/nosplit/probes" 2>&1)" +rc=$? +if [[ $rc -eq 1 ]] && grep -q 'split must be train or validation' <<<"$out"; then + pass "validate FAILs an unknown split" +else + fail "unknown split should FAIL (rc=$rc): $out" +fi + +score_json="$(run score "$TMP/probes" 2>"$TMP/score.err")" +rc=$? +printf '%s\n' "$score_json" >"$TMP/score.json" +if [[ $rc -eq 0 ]] && jq -e '.method == "listing-overlap" and (.skills | length == 1)' "$TMP/score.json" >/dev/null; then + pass "score emits a listing-overlap report" +else + fail "score should emit a report (rc=$rc): $(cat "$TMP/score.err") $score_json" +fi + +train_tr="$(jq -r '.skills[0].splits.train.trigger_rate' "$TMP/score.json")" +val_tr="$(jq -r '.skills[0].splits.validation.trigger_rate' "$TMP/score.json")" +train_ft="$(jq -r '.skills[0].splits.train.false_trigger_rate' "$TMP/score.json")" +val_ft="$(jq -r '.skills[0].splits.validation.false_trigger_rate' "$TMP/score.json")" +# Fixture descriptions are built so positives quote the target triggers and +# negatives quote the rival's. Expect a strong floor, not a perfect 1.0 on +# every token-overlap quirk. +awk -v t="$train_tr" -v v="$val_tr" 'BEGIN { exit !((t+0) >= 0.5 && (v+0) >= 0.5) }' +tr_ok=$? +awk -v t="$train_ft" -v v="$val_ft" 'BEGIN { exit !((t+0) <= 0.5 && (v+0) <= 0.5) }' +ft_ok=$? +if [[ $tr_ok -eq 0 && $ft_ok -eq 0 ]]; then + pass "score reports train and validation trigger_rate >= 0.5 and false_trigger_rate <= 0.5 (train=$train_tr/$train_ft val=$val_tr/$val_ft)" +else + fail "rates off the fixture floor (train=$train_tr/$train_ft val=$val_tr/$val_ft) stderr=$(cat "$TMP/score.err")" +fi + +n_train="$(jq '.skills[0].splits.train.n' "$TMP/score.json")" +n_val="$(jq '.skills[0].splits.validation.n' "$TMP/score.json")" +if [[ "$n_train" == "12" && "$n_val" == "8" ]]; then + pass "score keeps the 12/8 train/validation split" +else + fail "expected 12/8 split, got $n_train/$n_val" +fi + +cmp_json="$(run compare "$TMP/score.json" "$TMP/score.json")" +delta="$(jq -r '.skills[0].validation.trigger_rate_delta' <<<"$cmp_json")" +if awk -v d="$delta" 'BEGIN { exit !(d+0 == 0) }'; then + pass "compare against self is a zero validation delta" +else + fail "self-compare delta should be 0, got $delta ($cmp_json)" +fi + +# Treatment: flip one validation miss into a hit by copying baseline and +# forcing predicted=true on a false-trigger-negative if any, else bump rates +# via jq so compare shows a non-zero delta. +jq ' + .skills[0].splits.validation.trigger_rate = ((.skills[0].splits.validation.trigger_rate // 0) + 0.25) + | .skills[0].splits.validation.false_trigger_rate = ((.skills[0].splits.validation.false_trigger_rate // 0) * 0.5) +' "$TMP/score.json" >"$TMP/treatment.json" +cmp_json="$(run compare "$TMP/score.json" "$TMP/treatment.json")" +tdelta="$(jq -r '.skills[0].validation.trigger_rate_delta' <<<"$cmp_json")" +if awk -v d="$tdelta" 'BEGIN { exit !(d+0 > 0) }'; then + pass "compare reports a positive validation trigger_rate delta after a rewrite-shaped bump" +else + fail "expected positive delta, got $tdelta ($cmp_json)" +fi + +emit_dir="$TMP/eval-out" +run emit-plugin-eval "$TMP/probes" "$emit_dir" >/dev/null +n_cases="$(find "$emit_dir" -mindepth 1 -maxdepth 1 -type d | wc -l)" +if [[ "$n_cases" -eq 20 ]] && [[ -f "$emit_dir/fixture-target-pos-train-01/prompt.md" ]] && + [[ -f "$emit_dir/fixture-target-pos-train-01/graders/skill-fired.md" ]] && + [[ -f "$emit_dir/fixture-target-neg-train-01/graders/skill-quiet.md" ]]; then + pass "emit-plugin-eval writes 20 cases with fire and quiet graders" +else + fail "emit-plugin-eval layout wrong (n=$n_cases)" +fi + +if [[ $fails -gt 0 ]]; then + printf 'measure-invocation.test.sh: %s failed\n' "$fails" >&2 + exit 1 +fi +printf 'measure-invocation.test.sh: all passed\n' +exit 0 From 71d15dfcb307f29812076a2f1f813b7392448657 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 28 Sep 2026 04:10:57 +0000 Subject: [PATCH 2/7] feat(skill-quality): wire measure-invocation into check and CI Expose validate/score/compare as /skill-quality:check measure-invocation, document the command, and run the harness self-test plus probe validate on every CI event. Version 0.24.6. Co-authored-by: Kyle Sexton --- .github/workflows/ci.yml | 4 ++ .../skill-quality/.claude-plugin/plugin.json | 2 +- plugins/skill-quality/CHANGELOG.md | 14 +++++++ plugins/skill-quality/README.md | 22 ++++++++-- plugins/skill-quality/skills/check/SKILL.md | 42 +++++++++++++++++-- .../skills/check/evals/evals.json | 13 ++++++ 6 files changed, 90 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d4dc0050d9..50db962786d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1362,6 +1362,10 @@ jobs: run: scripts/check-skill-precompute-compose.sh --all - name: Test the shared listing-budget reporter run: bash plugins/skill-quality/scripts/check-listing-budget.test.sh + - name: Test the invocation-probe harness + run: bash plugins/skill-quality/scripts/measure-invocation.test.sh + - name: Validate invocation probes + run: bash plugins/skill-quality/scripts/measure-invocation.sh validate plugins/skill-quality/probes # Report-only (always exit 0 — see the script's own header): pools every # plugin's skills root into ONE shared aggregate, the shape a consumer who # installs the whole marketplace actually experiences. diff --git a/plugins/skill-quality/.claude-plugin/plugin.json b/plugins/skill-quality/.claude-plugin/plugin.json index 0cf96dcea8d..b23bafa74fe 100644 --- a/plugins/skill-quality/.claude-plugin/plugin.json +++ b/plugins/skill-quality/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "skill-quality", - "version": "0.24.10", + "version": "0.24.11", "description": "Skill-authoring QA tooling: a static contract checker that runs twenty-six deterministic checks over a Claude Code skill (frontmatter, explicit invocation mode, description/verb-contract polarity, per-skill listing-entry cap, trigger-keyword preservation, line caps, broken internal refs, markdownlint, gotchas surface, evals presence, precompute opportunity, completion-criteria signal, injection shell-declaration, fresh-eyes declaration conformance), a shared skill-listing budget reporter across a set of skills, and a bundled evals.json schema plus a deterministic eval-quality lint (duplicate case identities, missing fixtures, empty or vague grading criteria, set-coverage warnings). Runs against any repo's skills directory via the convention-resolution ladder, with no baked layout.", "author": { "name": "Melodic Software", diff --git a/plugins/skill-quality/CHANGELOG.md b/plugins/skill-quality/CHANGELOG.md index f455046b8fc..de6422bf8de 100644 --- a/plugins/skill-quality/CHANGELOG.md +++ b/plugins/skill-quality/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to the `skill-quality` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.24.11] - 2026-09-28 + +### Added + +- **`measure-invocation`: a repeatable description-invocation probe harness (#3526).** Scores + whether a skill's listing text would win the requests it should, with a train/validation split + and a false-trigger rate. Default method is a deterministic lexical listing-overlap floor that + CI re-runs without a model. `emit-plugin-eval` writes `claude plugin eval` cases for an + on-demand live run; `compare` prints the delta versus a baseline so a rewrite cannot hide a + validation drop behind a train gain. Seed probes (20 queries each) cover `skill-quality:check` + and `mcp-tools:audit`. Fleet-wide description rewrites stay attended. Command: + `bash plugins/skill-quality/scripts/measure-invocation.sh validate|score|compare|emit-plugin-eval`. + Contract: `reference/invocation-probes.md`. + ## [0.24.10] - 2026-09-28 ### Added diff --git a/plugins/skill-quality/README.md b/plugins/skill-quality/README.md index 60c9f2fecee..f5affbb18c7 100644 --- a/plugins/skill-quality/README.md +++ b/plugins/skill-quality/README.md @@ -2,8 +2,9 @@ A Claude Code plugin for **skill-authoring QA**: it runs a static, deterministic contract gate over a skill directory, reports the shared listing-budget estimate across a set of skills, and validates a -skill's `evals.json` against a bundled schema plus a deterministic eval-quality lint. No model -invocation anywhere. The same checks run identically in a session, a pre-commit hook, or CI. +skill's `evals.json` against a bundled schema plus a deterministic eval-quality lint, and scores +description-driven auto-invocation probes (`measure-invocation`). Default paths invoke no model. +The same checks run identically in a session, a pre-commit hook, or CI. The drift static analysis catches best is a rewrite silently dropping a `description` trigger phrase, which can degrade a skill's auto-invocation. Check 3 compares the trigger phrases against @@ -13,7 +14,7 @@ the reviewer to confirm the description still names that intent, or to restore t | Skill | What it does | |---|---| -| `/skill-quality:check` | Runs the contract gate (`check`), reports the shared listing budget (`listing-budget`), or schema-validates and quality-lints evals (`validate-evals`) for one skill, a set of roots, or every skill. | +| `/skill-quality:check` | Runs the contract gate (`check`), reports the shared listing budget (`listing-budget`), schema-validates and quality-lints evals (`validate-evals`), or scores description auto-invocation probes (`measure-invocation`). | | `/skill-quality:setup` | Check-only: resolves and verifies the skills directory and prints the guidance for routing a non-default `skills_root` change through Claude Code. | ## Checks @@ -123,6 +124,7 @@ because the budget it reports is the shared one. /skill-quality:check validate-evals my-skill # schema-check + quality-lint evals.json /skill-quality:check listing-budget # report the shared budget over the resolved root /skill-quality:check listing-budget plugins/*/skills # pool every plugin's root into one aggregate +/skill-quality:check measure-invocation # score description auto-invocation probes ``` ## Skills directory is never baked in @@ -169,6 +171,20 @@ stands alone. - `npx` (Node) is optional; without it the markdownlint check downgrades to a warning and the other twenty-five still gate. +## Description-invocation probes + +`measure-invocation` scores whether a skill's listing text would win the requests it should +(and stay quiet on the ones it should not). Default method is a deterministic lexical +listing-overlap floor; `emit-plugin-eval` writes `claude plugin eval` cases for a live run. +Contract: [`reference/invocation-probes.md`](reference/invocation-probes.md). + +```shell +bash plugins/skill-quality/scripts/measure-invocation.sh validate plugins/skill-quality/probes +bash plugins/skill-quality/scripts/measure-invocation.sh score plugins/skill-quality/probes +bash plugins/skill-quality/scripts/measure-invocation.sh compare \ + plugins/skill-quality/probes/baselines/listing-overlap.json /tmp/score.json +``` + ## Configuration diff --git a/plugins/skill-quality/skills/check/SKILL.md b/plugins/skill-quality/skills/check/SKILL.md index 0a008c304af..5edf183a0c9 100644 --- a/plugins/skill-quality/skills/check/SKILL.md +++ b/plugins/skill-quality/skills/check/SKILL.md @@ -1,6 +1,6 @@ --- -description: "Skill-authoring QA for Claude Code skills. Use when: 'check this skill', 'skill quality', 'lint my skill', 'is this SKILL.md valid', 'validate skill frontmatter', 'check skill before publishing', 'validate evals.json', 'shared listing budget', 'is the skill listing overflowing', or before shipping a skill or plugin. Actions: `check [| ...]` runs a twenty-six-check static contract gate over one skill, or over every skill under each given root, and reports PASS/FAIL with warnings; `validate-evals []` checks a skill's evals/evals.json against the bundled schema, then runs a deterministic eval-quality lint; `listing-budget [ ...]` reports the SHARED aggregate listing-budget estimate across every listing-eligible skill under the resolved root(s). Advisory only, never blocks. Not for: writing new skills, or running model-graded evals." -argument-hint: "[check|validate-evals|listing-budget] [ ...]. Omit the action for check; omit the name/root to run over every skill under the resolved root; give check one or more roots to gate several trees in one run" +description: "Skill-authoring QA for Claude Code skills. Use when: 'check this skill', 'skill quality', 'lint my skill', 'is this SKILL.md valid', 'validate skill frontmatter', 'check skill before publishing', 'validate evals.json', 'shared listing budget', 'is the skill listing overflowing', 'does this description actually trigger', 'invocation probe', 'trigger rate', or before shipping a skill or plugin. Actions: `check [| ...]` runs a twenty-six-check static contract gate over one skill, or over every skill under each given root, and reports PASS/FAIL with warnings; `validate-evals []` checks a skill's evals/evals.json against the bundled schema, then runs a deterministic eval-quality lint; `listing-budget [ ...]` reports the SHARED aggregate listing-budget estimate across every listing-eligible skill under the resolved root(s); `measure-invocation` scores description auto-invocation probes. Advisory only, never blocks. Not for: writing new skills." +argument-hint: "[check|validate-evals|listing-budget|measure-invocation] [ ...]. Omit the action for check; measure-invocation takes validate|score|compare|emit-plugin-eval" user-invocable: true disable-model-invocation: false shell: bash @@ -18,7 +18,9 @@ JSON schema, then runs the bundled `check-evals-quality.sh`, a deterministic eva (duplicate case ids/names, missing fixtures, empty or vague grading criteria, set-coverage warnings) that goes beyond structure without ever running a model-graded eval. The `listing-budget` action runs `check-listing-budget.sh`, a separate, always-advisory report on the SHARED listing budget every loaded skill draws from together (a different, cross-skill -limit from `check`'s per-skill entry cap). +limit from `check`'s per-skill entry cap). The `measure-invocation` action runs +`measure-invocation.sh`, a repeatable probe harness that scores whether a skill's listing text +would win the requests it should. ## Skills-directory resolution @@ -80,6 +82,8 @@ Parse `$ARGUMENTS`: - **`listing-budget [ ...]`**. Pool every listing-eligible skill under each given root into ONE shared aggregate (e.g. every plugin's skills dir in a marketplace repo). Every root given must exist. +- **`measure-invocation`** *(default `validate` then `score`)*. Run the description-invocation + probe harness. See Action: measure-invocation. ## Action: check @@ -188,6 +192,35 @@ Both claims are verified 2026-09-06 against Claude Code 2.1.263 and the skills p short", which names `"name-only"` as the way to free budget). Recheck when either section stops carrying its statement, or when a release note names skill listing budget or `skillOverrides`. +## Action: measure-invocation + +Repeatable probe harness for whether a skill's listing text would win the auto-invocation match. +Default method is a deterministic lexical floor (`listing-overlap`) that CI re-runs without a +model. Model-graded `claude plugin eval` cases are emitted on demand. Contract: +[reference/invocation-probes.md](../../reference/invocation-probes.md). + +1. Resolve the probes directory: `${CLAUDE_PLUGIN_ROOT}/probes` when present, else the + marketplace path `plugins/skill-quality/probes`. +2. Run, in this order unless the user named one sub-action: + + ```shell + bash "${CLAUDE_PLUGIN_ROOT}/scripts/measure-invocation.sh" validate + bash "${CLAUDE_PLUGIN_ROOT}/scripts/measure-invocation.sh" score + ``` + + `compare ` prints per-skill train and validation deltas. `emit-plugin-eval + ` writes `claude plugin eval` cases. +3. Report per skill, per split (`train` and `validation`): `trigger_rate` and + `false_trigger_rate`, sample size, and the method name. A rewrite is compared with `compare` + against `probes/baselines/listing-overlap.json` (or a later model-graded snapshot). The action + is complete when both splits are named; a single blended rate is not the done-condition. +4. `listing-overlap` is a floor, not a model-graded auto-invocation rate. Say so when reporting. + Do not treat a 1.0 positive rate on the floor as proof the description saturates live + auto-invocation. + +The seed probe set is two skills chosen for competitor density (`skill-quality:check`, +`mcp-tools:audit`). Fleet-wide description rewrites stay attended. + ## Cross-skill invocation (doctrine) The Skill tool executes one skill within the main conversation, so a step needing two skills is two @@ -202,6 +235,9 @@ tool. This gate does not automate that reachability check; author and review aga ## Gotchas +- `measure-invocation`'s default `listing-overlap` method is a lexical floor. A 1.0 positive + trigger rate means the description already contains the request's nouns, not that live + auto-invocation saturates. Report both splits and name the method. - A git repository is optional. Git-backed checks (trigger-keyword preservation, vendor byte-identity, stale-tracking metadata, committed-artifact scan) skip with a note when cwd is outside a repo. Marketplace plugin-cache installs are plain trees. Set diff --git a/plugins/skill-quality/skills/check/evals/evals.json b/plugins/skill-quality/skills/check/evals/evals.json index 8d91b809c3b..00a515593d5 100644 --- a/plugins/skill-quality/skills/check/evals/evals.json +++ b/plugins/skill-quality/skills/check/evals/evals.json @@ -109,6 +109,19 @@ "Output explains the mismatch is between the description/verb contract (read-only vs mutate) and is a factual-consistency defect in the listing surface", "Output does NOT treat the WARN as a mandate to add a --fix path or rename the skill" ] + }, + { + "id": 10, + "name": "measure-invocation-reports-both-splits", + "prompt": "measure-invocation", + "expected_output": "Routes to the measure-invocation action: validates the probe set, then scores it, and reports per skill a train trigger-rate and false-trigger-rate plus a validation trigger-rate and false-trigger-rate. Names listing-overlap as a lexical floor, not a model-graded auto-invocation rate. Does not rewrite any skill description.", + "files": [], + "expectations": [ + "Output runs the measure-invocation harness (validate and/or score), not the static contract gate", + "Output reports trigger_rate and false_trigger_rate for both the train split and the validation split", + "Output states that listing-overlap is a floor, not a live auto-invocation rate", + "Output does not rewrite skill descriptions" + ] } ] } From 9709d829bbc35c128dcec8f321621c984f162a8a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:46:57 -0400 Subject: [PATCH 3/7] fix(skill-quality): drop dead rate() helper from measure-invocation.sh shellcheck (SC2329) flagged rate() as never invoked. The scoring path already computes the same ratio inline via a jq rate(hits; n) function; the bash helper duplicated that math and nothing called it. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PT4esxbdC7eQiwQxy35Mie --- plugins/skill-quality/scripts/measure-invocation.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/plugins/skill-quality/scripts/measure-invocation.sh b/plugins/skill-quality/scripts/measure-invocation.sh index 6281bcffa21..bfef3eae777 100755 --- a/plugins/skill-quality/scripts/measure-invocation.sh +++ b/plugins/skill-quality/scripts/measure-invocation.sh @@ -222,15 +222,6 @@ cmd_validate() { note "validated $nfiles probe file(s)" } -rate() { - local hits="$1" n="$2" - if ((n == 0)); then - printf 'null' - return - fi - awk -v h="$hits" -v n="$n" 'BEGIN { printf "%.4f", h / n }' -} - cmd_score() { local method="listing-overlap" if [[ "${1:-}" == "--method" ]]; then From 653d592496296f3359df7b40be786234155a098e Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:55:13 -0400 Subject: [PATCH 4/7] fix(skill-quality): fix over-escaped \s and annotate the emitted grader regex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit shell-portability-lint flagged the bare \s escape in the generated input_match regex (a class no BSD grep/sed uses, but the literal token appears in this .sh regardless of context). It is data written into an eval grader's YAML frontmatter, consumed by the grader's own regex engine, never by shell grep/sed, so annotate it portability-ok at the site per the gate's documented escape. While there: the source had \\\\s (source-level double escaping), which bash's double-quote unescaping turns into a literal \\s in the emitted file instead of the single \s the sibling grader (plugins/evals/evals/measurable-criterion/graders/skill-fired.md) uses — reduced to match. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PT4esxbdC7eQiwQxy35Mie --- plugins/skill-quality/scripts/measure-invocation.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/skill-quality/scripts/measure-invocation.sh b/plugins/skill-quality/scripts/measure-invocation.sh index bfef3eae777..fedb6d003b0 100755 --- a/plugins/skill-quality/scripts/measure-invocation.sh +++ b/plugins/skill-quality/scripts/measure-invocation.sh @@ -463,7 +463,7 @@ ${request} printf '%s\n' "--- type: tool_used tool: Skill -input_match: '\"skill\"\\\\s*:\\\\s*\"(?:${plugin}:)?${leaf}\"' +input_match: '\"skill\"\\s*:\\s*\"(?:${plugin}:)?${leaf}\"' # portability-ok: \\s is read by the eval grader's own regex engine, never shell grep/sed min: 1 --- " >"$case_dir/graders/skill-fired.md" @@ -471,7 +471,7 @@ min: 1 printf '%s\n' "--- type: tool_used tool: Skill -input_match: '\"skill\"\\\\s*:\\\\s*\"(?:${plugin}:)?${leaf}\"' +input_match: '\"skill\"\\s*:\\s*\"(?:${plugin}:)?${leaf}\"' # portability-ok: \\s is read by the eval grader's own regex engine, never shell grep/sed min: 0 max: 0 arm: both From 9902d081729242c5bce6ac24244a62366b6ffa88 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:19:30 -0400 Subject: [PATCH 5/7] fix(skill-quality): keep the portability-ok annotation out of emitted grader YAML The previous commit's trailing comment shipped inside every emitted skill-fired.md/skill-quiet.md's input_match value. Move the regex into a match_re variable with the portability-ok annotation on its assignment line instead, so the emitted grader files stay byte-identical to the working sibling example (plugins/evals/evals/measurable-criterion/graders/skill-fired.md). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01PT4esxbdC7eQiwQxy35Mie --- plugins/skill-quality/scripts/measure-invocation.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/skill-quality/scripts/measure-invocation.sh b/plugins/skill-quality/scripts/measure-invocation.sh index fedb6d003b0..a9d16edc702 100755 --- a/plugins/skill-quality/scripts/measure-invocation.sh +++ b/plugins/skill-quality/scripts/measure-invocation.sh @@ -459,11 +459,14 @@ expected_outcome: Skill ${skill} $(if [[ "$expect" == "true" ]]; then echo fires ${request} " >"$case_dir/prompt.md" + # portability-ok: \s is read by the eval grader's own regex engine, never + # shell grep/sed; kept out of the emitted YAML so the grader files stay clean. + local match_re="\"skill\"\\s*:\\s*\"(?:${plugin}:)?${leaf}\"" if [[ "$expect" == "true" ]]; then printf '%s\n' "--- type: tool_used tool: Skill -input_match: '\"skill\"\\s*:\\s*\"(?:${plugin}:)?${leaf}\"' # portability-ok: \\s is read by the eval grader's own regex engine, never shell grep/sed +input_match: '${match_re}' min: 1 --- " >"$case_dir/graders/skill-fired.md" @@ -471,7 +474,7 @@ min: 1 printf '%s\n' "--- type: tool_used tool: Skill -input_match: '\"skill\"\\s*:\\s*\"(?:${plugin}:)?${leaf}\"' # portability-ok: \\s is read by the eval grader's own regex engine, never shell grep/sed +input_match: '${match_re}' min: 0 max: 0 arm: both From 1da45bc8713689e90a9e3ea66216bab0d89fcaa2 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 28 Sep 2026 14:29:55 -0400 Subject: [PATCH 6/7] fix(skill-quality): harden measure-invocation and restore check description Repo-relative listing_file in the baseline, real exit codes from compare and emit-plugin-eval, null-safe deltas, boolean expect_trigger check, sorted competitor order, docs no longer claim CI re-scores. The check skill description keeps main's text plus one action sentence; baseline regenerated against it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Hx63mvV8t39y644DQ3uLnn --- plugins/skill-quality/CHANGELOG.md | 3 +- .../probes/baselines/listing-overlap.json | 30 ++++++------ .../reference/invocation-probes.md | 4 +- .../scripts/measure-invocation.sh | 47 ++++++++++--------- .../scripts/measure-invocation.test.sh | 18 ++++++- plugins/skill-quality/skills/check/SKILL.md | 7 +-- 6 files changed, 64 insertions(+), 45 deletions(-) diff --git a/plugins/skill-quality/CHANGELOG.md b/plugins/skill-quality/CHANGELOG.md index de6422bf8de..de7b852070d 100644 --- a/plugins/skill-quality/CHANGELOG.md +++ b/plugins/skill-quality/CHANGELOG.md @@ -10,7 +10,8 @@ All notable changes to the `skill-quality` plugin are documented here. Format fo - **`measure-invocation`: a repeatable description-invocation probe harness (#3526).** Scores whether a skill's listing text would win the requests it should, with a train/validation split and a false-trigger rate. Default method is a deterministic lexical listing-overlap floor that - CI re-runs without a model. `emit-plugin-eval` writes `claude plugin eval` cases for an + runs without a model. CI validates the probe schema and runs the harness tests; `score` and + `compare` are run by hand. `emit-plugin-eval` writes `claude plugin eval` cases for an on-demand live run; `compare` prints the delta versus a baseline so a rewrite cannot hide a validation drop behind a train gain. Seed probes (20 queries each) cover `skill-quality:check` and `mcp-tools:audit`. Fleet-wide description rewrites stay attended. Command: diff --git a/plugins/skill-quality/probes/baselines/listing-overlap.json b/plugins/skill-quality/probes/baselines/listing-overlap.json index 7599e35a532..b9c0857b36c 100644 --- a/plugins/skill-quality/probes/baselines/listing-overlap.json +++ b/plugins/skill-quality/probes/baselines/listing-overlap.json @@ -1,12 +1,12 @@ { "method": "listing-overlap", - "generated_at": "2026-09-28T04:09:46Z", + "generated_at": "2026-09-28T18:27:15Z", "skills": [ { "skill": "mcp-tools:audit", "plugin": "mcp-tools", "method": "listing-overlap", - "listing_file": "/tmp/wt-3526/plugins/mcp-tools/skills/audit/SKILL.md", + "listing_file": "plugins/mcp-tools/skills/audit/SKILL.md", "splits": { "train": { "n": 12, @@ -234,16 +234,16 @@ "skill": "skill-quality:check", "plugin": "skill-quality", "method": "listing-overlap", - "listing_file": "/tmp/wt-3526/plugins/skill-quality/skills/check/SKILL.md", + "listing_file": "plugins/skill-quality/skills/check/SKILL.md", "splits": { "train": { "n": 12, "n_positive": 6, "n_negative": 6, "trigger_hits": 6, - "false_triggers": 4, + "false_triggers": 2, "trigger_rate": 1, - "false_trigger_rate": 0.6666666666666666 + "false_trigger_rate": 0.3333333333333333 }, "validation": { "n": 8, @@ -312,7 +312,7 @@ "expect_trigger": true, "predicted": true, "correct": true, - "target_score": 38, + "target_score": 39, "winner": "skill-quality:check", "request": "validate evals.json for my new skill" }, @@ -390,20 +390,20 @@ "id": "neg-train-04", "split": "train", "expect_trigger": false, - "predicted": true, - "correct": false, - "target_score": 1, - "winner": "skill-quality:check", + "predicted": false, + "correct": true, + "target_score": 0, + "winner": "none", "request": "what permissions are actually in effect on this machine" }, { "id": "neg-train-05", "split": "train", "expect_trigger": false, - "predicted": true, - "correct": false, - "target_score": 3, - "winner": "skill-quality:check", + "predicted": false, + "correct": true, + "target_score": 2, + "winner": "skill-quality:check+playbooks:skill-authoring", "request": "audit MCP tool descriptions against the spec" }, { @@ -422,7 +422,7 @@ "expect_trigger": false, "predicted": false, "correct": true, - "target_score": 18, + "target_score": 17, "winner": "playbooks:skill-authoring", "request": "skill best practices for writing a description" }, diff --git a/plugins/skill-quality/reference/invocation-probes.md b/plugins/skill-quality/reference/invocation-probes.md index e6c4668fe4f..7940dc3b704 100644 --- a/plugins/skill-quality/reference/invocation-probes.md +++ b/plugins/skill-quality/reference/invocation-probes.md @@ -11,7 +11,7 @@ that match. This harness does, in two layers: | Method | What it is | When to run | |---|---|---| -| `listing-overlap` | Deterministic floor: quoted-trigger hits plus content-token overlap against the target listing and named competitors | CI and every rewrite. The committed baseline uses this method. | +| `listing-overlap` | Deterministic floor: quoted-trigger hits plus content-token overlap against the target listing and named competitors | By hand, before and after a rewrite (`score` then `compare`). CI validates the probe schema and runs the harness tests only. The committed baseline uses this method. | | `emit-plugin-eval` | Writes `claude plugin eval` cases (`tool_used: Skill`) per probe | On demand, within one plugin. The CLI loads a single plugin, so cross-plugin competitors are out of scope for this method. | | Headless `claude -p` | Live cross-plugin selection | On demand, documented, not wrapped here. Replay its fire/quiet bits through `compare`. | @@ -76,7 +76,7 @@ nouns the seed positives use. False-trigger rates are the gap the floor can see: | Skill | Train false-trigger | Validation false-trigger | |---|---|---| | `mcp-tools:audit` | 0.00 | 0.25 | -| `skill-quality:check` | 0.67 | 0.50 | +| `skill-quality:check` | 0.33 | 0.50 | False positives on this floor are token-greedy: `check this SKILL.md before publishing` scores as `mcp-tools:audit` because that listing contains `check`; `write me a brand-new diff --git a/plugins/skill-quality/scripts/measure-invocation.sh b/plugins/skill-quality/scripts/measure-invocation.sh index a9d16edc702..f51c77a7f39 100755 --- a/plugins/skill-quality/scripts/measure-invocation.sh +++ b/plugins/skill-quality/scripts/measure-invocation.sh @@ -3,8 +3,8 @@ # # Measures whether a skill's listing text would win the requests it should # (and stay quiet on the ones it should not). Default method is a -# deterministic lexical listing-overlap floor that CI can re-run without a -# model. Model-graded `claude plugin eval` cases are emitted on demand; a +# deterministic lexical listing-overlap floor that runs without a model. +# Model-graded `claude plugin eval` cases are emitted on demand; a # replayed model run uses the same report shape so `compare` can show a # rewrite delta. # @@ -93,7 +93,7 @@ listing_tokens() { local text="$1" printf '%s\n' "$text" | skill_frontmatter::extract_triggers | sed "s/^'//;s/'$//" | tr '[:upper:]' '[:lower:]' printf '%s\n' "$text" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '\n' | - awk 'length($0) >= 3 && $0 !~ /^(the|and|for|with|when|use|this|that|from|into|not|your|our|are|was|were|has|have|had|its|but|or|nor|any|all|can|may|per|via|vs|than|then|also|just|only|each|both|same|such|into|over|under|after|before|about|into)$/' + awk 'length($0) >= 3 && $0 !~ /^(the|and|for|with|when|use|this|that|from|into|not|your|our|are|was|were|has|have|had|its|but|nor|any|all|can|may|per|via|than|then|also|just|only|each|both|same|such|over|under|after|before|about)$/' } score_request() { @@ -196,6 +196,11 @@ cmd_validate() { if [[ -n "$dup" ]]; then err "$f ($skill): duplicate query id(s): $(printf '%s' "$dup" | tr '\n' ' ')" fi + local bad_expect + bad_expect="$(jq -r '.queries[] | select(.expect_trigger | type != "boolean") | .id' "$f")" + if [[ -n "$bad_expect" ]]; then + err "$f ($skill): expect_trigger must be a boolean; bad id(s): $(printf '%s' "$bad_expect" | tr '\n' ' ')" + fi local bad_split bad_split="$(jq -r '.queries[] | select(.split != "train" and .split != "validation") | .id' "$f")" if [[ -n "$bad_split" ]]; then @@ -229,7 +234,7 @@ cmd_score() { shift 2 fi if [[ "$method" != "listing-overlap" ]]; then - printf 'Error: unknown method %s (listing-overlap is the CI floor; emit-plugin-eval for the live path)\n' "$method" >&2 + printf 'Error: unknown method %s (listing-overlap is the default floor; emit-plugin-eval for the live path)\n' "$method" >&2 exit 2 fi local dir="${1:-}" @@ -304,7 +309,7 @@ cmd_score() { best=0 fi local cname - for cname in "${!comp_listing[@]}"; do + for cname in $(printf '%s\n' "${!comp_listing[@]}" | sort); do local cscore cscore="$(score_request "$request" "${comp_listing[$cname]}")" if ((cscore > best)); then @@ -317,11 +322,6 @@ cmd_score() { winner="$skill+$cname" fi done - if [[ "$expect" == "true" && "$predicted" == "true" ]]; then - : - elif [[ "$expect" == "false" && "$predicted" == "false" ]]; then - : - fi local correct=false if [[ "$expect" == "true" && "$predicted" == "true" ]]; then correct=true @@ -362,7 +362,7 @@ cmd_score() { ' <<<"$cases_json")" jq -n --arg skill "$skill" --arg plugin "$plugin" --arg method "$method" \ - --arg listing_file "$md" --argjson splits "$split_json" --argjson cases "$cases_json" \ + --arg listing_file "${md#"$root"/}" --argjson splits "$split_json" --argjson cases "$cases_json" \ '{skill:$skill, plugin:$plugin, method:$method, listing_file:$listing_file, splits:$splits, cases:$cases}' \ >"$tmp/skill-$skill_idx.json" skill_idx=$((skill_idx + 1)) @@ -392,6 +392,7 @@ cmd_compare() { exit 2 fi jq -n --slurpfile b "$base" --slurpfile t "$treat" ' + def delta($t; $b): if $t == null or $b == null then null else $t - $b end; ($b[0].skills) as $bs | ($t[0].skills) as $ts | { method_baseline: $b[0].method, @@ -404,18 +405,18 @@ cmd_compare() { train: { trigger_rate_baseline: $bb.splits.train.trigger_rate, trigger_rate_treatment: $t.splits.train.trigger_rate, - trigger_rate_delta: (($t.splits.train.trigger_rate // 0) - ($bb.splits.train.trigger_rate // 0)), + trigger_rate_delta: delta($t.splits.train.trigger_rate; $bb.splits.train.trigger_rate), false_trigger_rate_baseline: $bb.splits.train.false_trigger_rate, false_trigger_rate_treatment: $t.splits.train.false_trigger_rate, - false_trigger_rate_delta: (($t.splits.train.false_trigger_rate // 0) - ($bb.splits.train.false_trigger_rate // 0)) + false_trigger_rate_delta: delta($t.splits.train.false_trigger_rate; $bb.splits.train.false_trigger_rate) }, validation: { trigger_rate_baseline: $bb.splits.validation.trigger_rate, trigger_rate_treatment: $t.splits.validation.trigger_rate, - trigger_rate_delta: (($t.splits.validation.trigger_rate // 0) - ($bb.splits.validation.trigger_rate // 0)), + trigger_rate_delta: delta($t.splits.validation.trigger_rate; $bb.splits.validation.trigger_rate), false_trigger_rate_baseline: $bb.splits.validation.false_trigger_rate, false_trigger_rate_treatment: $t.splits.validation.false_trigger_rate, - false_trigger_rate_delta: (($t.splits.validation.false_trigger_rate // 0) - ($bb.splits.validation.false_trigger_rate // 0)) + false_trigger_rate_delta: delta($t.splits.validation.false_trigger_rate; $bb.splits.validation.false_trigger_rate) } } ] @@ -429,7 +430,7 @@ cmd_emit() { printf 'Error: emit-plugin-eval needs \n' >&2 exit 2 fi - mkdir -p "$out" + mkdir -p "$out" || return 1 local f local -a files mapfile -t files < <(probe_files "$dir") @@ -447,7 +448,7 @@ cmd_emit() { expect="$(jq -r ".queries[$i].expect_trigger" "$f")" request="$(jq -r ".queries[$i].request" "$f")" local case_dir="$out/${plugin}-${leaf}-${id}" - mkdir -p "$case_dir/graders" + mkdir -p "$case_dir/graders" || return 1 printf '%s\n' "--- description: invocation probe ${skill} ${id} (${split}, expect_trigger=${expect}) tags: [invocation-probe, ${split}] @@ -458,7 +459,7 @@ expected_outcome: Skill ${skill} $(if [[ "$expect" == "true" ]]; then echo fires --- ${request} -" >"$case_dir/prompt.md" +" >"$case_dir/prompt.md" || return 1 # portability-ok: \s is read by the eval grader's own regex engine, never # shell grep/sed; kept out of the emitted YAML so the grader files stay clean. local match_re="\"skill\"\\s*:\\s*\"(?:${plugin}:)?${leaf}\"" @@ -469,7 +470,7 @@ tool: Skill input_match: '${match_re}' min: 1 --- -" >"$case_dir/graders/skill-fired.md" +" >"$case_dir/graders/skill-fired.md" || return 1 else printf '%s\n' "--- type: tool_used @@ -479,12 +480,12 @@ min: 0 max: 0 arm: both --- -" >"$case_dir/graders/skill-quiet.md" +" >"$case_dir/graders/skill-quiet.md" || return 1 fi done note "emitted $nq plugin-eval cases for $skill under $out" done - cat >"$out/README.md" <<'EOF' + cat >"$out/README.md" <<'EOF' || return 1 Plugin-eval cases generated by `measure-invocation.sh emit-plugin-eval`. Run within one plugin at a time (the CLI loads a single plugin): @@ -522,11 +523,11 @@ case "$ACTION" in ;; compare) cmd_compare "$@" - exit 0 + exit $? ;; emit-plugin-eval) cmd_emit "$@" - exit 0 + exit $? ;; *) printf 'Error: unknown action %s (validate|score|compare|emit-plugin-eval)\n' "$ACTION" >&2 diff --git a/plugins/skill-quality/scripts/measure-invocation.test.sh b/plugins/skill-quality/scripts/measure-invocation.test.sh index 8b5a286b739..ed75fe692bb 100755 --- a/plugins/skill-quality/scripts/measure-invocation.test.sh +++ b/plugins/skill-quality/scripts/measure-invocation.test.sh @@ -105,7 +105,6 @@ fi mkdir -p "$TMP/thin/probes" jq '.queries = .queries[0:3]' "$TMP/probes/target.json" >"$TMP/thin/probes/target.json" -# Fix skill_dir in the thin copy to still resolve. out="$(run validate "$TMP/thin/probes" 2>&1)" rc=$? if [[ $rc -eq 1 ]] && grep -q 'FAIL:' <<<"$out"; then @@ -124,6 +123,16 @@ else fail "unknown split should FAIL (rc=$rc): $out" fi +mkdir -p "$TMP/strexpect/probes" +jq '.queries[0].expect_trigger = "true"' "$TMP/probes/target.json" >"$TMP/strexpect/probes/target.json" +out="$(run validate "$TMP/strexpect/probes" 2>&1)" +rc=$? +if [[ $rc -eq 1 ]] && grep -q 'expect_trigger must be a boolean' <<<"$out"; then + pass "validate FAILs a string expect_trigger" +else + fail "string expect_trigger should FAIL (rc=$rc): $out" +fi + score_json="$(run score "$TMP/probes" 2>"$TMP/score.err")" rc=$? printf '%s\n' "$score_json" >"$TMP/score.json" @@ -181,6 +190,13 @@ else fail "expected positive delta, got $tdelta ($cmp_json)" fi +printf 'not json\n' >"$TMP/malformed.json" +if run compare "$TMP/score.json" "$TMP/malformed.json" >/dev/null 2>&1; then + fail "compare on a malformed report should exit non-zero" +else + pass "compare on a malformed report exits non-zero" +fi + emit_dir="$TMP/eval-out" run emit-plugin-eval "$TMP/probes" "$emit_dir" >/dev/null n_cases="$(find "$emit_dir" -mindepth 1 -maxdepth 1 -type d | wc -l)" diff --git a/plugins/skill-quality/skills/check/SKILL.md b/plugins/skill-quality/skills/check/SKILL.md index 5edf183a0c9..eb9a8cb9ef8 100644 --- a/plugins/skill-quality/skills/check/SKILL.md +++ b/plugins/skill-quality/skills/check/SKILL.md @@ -1,5 +1,5 @@ --- -description: "Skill-authoring QA for Claude Code skills. Use when: 'check this skill', 'skill quality', 'lint my skill', 'is this SKILL.md valid', 'validate skill frontmatter', 'check skill before publishing', 'validate evals.json', 'shared listing budget', 'is the skill listing overflowing', 'does this description actually trigger', 'invocation probe', 'trigger rate', or before shipping a skill or plugin. Actions: `check [| ...]` runs a twenty-six-check static contract gate over one skill, or over every skill under each given root, and reports PASS/FAIL with warnings; `validate-evals []` checks a skill's evals/evals.json against the bundled schema, then runs a deterministic eval-quality lint; `listing-budget [ ...]` reports the SHARED aggregate listing-budget estimate across every listing-eligible skill under the resolved root(s); `measure-invocation` scores description auto-invocation probes. Advisory only, never blocks. Not for: writing new skills." +description: "Skill-authoring QA for Claude Code skills. Use when: 'check this skill', 'skill quality', 'lint my skill', 'is this SKILL.md valid', 'validate skill frontmatter', 'check skill before publishing', 'validate evals.json', 'shared listing budget', 'is the skill listing overflowing', or before shipping a skill or plugin. Actions: `check [| ...]` runs a twenty-six-check static contract gate over one skill, or over every skill under each given root, and reports PASS/FAIL with warnings; `validate-evals []` checks a skill's evals/evals.json against the bundled schema, then runs a deterministic eval-quality lint; `listing-budget [ ...]` reports the SHARED aggregate listing-budget estimate across every listing-eligible skill under the resolved root(s). `measure-invocation` scores description auto-invocation probes. Advisory only, never blocks. Not for: writing new skills, or running model-graded evals." argument-hint: "[check|validate-evals|listing-budget|measure-invocation] [ ...]. Omit the action for check; measure-invocation takes validate|score|compare|emit-plugin-eval" user-invocable: true disable-model-invocation: false @@ -195,8 +195,9 @@ carrying its statement, or when a release note names skill listing budget or `sk ## Action: measure-invocation Repeatable probe harness for whether a skill's listing text would win the auto-invocation match. -Default method is a deterministic lexical floor (`listing-overlap`) that CI re-runs without a -model. Model-graded `claude plugin eval` cases are emitted on demand. Contract: +Default method is a deterministic lexical floor (`listing-overlap`) that runs without a model. CI +validates the probe schema and runs the harness tests; `score` and `compare` are run by hand. +Model-graded `claude plugin eval` cases are emitted on demand. Contract: [reference/invocation-probes.md](../../reference/invocation-probes.md). 1. Resolve the probes directory: `${CLAUDE_PLUGIN_ROOT}/probes` when present, else the From 3a0d7ea68fc4be9b832ff0fb9ec661ced4cd7586 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:44:00 -0400 Subject: [PATCH 7/7] fix(skill-quality): fail closed on partial competitors, mismatched compares, stale emit dirs Addresses the Codex findings on 1da45bc: score errors on a competitor it cannot load, compare requires matching non-empty skill sets, emit-plugin-eval refuses a non-empty out dir, and the SKILL.md plugin-eval layout claim carries its four-part verification record. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Hx63mvV8t39y644DQ3uLnn --- plugins/skill-quality/CHANGELOG.md | 4 +++- .../scripts/measure-invocation.sh | 13 ++++++++++-- .../scripts/measure-invocation.test.sh | 21 +++++++++++++++++++ plugins/skill-quality/skills/check/SKILL.md | 7 ++++++- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/plugins/skill-quality/CHANGELOG.md b/plugins/skill-quality/CHANGELOG.md index de7b852070d..03514deb24e 100644 --- a/plugins/skill-quality/CHANGELOG.md +++ b/plugins/skill-quality/CHANGELOG.md @@ -14,7 +14,9 @@ All notable changes to the `skill-quality` plugin are documented here. Format fo `compare` are run by hand. `emit-plugin-eval` writes `claude plugin eval` cases for an on-demand live run; `compare` prints the delta versus a baseline so a rewrite cannot hide a validation drop behind a train gain. Seed probes (20 queries each) cover `skill-quality:check` - and `mcp-tools:audit`. Fleet-wide description rewrites stay attended. Command: + and `mcp-tools:audit`. `score` fails on a competitor it cannot load, `compare` fails unless + both reports cover the same skills, and `emit-plugin-eval` refuses a non-empty output + directory. Fleet-wide description rewrites stay attended. Command: `bash plugins/skill-quality/scripts/measure-invocation.sh validate|score|compare|emit-plugin-eval`. Contract: `reference/invocation-probes.md`. diff --git a/plugins/skill-quality/scripts/measure-invocation.sh b/plugins/skill-quality/scripts/measure-invocation.sh index f51c77a7f39..b067dbd5abc 100755 --- a/plugins/skill-quality/scripts/measure-invocation.sh +++ b/plugins/skill-quality/scripts/measure-invocation.sh @@ -277,14 +277,14 @@ cmd_score() { local comp_dir comp_dir="$(jq -r --arg c "$comp" '.competitor_dirs[$c] // empty' "$f")" if [[ -z "$comp_dir" ]]; then - warn "$skill: competitor '$comp' has no competitor_dirs entry; skipped" + err "$skill: competitor '$comp' has no competitor_dirs entry; a partial rival set would inflate the rates" continue fi local cmd if cmd="$(resolve_skill_md "$root" "$comp_dir")"; then comp_listing["$comp"]="$(load_listing "$cmd")" else - warn "$skill: competitor_dirs '$comp' -> '$comp_dir' does not resolve" + err "$skill: competitor_dirs '$comp' -> '$comp_dir' does not resolve" fi done < <(jq -r '.competitors[]?' "$f") @@ -391,6 +391,11 @@ cmd_compare() { printf 'Error: compare needs two report JSON files\n' >&2 exit 2 fi + if ! jq -e -n --slurpfile b "$base" --slurpfile t "$treat" \ + '([$b[0].skills[].skill] | sort) == ([$t[0].skills[].skill] | sort) and ($b[0].skills | length) > 0' >/dev/null; then + printf 'Error: compare needs two reports over the same non-empty skill set\n' >&2 + return 1 + fi jq -n --slurpfile b "$base" --slurpfile t "$treat" ' def delta($t; $b): if $t == null or $b == null then null else $t - $b end; ($b[0].skills) as $bs | ($t[0].skills) as $ts | @@ -430,6 +435,10 @@ cmd_emit() { printf 'Error: emit-plugin-eval needs \n' >&2 exit 2 fi + if [[ -d "$out" && -n "$(ls -A "$out")" ]]; then + printf 'Error: emit-plugin-eval needs an empty or new ; stale cases would still run\n' >&2 + exit 2 + fi mkdir -p "$out" || return 1 local f local -a files diff --git a/plugins/skill-quality/scripts/measure-invocation.test.sh b/plugins/skill-quality/scripts/measure-invocation.test.sh index ed75fe692bb..d72d17986f1 100755 --- a/plugins/skill-quality/scripts/measure-invocation.test.sh +++ b/plugins/skill-quality/scripts/measure-invocation.test.sh @@ -208,6 +208,27 @@ else fail "emit-plugin-eval layout wrong (n=$n_cases)" fi +if run emit-plugin-eval "$TMP/probes" "$emit_dir" >/dev/null 2>&1; then + fail "emit-plugin-eval into a non-empty dir should refuse" +else + pass "emit-plugin-eval refuses a non-empty out dir" +fi + +jq '.skills = []' "$TMP/score.json" >"$TMP/empty.json" +if run compare "$TMP/score.json" "$TMP/empty.json" >/dev/null 2>&1; then + fail "compare over mismatched skill sets should exit non-zero" +else + pass "compare refuses mismatched skill sets" +fi + +mkdir -p "$TMP/badrival/probes" +jq '.competitor_dirs = {}' "$TMP/probes/target.json" >"$TMP/badrival/probes/target.json" +if run score "$TMP/badrival/probes" >/dev/null 2>&1; then + fail "score with an unloadable competitor should exit non-zero" +else + pass "score refuses an unloadable competitor" +fi + if [[ $fails -gt 0 ]]; then printf 'measure-invocation.test.sh: %s failed\n' "$fails" >&2 exit 1 diff --git a/plugins/skill-quality/skills/check/SKILL.md b/plugins/skill-quality/skills/check/SKILL.md index eb9a8cb9ef8..f001642975a 100644 --- a/plugins/skill-quality/skills/check/SKILL.md +++ b/plugins/skill-quality/skills/check/SKILL.md @@ -210,7 +210,12 @@ Model-graded `claude plugin eval` cases are emitted on demand. Contract: ``` `compare ` prints per-skill train and validation deltas. `emit-plugin-eval - ` writes `claude plugin eval` cases. + ` writes `claude plugin eval` cases into an empty or new ``. + Claim: each case is a directory holding `prompt.md` (frontmatter fields, body is the prompt) + and `graders/`, and a `tool_used` grader with `tool: Skill` and an `input_match` on the skill + name checks that the skill fired. Basis: , the + case layout and `tool_used` grader sections. As of: 2026-09-28. Recheck: that page changes the + case layout, the `prompt.md` fields, or the `tool_used` grader fields. 3. Report per skill, per split (`train` and `validation`): `trigger_rate` and `false_trigger_rate`, sample size, and the method name. A rewrite is compared with `compare` against `probes/baselines/listing-overlap.json` (or a later model-graded snapshot). The action