Repository navigation
Expand file tree
/
Copy pathdeterministic-suite.mjs
More file actions
executable file
·253 lines (243 loc) · 10.2 KB
/
Copy pathdeterministic-suite.mjs
File metadata and controls
executable file
·253 lines (243 loc) · 10.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
#!/usr/bin/env node
// deterministic-suite.mjs: copy a plugin's eval suite without the graders
// that need a paid judge, so a run spends nothing on grading.
//
// deterministic-suite.mjs <plugin-dir> <out-name>
//
// Reads the plugin's eval dir (the manifest's experimental.evals, else
// evals/) and writes <plugin-dir>/<out-name>/ holding every file of it except
// results/, the llm and baseline graders, and the cases left with no grader.
// `claude plugin eval --eval-dir <out-name>` then runs deterministic graders
// only. Prints {"evalDir", "kept", "dropped", "stripped"} as JSON: case paths
// relative to the eval dir, and stripped maps each kept case that lost graders
// to their names.
//
// A case is a directory holding prompt.md or case.yaml, found the way
// /evals:validate's validate-cases.py finds one. A grader is paid when its
// text names type llm or baseline, quoted or not: a graders/*.md file, or one
// item of case.yaml's block `graders:` list. Run after that validator passes,
// every grader has one of the six types, and a stray match only strips a
// grader, never keeps a paid one. Each case.yaml cut removes the item's lines
// and leaves every other byte; it is kept only when validate-cases.py's own
// parser reads the copy as the original minus those items and no paid line is
// left. Any other outcome (a flow-style list, a parse error, no python3) drops
// the case, with a note on stderr. A stripped grader's samples/<name>.json goes
// with it. The parser is loaded from this script's tree, never the plugin's.
//
// Exit 0 written; 2 a usage error, or the eval dir or output cannot be used.
import { spawnSync } from "node:child_process";
import { cpSync, existsSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
import { join, relative, sep } from "node:path";
import process from "node:process";
import { fileURLToPath, pathToFileURL } from "node:url";
import { isDeepStrictEqual } from "node:util";
const SKIP_DIRS = new Set(["results", "mocks", "graders", "node_modules", "__pycache__"]);
const PAID = /\btype["']?\s*:\s*["']?(?:llm|baseline)\b/i;
const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
const VALIDATOR = fileURLToPath(new URL("../../../plugins/evals/skills/validate/scripts/validate-cases.py", import.meta.url));
const PARSE = `
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("validate_cases", sys.argv[1])
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
print(json.dumps([module.parse_yaml(text) for text in json.load(sys.stdin)]))
`;
export function evalDirOf(pluginDir) {
try {
const manifest = JSON.parse(readFileSync(join(pluginDir, ".claude-plugin", "plugin.json"), "utf8"));
const named = manifest?.experimental?.evals;
if (typeof named === "string" && NAME.test(named)) {
return named;
}
} catch {
// No readable manifest: the default eval dir.
}
return "evals";
}
export function findCases(evalDir) {
const cases = [];
const walk = (dir) => {
for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
if (!entry.isDirectory() || SKIP_DIRS.has(entry.name) || entry.name.startsWith(".")) {
continue;
}
const path = join(dir, entry.name);
if (existsSync(join(path, "prompt.md")) || existsSync(join(path, "case.yaml"))) {
cases.push(path);
} else {
walk(path);
}
}
};
walk(evalDir);
return cases;
}
// The documents validate-cases.py's parse_yaml reads from texts, or throws.
function parseYaml(texts) {
const run = spawnSync("python3", ["-I", "-c", PARSE, VALIDATOR], { input: JSON.stringify(texts), encoding: "utf8" });
if (run.error || run.status !== 0) {
throw new Error(run.error?.message ?? (run.stderr.trim().split("\n").at(-1) || `python3 exit ${run.status}`));
}
return JSON.parse(run.stdout);
}
// Line ranges [start, end) of each item of the top-level block `graders:`
// list, each ending at its last non-blank, non-comment line; null for any
// other shape.
export function graderItems(lines) {
const header = lines.findIndex((line) => /^graders[ \t]*:[ \t]*(?:#.*)?\r?$/.test(line));
const body = (line) => line.trim() !== "" && !line.trim().startsWith("#");
const indent = (line) => line.length - line.trimStart().length;
const first = lines.findIndex((line, index) => index > header && body(line));
if (header === -1 || first === -1 || !lines[first].trimStart().startsWith("-")) {
return null;
}
const dash = indent(lines[first]);
const items = [];
for (let index = first; index < lines.length; index++) {
const line = lines[index];
if (!body(line)) {
continue;
}
if (indent(line) < dash || (indent(line) === dash && !line.slice(dash).startsWith("-"))) {
break;
}
if (indent(line) === dash) {
items.push([index, index + 1]);
} else {
items.at(-1)[1] = index + 1;
}
}
return { header, items };
}
// The copy of one case.yaml without its paid items, as {text, names, left}
// (left null when its graders are not a block list), or throws when the cut
// cannot be proven against the parser.
export function stripCaseYaml(text) {
const lines = text.split("\n");
const shape = graderItems(lines);
const paid = shape ? shape.items.map(([start, end]) => PAID.test(lines.slice(start, end).join("\n"))) : [];
if (!paid.includes(true)) {
if (PAID.test(text)) {
throw new Error("a paid grader outside a block graders list");
}
return { text, names: [], left: shape?.items.length ?? null };
}
// With every item paid, the cut takes the header and the whole list.
const cuts = paid.includes(false) ? shape.items.filter((_, index) => paid[index]) : [[shape.header, shape.items.at(-1)[1]]];
const stripped = lines.filter((_, index) => !cuts.some(([start, end]) => index >= start && index < end)).join("\n");
if (PAID.test(stripped)) {
throw new Error("a paid line is left after the cut");
}
const [before, after] = parseYaml([text, stripped]);
const graders = before?.graders;
if (!Array.isArray(graders) || graders.length !== shape.items.length) {
throw new Error("the graders list does not match its lines");
}
const expected = { ...before, graders: graders.filter((_, index) => !paid[index]) };
if (expected.graders.length === 0) {
delete expected.graders;
}
if (!isDeepStrictEqual(after, expected)) {
throw new Error("the cut changed more than the paid graders");
}
const names = graders.filter((_, index) => paid[index]).map((grader) => String(grader?.name));
const bad = names.find((name) => !NAME.test(name));
if (bad !== undefined) {
throw new Error(`the paid grader name ${JSON.stringify(bad)} is not a plain file name`);
}
return { text: stripped, names, left: expected.graders?.length ?? 0 };
}
// How one case is copied: null to drop it, else the case.yaml text to write
// (or null to leave it), and the graders/*.md files and names to strip.
export function planCase(caseDir) {
const graderDir = join(caseDir, "graders");
const files = existsSync(graderDir) ? readdirSync(graderDir).filter((name) => name.endsWith(".md")).sort() : [];
const paidFiles = files.filter((name) => PAID.test(readFileSync(join(graderDir, name), "utf8")));
const yamlPath = join(caseDir, "case.yaml");
const original = existsSync(yamlPath) ? readFileSync(yamlPath, "utf8") : null;
if (paidFiles.length === 0 && (original === null || !PAID.test(original))) {
return { yaml: null, files: [], names: [] };
}
const yaml = original === null ? { text: null, names: [], left: 0 } : stripCaseYaml(original);
if (yaml.left === null) {
yaml.left = parseYaml([original])[0]?.graders?.length ?? 0;
}
if (yaml.left + files.length - paidFiles.length === 0) {
return null;
}
return {
yaml: yaml.text === original ? null : yaml.text,
files: paidFiles,
names: [...paidFiles.map((name) => name.slice(0, -3)), ...yaml.names],
};
}
export function deterministicSuite(pluginDir, outName, warn = (line) => console.error(line)) {
const name = evalDirOf(pluginDir);
const evalDir = join(pluginDir, name);
const out = join(pluginDir, outName);
const kept = [];
const dropped = [];
const stripped = {};
const plans = new Map();
for (const caseDir of findCases(evalDir)) {
const path = relative(evalDir, caseDir).split(sep).join("/");
let plan = null;
try {
plan = planCase(caseDir);
} catch (error) {
warn(`deterministic-suite: ${path} dropped: ${error.message}`);
}
if (plan === null) {
dropped.push(path);
continue;
}
kept.push(path);
if (plan.names.length > 0) {
stripped[path] = plan.names;
plans.set(path, plan);
}
}
const skip = new Set([join(evalDir, "results"), ...dropped.map((path) => join(evalDir, path))]);
rmSync(out, { recursive: true, force: true });
cpSync(evalDir, out, { recursive: true, filter: (source) => !skip.has(source) });
for (const [path, plan] of plans) {
const copy = join(out, path);
if (plan.yaml !== null) {
writeFileSync(join(copy, "case.yaml"), plan.yaml);
}
for (const file of plan.files) {
rmSync(join(copy, "graders", file));
}
for (const grader of plan.names) {
rmSync(join(copy, "samples", `${grader}.json`), { force: true });
}
}
return { evalDir: name, kept, dropped, stripped };
}
export function main(argv = process.argv.slice(2)) {
const [pluginDir, outName] = argv;
if (argv.length !== 2 || !NAME.test(outName ?? "")) {
console.error("usage: deterministic-suite.mjs <plugin-dir> <out-name>");
return 2;
}
const name = evalDirOf(pluginDir);
const evalDir = join(pluginDir, name);
if (!existsSync(evalDir) || !statSync(evalDir).isDirectory()) {
console.error(`deterministic-suite: ${evalDir} is not a directory`);
return 2;
}
if (name === outName) {
console.error("deterministic-suite: the output would replace the eval dir");
return 2;
}
try {
console.log(JSON.stringify(deterministicSuite(pluginDir, outName)));
} catch (error) {
console.error(`deterministic-suite: ${error.message}`);
return 2;
}
return 0;
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
process.exitCode = main();
}