Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions .github/actions/report-check-run/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,25 @@ inputs:
description: The check-signed-commits all-verified output; empty when it did not run.
required: false
default: ""
lane-class:
description: >
The run job's lane-class output: report-lane-outcome's failure class,
empty on a success. With the three below it becomes one annotation on
the posted check run (lane-record.mjs).
required: false
default: ""
lane-resets:
description: The run job's lane-resets output, the ISO time a usage limit resets.
required: false
default: ""
lane-turns:
description: The run job's lane-turns output, the turns the skill took.
required: false
default: ""
lane-max-turns:
description: The run job's lane-max-turns output, the skill's --max-turns cap.
required: false
default: ""
verdict-path:
description: Path of the downloaded verdict.json. A missing file posts failure.
required: false
Expand Down Expand Up @@ -82,6 +101,10 @@ runs:
PR_HEAD_SHA: ${{ inputs.pr-head-sha }}
CAN_COMMIT: ${{ inputs.can-commit }}
ALL_VERIFIED: ${{ inputs.all-verified }}
LANE_CLASS: ${{ inputs.lane-class }}
LANE_RESETS: ${{ inputs.lane-resets }}
LANE_TURNS: ${{ inputs.lane-turns }}
LANE_MAX_TURNS: ${{ inputs.lane-max-turns }}
VERDICT_PATH: ${{ inputs.verdict-path }}
BASE_PATH: ${{ inputs.base-path }}
HEAD_SHA_FALLBACK: ${{ inputs.head-sha-fallback }}
Expand Down
64 changes: 64 additions & 0 deletions .github/actions/report-check-run/lane-record.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// The lane record annotation for a posted check run: the run's failure class,
// limit reset and turn headroom, which ci-workflows' report-lane-outcome
// computed in the run job from the execution file. Every field is a run job
// output and is checked against a fixed shape here; nothing the model wrote
// reaches the message. A rate-limit message keeps the exact shape
// .github/scripts/pr-unblock/limit-stop.mjs parses.

const CLASSES = new Set([
"auth",
"rate-limit",
"overloaded",
"max-turns",
"timeout",
"other",
"skipped-validation",
"incomplete",
]);
const RESETS = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?Z$/;
const COUNT = /^(0|[1-9][0-9]{0,5})$/;
const PATH = ".github/workflows/pr-run-activity-write.yml";

// One check-run annotation, or null when the run job left no record: the
// skill did not run, or the class is not one report-lane-outcome emits.
export function laneRecordAnnotation({
lane,
failureClass = "",
resets = "",
turns = "",
maxTurns = "",
}) {
if (failureClass !== "" && !CLASSES.has(failureClass)) {
return null;
}
const tokens = [
failureClass === "rate-limit" && RESETS.test(resets) && `resets=${resets}`,
COUNT.test(turns) && `turns=${turns}`,
COUNT.test(maxTurns) && `max-turns=${maxTurns}`,
].filter(Boolean);
if (failureClass === "" && tokens.length === 0) {
return null;
}
const record = tokens.map((token) => ` ${token}`).join("");
const at = { path: PATH, start_line: 1, end_line: 1 };
if (failureClass === "") {
return { ...at, annotation_level: "notice", message: `${lane} completed:${record}.` };
}
if (failureClass === "skipped-validation") {
return {
...at,
annotation_level: "warning",
message: `${lane} concluded: success class=skipped-validation${record} with no execution evidence.`,
};
}
const skipped = /review/iu.test(lane) ? "AI review" : lane;
return {
...at,
annotation_level: "failure",
message:
failureClass === "rate-limit"
? `${skipped} skipped: usage limit reached; re-run after it resets ` +
`(${lane} exited with: failure class=rate-limit${record}). Last SDK result: in the run log.`
: `${lane} exited with: failure class=${failureClass}${record}.`,
};
}
72 changes: 72 additions & 0 deletions .github/actions/report-check-run/lane-record.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import assert from "node:assert/strict";
import { test } from "node:test";

import { limitResetsAt } from "../../scripts/pr-unblock/limit-stop.mjs";
import { laneRecordAnnotation } from "./lane-record.mjs";

const ENDED = "2026-10-17T12:00:00Z";
const RESETS = "2026-10-17T15:30:00Z";
const run = { completed_at: ENDED };

test("a rate-limit record is the limit stop limit-stop.mjs reads, with its reset", () => {
const annotation = laneRecordAnnotation({
lane: "pr-unblock",
failureClass: "rate-limit",
resets: RESETS,
turns: "12",
maxTurns: "30",
});
assert.equal(annotation.annotation_level, "failure");
assert.equal(annotation.title, undefined);
assert.equal(
annotation.message,
"pr-unblock skipped: usage limit reached; re-run after it resets (pr-unblock exited with: failure " +
`class=rate-limit resets=${RESETS} turns=12 max-turns=30). Last SDK result: in the run log.`,
);
assert.equal(limitResetsAt(run, [annotation]), Date.parse(RESETS));
});

test("a rate-limit record with no reset reads as a limit stop on the fallback reset", () => {
const annotation = laneRecordAnnotation({ lane: "pr-unblock", failureClass: "rate-limit" });
assert.equal(limitResetsAt(run, [annotation]), Date.parse(ENDED) + 5 * 3_600_000);
});

test("a review lane's limit stop opens with AI review, as limit-stop.mjs expects", () => {
const annotation = laneRecordAnnotation({ lane: "pr-review", failureClass: "rate-limit", resets: RESETS });
assert.match(annotation.message, /^AI review skipped: /);
assert.equal(limitResetsAt(run, [annotation]), Date.parse(RESETS));
});

test("a malformed reset is dropped, never passed into the message", () => {
const annotation = laneRecordAnnotation({
lane: "pr-unblock",
failureClass: "rate-limit",
resets: "2099-01-01T00:00:00Z) x=(",
});
assert.doesNotMatch(annotation.message, /resets=/);
});

test("another class is a failure record that limit-stop.mjs does not read as a limit stop", () => {
const annotation = laneRecordAnnotation({ lane: "pr-unblock", failureClass: "max-turns", turns: "31", maxTurns: "30" });
assert.equal(annotation.annotation_level, "failure");
assert.equal(annotation.message, "pr-unblock exited with: failure class=max-turns turns=31 max-turns=30.");
assert.equal(limitResetsAt(run, [annotation]), null);
});

test("a success records turn headroom as a notice", () => {
const annotation = laneRecordAnnotation({ lane: "pr-unblock", turns: "4", maxTurns: "30" });
assert.equal(annotation.annotation_level, "notice");
assert.equal(annotation.message, "pr-unblock completed: turns=4 max-turns=30.");
});

test("a validation skip is a warning that names its success outcome", () => {
const annotation = laneRecordAnnotation({ lane: "pr-unblock", failureClass: "skipped-validation" });
assert.equal(annotation.annotation_level, "warning");
assert.match(annotation.message, /concluded: success class=skipped-validation/);
});

test("no record, an unknown class, or non-numeric turns add nothing", () => {
assert.equal(laneRecordAnnotation({ lane: "pr-unblock" }), null);
assert.equal(laneRecordAnnotation({ lane: "pr-unblock", failureClass: "forged class" }), null);
assert.equal(laneRecordAnnotation({ lane: "pr-unblock", turns: "4 x=1", maxTurns: "" }), null);
});
11 changes: 11 additions & 0 deletions .github/actions/report-check-run/report-check-run.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";

import { createGitHub } from "../check-trusted-trigger/check-trusted-trigger.mjs";
import { decide } from "./decide.mjs";
import { laneRecordAnnotation } from "./lane-record.mjs";

const HERE = path.dirname(fileURLToPath(import.meta.url));
const SCHEMA = "docs/conventions/pr-pipeline/pr-pipeline.schema.json";
Expand Down Expand Up @@ -89,6 +90,16 @@ export async function main({
return 1;
}
const body = { ...decision.report };
const annotation = laneRecordAnnotation({
lane: env.LANE,
failureClass: env.LANE_CLASS ?? "",
resets: env.LANE_RESETS ?? "",
turns: env.LANE_TURNS ?? "",
maxTurns: env.LANE_MAX_TURNS ?? "",
});
if (annotation) {
body.output = { ...body.output, annotations: [annotation] };
}
if (RUN_ID.test(env.GITHUB_RUN_ID ?? "")) {
body.details_url = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`;
}
Expand Down
18 changes: 18 additions & 0 deletions .github/actions/report-check-run/report-check-run.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,24 @@ test("a valid verdict POSTs one completed check run with the contract body", asy
);
});

test("a lane record from env POSTs as the check run's one annotation", async () => {
const { calls } = await run({
ACT_OUTCOME: "failure",
LANE_CLASS: "rate-limit",
LANE_RESETS: "2026-10-17T15:30:00Z",
LANE_TURNS: "3",
LANE_MAX_TURNS: "30",
});
assert.equal(calls[0].body.conclusion, "failure");
const [annotation] = calls[0].body.output.annotations;
assert.match(annotation.message, /class=rate-limit resets=2026-10-17T15:30:00Z turns=3 max-turns=30\)/);
});

test("no lane record POSTs no annotations", async () => {
const { calls } = await run();
assert.equal(calls[0].body.output.annotations, undefined);
});

test("an absent verdict file POSTs failure on the env head", async () => {
const { code, calls } = await run({
VERDICT_PATH: path.join(dir, "missing.json"),
Expand Down
14 changes: 9 additions & 5 deletions .github/scripts/work-resolve-issue/report-outcome.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@
// stopped comments why, adds the human-gated label, and names the branch
// that holds any partial work. A missing or invalid outcome file
// is stopped.
// limited a stopped run whose lane.json carries class rate-limit (the run
// hit its credential's usage limit): comments once with the
// limited a run that wrote no readable outcome file and whose lane.json
// carries class rate-limit (the run hit its credential's usage
// limit before it could report): comments once with the
// limited marker naming when the limit resets, keeps agent-ready
// and adds no label, so work-unblock restarts it after the reset.
//
Expand Down Expand Up @@ -139,10 +140,10 @@ export function readOutcome(path) {
try {
parsed = JSON.parse(readFileSync(path, "utf8"));
} catch {
return stopped("The run wrote no readable outcome file, so it stopped before it finished.");
return { ...stopped("The run wrote no readable outcome file, so it stopped before it finished."), unread: true };
}
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed) || !EXITS.has(parsed.exit)) {
return stopped("The run's outcome file names no known exit (done, pushback or stopped).");
return { ...stopped("The run's outcome file names no known exit (done, pushback or stopped)."), unread: true };
}
return {
exit: parsed.exit,
Expand Down Expand Up @@ -269,7 +270,10 @@ export function decide({
runId = "",
}) {
outcome = { changes: [], verification: [], notDone: [], ...outcome };
if (limit && outcome.exit !== "pushback" && !(outcome.exit === "done" && branch)) {
// A limit record counts only for a run that left no outcome of its own: any
// code the run executed can forge the record, so it must not hide a stop
// or pushback the skill reported from the person it goes to.
if (limit && outcome.unread) {
const resets = new Date(clampReset(limit.resets, now)).toISOString().replace(/\.\d+Z$/, "Z");
return limitedPlan({ issueNumber, branch, resets, runId, runUrl });
}
Expand Down
35 changes: 32 additions & 3 deletions .github/scripts/work-resolve-issue/report-outcome.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ test("main refuses a lane.json branch that is not this issue's lane branch", asy
// hours (the shortest plan limit window).
test("a limit-stopped run is limited: one marker comment, agent-ready kept, no label", () => {
const plan = decide({
outcome: { exit: "stopped", reason: "", title: "", summary: "" },
outcome: { exit: "stopped", reason: "", title: "", summary: "", unread: true },
issueNumber: 77,
issueLabels: ["agent-ready"],
branch: "",
Expand All @@ -499,7 +499,7 @@ test("a limit-stopped run is limited: one marker comment, agent-ready kept, no l

test("a limit stop with no reset time restarts five hours after the run ended", () => {
const plan = decide({
outcome: { exit: "stopped", reason: "", title: "", summary: "" },
outcome: { exit: "stopped", reason: "", title: "", summary: "", unread: true },
issueNumber: 77,
issueLabels: ["agent-ready"],
branch: "",
Expand All @@ -519,7 +519,7 @@ test("a limit stop's stated reset is held between one hour and seven days out",
["2027-01-01T00:00:00Z", "2026-10-24T12:00:00Z"],
]) {
const plan = decide({
outcome: { exit: "stopped", reason: "", title: "", summary: "" },
outcome: { exit: "stopped", reason: "", title: "", summary: "", unread: true },
issueNumber: 77,
issueLabels: ["agent-ready"],
branch: "",
Expand All @@ -532,6 +532,35 @@ test("a limit stop's stated reset is held between one hour and seven days out",
}
});

// The lane record can be forged by any code the run executed, so it never
// overrides an outcome the skill wrote.
test("a stop the skill reported goes to a person even with a limit record", () => {
const plan = decide({
outcome: { exit: "stopped", reason: "The tests cannot run here.", title: "", summary: "" },
issueNumber: 77,
issueLabels: ["agent-ready"],
branch: "",
changedFiles: [],
limit: { resets: null },
});
assert.notEqual(plan.action, "limited");
assert.deepEqual(plan.addLabels, ["needs-human"]);
assert.match(plan.comment, /The tests cannot run here\./);
});

test("an unreadable outcome file with a limit record reads as limited", () => {
const plan = decide({
outcome: readOutcome("/nonexistent/work-outcome.json"),
issueNumber: 77,
issueLabels: ["agent-ready"],
branch: "",
changedFiles: [],
limit: { resets: null },
now: Date.parse("2026-10-17T12:00:00Z"),
});
assert.equal(plan.action, "limited");
});
Comment thread
kyle-sexton marked this conversation as resolved.
Resolved

test("a pushback stays a pushback even on a limit-stopped run", () => {
const plan = decide({
outcome: { exit: "pushback", reason: "Unclear.", title: "", summary: "" },
Expand Down
Loading
Loading