diff --git a/.github/actions/report-check-run/action.yml b/.github/actions/report-check-run/action.yml index f77e6ba8fb0..8a7600c4bfd 100644 --- a/.github/actions/report-check-run/action.yml +++ b/.github/actions/report-check-run/action.yml @@ -51,6 +51,25 @@ inputs: description: The check-signed-commits all-verified output; empty when it did not run. required: false default: "" + lane-class: + description: > + The run job's lane-class output: report-lane-outcome's failure class, + empty on a success. With the three below it becomes one annotation on + the posted check run (lane-record.mjs). + required: false + default: "" + lane-resets: + description: The run job's lane-resets output, the ISO time a usage limit resets. + required: false + default: "" + lane-turns: + description: The run job's lane-turns output, the turns the skill took. + required: false + default: "" + lane-max-turns: + description: The run job's lane-max-turns output, the skill's --max-turns cap. + required: false + default: "" verdict-path: description: Path of the downloaded verdict.json. A missing file posts failure. required: false @@ -82,6 +101,10 @@ runs: PR_HEAD_SHA: ${{ inputs.pr-head-sha }} CAN_COMMIT: ${{ inputs.can-commit }} ALL_VERIFIED: ${{ inputs.all-verified }} + LANE_CLASS: ${{ inputs.lane-class }} + LANE_RESETS: ${{ inputs.lane-resets }} + LANE_TURNS: ${{ inputs.lane-turns }} + LANE_MAX_TURNS: ${{ inputs.lane-max-turns }} VERDICT_PATH: ${{ inputs.verdict-path }} BASE_PATH: ${{ inputs.base-path }} HEAD_SHA_FALLBACK: ${{ inputs.head-sha-fallback }} diff --git a/.github/actions/report-check-run/lane-record.mjs b/.github/actions/report-check-run/lane-record.mjs new file mode 100644 index 00000000000..94f9a435da2 --- /dev/null +++ b/.github/actions/report-check-run/lane-record.mjs @@ -0,0 +1,64 @@ +// The lane record annotation for a posted check run: the run's failure class, +// limit reset and turn headroom, which ci-workflows' report-lane-outcome +// computed in the run job from the execution file. Every field is a run job +// output and is checked against a fixed shape here; nothing the model wrote +// reaches the message. A rate-limit message keeps the exact shape +// .github/scripts/pr-unblock/limit-stop.mjs parses. + +const CLASSES = new Set([ + "auth", + "rate-limit", + "overloaded", + "max-turns", + "timeout", + "other", + "skipped-validation", + "incomplete", +]); +const RESETS = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?Z$/; +const COUNT = /^(0|[1-9][0-9]{0,5})$/; +const PATH = ".github/workflows/pr-run-activity-write.yml"; + +// One check-run annotation, or null when the run job left no record: the +// skill did not run, or the class is not one report-lane-outcome emits. +export function laneRecordAnnotation({ + lane, + failureClass = "", + resets = "", + turns = "", + maxTurns = "", +}) { + if (failureClass !== "" && !CLASSES.has(failureClass)) { + return null; + } + const tokens = [ + failureClass === "rate-limit" && RESETS.test(resets) && `resets=${resets}`, + COUNT.test(turns) && `turns=${turns}`, + COUNT.test(maxTurns) && `max-turns=${maxTurns}`, + ].filter(Boolean); + if (failureClass === "" && tokens.length === 0) { + return null; + } + const record = tokens.map((token) => ` ${token}`).join(""); + const at = { path: PATH, start_line: 1, end_line: 1 }; + if (failureClass === "") { + return { ...at, annotation_level: "notice", message: `${lane} completed:${record}.` }; + } + if (failureClass === "skipped-validation") { + return { + ...at, + annotation_level: "warning", + message: `${lane} concluded: success class=skipped-validation${record} with no execution evidence.`, + }; + } + const skipped = /review/iu.test(lane) ? "AI review" : lane; + return { + ...at, + annotation_level: "failure", + message: + failureClass === "rate-limit" + ? `${skipped} skipped: usage limit reached; re-run after it resets ` + + `(${lane} exited with: failure class=rate-limit${record}). Last SDK result: in the run log.` + : `${lane} exited with: failure class=${failureClass}${record}.`, + }; +} diff --git a/.github/actions/report-check-run/lane-record.test.mjs b/.github/actions/report-check-run/lane-record.test.mjs new file mode 100644 index 00000000000..762f96187c1 --- /dev/null +++ b/.github/actions/report-check-run/lane-record.test.mjs @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { limitResetsAt } from "../../scripts/pr-unblock/limit-stop.mjs"; +import { laneRecordAnnotation } from "./lane-record.mjs"; + +const ENDED = "2026-10-17T12:00:00Z"; +const RESETS = "2026-10-17T15:30:00Z"; +const run = { completed_at: ENDED }; + +test("a rate-limit record is the limit stop limit-stop.mjs reads, with its reset", () => { + const annotation = laneRecordAnnotation({ + lane: "pr-unblock", + failureClass: "rate-limit", + resets: RESETS, + turns: "12", + maxTurns: "30", + }); + assert.equal(annotation.annotation_level, "failure"); + assert.equal(annotation.title, undefined); + assert.equal( + annotation.message, + "pr-unblock skipped: usage limit reached; re-run after it resets (pr-unblock exited with: failure " + + `class=rate-limit resets=${RESETS} turns=12 max-turns=30). Last SDK result: in the run log.`, + ); + assert.equal(limitResetsAt(run, [annotation]), Date.parse(RESETS)); +}); + +test("a rate-limit record with no reset reads as a limit stop on the fallback reset", () => { + const annotation = laneRecordAnnotation({ lane: "pr-unblock", failureClass: "rate-limit" }); + assert.equal(limitResetsAt(run, [annotation]), Date.parse(ENDED) + 5 * 3_600_000); +}); + +test("a review lane's limit stop opens with AI review, as limit-stop.mjs expects", () => { + const annotation = laneRecordAnnotation({ lane: "pr-review", failureClass: "rate-limit", resets: RESETS }); + assert.match(annotation.message, /^AI review skipped: /); + assert.equal(limitResetsAt(run, [annotation]), Date.parse(RESETS)); +}); + +test("a malformed reset is dropped, never passed into the message", () => { + const annotation = laneRecordAnnotation({ + lane: "pr-unblock", + failureClass: "rate-limit", + resets: "2099-01-01T00:00:00Z) x=(", + }); + assert.doesNotMatch(annotation.message, /resets=/); +}); + +test("another class is a failure record that limit-stop.mjs does not read as a limit stop", () => { + const annotation = laneRecordAnnotation({ lane: "pr-unblock", failureClass: "max-turns", turns: "31", maxTurns: "30" }); + assert.equal(annotation.annotation_level, "failure"); + assert.equal(annotation.message, "pr-unblock exited with: failure class=max-turns turns=31 max-turns=30."); + assert.equal(limitResetsAt(run, [annotation]), null); +}); + +test("a success records turn headroom as a notice", () => { + const annotation = laneRecordAnnotation({ lane: "pr-unblock", turns: "4", maxTurns: "30" }); + assert.equal(annotation.annotation_level, "notice"); + assert.equal(annotation.message, "pr-unblock completed: turns=4 max-turns=30."); +}); + +test("a validation skip is a warning that names its success outcome", () => { + const annotation = laneRecordAnnotation({ lane: "pr-unblock", failureClass: "skipped-validation" }); + assert.equal(annotation.annotation_level, "warning"); + assert.match(annotation.message, /concluded: success class=skipped-validation/); +}); + +test("no record, an unknown class, or non-numeric turns add nothing", () => { + assert.equal(laneRecordAnnotation({ lane: "pr-unblock" }), null); + assert.equal(laneRecordAnnotation({ lane: "pr-unblock", failureClass: "forged class" }), null); + assert.equal(laneRecordAnnotation({ lane: "pr-unblock", turns: "4 x=1", maxTurns: "" }), null); +}); diff --git a/.github/actions/report-check-run/report-check-run.mjs b/.github/actions/report-check-run/report-check-run.mjs index ff4fe3d456b..359140c3f14 100644 --- a/.github/actions/report-check-run/report-check-run.mjs +++ b/.github/actions/report-check-run/report-check-run.mjs @@ -8,6 +8,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { createGitHub } from "../check-trusted-trigger/check-trusted-trigger.mjs"; import { decide } from "./decide.mjs"; +import { laneRecordAnnotation } from "./lane-record.mjs"; const HERE = path.dirname(fileURLToPath(import.meta.url)); const SCHEMA = "docs/conventions/pr-pipeline/pr-pipeline.schema.json"; @@ -89,6 +90,16 @@ export async function main({ return 1; } const body = { ...decision.report }; + const annotation = laneRecordAnnotation({ + lane: env.LANE, + failureClass: env.LANE_CLASS ?? "", + resets: env.LANE_RESETS ?? "", + turns: env.LANE_TURNS ?? "", + maxTurns: env.LANE_MAX_TURNS ?? "", + }); + if (annotation) { + body.output = { ...body.output, annotations: [annotation] }; + } if (RUN_ID.test(env.GITHUB_RUN_ID ?? "")) { body.details_url = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`; } diff --git a/.github/actions/report-check-run/report-check-run.test.mjs b/.github/actions/report-check-run/report-check-run.test.mjs index ad2ee605a2d..175254feac4 100644 --- a/.github/actions/report-check-run/report-check-run.test.mjs +++ b/.github/actions/report-check-run/report-check-run.test.mjs @@ -109,6 +109,24 @@ test("a valid verdict POSTs one completed check run with the contract body", asy ); }); +test("a lane record from env POSTs as the check run's one annotation", async () => { + const { calls } = await run({ + ACT_OUTCOME: "failure", + LANE_CLASS: "rate-limit", + LANE_RESETS: "2026-10-17T15:30:00Z", + LANE_TURNS: "3", + LANE_MAX_TURNS: "30", + }); + assert.equal(calls[0].body.conclusion, "failure"); + const [annotation] = calls[0].body.output.annotations; + assert.match(annotation.message, /class=rate-limit resets=2026-10-17T15:30:00Z turns=3 max-turns=30\)/); +}); + +test("no lane record POSTs no annotations", async () => { + const { calls } = await run(); + assert.equal(calls[0].body.output.annotations, undefined); +}); + test("an absent verdict file POSTs failure on the env head", async () => { const { code, calls } = await run({ VERDICT_PATH: path.join(dir, "missing.json"), diff --git a/.github/scripts/work-resolve-issue/report-outcome.mjs b/.github/scripts/work-resolve-issue/report-outcome.mjs index 2d2dafafe30..30ba37696ee 100755 --- a/.github/scripts/work-resolve-issue/report-outcome.mjs +++ b/.github/scripts/work-resolve-issue/report-outcome.mjs @@ -15,8 +15,9 @@ // stopped comments why, adds the human-gated label, and names the branch // that holds any partial work. A missing or invalid outcome file // is stopped. -// limited a stopped run whose lane.json carries class rate-limit (the run -// hit its credential's usage limit): comments once with the +// limited a run that wrote no readable outcome file and whose lane.json +// carries class rate-limit (the run hit its credential's usage +// limit before it could report): comments once with the // limited marker naming when the limit resets, keeps agent-ready // and adds no label, so work-unblock restarts it after the reset. // @@ -139,10 +140,10 @@ export function readOutcome(path) { try { parsed = JSON.parse(readFileSync(path, "utf8")); } catch { - return stopped("The run wrote no readable outcome file, so it stopped before it finished."); + return { ...stopped("The run wrote no readable outcome file, so it stopped before it finished."), unread: true }; } if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed) || !EXITS.has(parsed.exit)) { - return stopped("The run's outcome file names no known exit (done, pushback or stopped)."); + return { ...stopped("The run's outcome file names no known exit (done, pushback or stopped)."), unread: true }; } return { exit: parsed.exit, @@ -269,7 +270,10 @@ export function decide({ runId = "", }) { outcome = { changes: [], verification: [], notDone: [], ...outcome }; - if (limit && outcome.exit !== "pushback" && !(outcome.exit === "done" && branch)) { + // A limit record counts only for a run that left no outcome of its own: any + // code the run executed can forge the record, so it must not hide a stop + // or pushback the skill reported from the person it goes to. + if (limit && outcome.unread) { const resets = new Date(clampReset(limit.resets, now)).toISOString().replace(/\.\d+Z$/, "Z"); return limitedPlan({ issueNumber, branch, resets, runId, runUrl }); } diff --git a/.github/scripts/work-resolve-issue/report-outcome.test.mjs b/.github/scripts/work-resolve-issue/report-outcome.test.mjs index cd8306a22b4..444d760d705 100644 --- a/.github/scripts/work-resolve-issue/report-outcome.test.mjs +++ b/.github/scripts/work-resolve-issue/report-outcome.test.mjs @@ -482,7 +482,7 @@ test("main refuses a lane.json branch that is not this issue's lane branch", asy // hours (the shortest plan limit window). test("a limit-stopped run is limited: one marker comment, agent-ready kept, no label", () => { const plan = decide({ - outcome: { exit: "stopped", reason: "", title: "", summary: "" }, + outcome: { exit: "stopped", reason: "", title: "", summary: "", unread: true }, issueNumber: 77, issueLabels: ["agent-ready"], branch: "", @@ -499,7 +499,7 @@ test("a limit-stopped run is limited: one marker comment, agent-ready kept, no l test("a limit stop with no reset time restarts five hours after the run ended", () => { const plan = decide({ - outcome: { exit: "stopped", reason: "", title: "", summary: "" }, + outcome: { exit: "stopped", reason: "", title: "", summary: "", unread: true }, issueNumber: 77, issueLabels: ["agent-ready"], branch: "", @@ -519,7 +519,7 @@ test("a limit stop's stated reset is held between one hour and seven days out", ["2027-01-01T00:00:00Z", "2026-10-24T12:00:00Z"], ]) { const plan = decide({ - outcome: { exit: "stopped", reason: "", title: "", summary: "" }, + outcome: { exit: "stopped", reason: "", title: "", summary: "", unread: true }, issueNumber: 77, issueLabels: ["agent-ready"], branch: "", @@ -532,6 +532,35 @@ test("a limit stop's stated reset is held between one hour and seven days out", } }); +// The lane record can be forged by any code the run executed, so it never +// overrides an outcome the skill wrote. +test("a stop the skill reported goes to a person even with a limit record", () => { + const plan = decide({ + outcome: { exit: "stopped", reason: "The tests cannot run here.", title: "", summary: "" }, + issueNumber: 77, + issueLabels: ["agent-ready"], + branch: "", + changedFiles: [], + limit: { resets: null }, + }); + assert.notEqual(plan.action, "limited"); + assert.deepEqual(plan.addLabels, ["needs-human"]); + assert.match(plan.comment, /The tests cannot run here\./); +}); + +test("an unreadable outcome file with a limit record reads as limited", () => { + const plan = decide({ + outcome: readOutcome("/nonexistent/work-outcome.json"), + issueNumber: 77, + issueLabels: ["agent-ready"], + branch: "", + changedFiles: [], + limit: { resets: null }, + now: Date.parse("2026-10-17T12:00:00Z"), + }); + assert.equal(plan.action, "limited"); +}); + test("a pushback stays a pushback even on a limit-stopped run", () => { const plan = decide({ outcome: { exit: "pushback", reason: "Unclear.", title: "", summary: "" }, diff --git a/.github/workflows/pr-run-activity-write.yml b/.github/workflows/pr-run-activity-write.yml index 521dd582558..2dff62d77f7 100644 --- a/.github/workflows/pr-run-activity-write.yml +++ b/.github/workflows/pr-run-activity-write.yml @@ -270,6 +270,16 @@ jobs: dependency-run-sha256: ${{ steps.dependency-collect.outputs.run-sha256 }} dependency-run-attempt: ${{ steps.dependency-collect.outcome == 'success' && github.run_attempt || '' }} issue-run-started: ${{ inputs.issue-number != '' && steps.lane-token.outcome == 'success' }} + # The lane record, from steps that read the execution file after the + # skill. A forged limit stop can only restart the lane after a reset + # limit-stop.mjs and report-outcome clamp to at least an hour away, at + # most MAX_LIMIT_STOPS times. + lane-class: ${{ steps.lane-record.outputs.failure-class }} + lane-resets: ${{ steps.lane-record.outputs.resets-at }} + lane-turns: ${{ steps.lane-record.outcome == 'success' && steps.act-turns.outputs.turns || '' }} + lane-max-turns: >- + ${{ steps.lane-record.outcome == 'success' + && (steps.config.outputs.max-turns || inputs.default-max-turns) || '' }} # A gate skill succeeds only on a `pass` verdict, a skill that took no # model turn fails, and a skill whose commits could not be moved onto # the PR branch fails. In merge-base mode a clean merge skips the skill @@ -997,6 +1007,9 @@ jobs: .write(); // JSON escapes newlines, so no field can start a workflow command. core.info(JSON.stringify({ subtype: diag.subtype, is_error: diag.is_error, num_turns: diag.num_turns, models: diag.models.length })); + if (Number.isSafeInteger(diag.num_turns) && diag.num_turns >= 0) { + core.setOutput('turns', String(diag.num_turns)); + } if (!(diag.num_turns > 0) || diag.models.length === 0) { core.setFailed( 'no-model-turn: the skill ended without a model turn, so it did nothing; ' + @@ -1004,6 +1017,24 @@ jobs: ); } + # The lane record: the run's failure class and, on a usage limit, when it + # resets, read from the execution file's structured fields by + # ci-workflows' classifier, never from model text. The report job copies + # it onto the posted check run, and issue mode writes it into lane.json, + # where the limit-stop restarts read it. + - name: Record the lane outcome + id: lane-record + if: always() && (steps.act-skill.outcome == 'success' || steps.act-skill.outcome == 'failure') + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@64fd515245f2bdeea8dedca1a3cea9ca39804bec # v0.43.0 + with: + outcome: ${{ steps.act-skill.outcome }} + execution-file: >- + ${{ steps.act-skill.outputs.execution_file + || format('{0}/claude-execution-output.json', runner.temp) }} + lane: ${{ inputs.lane }} + credential: CLAUDE_CODE_OAUTH_TOKEN + max-turns: ${{ steps.config.outputs.max-turns || inputs.default-max-turns }} + - name: Upload the skill's final reply if: always() && (steps.act-skill.outcome == 'success' || steps.act-skill.outcome == 'failure') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -1514,8 +1545,7 @@ jobs: if-no-files-found: error # Issue mode: the lane branch is kept only when the skill committed to - # it, and the report-outcome run reads its name from lane.json, written - # here after the skill ended. + # it; the collect step writes its name into lane.json. - name: Keep the issue branch only when it holds commits id: issue-branch if: always() && inputs.issue-number != '' && steps.lane-branch.outputs.branch != '' @@ -1523,13 +1553,11 @@ jobs: env: BASE_SHA: ${{ steps.trusted-base.outputs.commit }} LANE_BRANCH: ${{ steps.lane-branch.outputs.branch }} - OUT_DIR: ${{ runner.temp }}/work-outcome with: github-token: ${{ steps.lane-token.outputs.token }} retries: 0 script: | - const fs = require('fs'); - const { BASE_SHA, LANE_BRANCH, OUT_DIR } = process.env; + const { BASE_SHA, LANE_BRANCH } = process.env; const ref = `heads/${LANE_BRANCH}`; const tip = (await github.request('GET /repos/{owner}/{repo}/git/ref/{ref}', { ...context.repo, ref })).data.object.sha; let branch = LANE_BRANCH; @@ -1540,9 +1568,7 @@ jobs: } else { core.info(`The skill's commits are on ${LANE_BRANCH} at ${tip}`); } - fs.mkdirSync(OUT_DIR, { recursive: true }); - fs.rmSync(`${OUT_DIR}/lane.json`, { force: true }); - fs.writeFileSync(`${OUT_DIR}/lane.json`, `${JSON.stringify({ branch })}\n`); + core.setOutput('branch', branch); # The skill's outcome file is model output, copied as data and capped. # Skill runs only: the report-outcome run must not overwrite the @@ -1554,19 +1580,21 @@ jobs: env: SCRATCH_OUTCOME: ${{ runner.temp }}/lane-scratch/work-outcome.json OUT_DIR: ${{ runner.temp }}/work-outcome - KEEP_OUTCOME: ${{ steps.issue-branch.outcome }} + # lane.json is built from step outputs, never from a file in the + # directory, which could have been planted during the run. Code the + # activity ran can still reach this step (BASH_ENV, PATH), so + # report-outcome re-checks every field. + LANE_BRANCH: ${{ steps.issue-branch.outcome == 'success' && steps.issue-branch.outputs.branch || '' }} + LANE_CLASS: ${{ steps.lane-record.outputs.failure-class }} + LANE_RESETS: ${{ steps.lane-record.outputs.resets-at }} run: | mkdir -p "$OUT_DIR" - rm -f "$OUT_DIR/work-outcome.json" + rm -f "$OUT_DIR/work-outcome.json" "$OUT_DIR/lane.json" if [ -f "$SCRATCH_OUTCOME" ] && [ ! -L "$SCRATCH_OUTCOME" ]; then head -c 65536 "$SCRATCH_OUTCOME" >"$OUT_DIR/work-outcome.json" fi - # Only the keep step's own write names a branch; anything else in - # the directory could have been planted during the run. - if [ "$KEEP_OUTCOME" != success ]; then - rm -f "$OUT_DIR/lane.json" - echo '{"branch":""}' >"$OUT_DIR/lane.json" - fi + jq -n --arg branch "$LANE_BRANCH" --arg class "$LANE_CLASS" --arg resets "$LANE_RESETS" \ + '{branch: $branch, class: $class, resets: $resets}' >"$OUT_DIR/lane.json" - name: Upload the issue outcome if: >- @@ -1909,6 +1937,10 @@ jobs: pr-head-sha: ${{ steps.pr-head.outputs.sha }} can-commit: ${{ needs.run.outputs.can-commit }} all-verified: ${{ steps.signed.outputs.all-verified }} + lane-class: ${{ needs.run.outputs.lane-class }} + lane-resets: ${{ needs.run.outputs.lane-resets }} + lane-turns: ${{ needs.run.outputs.lane-turns }} + lane-max-turns: ${{ needs.run.outputs.lane-max-turns }} verdict-path: ${{ runner.temp }}/verdict/verdict.json base-path: .base head-sha-fallback: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/docs/conventions/pr-pipeline/pr-run-activity.md b/docs/conventions/pr-pipeline/pr-run-activity.md index 3b52f4e3869..fbb623a37fa 100644 --- a/docs/conventions/pr-pipeline/pr-run-activity.md +++ b/docs/conventions/pr-pipeline/pr-run-activity.md @@ -251,6 +251,17 @@ PR branch, or when its merge commit failed or was not made; in `prepare: merge-b that skipped the skill is a success once its commit is promoted. These checks are steps that run after the skill. +The write file also outputs the lane record of a skill run: `lane-class` and `lane-resets`, the +failure class and usage-limit reset that ci-workflows' `report-lane-outcome` reads from the +execution file's structured fields, and `lane-turns` and `lane-max-turns`, the turns taken and the +cap. They come from steps after the skill. The report job passes them to `report-check-run`, which +checks each against a fixed shape and posts them as one annotation on the check run; a +`class=rate-limit` annotation is the limit stop the pr-unblock dispatchers read. In issue mode the +same `class` and `resets` go into `lane.json` beside the kept branch, for report-outcome's +`limited` exit. Any code the activity runs, not only the model, can forge this record, for +example by planting the execution file. A forged limit stop can only restart the lane, at least an +hour later and a bounded number of times. + ### Skill activities The prompt is `/: `, followed, for an activity with the `changed-paths`