Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/actions/report-check-run/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ and `head-sha` are cross-checks only. Any disagreement with the inputs is a fail
- Every neutral reason must be in `$defs/skip-reason` of `<base-path>/docs/conventions/pr-pipeline/pr-pipeline.schema.json`.
A reason outside it, or an unreadable schema, turns the neutral into a failure.
- `details_url` is this workflow run's page, built from the runner's own environment.
- When the run job's `lane-class`, `lane-resets`, `lane-turns` and `lane-max-turns` outputs make a
lane record ([lane-record.mjs](lane-record.mjs)), it opens `output.summary` as one line of
`key=value` tokens (`lane activity class resets turns max-turns`, absent ones omitted), goes into
`output.text` as one JSON block, sets `external_id` to `<lane>:<activity>:<GITHUB_RUN_ID>:<GITHUB_RUN_ATTEMPT>`
when both are positive integers, and is posted as one annotation for readers of older check
runs. It never changes the conclusion or title.
- Exit 0 after posting; exit 1 on a refusal, a malformed repository,
head SHA, fallback SHA or non-empty PR head SHA, or a failed POST.

Expand Down
5 changes: 3 additions & 2 deletions .github/actions/report-check-run/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,9 @@ inputs:
lane-class:
description: >
The run job's lane-class output: report-lane-outcome's failure class,
empty on a success. With the three below it becomes one annotation on
the posted check run (lane-record.mjs).
empty on a success. With the three below it becomes the first line of
the posted check run's output.summary, its output.text and external_id,
and one annotation (lane-record.mjs).
required: false
default: ""
lane-resets:
Expand Down
85 changes: 62 additions & 23 deletions .github/actions/report-check-run/lane-record.mjs
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
// The lane record annotation for a posted check run: the run's failure class,
// limit reset and turn headroom, which ci-workflows' report-lane-outcome
// computed in the run job from the execution file. Every field is a run job
// output and is checked against a fixed shape here; nothing the model wrote
// reaches the message. A rate-limit message keeps the exact shape
// .github/scripts/pr-unblock/limit-stop.mjs parses.
// The lane record for a posted check run: the run's failure class, limit reset
// and turn headroom, which ci-workflows' report-lane-outcome computed in the
// run job from the execution file. Every field is a run job output or the
// runner's own environment and is checked against a fixed shape here; nothing
// the model wrote reaches the record. It is posted twice: as the first line of
// output.summary (with the JSON form in output.text), which
// .github/scripts/pr-unblock/limit-stop.mjs reads first, and as one annotation
// in the shape limit-stop.mjs reads on check runs posted before the summary
// line existed.

const CLASSES = new Set([
"auth",
Expand All @@ -15,33 +18,69 @@ const CLASSES = new Set([
"skipped-validation",
"incomplete",
]);
const NAME = /^[a-z][a-z0-9]*(-[a-z0-9]+)*$/;
const RESETS = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?Z$/;
const COUNT = /^(0|[1-9][0-9]{0,5})$/;
const RUN_NUMBER = /^[1-9][0-9]*$/;
const PATH = ".github/workflows/pr-run-activity-write.yml";

// One check-run annotation, or null when the run job left no record: the
// skill did not run, or the class is not one report-lane-outcome emits.
export function laneRecordAnnotation({
lane,
failureClass = "",
resets = "",
turns = "",
maxTurns = "",
}) {
// The record's fields that passed their shape, or null when the run job left
// no record: the skill did not run, or the class is not one
// report-lane-outcome emits.
function laneRecord({ failureClass = "", resets = "", turns = "", maxTurns = "" }) {
if (failureClass !== "" && !CLASSES.has(failureClass)) {
return null;
}
const tokens = [
failureClass === "rate-limit" && RESETS.test(resets) && `resets=${resets}`,
COUNT.test(turns) && `turns=${turns}`,
COUNT.test(maxTurns) && `max-turns=${maxTurns}`,
].filter(Boolean);
if (failureClass === "" && tokens.length === 0) {
const fields = {
resets: failureClass === "rate-limit" && RESETS.test(resets) ? resets : null,
turns: COUNT.test(turns) ? turns : null,
"max-turns": COUNT.test(maxTurns) ? maxTurns : null,
};
if (failureClass === "" && fields.turns === null && fields["max-turns"] === null) {
return null;
}
return { class: failureClass || null, ...fields };
}

const tokensOf = (fields) =>
Object.entries(fields)
.filter(([, value]) => value !== null)
.map(([key, value]) => `${key}=${value}`);

// The record as the check run's output fields and external_id, or null when
// there is none. The summary line is `lane=<lane> activity=<activity>` then
// class, resets, turns and max-turns in that order, each omitted when absent.
export function laneRecordOutput({ lane, activity, runId = "", runAttempt = "", ...inputs }) {
const record = laneRecord(inputs);
if (!record || !NAME.test(lane ?? "") || !NAME.test(activity ?? "")) {
return null;
}
const fields = { lane, activity, ...record };
const json = Object.fromEntries(
Object.entries(fields)
.filter(([, value]) => value !== null)
.map(([key, value]) => [key, key.endsWith("turns") ? Number(value) : value]),
);
return {
line: tokensOf(fields).join(" "),
text: `\`\`\`json\n${JSON.stringify(json)}\n\`\`\``,
externalId:
RUN_NUMBER.test(runId) && RUN_NUMBER.test(runAttempt) ? `${lane}:${activity}:${runId}:${runAttempt}` : null,
};
}

// One check-run annotation, or null when there is no record.
export function laneRecordAnnotation({ lane, ...inputs }) {
const fields = laneRecord(inputs);
if (!fields) {
return null;
}
const record = tokens.map((token) => ` ${token}`).join("");
const { class: failureClass, ...rest } = fields;
const record = tokensOf(rest)
.map((token) => ` ${token}`)
.join("");
const at = { path: PATH, start_line: 1, end_line: 1 };
if (failureClass === "") {
if (failureClass === null) {
return { ...at, annotation_level: "notice", message: `${lane} completed:${record}.` };
}
if (failureClass === "skipped-validation") {
Expand Down
55 changes: 53 additions & 2 deletions .github/actions/report-check-run/lane-record.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ import assert from "node:assert/strict";
import { test } from "node:test";

import { limitResetsAt } from "../../scripts/pr-unblock/limit-stop.mjs";
import { laneRecordAnnotation } from "./lane-record.mjs";
import { laneRecordAnnotation, laneRecordOutput } from "./lane-record.mjs";

const ENDED = "2026-10-17T12:00:00Z";
const RESETS = "2026-10-17T15:30:00Z";
const run = { completed_at: ENDED };
const run = { conclusion: "failure", completed_at: ENDED };

test("a rate-limit record is the limit stop limit-stop.mjs reads, with its reset", () => {
const annotation = laneRecordAnnotation({
Expand Down Expand Up @@ -70,3 +70,54 @@ test("no record, an unknown class, or non-numeric turns add nothing", () => {
assert.equal(laneRecordAnnotation({ lane: "pr-unblock", failureClass: "forged class" }), null);
assert.equal(laneRecordAnnotation({ lane: "pr-unblock", turns: "4 x=1", maxTurns: "" }), null);
});

test("the output record is one line of key=value tokens in a fixed order, its JSON form, and a run-scoped external id", () => {
const record = laneRecordOutput({
lane: "pr-unblock",
activity: "resolve-conflicts",
failureClass: "rate-limit",
resets: RESETS,
turns: "12",
maxTurns: "30",
runId: "4242",
runAttempt: "2",
});
assert.equal(
record.line,
`lane=pr-unblock activity=resolve-conflicts class=rate-limit resets=${RESETS} turns=12 max-turns=30`,
);
assert.equal(
record.text,
"```json\n" +
`{"lane":"pr-unblock","activity":"resolve-conflicts","class":"rate-limit","resets":"${RESETS}","turns":12,"max-turns":30}` +
"\n```",
);
assert.equal(record.externalId, "pr-unblock:resolve-conflicts:4242:2");
});

test("absent fields are omitted from the output record, and a success carries no class", () => {
const record = laneRecordOutput({ lane: "pr-refine", activity: "simplify", turns: "4" });
assert.equal(record.line, "lane=pr-refine activity=simplify turns=4");
assert.equal(record.text, '```json\n{"lane":"pr-refine","activity":"simplify","turns":4}\n```');
assert.equal(record.externalId, null);
});

test("the output record drops what fails its shape: a reset off rate-limit, a forged reset, a malformed run id", () => {
assert.equal(
laneRecordOutput({ lane: "pr-unblock", activity: "x", failureClass: "other", resets: RESETS }).line,
"lane=pr-unblock activity=x class=other",
);
assert.equal(
laneRecordOutput({ lane: "pr-unblock", activity: "x", failureClass: "rate-limit", resets: `${RESETS} turns=1` }).line,
"lane=pr-unblock activity=x class=rate-limit",
);
const forgedRun = { lane: "pr-unblock", activity: "x", failureClass: "other", runAttempt: "1" };
assert.equal(laneRecordOutput({ ...forgedRun, runId: "1:2" }).externalId, null);
assert.equal(laneRecordOutput({ ...forgedRun, runId: "7", runAttempt: "0" }).externalId, null);
});

test("no record, an unknown class, or a malformed name add no output record", () => {
assert.equal(laneRecordOutput({ lane: "pr-unblock", activity: "x" }), null);
assert.equal(laneRecordOutput({ lane: "pr-unblock", activity: "x", failureClass: "forged class" }), null);
assert.equal(laneRecordOutput({ lane: "pr-unblock", activity: "x y=1", failureClass: "other" }), null);
});
22 changes: 18 additions & 4 deletions .github/actions/report-check-run/report-check-run.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";

import { createGitHub } from "../check-trusted-trigger/check-trusted-trigger.mjs";
import { decide } from "./decide.mjs";
import { laneRecordAnnotation } from "./lane-record.mjs";
import { laneRecordAnnotation, laneRecordOutput } from "./lane-record.mjs";

const HERE = path.dirname(fileURLToPath(import.meta.url));
const SCHEMA = "docs/conventions/pr-pipeline/pr-pipeline.schema.json";
Expand Down Expand Up @@ -90,15 +90,29 @@ export async function main({
return 1;
}
const body = { ...decision.report };
const annotation = laneRecordAnnotation({
const inputs = {
lane: env.LANE,
failureClass: env.LANE_CLASS ?? "",
resets: env.LANE_RESETS ?? "",
turns: env.LANE_TURNS ?? "",
maxTurns: env.LANE_MAX_TURNS ?? "",
};
const record = laneRecordOutput({
...inputs,
activity: env.ACTIVITY,
runId: env.GITHUB_RUN_ID ?? "",
runAttempt: env.GITHUB_RUN_ATTEMPT ?? "",
});
if (annotation) {
body.output = { ...body.output, annotations: [annotation] };
if (record) {
body.output = {
...body.output,
summary: `${record.line}\n\n${body.output.summary}`,
text: record.text,
annotations: [laneRecordAnnotation(inputs)],
};
if (record.externalId) {
body.external_id = record.externalId;
}
}
if (RUN_ID.test(env.GITHUB_RUN_ID ?? "")) {
body.details_url = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`;
Expand Down
75 changes: 64 additions & 11 deletions .github/actions/report-check-run/report-check-run.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import path from "node:path";
import { afterEach, beforeEach, test } from "node:test";
import { fileURLToPath } from "node:url";

import { anyHandles, limitResetsAt } from "../../scripts/pr-unblock/limit-stop.mjs";
import { main } from "./report-check-run.mjs";

const HERE = path.dirname(fileURLToPath(import.meta.url));
Expand Down Expand Up @@ -70,6 +71,7 @@ async function run(env = {}, status) {
GITHUB_API_URL: "https://api.example.test",
GITHUB_SERVER_URL: "https://github.com",
GITHUB_RUN_ID: "123",
GITHUB_RUN_ATTEMPT: "1",
...env,
},
fetchImpl,
Expand Down Expand Up @@ -109,22 +111,73 @@ test("a valid verdict POSTs one completed check run with the contract body", asy
);
});

test("a lane record from env POSTs as the check run's one annotation", async () => {
const { calls } = await run({
ACT_OUTCOME: "failure",
LANE_CLASS: "rate-limit",
LANE_RESETS: "2026-10-17T15:30:00Z",
LANE_TURNS: "3",
LANE_MAX_TURNS: "30",
const LIMIT = {
ACT_OUTCOME: "failure",
LANE_CLASS: "rate-limit",
LANE_RESETS: "2026-10-17T15:30:00Z",
LANE_TURNS: "3",
LANE_MAX_TURNS: "30",
};

test("a lane record from env opens the summary, fills output.text and external_id, and keeps the annotation", async () => {
const { calls } = await run({ ...LIMIT, GITHUB_RUN_ATTEMPT: "2" });
const { body } = calls[0];
assert.equal(body.conclusion, "failure");
const [line, blank, ...rest] = body.output.summary.split("\n");
assert.equal(line, "lane=pr-refine activity=simplify class=rate-limit resets=2026-10-17T15:30:00Z turns=3 max-turns=30");
assert.equal(blank, "");
assert.equal(rest.join("\n"), "The activity step failed.");
const json = /^```json\n(.*)\n```$/.exec(body.output.text);
assert.deepEqual(JSON.parse(json[1]), {
lane: "pr-refine",
activity: "simplify",
class: "rate-limit",
resets: "2026-10-17T15:30:00Z",
turns: 3,
"max-turns": 30,
});
assert.equal(calls[0].body.conclusion, "failure");
const [annotation] = calls[0].body.output.annotations;
assert.equal(body.external_id, "pr-refine:simplify:123:2");
const [annotation] = body.output.annotations;
assert.match(annotation.message, /class=rate-limit resets=2026-10-17T15:30:00Z turns=3 max-turns=30\)/);
});

test("no lane record POSTs no annotations", async () => {
test("the posted limit stop is the one limit-stop.mjs reads from the summary, with no annotations call", async () => {
const { calls } = await run(LIMIT);
const posted = { ...calls[0].body, id: 1, completed_at: "2026-10-17T12:00:00Z" };
assert.equal(limitResetsAt(posted), Date.parse("2026-10-17T15:30:00Z"));
const listAnnotations = async () => {
throw new Error("annotations read for a run whose summary carries the record");
};
const ids = [1, 2, 3].map((id) => ({ ...posted, id }));
const now = Date.parse("2026-10-17T16:00:00Z");
assert.equal(await anyHandles(ids.slice(0, 2), { listAnnotations, now }), false);
assert.equal(await anyHandles(ids, { listAnnotations, now }), true);
assert.equal(await anyHandles(ids.slice(0, 1), { listAnnotations, now: Date.parse("2026-10-17T15:00:00Z") }), true);
});

test("another posted failure class is a record limit-stop.mjs reads as no limit stop", async () => {
const { calls } = await run({ ACT_OUTCOME: "failure", LANE_CLASS: "max-turns", LANE_TURNS: "31", LANE_MAX_TURNS: "30" });
const posted = { ...calls[0].body, id: 1, completed_at: "2026-10-17T12:00:00Z" };
assert.equal(limitResetsAt(posted), null);
const listAnnotations = async () => {
throw new Error("annotations read for a run whose summary carries the record");
};
assert.equal(await anyHandles([posted], { listAnnotations, now: Date.parse("2026-10-20T00:00:00Z") }), true);
});

test("a malformed run attempt posts the record without an external_id", async () => {
const { calls } = await run({ ...LIMIT, GITHUB_RUN_ATTEMPT: "1:x" });
assert.equal("external_id" in calls[0].body, false);
assert.match(calls[0].body.output.summary, /^lane=pr-refine activity=simplify class=rate-limit /);
});

test("no lane record POSTs no annotations, text, external_id or record line", async () => {
const { calls } = await run();
assert.equal(calls[0].body.output.annotations, undefined);
const { output } = calls[0].body;
assert.equal(output.annotations, undefined);
assert.equal(output.text, undefined);
assert.equal("external_id" in calls[0].body, false);
assert.doesNotMatch(output.summary, /lane=/);
});

test("an absent verdict file POSTs failure on the env head", async () => {
Expand Down
Loading
Loading