diff --git a/.changes/pr-pipeline/fix-security-suggestion-dispute-route-7d2e9a41.md b/.changes/pr-pipeline/fix-security-suggestion-dispute-route-7d2e9a41.md new file mode 100644 index 00000000000..c37a38759e9 --- /dev/null +++ b/.changes/pr-pipeline/fix-security-suggestion-dispute-route-7d2e9a41.md @@ -0,0 +1,15 @@ +--- +bump: patch +--- + +### Fixed + +- A disputed security-review finding rated `SUGGESTION` no longer hands the pull request to a + person. address-feedback reads the lane's own `severity:` line (`severityOf` in + `address-feedback.mjs`: the body's only such line, outside any fence, first or last above the + footer) and holds for a person only at `IMPORTANT` or above, the rating this + repository counts as HIGH (`SECURITY_PERSON_SEVERITY`). A `SUGGESTION` is adjudicated as a + code-review finding is, and an adjudicator's `security` question on it ends the lane in a failing + status as `feedback-unadjudicated`, not `feedback-question-security`. Past the round cap it gets + the fresh fixer's round. An `IMPORTANT` or `CRITICAL` finding, one with no such line, an unknown + rating, a second rating-shaped line or one quoted from the pull request, and a thread a person directed still go to a person as before. diff --git a/.github/scripts/pr-unblock/address-feedback.mjs b/.github/scripts/pr-unblock/address-feedback.mjs index 9c453ce9096..cbc5cfb8021 100644 --- a/.github/scripts/pr-unblock/address-feedback.mjs +++ b/.github/scripts/pr-unblock/address-feedback.mjs @@ -100,22 +100,85 @@ const lastLine = (body) => export const laneOf = (body) => LANE_FOOTER.exec(lastLine(body))?.[1] ?? null; export const isLaneComment = (comment) => comment.authorId === MARKER_AUTHOR_ID && laneOf(comment.body) !== null; +// The security-review lane's ratings, lowest first (security-review SKILL.md, +// "Reporting": one `severity: ` line per inline comment), and the +// lowest one a person decides: this repository counts IMPORTANT as HIGH. +export const SECURITY_SEVERITIES = Object.freeze(["SUGGESTION", "IMPORTANT", "CRITICAL"]); +export const SECURITY_PERSON_SEVERITY = "IMPORTANT"; +// Any line shaped like a rating, wherever it sits; the lane's own line is +// unindented, outside a fence, and the comment's first line or the last one +// above its `claude-lane:` footer (blank lines skipped). +const SEVERITY_SHAPED = /^[ \t>]*severity:/i; +const SEVERITY_LINE = /^severity:[ \t]*([A-Za-z]+)[ \t]*$/i; +const FENCE = /^[ \t]*(```|~~~)/; +// A lane comment's one rating, upper-cased, or null (unrated, so a person +// decides) unless the body holds exactly one rating-shaped line and it is the +// lane's own: unindented, outside any fenced block, and the first non-blank +// line or the last non-blank line above the lane footer. A rating quoted from +// the pull request (fenced, indented, `>`-quoted or mid-body) never supplies +// it, and any second rating-shaped line voids it. +export function severityOf(body) { + const lines = String(body ?? "") + .split("\n") + .map((line) => line.replace(/\r$/, "")); + const content = []; + let fence = null; + for (const line of lines) { + const open = FENCE.exec(line)?.[1]; + if (fence === null && open) { + fence = open; + content.push({ line, fenced: true }); + continue; + } + if (fence !== null) { + if (open === fence) { + fence = null; + } + content.push({ line, fenced: true }); + continue; + } + content.push({ line, fenced: false }); + } + const shaped = content.filter(({ line }) => SEVERITY_SHAPED.test(line)); + if (shaped.length !== 1 || shaped[0].fenced) { + return null; + } + const nonBlank = content.filter(({ line }) => line.trim() !== ""); + const footer = nonBlank.length > 0 && laneOf(nonBlank.at(-1).line) !== null; + const positions = [nonBlank[0], footer ? nonBlank.at(-2) : undefined]; + const match = positions.includes(shaped[0]) ? SEVERITY_LINE.exec(shaped[0].line) : null; + const rating = match?.[1].toUpperCase(); + return SECURITY_SEVERITIES.includes(rating) ? rating : null; +} +// Whether a security-review lane comment is one a person decides: rated at or +// above SECURITY_PERSON_SEVERITY, or with no rating it can read (fail closed). +export const securityNeedsPerson = (body) => { + const severity = severityOf(body); + return severity === null || SECURITY_SEVERITIES.indexOf(severity) >= SECURITY_SEVERITIES.indexOf(SECURITY_PERSON_SEVERITY); +}; + // A thread whose first comment is a pr-verify verifier note (threadAuthors). const isVerifierNote = (thread) => thread.comments[0]?.author === "verifier"; // Past the round cap, the threads only a person may still decide: a security -// lane finding, a verifier note a trusted person directed, and a finding -// whose lane no footer pr-review-cli appended names: only on a -// github-actions[bot] root (`rootAuthorId`) is the footer the reusable's, -// never model text. The code lane's finding and an undirected verifier note -// go to a fresh fixer: a round cap is a lane bound, and pr-verify's FAIL -// already holds the merge and judges the next head afresh. -export const needsPerson = (thread, rootAuthorId) => - isVerifierNote(thread) - ? thread.comments.some((comment) => comment.direction === true) - : rootAuthorId !== MARKER_AUTHOR_ID || - thread.comments[0]?.author !== "reviewer" || - laneOf(thread.comments[0].body) !== "claude-review"; +// lane finding a person decides (securityNeedsPerson), a verifier note a +// trusted person directed, and a finding whose lane no footer pr-review-cli +// appended names: only on a github-actions[bot] root (`rootAuthorId`) is the +// footer the reusable's, never model text. The code lane's finding, a +// security lane finding rated below SECURITY_PERSON_SEVERITY and an undirected +// verifier note go to a fresh fixer: a round cap is a lane bound, and +// pr-verify's FAIL already holds the merge and judges the next head afresh. +export const needsPerson = (thread, rootAuthorId) => { + if (isVerifierNote(thread)) { + return thread.comments.some((comment) => comment.direction === true); + } + const root = thread.comments[0]; + if (rootAuthorId !== MARKER_AUTHOR_ID || root?.author !== "reviewer") { + return true; + } + const lane = laneOf(root.body); + return lane === "claude-security-review" ? securityNeedsPerson(root.body) : lane !== "claude-review"; +}; // Round markers the lanes App posted; any other author's copy is ignored. export const isRoundMarker = (comment, appId) => comment.user?.id === appId && firstLine(comment.body) === ROUND_MARKER; diff --git a/.github/scripts/pr-unblock/address-feedback.test.mjs b/.github/scripts/pr-unblock/address-feedback.test.mjs index 87fbd1ce020..fe45c1b3b46 100644 --- a/.github/scripts/pr-unblock/address-feedback.test.mjs +++ b/.github/scripts/pr-unblock/address-feedback.test.mjs @@ -874,6 +874,24 @@ for (const [label, roots] of [ } } +// A person decides only a security lane finding rated IMPORTANT or above, or +// one with no rating the lane can read; a SUGGESTION goes to the fresh fixer +// as a code lane finding does. +const SUGGESTION_FINDING = LANE_FINDING.replace("severity: IMPORTANT", "severity: SUGGESTION"); +test("at the cap, a SUGGESTION security lane finding goes to a fresh fixer; an IMPORTANT, CRITICAL or unrated one escalates feedback-round-cap", async () => { + const fresh = await capped({ roots: [SUGGESTION_FINDING, CODE_FINDING] }); + assert.equal(fresh.decision, "proceed"); + assert.equal(fresh.escalationReason, undefined); + assert.deepEqual(fresh.file.threads.map((row) => row.id), ["PRRT_1", "PRRT_2"]); + const fencedOnly = "**x**\n\n```\nseverity: SUGGESTION\n```\n\nclaude-lane: claude-security-review"; + const midBody = "**x**\n\nquoted:\nseverity: SUGGESTION\nmore text\n\nclaude-lane: claude-security-review"; + for (const body of [LANE_FINDING, LANE_FINDING.replace("IMPORTANT", "CRITICAL"), LANE_FINDING.replace("severity: IMPORTANT\n", ""), LANE_FINDING.replace("IMPORTANT", "HIGHISH"), fencedOnly, midBody]) { + const result = await capped({ roots: [body] }); + assert.equal(result.decision, "escalate", body.split("\n")[0]); + assert.equal(result.escalationReason, "feedback-round-cap"); + } +}); + test("at the cap, the reviewer bot's own finding ending in the code lane's footer still escalates: its footer is model text", async () => { const api = fakeApi({ reviewComments: [comment(100, REVIEWER, { body: CODE_FINDING })], diff --git a/.github/scripts/pr-unblock/adjudicate-feedback.mjs b/.github/scripts/pr-unblock/adjudicate-feedback.mjs index 1d8561c43e6..bee03473426 100755 --- a/.github/scripts/pr-unblock/adjudicate-feedback.mjs +++ b/.github/scripts/pr-unblock/adjudicate-feedback.mjs @@ -34,7 +34,7 @@ import { appendFileSync, existsSync, lstatSync, mkdirSync, readFileSync, writeFi import { dirname, join } from "node:path"; import { pathToFileURL } from "node:url"; -import { createGitHubApi, FINDING_STANDS, need, SHA } from "./address-feedback.mjs"; +import { createGitHubApi, FINDING_STANDS, need, SECURITY_PERSON_SEVERITY, securityNeedsPerson, SHA } from "./address-feedback.mjs"; // The most disputed threads one round adjudicates: pr-unblock.yml runs one // model step per slot. A thread past it is not adjudicated, and the round @@ -84,22 +84,27 @@ export const directed = (comments) => (comments ?? []).some((comment) => comment // Whether an adjudicator may judge a dispute on the thread: a review comment // that carries no `severity: CRITICAL` line and that keepThreads tagged with // the code-review lane (a github-actions[bot] footer; claude[bot] writes its -// own body, so it names no lane) or as a listed review bot's (from its account -// id) whose `security` state is clear: the security-review lane reviewed the -// head the bot reviewed and holds nothing there or since. The bot's own -// severity badge decides nothing. A listed bot's finding the lane held goes to -// a person (feedback-question-security); one the lane has not reviewed yet -// waits for the next round (settleDisputes). A CRITICAL finding, a -// security-review finding, one of no tagged lane, and a pr-verify verifier -// note (resolving it leaves the verifier's FAIL holding the merge, and -// pr-verify judges each head once) always go to a person. +// own body, so it names no lane), with the security-review lane and a rating +// below the one a person decides (securityNeedsPerson), or as a listed review +// bot's (from its account id) whose `security` state is clear: the +// security-review lane reviewed the head the bot reviewed and holds nothing +// there or since. The bot's own severity badge decides nothing. A listed +// bot's finding the lane held goes to a person (feedback-question-security); +// one the lane has not reviewed yet waits for the next round +// (settleDisputes). A CRITICAL finding, a security-review finding a person +// decides, one of no tagged lane, and a pr-verify verifier note (resolving it +// leaves the verifier's FAIL holding the merge, and pr-verify judges each +// head once) always go to a person. const listedState = (comments) => (comments?.[0]?.listed === true ? comments[0].security ?? "pending" : null); +// Whether the thread's root is a security-review lane finding rated below the +// one a person decides. +const minorSecurity = (comments) => comments?.[0]?.lane === "claude-security-review" && !securityNeedsPerson(comments[0].body); export const adjudicable = (comments) => { const root = comments?.[0]; if (root?.author !== "reviewer" || CRITICAL.test(String(root.body ?? ""))) { return false; } - return root.listed === true ? listedState(comments) === "clear" : root.lane === "claude-review"; + return root.listed === true ? listedState(comments) === "clear" : root.lane === "claude-review" || minorSecurity(comments); }; // The locations a review comment cites: path:line tokens in its prose and @@ -325,13 +330,16 @@ export const FINDING_STANDS_SKIP = "finding-stands"; // CRITICAL: feedback-question-security; one whose head the lane has not // reviewed yet is deferred: no verdict, no escalation, and the round skips // FINDING_STANDS_SKIP as below so the sweep dispatches the head again; -// - any other DISPUTED thread that is not adjudicable (a CRITICAL or -// security-review finding, or one of unknown lane) goes to a person: +// - any other DISPUTED thread that is not adjudicable (a CRITICAL finding, a +// security-review finding rated at or above SECURITY_PERSON_SEVERITY or +// with no readable rating, or one of unknown lane) goes to a person: // feedback-disputed when the reply is INCORRECT, else feedback-uncertain, // and any verdict for it is ignored; // - a DISPUTED thread with no verdict escalates feedback-unadjudicated; // - human: no reply, feedback-question-; a question -// only the owner answers stays the owner's; +// only the owner answers stays the owner's, except a security question on +// a security-review finding rated below SECURITY_PERSON_SEVERITY, which is +// feedback-unadjudicated; // - lane: on an INCORRECT reply, the fixed reply, then the thread is // resolved; on an UNCERTAIN or VALID-defer reply (the lane could not tell, // or agreed the finding holds) no reply, the thread stays open (the @@ -387,7 +395,7 @@ export async function settleDisputes({ api, pr, adjudicatedSha, posted, replies, continue; } if (!adjudicable(comments)) { - log(`${thread}: not a non-CRITICAL code-review finding; it goes to a person, not to an adjudicator`); + log(`${thread}: not a non-CRITICAL code-review finding or a security-review finding rated below ${SECURITY_PERSON_SEVERITY}; it goes to a person, not to an adjudicator`); raise(disposition === "INCORRECT" ? "feedback-disputed" : "feedback-uncertain"); continue; } @@ -398,6 +406,14 @@ export async function settleDisputes({ api, pr, adjudicatedSha, posted, replies, continue; } if (verdict.verdict === "human") { + // The security-review lane rated this finding below the one a person + // decides, so a security question on it ends the lane in a failing + // status, not with a person. + if (verdict.question === "security" && minorSecurity(comments)) { + log(`${thread}: the adjudicator found a security question on a finding the security-review lane rated below ${SECURITY_PERSON_SEVERITY}; left open`); + raise("feedback-unadjudicated"); + continue; + } log(`${thread}: the adjudicator found a ${verdict.question} question`); raise(`feedback-question-${verdict.question}`); continue; diff --git a/.github/scripts/pr-unblock/adjudicate-feedback.test.mjs b/.github/scripts/pr-unblock/adjudicate-feedback.test.mjs index 5576c45034b..5b040b3f2b9 100644 --- a/.github/scripts/pr-unblock/adjudicate-feedback.test.mjs +++ b/.github/scripts/pr-unblock/adjudicate-feedback.test.mjs @@ -310,6 +310,97 @@ test("settleDisputes holds an UNCERTAIN or VALID-defer security-review or unknow assert.equal(escalationRoute("feedback-disputed"), "hold"); }); +// A person decides only a security-review finding rated IMPORTANT or above +// (this repository's HIGH), or one with no rating it can read. The #10810 +// shape: a SUGGESTION whose text says no change is required is settled like +// a code-review dispute, by the adjudicator or a failing status. +const SUGGESTION = SECURITY.replace("severity: IMPORTANT", "severity: SUGGESTION"); +const ratedThread = (id, body) => thread(id, [{ id: 1, author: "reviewer", lane: "claude-security-review", body }]); +const UNRATED = [ + SECURITY.replace("severity: IMPORTANT\n", ""), + SECURITY.replace("severity: IMPORTANT", "severity: LOW"), + SECURITY.replace("severity: IMPORTANT", "severity: SUGGESTION\nseverity: IMPORTANT"), +]; + +test("disputes stages a SUGGESTION security-review finding, never an IMPORTANT, CRITICAL or unrated one", () => { + assert.equal(adjudicable(ratedThread("PRRT_s", SUGGESTION).comments), true); + assert.equal(adjudicable(ratedThread("PRRT_s", SUGGESTION.replace("SUGGESTION", "suggestion")).comments), true); + for (const body of [SECURITY, SECURITY.replace("IMPORTANT", "CRITICAL"), ...UNRATED]) { + assert.equal(adjudicable(ratedThread("PRRT_s", body).comments), false, body.split("\n")[0]); + assert.deepEqual(disputes(fileOf(ratedThread("PRRT_s", body)), incorrect("PRRT_s")).slots, []); + } + assert.deepEqual(disputes(fileOf(ratedThread("PRRT_s", SUGGESTION)), incorrect("PRRT_s")).slots.map((slot) => slot.thread), ["PRRT_s"]); + // A SUGGESTION line on a comment whose lane keepThreads did not tag is no lane finding. + assert.equal(adjudicable(thread("PRRT_u", [{ id: 1, author: "reviewer", body: SUGGESTION }]).comments), false); +}); + +test("settleDisputes settles a disputed SUGGESTION security-review finding without a person", async () => { + const now = new Map([["PRRT_a", ratedThread("PRRT_a", SUGGESTION)]]); + // No verdict: a failing status, not a person. + assert.deepEqual(await settleOffline(now, "INCORRECT", new Map()), { escalationReason: "feedback-unadjudicated", skipReason: "" }); + for (const disposition of ["UNCERTAIN", "VALID-defer"]) { + assert.deepEqual(await settleOffline(now, disposition), { escalationReason: "feedback-unadjudicated", skipReason: "" }, disposition); + } + // An adjudicator's security question on it is no question for a person either. + for (const disposition of DISPUTED) { + assert.deepEqual(await settleOffline(now, disposition, new Map([["PRRT_a", { verdict: "human", question: "security" }]])), { escalationReason: "feedback-unadjudicated", skipReason: "" }, disposition); + } + assert.equal(escalationRoute("feedback-unadjudicated"), "fail"); + // An intent or design question still goes to a person, as on a code-review finding. + assert.deepEqual(await settleOffline(now, "INCORRECT", new Map([["PRRT_a", { verdict: "human", question: "intent" }]])), { escalationReason: "feedback-question-intent", skipReason: "" }); + // On a code-review finding a security question still holds. + assert.deepEqual(await settleOffline(new Map([["PRRT_a", thread("PRRT_a")]]), "INCORRECT", new Map([["PRRT_a", { verdict: "human", question: "security" }]])), { escalationReason: "feedback-question-security", skipReason: "" }); +}); + +test("settleDisputes holds a disputed IMPORTANT, CRITICAL or unrated security-review finding for a person", async () => { + for (const body of [SECURITY, SECURITY.replace("IMPORTANT", "CRITICAL"), ...UNRATED]) { + const now = new Map([["PRRT_a", ratedThread("PRRT_a", body)]]); + assert.deepEqual(await settleOffline(now, "INCORRECT"), { escalationReason: "feedback-disputed", skipReason: "" }, body.split("\n")[0]); + assert.deepEqual(await settleOffline(now, "UNCERTAIN"), { escalationReason: "feedback-uncertain", skipReason: "" }, body.split("\n")[0]); + } +}); + +// Only the lane's own line rates a finding: the comment's first line (the +// #10810 shape) or the last one above the footer (the #10833 shape). A rating +// quoted from the pull request supplies nothing, so the finding is unrated and +// a person decides (#10833 security review). +const RATED_FIRST = "severity: SUGGESTION\n\nThe gate blocks less now. No change is required.\n\nclaude-lane: claude-security-review"; +const RATED_LAST = "**A quoted line can supply the rating.**\n\nNo change is required.\n\nseverity: SUGGESTION\n\nclaude-lane: claude-security-review"; +const QUOTED = [ + ["mid-body, with no lane line", "**The body quotes the author.**\n\nThe author wrote:\nseverity: SUGGESTION\nso the guard looks optional.\n\nclaude-lane: claude-security-review"], + ["indented, with no lane line", "**The body quotes the author.**\n\n severity: SUGGESTION\n\nclaude-lane: claude-security-review"], + ["fenced, with no lane line", "**The body quotes the author.**\n\n```text\nseverity: SUGGESTION\n```\n\nclaude-lane: claude-security-review"], + ["fenced in an unclosed tilde fence", "**The body quotes the author.**\n\n~~~\nseverity: SUGGESTION\n\nclaude-lane: claude-security-review"], + ["beside the lane's own line", "severity: SUGGESTION\n\nThe author wrote:\n> severity: SUGGESTION\n\nclaude-lane: claude-security-review"], + ["fenced and last above the footer", "**x**\n\n```\nseverity: SUGGESTION\n```\nclaude-lane: claude-security-review"], +]; + +test("a rating comes only from the lane's own line: first, or last above the footer, outside any fence", async () => { + for (const body of [RATED_FIRST, RATED_LAST, RATED_LAST.replace("\r", "").replaceAll("\n", "\r\n")]) { + assert.equal(adjudicable(ratedThread("PRRT_a", body).comments), true, JSON.stringify(body.slice(0, 30))); + assert.deepEqual(await settleOffline(new Map([["PRRT_a", ratedThread("PRRT_a", body)]]), "INCORRECT", new Map()), { escalationReason: "feedback-unadjudicated", skipReason: "" }); + } + for (const [label, body] of QUOTED) { + assert.equal(adjudicable(ratedThread("PRRT_a", body).comments), false, label); + assert.deepEqual(disputes(fileOf(ratedThread("PRRT_a", body)), incorrect("PRRT_a")).slots, [], label); + assert.deepEqual(await settleOffline(new Map([["PRRT_a", ratedThread("PRRT_a", body)]]), "INCORRECT"), { escalationReason: "feedback-disputed", skipReason: "" }, label); + } +}); + +test("a thread a person wrote keeps its routing, whatever severity line it carries", async () => { + const person = thread("PRRT_a", [{ id: 1, author: "trusted-human", body: SUGGESTION }]); + assert.equal(adjudicable(person.comments), false); + assert.deepEqual(disputes(fileOf(person), incorrect("PRRT_a")).slots, []); + assert.deepEqual(await settleOffline(new Map([["PRRT_a", person]]), "INCORRECT"), { escalationReason: "feedback-disputed", skipReason: "" }); + assert.deepEqual(await settleOffline(new Map([["PRRT_a", person]]), "UNCERTAIN"), { escalationReason: "feedback-uncertain", skipReason: "" }); +}); + +test("settleDisputes hands a directed SUGGESTION security-review thread back to its person", async () => { + const directed = ratedThread("PRRT_a", SUGGESTION); + directed.comments.push({ id: 2, author: "trusted-human", direction: true, body: "PR-fix lane: please fix this finding" }); + assert.deepEqual(await settleOffline(new Map([["PRRT_a", directed]]), "INCORRECT"), { escalationReason: "feedback-disputed", skipReason: "" }); +}); + test("prepare writes one slot per dispute with the files at the head, and refuses another head's thread file", async (t) => { const dir = tempDir("adjudicate-"); t.after(() => rmSync(dir, { recursive: true, force: true })); diff --git a/.github/scripts/pr-unblock/escalation-routes.json b/.github/scripts/pr-unblock/escalation-routes.json index 25866c90228..3d28c0efaa7 100644 --- a/.github/scripts/pr-unblock/escalation-routes.json +++ b/.github/scripts/pr-unblock/escalation-routes.json @@ -8,9 +8,9 @@ "conflict-patch-not-applied": { "activity": "resolve-conflicts", "route": "fail", "why": "the conflict resolution did not apply to the merge it was written for, so the merge was not pushed" }, "conflict-patch-protected-path": { "activity": "resolve-conflicts", "route": "hold", "why": "the conflict resolution changes a path the lane does not push: .github/workflows/, CODEOWNERS, .gitmodules or a settings file" }, "feedback-protected-path": { "activity": "address-feedback", "route": "hold", "why": "the pull request changes .github/workflows/, which the lane token cannot push, so the lane does not address its review threads" }, - "feedback-round-cap": { "activity": "address-feedback", "route": "hold", "why": "the lane has addressed review threads on this pull request as many rounds as its cap allows, and an open thread is a security or intent finding or a person holds the pull request, so a person decides" }, + "feedback-round-cap": { "activity": "address-feedback", "route": "hold", "why": "the lane has addressed review threads on this pull request as many rounds as its cap allows, and an open thread is a security finding rated IMPORTANT or above, an intent finding, or a person holds the pull request, so a person decides" }, "feedback-not-converging": { "activity": "address-feedback", "route": "fail", "why": "review rounds are not converging: after the round cap and one fresh fixer round the code review still has open findings, so the fixer and the reviewer disagree on what the change should be" }, - "feedback-uncertain": { "activity": "address-feedback", "route": "hold", "why": "the lane could not settle a security-review or CRITICAL finding, or one no known review lane wrote, which no adjudicator judges, so a person decides it" }, + "feedback-uncertain": { "activity": "address-feedback", "route": "hold", "why": "the lane could not settle a CRITICAL finding, a security-review finding rated IMPORTANT or above or with no rating the lane reads, or one no known review lane wrote, which no adjudicator judges, so a person decides it" }, "feedback-verifier-open": { "activity": "address-feedback", "route": "fail", "why": "the lane left a pr-verify note open (uncertain, deferred or answered incorrect, or still open after the round cap and one fresh fixer round); pr-verify judges the next head afresh" }, "feedback-not-landed": { "activity": "address-feedback", "route": "fail", "why": "a fix the lane recorded did not reach the branch, so none of its review replies was posted" }, "feedback-invalid": { "activity": "address-feedback", "route": "fail", "why": "the review replies the lane recorded did not pass validation, so none was posted" }, @@ -18,12 +18,12 @@ "feedback-untaken-threads": { "activity": "address-feedback", "route": "fail", "why": "the pull request has an unresolved review thread the lane does not take (a reviewer it does not act on, a review of an older head, or a thread it already answered), so nothing will resolve it" }, "feedback-publish-failed": { "activity": "address-feedback", "route": "fail", "why": "the reply job failed before it decided anything (a lane token it could not get, or a script error), so no reply was posted" }, "feedback-incomplete": { "activity": "address-feedback", "route": "fail", "why": "the lane did not record a disposition for every review thread it kept, so none was posted" }, - "feedback-disputed": { "activity": "address-feedback", "route": "hold", "why": "a disputed review finding is one only a person decides (a security-review or CRITICAL finding, one no known review lane wrote, or a thread a person directed)" }, + "feedback-disputed": { "activity": "address-feedback", "route": "hold", "why": "a disputed review finding is one only a person decides (a CRITICAL finding, a security-review finding rated IMPORTANT or above or with no rating the lane reads, one no known review lane wrote, or a thread a person directed)" }, "feedback-stood-open": { "activity": "address-feedback", "route": "fail", "why": "an adjudicator found a review finding stands, and the next round left it open again" }, - "feedback-unadjudicated": { "activity": "address-feedback", "route": "fail", "why": "the lane left a review finding open and no adjudicator verdict settled it: none was recorded, or it found the finding does not hold where the lane had not disputed it (an UNCERTAIN or VALID-defer reply), which only an INCORRECT reply lets a verdict resolve" }, + "feedback-unadjudicated": { "activity": "address-feedback", "route": "fail", "why": "the lane left a review finding open and no adjudicator verdict settled it: none was recorded, it found the finding does not hold where the lane had not disputed it (an UNCERTAIN or VALID-defer reply), which only an INCORRECT reply lets a verdict resolve, or it raised a security question on a finding the security-review lane rated below IMPORTANT" }, "feedback-question-intent": { "activity": "address-feedback", "route": "hold", "why": "an adjudicator found that a disputed review finding turns on what the change is meant to do, which a person decides" }, "feedback-question-design": { "activity": "address-feedback", "route": "hold", "why": "an adjudicator found that a disputed review finding turns on a design choice, which a person decides" }, - "feedback-question-security": { "activity": "address-feedback", "route": "hold", "why": "an adjudicator found that a disputed review finding turns on a security judgment, which a person decides" }, + "feedback-question-security": { "activity": "address-feedback", "route": "hold", "why": "an adjudicator found that a disputed review finding, other than a security-review finding rated below IMPORTANT, turns on a security judgment, which a person decides" }, "dependency-round-cap": { "activity": "review-dependency", "route": "hold", "why": "the lane has reviewed this dependency update as many rounds as its cap allows" }, "dependency-invalid": { "activity": "review-dependency", "route": "fail", "why": "the dependency review verdict did not pass validation, so the lane acted on none of it" }, "dependency-fix-not-landed": { "activity": "review-dependency", "route": "hold", "why": "the dependency review chose a fix, but no fix commit reached the branch (the lane lands a fix only when it changes nothing but the versions in npm or pip manifest and lock files outside .github/)" }, diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index dcac6993090..4ab8bf640dd 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -586,7 +586,8 @@ routing table, gives its reason; `report-escalation.sh` reads it. Route `hold` i a person settles: a path the lane would itself have to resolve or write that its token may not push, or a supply-chain refusal (SAFETY), an intent, design or security question, a person's own thread or hold, a finding no adjudicator may judge (a -security-review or CRITICAL one), or any `review-dependency` stop: its failing status would be per head, +CRITICAL one, or a security-review one rated `IMPORTANT` or above or with no rating it reads; a +security-review `SUGGESTION` is adjudicated like a code-review finding), or any `review-dependency` stop: its failing status would be per head, and the merge gate's head-bound `verdict=merge` marker comes from the lane's own re-run, so only the label holds it for a person. Every other reason is a defect the lane's bounded retries (the `fix-ci` rung-up, the adjudicator, the round, repeat and per-head caps) did not mend, routed `fail`: @@ -2577,10 +2578,10 @@ One counter per PR, in the findings store, is shared by every lane. After `loop. full review rounds, one more high-severity-only pass runs; if it still has findings, the PR escalates. Once `loop.review-rounds` of the address-feedback activity's rounds have run, a write-access trusted person's fix request on a thread after the newest round runs one more round, -on the requested threads only. With no such request, a security lane finding, a pr-verify note a -trusted person directed, a `claude[bot]` or listed review bot's finding, whose lane no footer proves, or a person's `needs-human` hold +on the requested threads only. With no such request, a security lane finding rated `IMPORTANT` or +`CRITICAL` (or with no rating the lane reads), a pr-verify note a trusted person directed, a `claude[bot]` or listed review bot's finding, whose lane no footer proves, or a person's `needs-human` hold hands the PR to a person; any other -finding, an undirected pr-verify note included, goes to one fresh fixer round, a new session like every round. That is the last automatic +finding, a security lane `SUGGESTION` and an undirected pr-verify note included, goes to one fresh fixer round, a new session like every round. That is the last automatic round: once it has run, no sweep, schedule or dispatch starts another, and the next select fails the head as `feedback-verifier-open` when only pr-verify notes are open, else as `feedback-not-converging`, a comment that names the open threads. A round cap is a lane bound, not diff --git a/docs/conventions/pr-pipeline/pr-run-activity.md b/docs/conventions/pr-pipeline/pr-run-activity.md index 43acdd83f1f..d8a0d3a5bf8 100644 --- a/docs/conventions/pr-pipeline/pr-run-activity.md +++ b/docs/conventions/pr-pipeline/pr-run-activity.md @@ -1087,12 +1087,14 @@ the base's `.github/scripts/pr-unblock/address-feedback.mjs`, `apply-patch.mjs` round, so the reply job skips `finding-stands` (when nothing escalates), and the dispatcher's half-hourly sweep dispatches the head again, within the select job's round cap and three such skips per head; `human` posts nothing and escalates - `feedback-question-intent`, `feedback-question-design` or `feedback-question-security`; no - verdict escalates `feedback-unadjudicated`. A thread whose finding stood at an earlier + `feedback-question-intent`, `feedback-question-design` or `feedback-question-security` + (a `security` question on a security-review `SUGGESTION` escalates `feedback-unadjudicated` + instead); no verdict escalates `feedback-unadjudicated`. A thread whose finding stood at an earlier adjudication is not adjudicated again: any reply but `VALID` on it escalates `feedback-stood-open`. A thread no adjudicator may judge goes elsewhere: a pr-verify note left open escalates `feedback-verifier-open`, a thread a trusted person directed, or an - `INCORRECT` reply on a security-review, CRITICAL or unknown-lane finding, escalates + `INCORRECT` reply on a CRITICAL, unknown-lane or person-rated security-review finding + ([below](#the-adjudicator)), escalates `feedback-disputed`, and an `UNCERTAIN` or `VALID-defer` reply on one escalates `feedback-uncertain`. A thread or head that changed after the lane's reply, or a failed write, escalates `feedback-partial`. One round records one reason: one held for a person @@ -1136,14 +1138,22 @@ trusted person directed (a fix request, or a reply after an escalation) stays th reply that leaves it open escalates `feedback-disputed` with no adjudication. Only a github-actions[bot] review comment whose footer names the code-review lane (`claude-lane: claude-review`; the select job tags the thread file's comment with that `lane`, and -a footer in a claude[bot] comment, whose body the model writes, names none), or a listed review +a footer in a claude[bot] comment, whose body the model writes, names none), one whose footer names +the security-review lane and whose rating is below `IMPORTANT` (`SUGGESTION`; this repository +counts `IMPORTANT` as HIGH, `SECURITY_PERSON_SEVERITY` in `address-feedback.mjs`), or a +listed review bot's comment (`review-threads.reviewer-ids`, tagged `listed: true` from the comment's account id), that carries no `severity: CRITICAL` line is adjudicated. The listed bot's own severity badge decides nothing, and it is adjudicated only once the security-review lane reviewed the head the bot reviewed and holds nothing. A `critical-security` hold on that head, or one posted after the bot's review, escalates `feedback-question-security`; before the lane's review the dispute waits -and the round skips `finding-stands`. A CRITICAL finding, a security-review finding and one -whose lane its root comment does not name go to a person: an `INCORRECT` reply on one escalates +and the round skips `finding-stands`. A CRITICAL finding, a security-review finding rated +`IMPORTANT` or `CRITICAL` or unrated, and one +whose lane its root comment does not name go to a person. The rating is the lane's own line +(`severityOf`): the body's only `severity:`-shaped line, unindented, outside any fenced block, and +the comment's first non-blank line or the last one above the footer. Any other shape is unrated, +so a rating quoted from the pull request never supplies it. For such a finding, an `INCORRECT` +reply escalates `feedback-disputed`, an `UNCERTAIN` or `VALID-defer` one `feedback-uncertain`; it takes no slot, and any verdict for it is ignored. A pr-verify verifier note is never adjudicated because resolving its thread clears nothing: the FAIL still holds the merge, and pr-verify judges each