Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
---
bump: patch
---

### Fixed

- A disputed security-review finding rated `SUGGESTION` no longer hands the pull request to a
person. address-feedback reads the lane's own `severity:` line (`severityOf` in
`address-feedback.mjs`: the body's only such line, outside any fence, first or last above the
footer) and holds for a person only at `IMPORTANT` or above, the rating this
repository counts as HIGH (`SECURITY_PERSON_SEVERITY`). A `SUGGESTION` is adjudicated as a
code-review finding is, and an adjudicator's `security` question on it ends the lane in a failing
status as `feedback-unadjudicated`, not `feedback-question-security`. Past the round cap it gets
the fresh fixer's round. An `IMPORTANT` or `CRITICAL` finding, one with no such line, an unknown
rating, a second rating-shaped line or one quoted from the pull request, and a thread a person directed still go to a person as before.
87 changes: 75 additions & 12 deletions .github/scripts/pr-unblock/address-feedback.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -100,22 +100,85 @@ const lastLine = (body) =>
export const laneOf = (body) => LANE_FOOTER.exec(lastLine(body))?.[1] ?? null;
export const isLaneComment = (comment) => comment.authorId === MARKER_AUTHOR_ID && laneOf(comment.body) !== null;

// The security-review lane's ratings, lowest first (security-review SKILL.md,
// "Reporting": one `severity: <rating>` line per inline comment), and the
// lowest one a person decides: this repository counts IMPORTANT as HIGH.
export const SECURITY_SEVERITIES = Object.freeze(["SUGGESTION", "IMPORTANT", "CRITICAL"]);
export const SECURITY_PERSON_SEVERITY = "IMPORTANT";
// Any line shaped like a rating, wherever it sits; the lane's own line is
// unindented, outside a fence, and the comment's first line or the last one
// above its `claude-lane:` footer (blank lines skipped).
const SEVERITY_SHAPED = /^[ \t>]*severity:/i;
const SEVERITY_LINE = /^severity:[ \t]*([A-Za-z]+)[ \t]*$/i;
const FENCE = /^[ \t]*(```|~~~)/;
// A lane comment's one rating, upper-cased, or null (unrated, so a person
// decides) unless the body holds exactly one rating-shaped line and it is the
// lane's own: unindented, outside any fenced block, and the first non-blank
// line or the last non-blank line above the lane footer. A rating quoted from
// the pull request (fenced, indented, `>`-quoted or mid-body) never supplies
// it, and any second rating-shaped line voids it.
export function severityOf(body) {
const lines = String(body ?? "")
.split("\n")
.map((line) => line.replace(/\r$/, ""));
const content = [];
let fence = null;
for (const line of lines) {
const open = FENCE.exec(line)?.[1];
if (fence === null && open) {
fence = open;
content.push({ line, fenced: true });
continue;
}
if (fence !== null) {
if (open === fence) {
fence = null;
}
content.push({ line, fenced: true });
continue;
}
content.push({ line, fenced: false });
}
const shaped = content.filter(({ line }) => SEVERITY_SHAPED.test(line));
if (shaped.length !== 1 || shaped[0].fenced) {
return null;
}
const nonBlank = content.filter(({ line }) => line.trim() !== "");
const footer = nonBlank.length > 0 && laneOf(nonBlank.at(-1).line) !== null;
const positions = [nonBlank[0], footer ? nonBlank.at(-2) : undefined];
const match = positions.includes(shaped[0]) ? SEVERITY_LINE.exec(shaped[0].line) : null;
const rating = match?.[1].toUpperCase();
return SECURITY_SEVERITIES.includes(rating) ? rating : null;
}
// Whether a security-review lane comment is one a person decides: rated at or
// above SECURITY_PERSON_SEVERITY, or with no rating it can read (fail closed).
export const securityNeedsPerson = (body) => {
const severity = severityOf(body);
return severity === null || SECURITY_SEVERITIES.indexOf(severity) >= SECURITY_SEVERITIES.indexOf(SECURITY_PERSON_SEVERITY);
};

// A thread whose first comment is a pr-verify verifier note (threadAuthors).
const isVerifierNote = (thread) => thread.comments[0]?.author === "verifier";

// Past the round cap, the threads only a person may still decide: a security
// lane finding, a verifier note a trusted person directed, and a finding
// whose lane no footer pr-review-cli appended names: only on a
// github-actions[bot] root (`rootAuthorId`) is the footer the reusable's,
// never model text. The code lane's finding and an undirected verifier note
// go to a fresh fixer: a round cap is a lane bound, and pr-verify's FAIL
// already holds the merge and judges the next head afresh.
export const needsPerson = (thread, rootAuthorId) =>
isVerifierNote(thread)
? thread.comments.some((comment) => comment.direction === true)
: rootAuthorId !== MARKER_AUTHOR_ID ||
thread.comments[0]?.author !== "reviewer" ||
laneOf(thread.comments[0].body) !== "claude-review";
// lane finding a person decides (securityNeedsPerson), a verifier note a
// trusted person directed, and a finding whose lane no footer pr-review-cli
// appended names: only on a github-actions[bot] root (`rootAuthorId`) is the
// footer the reusable's, never model text. The code lane's finding, a
// security lane finding rated below SECURITY_PERSON_SEVERITY and an undirected
// verifier note go to a fresh fixer: a round cap is a lane bound, and
// pr-verify's FAIL already holds the merge and judges the next head afresh.
export const needsPerson = (thread, rootAuthorId) => {
if (isVerifierNote(thread)) {
return thread.comments.some((comment) => comment.direction === true);
}
const root = thread.comments[0];
if (rootAuthorId !== MARKER_AUTHOR_ID || root?.author !== "reviewer") {
return true;
}
const lane = laneOf(root.body);
return lane === "claude-security-review" ? securityNeedsPerson(root.body) : lane !== "claude-review";
};

// Round markers the lanes App posted; any other author's copy is ignored.
export const isRoundMarker = (comment, appId) => comment.user?.id === appId && firstLine(comment.body) === ROUND_MARKER;
Expand Down
18 changes: 18 additions & 0 deletions .github/scripts/pr-unblock/address-feedback.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -874,6 +874,24 @@ for (const [label, roots] of [
}
}

// A person decides only a security lane finding rated IMPORTANT or above, or
// one with no rating the lane can read; a SUGGESTION goes to the fresh fixer
// as a code lane finding does.
const SUGGESTION_FINDING = LANE_FINDING.replace("severity: IMPORTANT", "severity: SUGGESTION");
test("at the cap, a SUGGESTION security lane finding goes to a fresh fixer; an IMPORTANT, CRITICAL or unrated one escalates feedback-round-cap", async () => {
const fresh = await capped({ roots: [SUGGESTION_FINDING, CODE_FINDING] });
assert.equal(fresh.decision, "proceed");
assert.equal(fresh.escalationReason, undefined);
assert.deepEqual(fresh.file.threads.map((row) => row.id), ["PRRT_1", "PRRT_2"]);
const fencedOnly = "**x**\n\n```\nseverity: SUGGESTION\n```\n\nclaude-lane: claude-security-review";
const midBody = "**x**\n\nquoted:\nseverity: SUGGESTION\nmore text\n\nclaude-lane: claude-security-review";
for (const body of [LANE_FINDING, LANE_FINDING.replace("IMPORTANT", "CRITICAL"), LANE_FINDING.replace("severity: IMPORTANT\n", ""), LANE_FINDING.replace("IMPORTANT", "HIGHISH"), fencedOnly, midBody]) {
const result = await capped({ roots: [body] });
assert.equal(result.decision, "escalate", body.split("\n")[0]);
assert.equal(result.escalationReason, "feedback-round-cap");
}
});

test("at the cap, the reviewer bot's own finding ending in the code lane's footer still escalates: its footer is model text", async () => {
const api = fakeApi({
reviewComments: [comment(100, REVIEWER, { body: CODE_FINDING })],
Expand Down
46 changes: 31 additions & 15 deletions .github/scripts/pr-unblock/adjudicate-feedback.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import { appendFileSync, existsSync, lstatSync, mkdirSync, readFileSync, writeFi
import { dirname, join } from "node:path";
import { pathToFileURL } from "node:url";

import { createGitHubApi, FINDING_STANDS, need, SHA } from "./address-feedback.mjs";
import { createGitHubApi, FINDING_STANDS, need, SECURITY_PERSON_SEVERITY, securityNeedsPerson, SHA } from "./address-feedback.mjs";

// The most disputed threads one round adjudicates: pr-unblock.yml runs one
// model step per slot. A thread past it is not adjudicated, and the round
Expand Down Expand Up @@ -84,22 +84,27 @@ export const directed = (comments) => (comments ?? []).some((comment) => comment
// Whether an adjudicator may judge a dispute on the thread: a review comment
// that carries no `severity: CRITICAL` line and that keepThreads tagged with
// the code-review lane (a github-actions[bot] footer; claude[bot] writes its
// own body, so it names no lane) or as a listed review bot's (from its account
// id) whose `security` state is clear: the security-review lane reviewed the
// head the bot reviewed and holds nothing there or since. The bot's own
// severity badge decides nothing. A listed bot's finding the lane held goes to
// a person (feedback-question-security); one the lane has not reviewed yet
// waits for the next round (settleDisputes). A CRITICAL finding, a
// security-review finding, one of no tagged lane, and a pr-verify verifier
// note (resolving it leaves the verifier's FAIL holding the merge, and
// pr-verify judges each head once) always go to a person.
// own body, so it names no lane), with the security-review lane and a rating
// below the one a person decides (securityNeedsPerson), or as a listed review
// bot's (from its account id) whose `security` state is clear: the
// security-review lane reviewed the head the bot reviewed and holds nothing
// there or since. The bot's own severity badge decides nothing. A listed
// bot's finding the lane held goes to a person (feedback-question-security);
// one the lane has not reviewed yet waits for the next round
// (settleDisputes). A CRITICAL finding, a security-review finding a person
// decides, one of no tagged lane, and a pr-verify verifier note (resolving it
// leaves the verifier's FAIL holding the merge, and pr-verify judges each
// head once) always go to a person.
const listedState = (comments) => (comments?.[0]?.listed === true ? comments[0].security ?? "pending" : null);
// Whether the thread's root is a security-review lane finding rated below the
// one a person decides.
const minorSecurity = (comments) => comments?.[0]?.lane === "claude-security-review" && !securityNeedsPerson(comments[0].body);
export const adjudicable = (comments) => {
const root = comments?.[0];
if (root?.author !== "reviewer" || CRITICAL.test(String(root.body ?? ""))) {
return false;
}
return root.listed === true ? listedState(comments) === "clear" : root.lane === "claude-review";
return root.listed === true ? listedState(comments) === "clear" : root.lane === "claude-review" || minorSecurity(comments);
};

// The locations a review comment cites: path:line tokens in its prose and
Expand Down Expand Up @@ -325,13 +330,16 @@ export const FINDING_STANDS_SKIP = "finding-stands";
// CRITICAL: feedback-question-security; one whose head the lane has not
// reviewed yet is deferred: no verdict, no escalation, and the round skips
// FINDING_STANDS_SKIP as below so the sweep dispatches the head again;
// - any other DISPUTED thread that is not adjudicable (a CRITICAL or
// security-review finding, or one of unknown lane) goes to a person:
// - any other DISPUTED thread that is not adjudicable (a CRITICAL finding, a
// security-review finding rated at or above SECURITY_PERSON_SEVERITY or
// with no readable rating, or one of unknown lane) goes to a person:
// feedback-disputed when the reply is INCORRECT, else feedback-uncertain,
// and any verdict for it is ignored;
// - a DISPUTED thread with no verdict escalates feedback-unadjudicated;
// - human: no reply, feedback-question-<intent|design|security>; a question
// only the owner answers stays the owner's;
// only the owner answers stays the owner's, except a security question on
// a security-review finding rated below SECURITY_PERSON_SEVERITY, which is
// feedback-unadjudicated;
// - lane: on an INCORRECT reply, the fixed reply, then the thread is
// resolved; on an UNCERTAIN or VALID-defer reply (the lane could not tell,
// or agreed the finding holds) no reply, the thread stays open (the
Expand Down Expand Up @@ -387,7 +395,7 @@ export async function settleDisputes({ api, pr, adjudicatedSha, posted, replies,
continue;
}
if (!adjudicable(comments)) {
log(`${thread}: not a non-CRITICAL code-review finding; it goes to a person, not to an adjudicator`);
log(`${thread}: not a non-CRITICAL code-review finding or a security-review finding rated below ${SECURITY_PERSON_SEVERITY}; it goes to a person, not to an adjudicator`);
raise(disposition === "INCORRECT" ? "feedback-disputed" : "feedback-uncertain");
continue;
}
Expand All @@ -398,6 +406,14 @@ export async function settleDisputes({ api, pr, adjudicatedSha, posted, replies,
continue;
}
if (verdict.verdict === "human") {
// The security-review lane rated this finding below the one a person
// decides, so a security question on it ends the lane in a failing
// status, not with a person.
if (verdict.question === "security" && minorSecurity(comments)) {
log(`${thread}: the adjudicator found a security question on a finding the security-review lane rated below ${SECURITY_PERSON_SEVERITY}; left open`);
raise("feedback-unadjudicated");
continue;
}
log(`${thread}: the adjudicator found a ${verdict.question} question`);
raise(`feedback-question-${verdict.question}`);
continue;
Expand Down
Loading
Loading