Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changes/harness-ops/feat-lane-limit-restarts-361b393d.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
bump: minor
---

### Added

- `/harness-ops:morning-brief` gains a Lanes section: per lane workflow over the last 7 days, runs, failures by class (rate-limit, max-turns, auth, other), limit stops, restarts and items with 3 or more limit stops (an item is a pull request run's head, else the event and the `#N` that ends the run title), then turn headroom (max and p90 turns against the max-turns cap) with a `CHECK` line and run links when a run hit max-turns or the p90 is at or above 80% of the cap. A run the action stopped for exceeding its turn cap counts as max-turns whatever class the lane record gives. New flags: `--lane-workflows`, `--lane-read-limit`, and the fixture flags `--lane-runs-json` and `--lane-annotations-json`.
23 changes: 9 additions & 14 deletions .github/scripts/pr-unblock/dispatch.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
// It lists open PRs into main, keeps the non-draft, same-repository ones
// whose mergeable state is CONFLICTING (re-querying UNKNOWN a bounded number
// of times), skips a PR whose current head already carries a pr-unblock check
// run, and dispatches pr-unblock.yml on main for at most maxDispatch of them,
// run (one the usage limit stopped stops counting once its limit resets, see
// limit-stop.mjs), and dispatches pr-unblock.yml on main for at most maxDispatch of them,
// lowest PR number first. Labels and authors are left to the lane's own
// config and gate. Logs carry PR numbers, SHAs and run ids only. The run ids
// it dispatched become the step output run-ids, which the workflow records.
Expand All @@ -19,6 +20,7 @@ import { pathToFileURL } from "node:url";

import { POLICY_PATH } from "./dependency-classify.mjs";
import { dispatchDependencyReviews } from "./dependency-dispatch.mjs";
import { anyHandles } from "./limit-stop.mjs";
import { readConfig, readPolicy } from "./review-dependency.mjs";

export const DEFAULTS = Object.freeze({
Expand Down Expand Up @@ -57,16 +59,6 @@ function isPending(node, base) {
);
}

// A check run handles the head unless it is the gate's completed neutral skip
// for an untrusted trigger: a trusted dispatch may still resolve that head.
function handles(run) {
return !(
run.status === "completed" &&
run.conclusion === "neutral" &&
String(run.output?.title ?? "").startsWith("Skipped: untrusted-trigger")
);
}

async function listAll(api, base) {
const nodes = [];
let after;
Expand All @@ -92,6 +84,7 @@ export async function dispatchConflicting({
maxDispatch = DEFAULTS.maxDispatch,
unknownRetries = DEFAULTS.unknownRetries,
unknownDelayMs = DEFAULTS.unknownDelayMs,
now = Date.now(),
}) {
const listed = await listAll(api, base);
const candidates = listed.filter((node) => isCandidate(node, base));
Expand Down Expand Up @@ -120,17 +113,18 @@ export async function dispatchConflicting({
log(`defer #${node.number}: ${maxDispatch} dispatches this run; left for a later run`);
continue;
}
let existing;
let handled;
try {
existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId });
const existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId });
handled = await anyHandles(existing, { listAnnotations: api.listAnnotations, now });
} catch (error) {
failed += 1;
log(
`::error::#${node.number}: could not read check runs for ${node.headRefOid}: ${error.message}`,
);
continue;
}
if (existing.some(handles)) {
if (handled) {
log(`skip #${node.number}: head ${node.headRefOid} already has a ${checkName} check run`);
continue;
}
Expand Down Expand Up @@ -259,6 +253,7 @@ export function createGitHubApi({
);
return result.check_runs;
},
listAnnotations: (checkRunId) => paginate(`check-runs/${Number(checkRunId)}/annotations`),
async dispatchWorkflow({ workflow, ref, inputs }) {
const result = await request(
`${apiUrl}/repos/${owner}/${name}/actions/workflows/${workflow}/dispatches`,
Expand Down
39 changes: 39 additions & 0 deletions .github/scripts/pr-unblock/dispatch.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@ import { createGitHubApi, dispatchConflicting, main } from "./dispatch.mjs";

const CHECK_NAME = "pr-unblock / resolve-conflicts";
const SUCCESS_RUN = { status: "completed", conclusion: "success", output: { title: "Resolved" } };
// The lane record report-lane-outcome (ci-workflows) writes on a limit stop.
const limitRecord = (tokens = "") => ({
annotation_level: "failure",
title: "",
message: `pr-unblock skipped: usage limit reached; re-run after it resets (pr-unblock exited with: failure class=rate-limit${tokens}). Last SDK result: {}`,
});
const UNTRUSTED_SKIP = {
status: "completed",
conclusion: "neutral",
Expand Down Expand Up @@ -39,6 +45,7 @@ function fakeApi({
failDispatch = new Set(),
failChecks = new Set(),
runIds = new Map(),
annotations = new Map(),
} = {}) {
const calls = { list: [], get: [], checks: [], dispatch: [] };
let round = 0;
Expand Down Expand Up @@ -77,6 +84,9 @@ function fakeApi({
}
return checked.has(sha) ? [SUCCESS_RUN] : [];
},
async listAnnotations(checkRunId) {
return annotations.get(checkRunId) ?? [];
},
async dispatchWorkflow({ workflow, ref, inputs }) {
calls.dispatch.push({ workflow, ref, inputs });
if (failDispatch.has(inputs["pr-number"])) {
Expand Down Expand Up @@ -233,6 +243,35 @@ test("an in-progress run or any other completed conclusion counts as handled", a
assert.deepEqual(api.calls.dispatch, []);
});

// A run the usage limit stopped (FINAL-DESIGN C5) stops counting as handled
// once its limit resets: at its `resets=` time, else five hours after it
// ended. Until then, and for any other failure, it still counts.
test("a limit-stopped run counts as handled only until its limit resets", async () => {
const now = Date.parse("2026-10-17T12:00:00Z");
const stop = (id, completed) => ({ id, status: "completed", conclusion: "failure", completed_at: completed, output: { title: "x" } });
const [statedPast, statedFuture, fallbackPast, fallbackFuture] = [pr(21), pr(22), pr(23), pr(24)];
const { api, run } = harness(
{
pages: [[statedPast, statedFuture, fallbackPast, fallbackFuture]],
checked: new Map([
[statedPast.headRefOid, [stop(1, "2026-10-17T10:00:00Z")]],
[statedFuture.headRefOid, [stop(2, "2026-10-17T10:00:00Z")]],
[fallbackPast.headRefOid, [stop(3, "2026-10-17T06:00:00Z")]],
[fallbackFuture.headRefOid, [stop(4, "2026-10-17T08:00:00Z")]],
]),
annotations: new Map([
[1, [{ annotation_level: "failure", message: "Process completed with exit code 1." }, limitRecord(" resets=2026-10-17T11:00:00Z turns=3")]],
[2, [limitRecord(" resets=2026-10-17T13:00:00Z resets-window=five_hour")]],
[3, [limitRecord()]],
[4, [limitRecord(" turns=2")]],
]),
},
{ now, maxDispatch: 10 },
);
await run();
assert.deepEqual(dispatchedNumbers(api), ["21", "23"]);
});

test("maxDispatch defaults to 3, in ascending PR number, logging the PRs left for a later run", async () => {
const { api, lines, run } = harness({ pages: [[pr(25), pr(23), pr(21), pr(24), pr(22)]] });
await run();
Expand Down
20 changes: 8 additions & 12 deletions .github/scripts/pr-unblock/feedback-dispatch.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
// github-actions[bot], name that head;
// - an unresolved review thread exists whose every comment is the reviewer
// bot's;
// - that head has no pr-unblock / address-feedback check run yet, and no
// - that head has no pr-unblock / address-feedback check run yet (one the
// usage limit stopped stops counting once its limit resets), and no
// pr-unblock.yml address-feedback run for the PR is still queued or going;
// - the base config's pr-unblock lane is enabled, holds enabled
// address-feedback and publish-feedback slots, and address-feedback applies
Expand All @@ -27,6 +28,8 @@
import { appendFileSync } from "node:fs";
import { pathToFileURL } from "node:url";

import { anyHandles } from "./limit-stop.mjs";

export const DEFAULTS = Object.freeze({
base: "main",
workflow: "pr-unblock.yml",
Expand Down Expand Up @@ -77,16 +80,6 @@ export function hasBotThread(threads, reviewerId = DEFAULTS.reviewerId) {
// pr-unblock.yml's run-name, which names the activity and the PR.
export const runTitle = (activity, number) => `pr-unblock ${activity} #${number}`;

// A check run handles the head unless it is the gate's completed neutral skip
// for an untrusted trigger.
function handles(run) {
return !(
run.status === "completed" &&
run.conclusion === "neutral" &&
String(run.output?.title ?? "").startsWith("Skipped: untrusted-trigger")
);
}

export async function dispatchFeedback({
api,
repository,
Expand All @@ -96,6 +89,7 @@ export async function dispatchFeedback({
resolveSlot,
log,
settings = DEFAULTS,
now = Date.now(),
}) {
const skip = (why) => {
log(`skip: ${why}`);
Expand Down Expand Up @@ -151,7 +145,7 @@ export async function dispatchFeedback({
return skip(`#${number} has no unresolved thread left only by the reviewer bot`);
}
const existing = await api.listCheckRuns({ sha: head, checkName: settings.checkName, appId: settings.appId });
if (existing.some(handles)) {
if (await anyHandles(existing, { listAnnotations: api.listAnnotations, now })) {
return skip(`#${number}: head ${head} already has a ${settings.checkName} check run`);
}
// Both review lanes finish on one head, and their dispatchers queue in one
Expand Down Expand Up @@ -277,6 +271,8 @@ export function createGitHubApi({ token, repository, apiUrl, graphqlUrl, fetch =
});
return (await request(`${repo}/commits/${sha}/check-runs?${query}`)).check_runs;
},
// A limit stop's annotations are a handful, well inside one page.
listAnnotations: (checkRunId) => request(`${repo}/check-runs/${Number(checkRunId)}/annotations?per_page=100`),
// The newest workflow_dispatch runs: a run still queued or going is
// among them, since a dispatcher waits at most minutes behind it.
async listWorkflowRuns(workflow) {
Expand Down
39 changes: 39 additions & 0 deletions .github/scripts/pr-unblock/feedback-dispatch.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ const marker = (lane, sha = HEAD, user = ACTIONS_BOT) => ({
user,
body: `<!-- claude-lane-reviewed-head lane=${lane} sha=${sha} -->\n${lane} has reviewed this pull request through ${sha}.`,
});
// The lane record report-lane-outcome (ci-workflows) writes on a limit stop.
const limitRecord = (tokens = "") => ({
annotation_level: "failure",
title: "",
message: `pr-unblock skipped: usage limit reached; re-run after it resets (pr-unblock exited with: failure class=rate-limit${tokens}). Last SDK result: {}`,
});
const thread = (authors, isResolved = false) => ({
isResolved,
comments: { nodes: authors.map((author) => ({ author })), pageInfo: { hasNextPage: false } },
Expand All @@ -44,6 +50,7 @@ function fakeApi({
comments = [marker("claude-review"), marker("claude-security-review")],
threads = [thread([CLAUDE])],
checkRuns = [],
annotations = [],
workflowRuns = [],
workflowRunId = 38779200001,
} = {}) {
Expand Down Expand Up @@ -80,6 +87,7 @@ function fakeApi({
listIssueComments: async () => comments,
listReviewThreads: async () => threads,
listCheckRuns: async () => checkRuns,
listAnnotations: async () => annotations,
listWorkflowRuns: async () => workflowRuns,
async dispatchWorkflow(request) {
calls.dispatch.push(request);
Expand Down Expand Up @@ -251,6 +259,37 @@ test("the gate's untrusted-trigger skip on the head does not block a dispatch",
assert.equal((await decide(api)).result.number, 42);
});

// FINAL-DESIGN C5: a limit-stopped address-feedback run holds the head only
// until its limit resets; the next review completion then dispatches again.
test("a limit-stopped check on the head blocks a dispatch only until its reset", async () => {
const stopped = [{ id: 7, status: "completed", conclusion: "failure", completed_at: "2026-10-17T10:00:00Z", output: { title: "x" } }];
for (const [resets, expected] of [
["2026-10-17T11:00:00Z", 42],
["2026-10-17T13:00:00Z", undefined],
]) {
const api = fakeApi({ checkRuns: stopped, annotations: [limitRecord(` resets=${resets}`)] });
const result = await dispatchFeedback({
api,
repository: REPOSITORY,
runId: RUN_ID,
upstreamRunId: UPSTREAM_RUN_ID,
appSenderId: APP_ID,
resolveSlot: async () => resolvedLane(),
log: () => {},
now: Date.parse("2026-10-17T12:00:00Z"),
});
assert.equal(result?.number, expected, resets);
}
});

test("a failed check on the head that is not a limit stop blocks a dispatch", async () => {
const api = fakeApi({
checkRuns: [{ id: 7, status: "completed", conclusion: "failure", completed_at: "2026-10-01T00:00:00Z", output: { title: "x" } }],
annotations: [{ annotation_level: "failure", title: "", message: "pr-unblock exited with: failure class=other (infrastructure error)" }],
});
assert.equal((await decide(api)).result, null);
});

test("a dispatch with no usable run id throws", async () => {
await assert.rejects(decide(fakeApi({ workflowRunId: null })), /no valid workflow_run_id/);
});
Expand Down
75 changes: 75 additions & 0 deletions .github/scripts/pr-unblock/limit-stop.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Whether an existing pr-unblock check run already handles a head, shared by
// dispatch.mjs and feedback-dispatch.mjs. A check run handles the head unless
// it is the gate's completed neutral skip for an untrusted trigger, or a run
// the usage limit stopped whose limit has reset: the lane's next trigger then
// dispatches it again, at most MAX_LIMIT_STOPS times on one head.
//
// A limit stop is read only from the lane record annotation that ci-workflows'
// report-lane-outcome writes with core.error: failure level, no title, and a
// message that opens "<AI review|lane> skipped: usage limit reached; re-run
// after it resets (<lane> exited with: <outcome> class=rate-limit <record
// tokens>). Last SDK result: ", matched from its start, so text a step or the
// model prints elsewhere in the job cannot pass for one.

// When a limit stop names no `resets=` time, it is taken to reset this long
// after the run ended: the shortest plan limit is a five-hour session window
// (https://support.claude.com/en/articles/11145838).
// Hard-coded until pr-pipeline.yaml has a limits.fallback-reset leaf:
// https://github.com/melodic-software/claude-code-plugins/issues/7764
export const FALLBACK_RESET_MS = 5 * 3_600_000;
const HOUR_MS = 3_600_000;
// Limit stops on one head after which the dispatchers stop re-dispatching it
// (judgment); the next push makes a new head.
export const MAX_LIMIT_STOPS = 3;
const RECORD =
/^([^\n()]+) skipped: usage limit reached; re-run after it resets \(([^\n()]+) exited with: [a-z_]+ class=rate-limit((?: [a-z-]+=[^\s()]+)*)\)\. Last SDK result: /;
const RESETS = /^resets=(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?Z)$/;

// When the limit that stopped `run` resets (epoch ms), or null when no lane
// record marks it a limit stop. The stated reset is held between one hour and
// seven days after the run ended.
export function limitResetsAt(run, annotations) {
const ended = Date.parse(run.completed_at ?? "");
if (!Number.isFinite(ended)) {
return null;
}
const record = annotations
.filter((annotation) => annotation?.annotation_level === "failure" && !annotation.title)
.map((annotation) => RECORD.exec(String(annotation.message ?? "")))
.find((match) => match && match[1] === (/review/i.test(match[2]) ? "AI review" : match[2]));
if (!record) {
return null;
}
const stated = record[3]
.split(" ")
.map((token) => RESETS.exec(token)?.[1])
.find(Boolean);
const resets = stated ? Date.parse(stated) : ended + FALLBACK_RESET_MS;
return Math.min(Math.max(resets, ended + HOUR_MS), ended + 7 * 24 * HOUR_MS);
}

const untrustedSkip = (run) =>
run.status === "completed" &&
run.conclusion === "neutral" &&
String(run.output?.title ?? "").startsWith("Skipped: untrusted-trigger");

// Whether `runs` handle the head: any run that is not an untrusted skip or a
// reset limit stop does, and so do MAX_LIMIT_STOPS limit stops. Annotations
// are read only for a failed run.
export async function anyHandles(runs, { listAnnotations, now }) {
let stops = 0;
for (const run of runs) {
if (untrustedSkip(run)) {
continue;
}
if (run.status !== "completed" || run.conclusion !== "failure") {
return true;
}
const resets = limitResetsAt(run, await listAnnotations(run.id));
if (resets === null || now < resets) {
return true;
}
stops += 1;
}
return stops >= MAX_LIMIT_STOPS;
}
Loading
Loading