Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
bump: patch
---

### Changed

- The merge gate runs the shared YAML reader once per config text and key list in a process, not once per commit: the post-merge check reads the pipeline config at every merge since its floor, and those reads spawned about 14,600 reader processes per candidate. A replayed gate run on one candidate drops from 138 to 44 seconds of local work, with the same verdict.
- A gate candidate no longer re-runs a post-merge check that already came back unreadable at main's current head, from the same floor, identity and lane, within the shared answer's age: it is refused with that check's reason. The merge job's own exit-3 check counts as that one check, so an unreadable day costs one check per sweep instead of one per candidate (about 50 gh calls and 4 minutes each). A check that comes back unreadable is still a hold, never a pass.
Original file line number Diff line number Diff line change
Expand Up @@ -3698,16 +3698,29 @@ def assess_unattended(
shared: merge_policy.SharedPostMerge | None = None,
share: Callable[[merge_policy.SharedPostMerge], object] | None = None,
stop_when_closed: bool = False,
postmerge_result: str | None = None,
) -> merge_policy.UnattendedPolicy:
"""The unattended decision for one pull request, in the wiring order:
identity, the credential chokepoint, the kill switch, a default-branch
base, the post-merge check (or the run's `shared` answer), demotion and cap, in flight, promotion evidence, eligibility, then
the review evidence and the score. Raises `PolicyReadError` when the
pull request or its pipeline config cannot be read."""

post_merge = postmerge.gate_reader(
repo, self_logins, lane_merges_only=authority == merge_policy.PR_MERGE
)
lane = authority == merge_policy.PR_MERGE
post_merge = postmerge.gate_reader(repo, self_logins, lane_merges_only=lane)
if postmerge_result is not None:

def main_head() -> str | None:
try:
owner, name = merge_policy._split_repo(repo)
branch = merge_policy._default_branch(repo, shared=True)
return merge_policy.base_tip(owner, name, branch)
except merge_policy.PolicyReadError:
return None

post_merge = postmerge.reuse_unreadable(
postmerge_result, post_merge, main_head, lane_merges_only=lane
)

def promotion(window_start: datetime | None) -> merge_policy.Promotion:
return merge_policy.read_promotion(
Expand Down Expand Up @@ -5426,6 +5439,7 @@ def _deadline_stop() -> int:
# request whose read-back needs the policy of a merged
# pull request, so a merged one skips the post-merge check.
stop_when_closed=pending_path is None,
postmerge_result=args.postmerge_result,
# Each recompute goes back to the file for the next candidate.
share=(
None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,13 @@ def resolve_bash(
return next((c for c in candidates if _accept_bash(c, windows, exists)), None)


# The shared reader's answers by document, keys, reader script, command
# runner and environment, of which it is a pure function: `read_config_at`
# reads the same few config texts at hundreds of commits per gate run, and
# one reader process per key made that most of a candidate's time.
_CONCERN_ANSWERS: dict[Any, dict[str, list[str]] | None] = {}


def _concern_values(
text: str,
keys: list[tuple[str, bool]],
Expand All @@ -328,8 +335,32 @@ def _concern_values(
run through `run` (default `run_command`).

None when the reader rejects the document (`--strict` exit 3). Any other
failure raises.
failure raises. With the default runner an answer is read once per
process (`_CONCERN_ANSWERS`); a rejection or failure is read again, so a
one-off reader failure never sticks.
"""
if run is not None:
return _read_concern_values(text, keys, run)
key = (
text,
tuple(keys),
str(PARSER),
run_command,
tuple(sorted(os.environ.items())),
)
found = _CONCERN_ANSWERS.get(key)
if found is None:
found = _read_concern_values(text, keys, run_command)
if found is None:
return None
_CONCERN_ANSWERS[key] = found
return {k: list(v) for k, v in found.items()}


def _read_concern_values(
text: str, keys: list[tuple[str, bool]], run: Callable[..., Any]
) -> dict[str, list[str]] | None:
"""`_concern_values`, read through the shared reader every time."""
platform = "win32" if os.name == "nt" else sys.platform
bash = resolve_bash(os.environ, platform, os.path.isfile)
if bash is None:
Expand All @@ -341,7 +372,7 @@ def _concern_values(
for key, as_list in keys:
argv = [bash, str(PARSER), "--strict", *(["--list"] if as_list else [])]
try:
proc = (run or run_command)(
proc = run(
[*argv, str(document), key],
allowed_executables=(bash,),
check=False,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3896,7 +3896,11 @@ def read_shared(path: str) -> policy.SharedPostMerge | None:
demotion = policy.Demotion(
record["active"], fields[3], tuple(records), newest, cleared
)
report = {k: v for k, v in payload.items() if k not in ("since", "shared")}
report = {
k: v
for k, v in payload.items()
if k not in ("since", "shared", UNREADABLE_KEY)
}
return policy.SharedPostMerge(
main_sha=fields[0],
config_sha=fields[1],
Expand Down Expand Up @@ -3941,6 +3945,13 @@ def write_shared(path: str, shared: policy.SharedPostMerge) -> bool:
"computedAt": shared.computed_at.isoformat(),
},
}
return _replace_json(path, payload)


def _replace_json(path: str, payload: Mapping[str, Any]) -> bool:
"""Write `payload` to a temp file beside `path`, then `os.replace` it,
so a reader sees the old file or the new one, never part of one. Best
effort: a failure is logged and the old file stays."""
tmp: Path | None = None
try:
target = Path(path)
Expand All @@ -3956,6 +3967,109 @@ def write_shared(path: str, shared: policy.SharedPostMerge) -> bool:
return True


# The run report's record of a gate's own check that came back unreadable.
UNREADABLE_KEY = "unreadableRecompute"


def _unreadable(report: Mapping[str, Any]) -> bool:
return (
report.get("ok") is not True
and report.get("readable") is False
and str(report.get("reason") or "").startswith("unreadable")
)
Comment thread
kyle-sexton marked this conversation as resolved.
Resolved


def _unreadable_at(
path: str,
head: str,
floor: datetime,
identity: str,
lane_merges_only: bool,
) -> dict[str, Any] | None:
"""The report of an unreadable check at `path` made at main's `head` from
`floor` for this identity and lane within `SHARED_MAX_AGE`, or None: the
gate's own (`UNREADABLE_KEY`), else the job's check the file holds."""
try:
with open(path, encoding="utf-8") as handle:
payload = json_object(json.load(handle))
except (OSError, ValueError):
return None
shared = json_object(payload.get("shared"))
job = {
"mainSha": shared.get("mainSha"),
"since": payload.get("since"),
"identity": shared.get("identity"),
"laneMergesOnly": shared.get("laneMergesOnly"),
"computedAt": shared.get("computedAt"),
"report": {
k: v
for k, v in payload.items()
if k not in ("since", "shared", UNREADABLE_KEY)
},
}
now = datetime.now(UTC)
for record in (json_object(payload.get(UNREADABLE_KEY)), job):
report = json_object(record.get("report"))
try:
age = now - policy._time(record.get("computedAt"))
if (
record.get("mainSha") == head
and policy._time(record.get("since")) == floor
and _login_key(record.get("identity")) == _login_key(identity)
and record.get("laneMergesOnly") is lane_merges_only
and timedelta(0) <= age <= policy.SHARED_MAX_AGE
and _unreadable(report)
):
return report
except policy.PolicyReadError:
continue
return None


def reuse_unreadable(
path: str,
post_merge: Callable[[datetime, str], policy.PostMergeGate],
head: Callable[[], str | None],
*,
lane_merges_only: bool,
) -> Callable[[datetime, str], policy.PostMergeGate]:
"""`post_merge` for one gate candidate of a run whose report is at
`path`. A check this run already made at main's head now (`head()`),
from the same floor, identity and lane, that came back unreadable is
refused again with its reason and not run again: the same reads at the
same head answer the same, and each run costs minutes of the gate's
budget. A head that cannot be read runs the check. A check that comes
back unreadable is recorded in the file for the next candidate."""

def gate(floor: datetime, identity: str) -> policy.PostMergeGate:
sha = head()
if not (isinstance(sha, str) and policy.SHA_RE.fullmatch(sha)):
return post_merge(floor, identity)
known = _unreadable_at(path, sha, floor, identity, lane_merges_only)
if known is not None:
return gate_from_report({**known, "unreadableReusedAt": sha})
computed_at = datetime.now(UTC)
answer = post_merge(floor, identity)
if _unreadable(answer.report):
try:
with open(path, encoding="utf-8") as handle:
payload = json_object(json.load(handle))
except (OSError, ValueError):
payload = {}
payload[UNREADABLE_KEY] = {
"mainSha": sha,
"since": floor.isoformat(),
"identity": identity,
"laneMergesOnly": lane_merges_only,
"computedAt": computed_at.isoformat(),
"report": dict(answer.report),
}
_replace_json(path, payload)
return answer

return gate


def _since(value: str) -> datetime:
parsed = datetime.fromisoformat(value)
if parsed.utcoffset() is None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9444,39 +9444,79 @@ def test_a_permission_past_the_bound_is_read_again(self) -> None:
)


_read_concern_values = policy._concern_values
_real_run_command = policy.run_command
_READER = policy.PARSER
_concern_answers: dict[Any, dict[str, list[str]] | None] = {}
def setUpModule() -> None:
_role_label.start()


def _cached_concern_values(
text: str, keys: list[tuple[str, bool]]
) -> dict[str, list[str]] | None:
"""The shared YAML reader's answer, read once per document, key list and
environment: the reader is a pure function of them, and spawning it for
every config read dominated this module's run time. A test that swaps
the command runner or the reader script reads through."""
if policy.run_command is not _real_run_command or policy.PARSER != _READER:
return _read_concern_values(text, keys)
key = (text, tuple(keys), tuple(sorted(os.environ.items())))
if key not in _concern_answers:
_concern_answers[key] = _read_concern_values(text, keys)
found = _concern_answers[key]
return None if found is None else {k: list(v) for k, v in found.items()}
def tearDownModule() -> None:
_role_label.stop()


_concern_cache = mock.patch.object(policy, "_concern_values", _cached_concern_values)
class ConcernValuesAnswers(unittest.TestCase):
"""The shared YAML reader runs once per document and key list in a
process: `read_config_at` reads the same few config texts at hundreds of
commits per gate run."""

def setUp(self) -> None:
self.argvs: list[list[str]] = []
self.fail = False
real = policy.run_command

def setUpModule() -> None:
_role_label.start()
_concern_cache.start()
def counting(argv: list[str], **kwargs: Any) -> Any:
self.argvs.append(argv)
if self.fail:
raise OSError("reader unavailable")
return real(argv, **kwargs)

patcher = mock.patch.object(policy, "run_command", counting)
patcher.start()
self.addCleanup(patcher.stop)

def tearDownModule() -> None:
_concern_cache.stop()
_role_label.stop()
def test_a_document_is_read_once(self) -> None:
keys = [("merge.rung", False), ("merge.authority", False)]
text = "merge:\n rung: off\n authority: pr-merge\n"
first = policy._concern_values(text, keys)
second = policy._concern_values(text, keys)
self.assertEqual(
first, {"merge.rung": ["off"], "merge.authority": ["pr-merge"]}
)
self.assertEqual(second, first)
self.assertEqual(len(self.argvs), 2)
second["merge.rung"].append("x")
self.assertEqual(policy._concern_values(text, keys), first)
policy._concern_values("merge:\n rung: on\n", keys)
self.assertEqual(len(self.argvs), 4)

def test_a_rejected_read_is_read_again(self) -> None:
keys = [("merge.rung", False)]
text = "merge:\n rung: rejected-once\n"
answers = iter([None, {"merge.rung": ["rejected-once"]}])
with mock.patch.object(
policy, "_read_concern_values", lambda *_: next(answers)
):
self.assertIsNone(policy._concern_values(text, keys))
self.assertEqual(
policy._concern_values(text, keys), {"merge.rung": ["rejected-once"]}
)

def test_a_failed_read_is_read_again(self) -> None:
keys = [("merge.rung", False)]
text = "merge:\n rung: retried\n"
self.fail = True
with self.assertRaises(policy.PolicyReadError):
policy._concern_values(text, keys)
self.fail = False
self.assertEqual(
policy._concern_values(text, keys), {"merge.rung": ["retried"]}
)
self.assertEqual(len(self.argvs), 2)

def test_a_given_runner_reads_every_time(self) -> None:
keys = [("merge.rung", False)]
text = "merge:\n rung: given\n"
policy._concern_values(text, keys, run=policy.run_command)
policy._concern_values(text, keys, run=policy.run_command)
self.assertEqual(len(self.argvs), 2)


class FileContents(unittest.TestCase):
Expand Down
Loading
Loading