From cb9f4bacb77ad547eae6e0415bb2b722fde30370 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:28:54 -0400 Subject: [PATCH 01/73] fix(knowledge): drop rolling-caption repeats at video-digest paragraph boundaries On the auto-caption path, a paragraph that opens with the previous paragraph's trailing 2-12 words (compared case- and punctuation-insensitively) now drops them, longest match first. Co-Authored-By: Claude Opus 5.5 --- .../extraction/transcript/write-transcript.js | 51 ++++++++++++++++++- .../transcript/write-transcript.test.js | 49 ++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index 1415b26dd74..a1967cfa6be 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -34,6 +34,54 @@ function countParagraphs(transcript) { return transcript ? transcript.split("\n\n").length : 0; } +const BOUNDARY_REPEAT_MIN_WORDS = 2; +const BOUNDARY_REPEAT_MAX_WORDS = 12; +const PARAGRAPH_STAMP = /^(\[[^\]]*\]\s*)([\s\S]*)$/; + +/** @param {string} word */ +function comparableWord(word) { + return word.toLowerCase().replace(/[^\p{L}\p{N}']/gu, ""); +} + +/** @param {string[]} words */ +function comparableRun(words) { + return words.map(comparableWord).join(" "); +} + +/** + * Drop the words a paragraph opens with when they repeat the previous + * paragraph's closing words: the longest run of 2 to 12 words, compared + * case- and punctuation-insensitively. A rolling auto caption carries its + * tail into the next cue, and the vendor formatter's case-only overlap check + * misses a repeat that differs in punctuation. A paragraph always keeps at + * least one word. + * + * @param {string} transcript - paragraphs joined by a blank line, each `[M:SS] text` + * @returns {string} + */ +export function dropParagraphBoundaryRepeats(transcript) { + const paragraphs = transcript.split("\n\n"); + for (let index = 1; index < paragraphs.length; index++) { + const current = PARAGRAPH_STAMP.exec(paragraphs[index]); + const previous = PARAGRAPH_STAMP.exec(paragraphs[index - 1]); + if (!current || !previous) continue; + const previousWords = previous[2].split(/\s+/).filter(Boolean); + const currentWords = current[2].split(/\s+/).filter(Boolean); + const longest = Math.min( + BOUNDARY_REPEAT_MAX_WORDS, + previousWords.length, + currentWords.length - 1, + ); + for (let length = longest; length >= BOUNDARY_REPEAT_MIN_WORDS; length--) { + if (comparableRun(previousWords.slice(-length)) === comparableRun(currentWords.slice(0, length))) { + paragraphs[index] = current[1] + currentWords.slice(length).join(" "); + break; + } + } + } + return paragraphs.join("\n\n"); +} + /** * Build transcript text from a caption file. * @@ -66,7 +114,8 @@ export function buildTranscriptText(vttText, isAutoCaption, { repairLexicon = nu repairedTermCount = repaired.replacementCount; } - const transcript = formatTranscript(cues); + const formatted = formatTranscript(cues); + const transcript = isAutoCaption ? dropParagraphBoundaryRepeats(formatted) : formatted; return { transcript, diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index b2353bfe5c9..b6384f6295d 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -41,6 +41,55 @@ describe("buildTranscriptText", () => { expect(result.transcript).toBe("[0:04] [music] >> Hi everybody."); }); + it("drops auto-caption words a paragraph repeats from the previous paragraph's tail", () => { + // The rolling caption carries "benchmarks, too." into the next cue, which + // opens a new paragraph after the sentence end (#6992). + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +we ran the + +00:00:03.000 --> 00:00:05.000 +numbers on benchmarks, too. + +00:00:05.000 --> 00:00:07.000 +Benchmarks too and then we moved on. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe( + "[0:01] we ran the numbers on benchmarks, too.\n\n[0:05] and then we moved on.", + ); + }); + + it("keeps a single repeated word at a paragraph boundary", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +we ran the + +00:00:03.000 --> 00:00:05.000 +numbers again. + +00:00:05.000 --> 00:00:07.000 +Again we moved on. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe("[0:01] we ran the numbers again.\n\n[0:05] Again we moved on."); + }); + + it("leaves a manual-caption paragraph boundary repeat in place", () => { + const manualVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +We ran the numbers on benchmarks, too. + +00:00:05.000 --> 00:00:07.000 +Benchmarks too, and then we moved on. +`; + const result = buildTranscriptText(manualVtt, false); + expect(result.transcript).toContain("Benchmarks too, and then we moved on."); + }); + it("parses manual captions without auto-clean pass", () => { const manualVtt = `WEBVTT From 3509038e16510e16b0299eca55cdcf2eba99157e Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:34:38 -0400 Subject: [PATCH 02/73] feat(knowledge): re-lay out a closed video-digest slice into the corpus layout watch/relayout-slice.js copies a closed slice into the knowledge-corpus layout (transcript/, metadata/ with a trimmed info.json, frames/all/, frames/key/, media/, analysis/, a provenance README when absent), rewrites every slice path its markdown names through a path map, and link-checks the result, exiting 1 on unresolved slice-internal paths. It refuses an unclosed slice, and a slice whose temp media is gone unless --no-media. output-contract.md documents it in place of the by-hand re-layout. Co-Authored-By: Claude Opus 5.5 --- ...st-relayout-and-caption-dedupe-59495b96.md | 11 + .../knowledge/skills/video-digest/SKILL.md | 2 +- .../video-digest/context/output-contract.md | 25 +- .../extraction/transcript/write-transcript.js | 4 +- .../extraction/watch/relayout-slice.js | 517 ++++++++++++++++++ .../extraction/watch/relayout-slice.test.js | 248 +++++++++ 6 files changed, 802 insertions(+), 5 deletions(-) create mode 100644 .changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md create mode 100644 plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js create mode 100644 plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js diff --git a/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md new file mode 100644 index 00000000000..ad36fcd3abc --- /dev/null +++ b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md @@ -0,0 +1,11 @@ +--- +bump: minor +--- + +### Added + +- video-digest: `watch/relayout-slice.js` copies a closed slice into the knowledge-corpus layout (`transcript/`, `metadata/`, `frames/all/`, `frames/key/`, `media/`, `analysis/`, provenance `README.md`), rewrites the slice paths its markdown names through a path map, and exits 1 listing every slice-internal path that still does not resolve. It refuses an unclosed slice, and a slice whose temp media is gone unless `--no-media` ([#6902](https://github.com/melodic-software/claude-code-plugins/issues/6902)). + +### Fixed + +- video-digest: an auto-caption transcript paragraph no longer repeats the previous paragraph's closing words (2 to 12 words, compared case- and punctuation-insensitively) that a rolling caption carried over ([#6992](https://github.com/melodic-software/claude-code-plugins/issues/6992)). diff --git a/plugins/knowledge/skills/video-digest/SKILL.md b/plugins/knowledge/skills/video-digest/SKILL.md index 3df19ea6a34..b1621537d84 100644 --- a/plugins/knowledge/skills/video-digest/SKILL.md +++ b/plugins/knowledge/skills/video-digest/SKILL.md @@ -43,7 +43,7 @@ Read a spoke **only when its condition holds**. These are mutually exclusive by | `context/watch-pipeline.md` | when running the **watch** action (or `resume` into it), the full per-phase procedure | | `context/workflow.md` | when you want the watch phase-flow diagram + phase summary table | | `context/watch-queue.md` | when running any **queue** action, or dequeuing via `watch` / `watch `. Claim protocol and preflight decision table | -| `context/output-contract.md` | when about to write or stage slice artifacts, or when a non-default `library_dir` work root is configured | +| `context/output-contract.md` | when about to write or stage slice artifacts, when a non-default `library_dir` work root is configured, or when re-laying out a closed slice into a corpus layout | | `context/quality-gates.md` | when grading a phase or a finished slice against binary criteria | | `context/synthesis-contract.md` | when harvesting decks, or when applying the frame promotion bar | | `templates/vision-json-shapes.md` | when briefing a vision subagent to write a sheet triage, promotion-decisions or quality-audit JSON file. Each file's keys and the rules its validators enforce | diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 988b6fa4396..6bcd390ce97 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -55,9 +55,28 @@ once per process, and the new name wins when both are set. **Scope of the setting.** `library_dir` relocates the work *root*; it does not reshape the `//` sub-path itself. A consumer whose own convention lands source -material at a differently-shaped path (for example `sources///`) does not get that -shape from this skill: land under `library_dir` as-written and re-lay-out by hand, or fork -the sub-path in your own automation. This skill's contract is root relocation only. +material at a differently-shaped path (for example `sources///`) lands under +`library_dir` as written, then re-lays out the closed slice with `relayout-slice.js`: + +```bash +node "/extraction/run.mjs" --data-dir "" watch/relayout-slice.js "" "" [--no-media] +``` + +It writes the knowledge-corpus layout: `transcript/`, `metadata/` (with a trimmed `info.json`), +`frames/all/`, `frames/key/` beside the three frame logs in `frames/`, `media/.`, +`analysis/` (`RESEARCH.md`, `research/`, `recommendations/`), and a provenance `README.md` when +the target has none. The script's header comment is the authoritative mapping. It rewrites each +slice path the copied markdown names (links and backticked paths) through a path map, relative to +the file's new location, then link-checks the target and exits 1 listing every slice-internal path +that still does not resolve; paths under `plugins/`, `docs/`, `templates/` and `.claude/`, +`AGENTS.md`, and paths that leave the target are not checked. Fix those by hand and re-run the +script, which overwrites the copied files. + +It refuses a slice whose `run-state/watch.json` status is not `complete`. The media, every +extracted frame, the served caption tracks and the info JSON come from the temp session that +`watch.json` records, and it refuses a slice whose temp media is gone unless `--no-media` is +passed, which re-lays out the slice files alone. `close` removes that temp session when it can, +so after a normal close only `--no-media` runs. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index a1967cfa6be..bb8f35b88e8 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -73,7 +73,9 @@ export function dropParagraphBoundaryRepeats(transcript) { currentWords.length - 1, ); for (let length = longest; length >= BOUNDARY_REPEAT_MIN_WORDS; length--) { - if (comparableRun(previousWords.slice(-length)) === comparableRun(currentWords.slice(0, length))) { + if ( + comparableRun(previousWords.slice(-length)) === comparableRun(currentWords.slice(0, length)) + ) { paragraphs[index] = current[1] + currentWords.slice(length).join(" "); break; } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js new file mode 100644 index 00000000000..f7dde2acbfb --- /dev/null +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -0,0 +1,517 @@ +#!/usr/bin/env node +/** + * Copy a closed video-digest slice into a target directory in the + * knowledge-corpus layout, rewrite the slice-internal paths its markdown names, + * then link-check the result. + * + * Target layout: `transcript/` (transcript text and the served caption tracks), + * `metadata/` (trimmed info JSON, harvested links and repo analysis, deck + * inventory), `frames/all/` (every extracted frame plus `frame-times.json`), + * `frames/key/` (promoted key frames) beside the frame logs in `frames/`, + * `media/.`, `analysis/` (`RESEARCH.md`, `research/`, + * `recommendations/`) and a provenance `README.md` when the target has none. + * + * Every reference is rewritten through a path map built from the copy plan: + * a reference resolves first against the naming file's own directory, then + * against the slice root, and is left alone when neither lands on a mapped + * path. The link check then reports what still does not resolve. + * + * Usage: node watch/relayout-slice.js [--no-media] + * Exit: 0 copied and every slice-internal path resolves; 1 refused or + * unresolved paths (listed on stderr); 2 usage. + */ + +import fs from "node:fs"; +import path from "node:path"; + +import { isMainModule } from "@melodic/video-digestion/shared/main-module"; +import { writeStderr, writeStdout } from "@melodic/video-digestion/shared/terminal"; + +import { listWorkDirFiles, resolveMediaArtifacts } from "../acquisition/acquire.js"; +import { LANES } from "../lib/slice-lanes.js"; +import { resolveTempSession } from "../lib/temp-session-paths.js"; +import { watchStatePath } from "./watch-state.js"; + +/** info.json keys the corpus copy drops: format, caption-URL, heatmap and request lists. */ +export const INFO_JSON_DROPPED_KEYS = Object.freeze([ + "formats", + "requested_formats", + "thumbnails", + "automatic_captions", + "subtitles", + "heatmap", + "http_headers", + "requested_subtitles", + "_format_sort_fields", + "requested_downloads", +]); + +export const INFO_JSON_COMMENT_CAP = 20; + +/** Path prefixes that name files in another repository, never in the slice. */ +export const ALLOWED_MISS_PREFIXES = Object.freeze([ + "plugins/", + "docs/", + "templates/", + ".claude/", + "AGENTS.md", +]); + +const KEY_FRAME_LOGS = new Set([ + "key-frames-manifest.md", + "visual-frames.md", + "key-frame-quality-audit.md", +]); +const SOURCE_METADATA = new Set([ + "harvested-links.json", + "harvested-repo-analysis.json", + "deck-inventory.md", +]); +const TRANSCRIPT_TEXT = /^transcript(?:-\d+)?\.txt$/; + +/** First path segments that name a slice or corpus-layout location. */ +const LAYOUT_SEGMENTS = new Set([ + "transcript", + "metadata", + "frames", + "media", + "analysis", + "SLICE", + ...Object.values(LANES), +]); + +const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; +const PLACEHOLDER = /[<>{}*$\s]/; +const LINE_SUFFIX = /:\d+(?:[-,]\d+)*$/; +const MARKDOWN_LINK = /(\]\()(<[^>]+>|[^)\s]+)/g; +const BACKTICK_SPAN = /`([^`\s]+)`/g; +/** A backticked span names a path when it holds a separator or ends in an extension. */ +const PATH_LIKE = /\/|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)?(?:#.*)?$/; + +/** + * Corpus-layout path for a slice-relative file, or null when the corpus keeps + * no copy of it. + * + * @param {string} sliceRel - posix path relative to the slice root + * @returns {string|null} + */ +export function corpusPathForSliceFile(sliceRel) { + const [lane, ...rest] = sliceRel.split("/"); + const name = rest.join("/"); + if (sliceRel === "RESEARCH.md") return "analysis/RESEARCH.md"; + if (lane === LANES.research || lane === LANES.recommendations) return `analysis/${sliceRel}`; + if (lane === LANES.source && rest.length === 1) { + if (TRANSCRIPT_TEXT.test(name)) return `transcript/${name}`; + if (SOURCE_METADATA.has(name)) return `metadata/${name}`; + } + if (lane === LANES.keyFrames) { + if (rest[0] === "frames" && rest.length > 1) return `frames/key/${rest.slice(1).join("/")}`; + if (rest.length === 1 && KEY_FRAME_LOGS.has(name)) return `frames/${name}`; + } + return null; +} + +/** + * @param {string} dir + * @param {string} [prefix] + * @returns {string[]} posix paths of every file under dir, relative to it + */ +function listFiles(dir, prefix = "") { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + return entry.isDirectory() ? listFiles(path.join(dir, entry.name), rel) : [rel]; + }); +} + +/** + * Directory mappings implied by the file mappings: a source directory maps to + * a target directory only when every mapped file under it agrees. + * + * @param {Map} fileMap + * @returns {Map} + */ +function deriveDirectoryMap(fileMap) { + /** @type {Map} */ + const candidates = new Map(); + for (const [from, to] of fileMap) { + const fromParts = from.split("/"); + const toParts = to.split("/"); + for (let depth = 1; depth < fromParts.length && depth < toParts.length; depth++) { + if (fromParts.at(-depth) !== toParts.at(-depth)) break; + const fromDir = fromParts.slice(0, -depth).join("/"); + const toDir = toParts.slice(0, -depth).join("/"); + const seen = candidates.get(fromDir); + candidates.set(fromDir, seen === undefined || seen === toDir ? toDir : null); + } + } + return new Map( + [...candidates].filter( + /** @returns {entry is [string, string]} */ (entry) => entry[1] !== null, + ), + ); +} + +/** + * @typedef {Object} PathMap + * @property {Map} files - slice-relative file → target-relative file + * @property {Map} dirs - slice-relative directory → target-relative directory + * @property {Map} tempFrames - temp frame file name → target-relative file + * @property {string} sliceName - the slice directory's own name + */ + +/** + * Split a reference into its path and the suffix to carry over unchanged + * (`#anchor`, `?query`, `:line`). + * + * @param {string} ref + * @returns {{ refPath: string, suffix: string }} + */ +function splitReference(ref) { + const hash = ref.search(/[#?]/); + let refPath = hash === -1 ? ref : ref.slice(0, hash); + let suffix = hash === -1 ? "" : ref.slice(hash); + const line = LINE_SUFFIX.exec(refPath); + if (line) { + suffix = line[0] + suffix; + refPath = refPath.slice(0, line.index); + } + return { refPath, suffix }; +} + +/** + * Target-relative location a reference names, or null when the path map does + * not hold it. + * + * @param {string} refPath + * @param {string} sourceFile - slice-relative path of the file naming it + * @param {PathMap} map + * @returns {{ target: string, isDir: boolean }|null} + */ +function lookUpReference(refPath, sourceFile, map) { + const bare = refPath.replace(/\/+$/, ""); + if (!bare || URL_SCHEME.test(bare) || PLACEHOLDER.test(bare) || bare.startsWith("/")) { + return null; + } + const lookup = (/** @type {string} */ key) => { + if (map.files.has(key)) + return { target: /** @type {string} */ (map.files.get(key)), isDir: false }; + if (map.dirs.has(key)) + return { target: /** @type {string} */ (map.dirs.get(key)), isDir: true }; + return null; + }; + // `SLICE/` and `.work///` name the slice root explicitly. + if (bare.startsWith("SLICE/")) return lookup(path.posix.normalize(bare.slice("SLICE/".length))); + const marker = `/${map.sliceName}/`; + if (bare.includes(marker)) { + return lookup(path.posix.normalize(bare.slice(bare.indexOf(marker) + marker.length))); + } + + const fromFile = path.posix.normalize(path.posix.join(path.posix.dirname(sourceFile), bare)); + const fromRoot = bare.startsWith(".") ? null : path.posix.normalize(bare); + const found = lookup(fromFile) ?? (fromRoot === null ? null : lookup(fromRoot)); + if (found) return found; + const frame = bare.includes("/") ? undefined : map.tempFrames.get(bare); + return frame ? { target: frame, isDir: false } : null; +} + +/** + * Rewrite every markdown link and backticked path in `text` that names a + * mapped slice path, relative to the file's target location. + * + * @param {string} text + * @param {string} sourceFile - slice-relative path of the file + * @param {string} targetFile - target-relative path the file is copied to + * @param {PathMap} map + * @returns {{ text: string, rewrites: number }} + */ +export function rewriteSliceReferences(text, sourceFile, targetFile, map) { + let rewrites = 0; + const rewrite = (/** @type {string} */ ref) => { + const { refPath, suffix } = splitReference(ref); + const found = lookUpReference(refPath, sourceFile, map); + if (!found) return ref; + const relative = path.posix.relative(path.posix.dirname(targetFile), found.target) || "."; + const trailing = found.isDir && refPath.endsWith("/") ? "/" : ""; + const next = `${relative}${trailing}${suffix}`; + if (next !== ref) rewrites += 1; + return next; + }; + const rewritten = text + .replace(MARKDOWN_LINK, (_match, open, target) => + target.startsWith("<") + ? `${open}<${rewrite(target.slice(1, -1))}>` + : `${open}${rewrite(target)}`, + ) + .replace(BACKTICK_SPAN, (match, span) => + PATH_LIKE.test(span) ? `\`${rewrite(span)}\`` : match, + ); + return { text: rewritten, rewrites }; +} + +/** + * True when a backticked span reads as a path into the slice: an explicit + * `./` or `../` path, or one whose first segment is a layout directory or an + * entry beside the naming file. + * + * @param {string} refPath + * @param {string} fileDir - absolute directory of the naming file + */ +function looksSliceInternal(refPath, fileDir) { + if (!refPath.includes("/")) return false; + if (refPath.startsWith("./") || refPath.startsWith("../")) return true; + const first = refPath.split("/")[0]; + return LAYOUT_SEGMENTS.has(first) || fs.existsSync(path.join(fileDir, first)); +} + +/** + * Link check over a re-laid-out slice: every markdown link and every + * backticked slice path in its markdown must resolve against the naming + * file's directory. URLs, placeholders, absolute paths, paths that leave the + * slice (another repository) and the allowed-miss prefixes are skipped. + * + * @param {string} rootDir + * @returns {{ file: string, ref: string }[]} unresolved references + */ +export function checkLayoutLinks(rootDir) { + const root = path.resolve(rootDir); + /** @type {{ file: string, ref: string }[]} */ + const unresolved = []; + const markdown = listFiles(root).filter( + (file) => file.endsWith(".md") && !file.startsWith("frames/all/") && !file.startsWith("media/"), + ); + for (const file of markdown) { + const fileDir = path.dirname(path.join(root, file)); + const text = fs.readFileSync(path.join(root, file), "utf8"); + const check = (/** @type {string} */ ref, /** @type {boolean} */ isLink) => { + const { refPath } = splitReference(ref); + if (!refPath || URL_SCHEME.test(refPath) || PLACEHOLDER.test(refPath)) return; + if (refPath.startsWith("/") || refPath.startsWith("~")) return; + if (ALLOWED_MISS_PREFIXES.some((prefix) => refPath.startsWith(prefix))) return; + if (!isLink && !looksSliceInternal(refPath, fileDir)) return; + const resolved = path.resolve(fileDir, refPath); + const rel = path.relative(root, resolved); + if (rel.startsWith("..") || path.isAbsolute(rel)) return; + if (!fs.existsSync(resolved)) unresolved.push({ file, ref }); + }; + for (const match of text.matchAll(MARKDOWN_LINK)) { + check(match[2].replace(/^<|>$/g, ""), true); + } + for (const match of text.matchAll(BACKTICK_SPAN)) { + check(match[1], false); + } + } + return unresolved; +} + +/** + * @param {Record} info + * @returns {Record} + */ +export function trimInfoJson(info) { + const trimmed = Object.fromEntries( + Object.entries(info).filter(([key]) => !INFO_JSON_DROPPED_KEYS.includes(key)), + ); + if (Array.isArray(trimmed.comments)) { + trimmed.comments = trimmed.comments.slice(0, INFO_JSON_COMMENT_CAP); + } + return trimmed; +} + +const CONTENTS_LINES = /** @type {const} */ ([ + ["media/", "source video as downloaded"], + ["transcript/", "timestamped transcript, plus the caption tracks as served when kept"], + ["metadata/", "source metadata: trimmed info JSON, harvested links, repo analysis, decks"], + ["frames/all/", "every frame the pipeline extracted, with their times in `frame-times.json`"], + ["frames/key/", "the promoted key frames; the logs beside them in `frames/` describe them"], + ["analysis/", "the digest's research and recommendations"], +]); + +/** + * @param {{ title: string, sourceUrl: string, videoId: string, acquiredAt?: string }} meta + * @param {Set} presentTopDirs - target directories that received files + */ +function provenanceReadme({ title, sourceUrl, videoId, acquiredAt }, presentTopDirs) { + const fetched = acquiredAt ? ` on ${acquiredAt.slice(0, 10)}` : ""; + const contents = CONTENTS_LINES.filter(([dir]) => presentTopDirs.has(dir)).map( + ([dir, label]) => `- \`${dir}\`: ${label}`, + ); + return [ + `# ${title}`, + "", + "## Provenance", + "", + `- Canonical origin: <${sourceUrl}>`, + `- Video id: \`${videoId}\``, + `- Fetched${fetched} through \`/knowledge:video-digest\` and re-laid out from its closed slice.`, + "", + "## Contents", + "", + ...contents, + "", + ].join("\n"); +} + +/** + * @typedef {Object} RelayoutResult + * @property {number} exitCode + * @property {number} copied - files written to the target + * @property {number} rewrites - references rewritten + * @property {{ file: string, ref: string }[]} unresolved + */ + +/** + * @param {{ sliceDir: string, targetDir: string, noMedia?: boolean }} options + * @returns {Promise} + */ +export async function relayoutSlice({ sliceDir, targetDir, noMedia = false }) { + const slice = path.resolve(sliceDir); + const target = path.resolve(targetDir); + const refuse = (/** @type {string} */ message) => { + writeStderr(`relayout: ${message}`); + return { exitCode: 1, copied: 0, rewrites: 0, unresolved: [] }; + }; + + let state; + try { + state = JSON.parse(fs.readFileSync(watchStatePath(slice), "utf8")); + } catch { + return refuse(`no readable run-state/watch.json under ${slice}`); + } + if (state.status !== "complete") { + return refuse(`slice status is "${state.status}", not "complete"; close the slice first`); + } + + const temp = resolveTempSession(state.tempSession ?? {}); + const workFiles = temp.workDir ? await listWorkDirFiles(temp.workDir) : []; + const media = resolveMediaArtifacts(workFiles, state.videoId); + if (!media.videoPath && !noMedia) { + return refuse( + `the temp session's media is gone (${temp.workDir ?? "no tempSession.workDir"}); ` + + "pass --no-media to re-lay out without media/, frames/all/, the caption tracks and info.json", + ); + } + + /** @type {{ from: string, to: string, sourceFile?: string }[]} */ + const plan = []; + for (const file of listFiles(slice)) { + const to = corpusPathForSliceFile(file); + if (to) plan.push({ from: path.join(slice, file), to, sourceFile: file }); + } + for (const caption of media.captionPaths) { + const name = path.basename(caption).replace(`${state.videoId}.`, ""); + plan.push({ from: caption, to: `transcript/${name}` }); + } + /** @type {Map} */ + const tempFrames = new Map(); + if (temp.framesDir) { + for (const name of listFiles(temp.framesDir).filter((file) => !file.includes("/"))) { + plan.push({ from: path.join(temp.framesDir, name), to: `frames/all/${name}` }); + tempFrames.set(name, `frames/all/${name}`); + } + } + if (media.videoPath && !noMedia) { + plan.push({ + from: media.videoPath, + to: `media/${state.videoId}${path.extname(media.videoPath)}`, + }); + } + + // A reference to the slice README lands on the target README, provenance or existing. + const files = new Map([ + ["README.md", "README.md"], + ...plan.flatMap((entry) => + entry.sourceFile ? [/** @type {[string, string]} */ ([entry.sourceFile, entry.to])] : [], + ), + ]); + /** @type {PathMap} */ + const map = { + files, + dirs: deriveDirectoryMap(files), + tempFrames, + sliceName: path.basename(slice), + }; + + let rewrites = 0; + const write = (/** @type {string} */ to, /** @type {string|Buffer} */ body) => { + const dest = path.join(target, to); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.writeFileSync(dest, body); + }; + for (const entry of plan) { + if (entry.sourceFile?.endsWith(".md")) { + const result = rewriteSliceReferences( + fs.readFileSync(entry.from, "utf8"), + entry.sourceFile, + entry.to, + map, + ); + rewrites += result.rewrites; + write(entry.to, result.text); + } else { + fs.mkdirSync(path.dirname(path.join(target, entry.to)), { recursive: true }); + fs.copyFileSync(entry.from, path.join(target, entry.to)); + } + } + let copied = plan.length; + if (media.metadataPath) { + const info = JSON.parse(fs.readFileSync(media.metadataPath, "utf8")); + write("metadata/info.json", `${JSON.stringify(trimInfoJson(info), null, 2)}\n`); + copied += 1; + } + + const readmePath = path.join(target, "README.md"); + if (!fs.existsSync(readmePath)) { + const present = new Set( + CONTENTS_LINES.map(([dir]) => dir).filter((dir) => fs.existsSync(path.join(target, dir))), + ); + write( + "README.md", + provenanceReadme( + { + title: state.title, + sourceUrl: state.sourceUrl, + videoId: state.videoId, + acquiredAt: state.tempSession?.acquiredAt, + }, + present, + ), + ); + copied += 1; + } + + const unresolved = checkLayoutLinks(target); + return { exitCode: unresolved.length > 0 ? 1 : 0, copied, rewrites, unresolved }; +} + +/** + * @param {string[]} args - argv after the script path + * @returns {Promise} + */ +export async function runRelayoutCli(args) { + const noMedia = args.includes("--no-media"); + const positional = args.filter((arg) => arg !== "--no-media"); + if (positional.length !== 2 || positional.some((arg) => arg.startsWith("--"))) { + writeStderr("Usage: node watch/relayout-slice.js [--no-media]"); + return 2; + } + const [sliceDir, targetDir] = positional; + const result = await relayoutSlice({ sliceDir, targetDir, noMedia }); + if (result.copied > 0) { + writeStdout( + `relayout: ${result.copied} files written to ${path.resolve(targetDir)}, ` + + `${result.rewrites} references rewritten`, + ); + } + for (const { file, ref } of result.unresolved) { + writeStderr(`unresolved: ${file} -> ${ref}`); + } + if (result.unresolved.length > 0) { + writeStderr(`relayout: ${result.unresolved.length} slice-internal paths do not resolve`); + } + return result.exitCode; +} + +if (isMainModule(import.meta.url)) { + process.exit(await runRelayoutCli(process.argv.slice(2))); +} diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js new file mode 100644 index 00000000000..14f61d697a3 --- /dev/null +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -0,0 +1,248 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { checkLayoutLinks, relayoutSlice } from "./relayout-slice.js"; + +/** @type {string[]} */ +const scratch = []; + +function makeDir(prefix) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + scratch.push(dir); + return dir; +} + +/** @param {string} root @param {Record} files */ +function writeTree(root, files) { + for (const [rel, body] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), body); + } +} + +/** @param {string} root @param {string} rel */ +function read(root, rel) { + return fs.readFileSync(path.join(root, rel), "utf8"); +} + +const VIDEO_ID = "abc123XYZ_0"; + +/** + * A small closed slice plus its temp session (media, caption tracks, info JSON + * and two extracted frames). + * + * @param {{ status?: string, withMedia?: boolean, extraFiles?: Record }} [options] + */ +function makeFixture({ status = "complete", withMedia = true, extraFiles = {} } = {}) { + const parent = makeDir("relayout-slice-"); + const sliceDir = path.join(parent, "sample-talk-abc123XYZ_0"); + const workDir = makeDir("relayout-work-"); + const framesDir = makeDir("relayout-frames-"); + if (withMedia) { + writeTree(workDir, { + [`${VIDEO_ID}.mp4`]: "video-bytes", + [`${VIDEO_ID}.en.vtt`]: "WEBVTT\n", + [`${VIDEO_ID}.en-orig.vtt`]: "WEBVTT\n", + [`${VIDEO_ID}.info.json`]: JSON.stringify({ + id: VIDEO_ID, + title: "Sample Talk", + formats: [{}], + requested_formats: [{}], + thumbnails: [{}], + automatic_captions: {}, + subtitles: {}, + heatmap: [], + http_headers: {}, + requested_subtitles: {}, + _format_sort_fields: [], + requested_downloads: [{}], + comments: Array.from({ length: 25 }, (_, index) => ({ id: `c${index}` })), + }), + }); + writeTree(framesDir, { + "scene_0001.png": "png", + "anchor_00012000_0001.png": "png", + "frame-times.json": "{}", + }); + } else { + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(framesDir, { recursive: true, force: true }); + } + writeTree(sliceDir, { + "run-state/watch.json": JSON.stringify({ + videoId: VIDEO_ID, + videoSlug: "sample-talk-abc123XYZ_0", + sourceUrl: `https://www.youtube.com/watch?v=${VIDEO_ID}`, + title: "Sample Talk", + status, + tempSession: { workDir, framesDir, acquiredAt: "2026-10-10T12:00:00.000Z" }, + }), + "README.md": "# Sample Talk journey\n", + "RESEARCH.md": "Clusters from `research/claim-inventory.md`; shards in `research/findings/`.\n", + "source/transcript.txt": "[0:01] hello\n", + "source/harvested-links.json": "[]", + "key-frames/frames/title-slide.png": "png", + "key-frames/visual-frames.md": + "| `frames/title-slide.png` | `scene_0001.png` |\n\nTiers: [manifest](key-frames-manifest.md)\n", + "key-frames/key-frames-manifest.md": "| title-slide.png |\n", + "key-frames/key-frame-quality-audit.md": "| title-slide.png | yes |\n", + "key-frames/selection.json": "{}", + "research/claim-inventory.md": "Transcript: `source/transcript.txt`.\n", + "research/findings/topic.md": + "Quote at `SLICE/source/transcript.txt:3`; see [agenda](../claim-inventory.md#sessions).\n", + "recommendations/menu.md": [ + "Evidence: `source/transcript.txt` and ![slide](../key-frames/frames/title-slide.png).", + "Detail: `research/findings/topic.md`, the hub [README](README.md), the [journey](../README.md).", + "Target file: `plugins/knowledge/skills/video-digest/SKILL.md`.", + "", + ].join("\n"), + "recommendations/README.md": "# Hub\n", + ...extraFiles, + }); + return { sliceDir, targetDir: path.join(makeDir("relayout-target-"), "out") }; +} + +afterEach(() => { + for (const dir of scratch.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe("relayoutSlice", () => { + it("copies a closed slice into the corpus layout", async () => { + const { sliceDir, targetDir } = makeFixture(); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + for (const rel of [ + "transcript/transcript.txt", + "transcript/en.vtt", + "transcript/en-orig.vtt", + "metadata/info.json", + "metadata/harvested-links.json", + "frames/all/scene_0001.png", + "frames/all/anchor_00012000_0001.png", + "frames/all/frame-times.json", + "frames/key/title-slide.png", + "frames/visual-frames.md", + "frames/key-frames-manifest.md", + "frames/key-frame-quality-audit.md", + `media/${VIDEO_ID}.mp4`, + "analysis/RESEARCH.md", + "analysis/research/claim-inventory.md", + "analysis/research/findings/topic.md", + "analysis/recommendations/menu.md", + "analysis/recommendations/README.md", + "README.md", + ]) { + expect(fs.existsSync(path.join(targetDir, rel)), rel).toBe(true); + } + expect(fs.existsSync(path.join(targetDir, "key-frames"))).toBe(false); + expect(fs.existsSync(path.join(targetDir, "run-state"))).toBe(false); + }); + + it("trims info.json to the corpus copy", async () => { + const { sliceDir, targetDir } = makeFixture(); + + await relayoutSlice({ sliceDir, targetDir }); + + const info = JSON.parse(read(targetDir, "metadata/info.json")); + expect(Object.keys(info).sort()).toEqual(["comments", "id", "title"]); + expect(info.comments).toHaveLength(20); + }); + + it("rewrites slice-internal references relative to each file's new location", async () => { + const { sliceDir, targetDir } = makeFixture(); + + await relayoutSlice({ sliceDir, targetDir }); + + expect(read(targetDir, "analysis/recommendations/menu.md")).toBe( + [ + "Evidence: `../../transcript/transcript.txt` and ![slide](../../frames/key/title-slide.png).", + "Detail: `../research/findings/topic.md`, the hub [README](README.md), the [journey](../../README.md).", + "Target file: `plugins/knowledge/skills/video-digest/SKILL.md`.", + "", + ].join("\n"), + ); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "| `key/title-slide.png` | `all/scene_0001.png` |\n\nTiers: [manifest](key-frames-manifest.md)\n", + ); + expect(read(targetDir, "analysis/research/findings/topic.md")).toBe( + "Quote at `../../../transcript/transcript.txt:3`; see [agenda](../claim-inventory.md#sessions).\n", + ); + expect(read(targetDir, "analysis/research/claim-inventory.md")).toBe( + "Transcript: `../../transcript/transcript.txt`.\n", + ); + expect(read(targetDir, "analysis/RESEARCH.md")).toBe( + "Clusters from `research/claim-inventory.md`; shards in `research/findings/`.\n", + ); + }); + + it("keeps a README the target already has", async () => { + const { sliceDir, targetDir } = makeFixture(); + fs.mkdirSync(targetDir, { recursive: true }); + fs.writeFileSync(path.join(targetDir, "README.md"), "# Curated\n"); + + await relayoutSlice({ sliceDir, targetDir }); + + expect(read(targetDir, "README.md")).toBe("# Curated\n"); + }); + + it("fails the link check on a slice path the corpus layout does not keep", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "recommendations/questions.md": "Plan: `key-frames/selection.json`.\n" }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/recommendations/questions.md", ref: "key-frames/selection.json" }, + ]); + }); + + it("refuses a slice that is not closed", async () => { + const { sliceDir, targetDir } = makeFixture({ status: "synthesizing" }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("refuses when the temp session's media is gone, unless --no-media", async () => { + const { sliceDir, targetDir } = makeFixture({ withMedia: false }); + + const refused = await relayoutSlice({ sliceDir, targetDir }); + expect(refused.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + + const result = await relayoutSlice({ sliceDir, targetDir, noMedia: true }); + expect(result.exitCode).toBe(0); + expect(fs.existsSync(path.join(targetDir, "media"))).toBe(false); + expect(fs.existsSync(path.join(targetDir, "analysis/recommendations/menu.md"))).toBe(true); + }); +}); + +describe("checkLayoutLinks", () => { + it("stays quiet on URLs, placeholders, other-repository paths and resolvable paths", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "transcript/transcript.txt": "", + "analysis/RESEARCH.md": [ + "[docs](https://code.claude.com/docs/en/hooks) and .", + "Paths: `plugins/knowledge/SKILL.md`, `docs/adr/0001.md`, `templates/x.md`, `.claude/rules/a.md`.", + "Links: [skill](plugins/knowledge/SKILL.md), [adr](docs/adr/0001.md), [rules](AGENTS.md).", + "[agents](../../../AGENTS.md), `AGENTS.md`, `mattpocock/skills`, `/code-review`.", + "Temp: `{tmp}/video-frames-x/scene_0001.png`, `/report-.html`.", + "Outside the slice: `../../other-repo/README.md`. Inside: `../transcript/transcript.txt`.", + "[anchor](#verdicts)", + "", + ].join("\n"), + }); + + expect(checkLayoutLinks(root)).toEqual([]); + }); +}); From 271fb465257edfbd9df9fb8156f7f26809f3be2c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:39:25 -0400 Subject: [PATCH 03/73] feat(knowledge): re-lay out a video-digest slice before close and retire unkept paths relayout-slice.js now also runs on an unclosed slice whose outcome checks pass, so the temp media is still there; the documented order is outcome checks, relayout with media, then close. It copies the frame triage log and the companion sources and digest, turns references to pipeline files the layout does not keep into plain text saying so, moves unknown files in copied directories with their directory, and treats a backticked path under no target directory as a path into another repo. Co-Authored-By: Claude Opus 5.5 --- ...st-relayout-and-caption-dedupe-59495b96.md | 2 +- .../video-digest/context/output-contract.md | 42 ++-- .../extraction/watch/relayout-slice.js | 183 ++++++++++++------ .../extraction/watch/relayout-slice.test.js | 60 +++++- 4 files changed, 210 insertions(+), 77 deletions(-) diff --git a/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md index ad36fcd3abc..fd79ef46b36 100644 --- a/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md +++ b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md @@ -4,7 +4,7 @@ bump: minor ### Added -- video-digest: `watch/relayout-slice.js` copies a closed slice into the knowledge-corpus layout (`transcript/`, `metadata/`, `frames/all/`, `frames/key/`, `media/`, `analysis/`, provenance `README.md`), rewrites the slice paths its markdown names through a path map, and exits 1 listing every slice-internal path that still does not resolve. It refuses an unclosed slice, and a slice whose temp media is gone unless `--no-media` ([#6902](https://github.com/melodic-software/claude-code-plugins/issues/6902)). +- video-digest: `watch/relayout-slice.js` copies a slice into the knowledge-corpus layout (`transcript/`, `metadata/`, `frames/all/`, `frames/key/`, `media/`, `analysis/`, provenance `README.md`), rewrites the slice paths its markdown names through a path map (a pipeline file the layout does not keep becomes plain text saying so), and exits 1 listing every slice-internal path that still does not resolve. It runs on a closed slice, or before `close` on one whose outcome checks pass so the temp media is still there, and refuses a slice whose temp media is gone unless `--no-media` ([#6902](https://github.com/melodic-software/claude-code-plugins/issues/6902)). ### Fixed diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 6bcd390ce97..daca1e01066 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -56,27 +56,37 @@ once per process, and the new name wins when both are set. **Scope of the setting.** `library_dir` relocates the work *root*; it does not reshape the `//` sub-path itself. A consumer whose own convention lands source material at a differently-shaped path (for example `sources///`) lands under -`library_dir` as written, then re-lays out the closed slice with `relayout-slice.js`: +`library_dir` as written, then re-lays out the slice with `relayout-slice.js`: ```bash node "/extraction/run.mjs" --data-dir "" watch/relayout-slice.js "" "" [--no-media] ``` -It writes the knowledge-corpus layout: `transcript/`, `metadata/` (with a trimmed `info.json`), -`frames/all/`, `frames/key/` beside the three frame logs in `frames/`, `media/.`, -`analysis/` (`RESEARCH.md`, `research/`, `recommendations/`), and a provenance `README.md` when -the target has none. The script's header comment is the authoritative mapping. It rewrites each -slice path the copied markdown names (links and backticked paths) through a path map, relative to -the file's new location, then link-checks the target and exits 1 listing every slice-internal path -that still does not resolve; paths under `plugins/`, `docs/`, `templates/` and `.claude/`, -`AGENTS.md`, and paths that leave the target are not checked. Fix those by hand and re-run the -script, which overwrites the copied files. - -It refuses a slice whose `run-state/watch.json` status is not `complete`. The media, every -extracted frame, the served caption tracks and the info JSON come from the temp session that -`watch.json` records, and it refuses a slice whose temp media is gone unless `--no-media` is -passed, which re-lays out the slice files alone. `close` removes that temp session when it can, -so after a normal close only `--no-media` runs. +The media, every extracted frame, the served caption tracks and the info JSON come from the temp +session `watch.json` records, which `close` removes. To keep them, run in this order: + +1. `evals/check-watch-outcomes.js ""` passes. +2. `watch/relayout-slice.js "" ""`, with media. +3. `watch/watch-state.js close ""`. + +The script runs on a closed slice, or on an unclosed one whose outcome checks pass (it runs them +itself and refuses when they fail). It refuses any slice whose temp media is gone unless +`--no-media` is passed, which re-lays out the slice files alone; after a normal close that is the +only way it runs. + +It writes the knowledge-corpus layout: `transcript/`, `metadata/` (with a trimmed `info.json` +and the companion sources brief), `frames/all/`, `frames/key/` beside the frame logs in +`frames/`, `media/.`, `analysis/` (`RESEARCH.md`, `research/`, `recommendations/`, the +companion digest), and a provenance `README.md` when the target has none. The script's header +comment is the authoritative mapping. Each slice path the copied markdown names (links and +backticked paths) is rewritten through a path map relative to the file's new location, and one +naming a pipeline file the layout does not keep (such as `run-state/watch.json`) becomes plain +text saying it is not retained. It then link-checks the target and exits 1 listing every +slice-internal path that still does not resolve. Not checked: paths under `plugins/`, `docs/`, +`templates/` and `.claude/`, `AGENTS.md`, paths that leave the target, and backticked paths whose +first segment is neither a top-level directory of the target nor an entry beside the naming file +(paths into another repository). Fix what it lists by hand and re-run; it overwrites the copied +files. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index f7dde2acbfb..2cd0f296247 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -1,20 +1,25 @@ #!/usr/bin/env node /** - * Copy a closed video-digest slice into a target directory in the + * Copy a finished video-digest slice into a target directory in the * knowledge-corpus layout, rewrite the slice-internal paths its markdown names, - * then link-check the result. + * then link-check the result. A slice is finished when it is closed, or when + * it is not closed yet and its outcome checks pass (the pre-close run, while + * the temp session still holds the media). * * Target layout: `transcript/` (transcript text and the served caption tracks), * `metadata/` (trimmed info JSON, harvested links and repo analysis, deck - * inventory), `frames/all/` (every extracted frame plus `frame-times.json`), - * `frames/key/` (promoted key frames) beside the frame logs in `frames/`, - * `media/.`, `analysis/` (`RESEARCH.md`, `research/`, - * `recommendations/`) and a provenance `README.md` when the target has none. + * inventory, companion sources), `frames/all/` (every extracted frame plus + * `frame-times.json`), `frames/key/` (promoted key frames) beside the frame + * logs in `frames/`, `media/.`, `analysis/` (`RESEARCH.md`, + * `research/`, `recommendations/`, `companion-digest/`) and a provenance + * `README.md` when the target has none. * * Every reference is rewritten through a path map built from the copy plan: * a reference resolves first against the naming file's own directory, then - * against the slice root, and is left alone when neither lands on a mapped - * path. The link check then reports what still does not resolve. + * against the slice root. One landing on a copied path is rewritten relative + * to the file's new location; one landing on a slice file or directory the + * layout does not keep becomes plain text naming it as not retained; anything + * else is left alone. The link check then reports what still does not resolve. * * Usage: node watch/relayout-slice.js [--no-media] * Exit: 0 copied and every slice-internal path resolves; 1 refused or @@ -28,6 +33,7 @@ import { isMainModule } from "@melodic/video-digestion/shared/main-module"; import { writeStderr, writeStdout } from "@melodic/video-digestion/shared/terminal"; import { listWorkDirFiles, resolveMediaArtifacts } from "../acquisition/acquire.js"; +import { runCheckWatchOutcomes } from "../evals/check-watch-outcomes.js"; import { LANES } from "../lib/slice-lanes.js"; import { resolveTempSession } from "../lib/temp-session-paths.js"; import { watchStatePath } from "./watch-state.js"; @@ -61,29 +67,21 @@ const KEY_FRAME_LOGS = new Set([ "key-frames-manifest.md", "visual-frames.md", "key-frame-quality-audit.md", + "frame-triage-log.md", ]); const SOURCE_METADATA = new Set([ "harvested-links.json", "harvested-repo-analysis.json", "deck-inventory.md", + "companion-sources.md", ]); const TRANSCRIPT_TEXT = /^transcript(?:-\d+)?\.txt$/; -/** First path segments that name a slice or corpus-layout location. */ -const LAYOUT_SEGMENTS = new Set([ - "transcript", - "metadata", - "frames", - "media", - "analysis", - "SLICE", - ...Object.values(LANES), -]); - const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$\s]/; const LINE_SUFFIX = /:\d+(?:[-,]\d+)*$/; const MARKDOWN_LINK = /(\]\()(<[^>]+>|[^)\s]+)/g; +const FULL_LINK = /!?\[([^\]\n]*)\]\((<[^>]+>|[^)\s]+)[^)\n]*\)/g; const BACKTICK_SPAN = /`([^`\s]+)`/g; /** A backticked span names a path when it holds a separator or ends in an extension. */ const PATH_LIKE = /\/|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)?(?:#.*)?$/; @@ -100,6 +98,9 @@ export function corpusPathForSliceFile(sliceRel) { const name = rest.join("/"); if (sliceRel === "RESEARCH.md") return "analysis/RESEARCH.md"; if (lane === LANES.research || lane === LANES.recommendations) return `analysis/${sliceRel}`; + if (lane === LANES.source && rest[0] === "companion-digest" && rest.length > 1) { + return `analysis/${name}`; + } if (lane === LANES.source && rest.length === 1) { if (TRANSCRIPT_TEXT.test(name)) return `transcript/${name}`; if (SOURCE_METADATA.has(name)) return `metadata/${name}`; @@ -158,8 +159,11 @@ function deriveDirectoryMap(fileMap) { * @property {Map} dirs - slice-relative directory → target-relative directory * @property {Map} tempFrames - temp frame file name → target-relative file * @property {string} sliceName - the slice directory's own name + * @property {Set} sliceEntries - every slice-relative file and directory */ +/** @typedef {{ target: string, isDir: boolean } | { retired: string }} Lookup */ + /** * Split a reference into its path and the suffix to carry over unchanged * (`#anchor`, `?query`, `:line`). @@ -180,44 +184,66 @@ function splitReference(ref) { } /** - * Target-relative location a reference names, or null when the path map does - * not hold it. + * What a reference names: a copied path (its target), a slice file or + * directory the layout does not keep (`retired`), or null when it names + * neither. * * @param {string} refPath * @param {string} sourceFile - slice-relative path of the file naming it * @param {PathMap} map - * @returns {{ target: string, isDir: boolean }|null} + * @returns {Lookup|null} */ function lookUpReference(refPath, sourceFile, map) { const bare = refPath.replace(/\/+$/, ""); if (!bare || URL_SCHEME.test(bare) || PLACEHOLDER.test(bare) || bare.startsWith("/")) { return null; } - const lookup = (/** @type {string} */ key) => { - if (map.files.has(key)) - return { target: /** @type {string} */ (map.files.get(key)), isDir: false }; - if (map.dirs.has(key)) - return { target: /** @type {string} */ (map.dirs.get(key)), isDir: true }; + const lookup = (/** @type {string|null} */ key) => { + if (key === null) return null; + const file = map.files.get(key); + if (file !== undefined) return { target: file, isDir: false }; + const dir = map.dirs.get(key); + if (dir !== undefined) return { target: dir, isDir: true }; return null; }; + /** @type {(string|null)[]} */ + let keys; // `SLICE/` and `.work///` name the slice root explicitly. - if (bare.startsWith("SLICE/")) return lookup(path.posix.normalize(bare.slice("SLICE/".length))); const marker = `/${map.sliceName}/`; - if (bare.includes(marker)) { - return lookup(path.posix.normalize(bare.slice(bare.indexOf(marker) + marker.length))); + if (bare.startsWith("SLICE/")) { + keys = [path.posix.normalize(bare.slice("SLICE/".length))]; + } else if (bare.includes(marker)) { + keys = [path.posix.normalize(bare.slice(bare.indexOf(marker) + marker.length))]; + } else { + keys = [ + path.posix.normalize(path.posix.join(path.posix.dirname(sourceFile), bare)), + bare.startsWith(".") ? null : path.posix.normalize(bare), + ]; + } + for (const key of keys) { + const found = lookup(key); + if (found) return found; + } + const retired = keys.find((key) => key !== null && map.sliceEntries.has(key)); + if (retired) return { retired }; + // An unknown file in a copied directory moves with the directory, so the + // link check flags it at its new location. + for (const key of bare.includes("/") ? keys : []) { + const dir = key === null ? undefined : map.dirs.get(path.posix.dirname(key)); + if (dir !== undefined && key !== null) { + return { target: `${dir}/${path.posix.basename(key)}`, isDir: false }; + } } - - const fromFile = path.posix.normalize(path.posix.join(path.posix.dirname(sourceFile), bare)); - const fromRoot = bare.startsWith(".") ? null : path.posix.normalize(bare); - const found = lookup(fromFile) ?? (fromRoot === null ? null : lookup(fromRoot)); - if (found) return found; const frame = bare.includes("/") ? undefined : map.tempFrames.get(bare); return frame ? { target: frame, isDir: false } : null; } +const NOT_RETAINED = "not retained in this copy"; + /** * Rewrite every markdown link and backticked path in `text` that names a - * mapped slice path, relative to the file's target location. + * slice path: a copied one relative to the file's target location, one the + * layout does not keep as plain text naming it as not retained. * * @param {string} text * @param {string} sourceFile - slice-relative path of the file @@ -227,10 +253,12 @@ function lookUpReference(refPath, sourceFile, map) { */ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { let rewrites = 0; + const unwrap = (/** @type {string} */ target) => + target.startsWith("<") ? target.slice(1, -1) : target; const rewrite = (/** @type {string} */ ref) => { const { refPath, suffix } = splitReference(ref); const found = lookUpReference(refPath, sourceFile, map); - if (!found) return ref; + if (!found || "retired" in found) return ref; const relative = path.posix.relative(path.posix.dirname(targetFile), found.target) || "."; const trailing = found.isDir && refPath.endsWith("/") ? "/" : ""; const next = `${relative}${trailing}${suffix}`; @@ -238,30 +266,43 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { return next; }; const rewritten = text + .replace(FULL_LINK, (match, label, target) => { + const found = lookUpReference(splitReference(unwrap(target)).refPath, sourceFile, map); + if (!found || !("retired" in found)) return match; + rewrites += 1; + return `${label} (${found.retired}, ${NOT_RETAINED})`; + }) .replace(MARKDOWN_LINK, (_match, open, target) => - target.startsWith("<") - ? `${open}<${rewrite(target.slice(1, -1))}>` - : `${open}${rewrite(target)}`, + target.startsWith("<") ? `${open}<${rewrite(unwrap(target))}>` : `${open}${rewrite(target)}`, ) - .replace(BACKTICK_SPAN, (match, span) => - PATH_LIKE.test(span) ? `\`${rewrite(span)}\`` : match, - ); + .replace(BACKTICK_SPAN, (match, span) => { + if (!PATH_LIKE.test(span)) return match; + const found = lookUpReference(splitReference(span).refPath, sourceFile, map); + if (found && "retired" in found) { + rewrites += 1; + return `${found.retired} (${NOT_RETAINED})`; + } + return `\`${rewrite(span)}\``; + }); return { text: rewritten, rewrites }; } /** - * True when a backticked span reads as a path into the slice: an explicit - * `./` or `../` path, or one whose first segment is a layout directory or an - * entry beside the naming file. + * True when a backticked span reads as a path into the slice: `SLICE/`, an + * explicit `./` or `../` path, or one whose first segment is a top-level + * entry of the re-laid-out slice or an entry beside the naming file. Any + * other path is one the path map did not know under no slice directory, a + * path into another repository. * * @param {string} refPath * @param {string} fileDir - absolute directory of the naming file + * @param {Set} topLevel - top-level entries of the re-laid-out slice */ -function looksSliceInternal(refPath, fileDir) { +function looksSliceInternal(refPath, fileDir, topLevel) { if (!refPath.includes("/")) return false; - if (refPath.startsWith("./") || refPath.startsWith("../")) return true; + if (/^(?:\.\.?|SLICE)\//.test(refPath)) return true; const first = refPath.split("/")[0]; - return LAYOUT_SEGMENTS.has(first) || fs.existsSync(path.join(fileDir, first)); + return topLevel.has(first) || fs.existsSync(path.join(fileDir, first)); } /** @@ -275,6 +316,7 @@ function looksSliceInternal(refPath, fileDir) { */ export function checkLayoutLinks(rootDir) { const root = path.resolve(rootDir); + const topLevel = new Set(fs.existsSync(root) ? fs.readdirSync(root) : []); /** @type {{ file: string, ref: string }[]} */ const unresolved = []; const markdown = listFiles(root).filter( @@ -288,7 +330,7 @@ export function checkLayoutLinks(rootDir) { if (!refPath || URL_SCHEME.test(refPath) || PLACEHOLDER.test(refPath)) return; if (refPath.startsWith("/") || refPath.startsWith("~")) return; if (ALLOWED_MISS_PREFIXES.some((prefix) => refPath.startsWith(prefix))) return; - if (!isLink && !looksSliceInternal(refPath, fileDir)) return; + if (!isLink && !looksSliceInternal(refPath, fileDir, topLevel)) return; const resolved = path.resolve(fileDir, refPath); const rel = path.relative(root, resolved); if (rel.startsWith("..") || path.isAbsolute(rel)) return; @@ -343,7 +385,7 @@ function provenanceReadme({ title, sourceUrl, videoId, acquiredAt }, presentTopD "", `- Canonical origin: <${sourceUrl}>`, `- Video id: \`${videoId}\``, - `- Fetched${fetched} through \`/knowledge:video-digest\` and re-laid out from its closed slice.`, + `- Fetched${fetched} through \`/knowledge:video-digest\` and re-laid out from its finished slice.`, "", "## Contents", "", @@ -361,10 +403,21 @@ function provenanceReadme({ title, sourceUrl, videoId, acquiredAt }, presentTopD */ /** - * @param {{ sliceDir: string, targetDir: string, noMedia?: boolean }} options + * @param {{ + * sliceDir: string, + * targetDir: string, + * noMedia?: boolean, + * verifyOutcomes?: (sliceDir: string) => number, + * }} options - `verifyOutcomes` gates a slice that is not closed yet; it + * returns 0 when the slice's outcome checks pass * @returns {Promise} */ -export async function relayoutSlice({ sliceDir, targetDir, noMedia = false }) { +export async function relayoutSlice({ + sliceDir, + targetDir, + noMedia = false, + verifyOutcomes = (dir) => runCheckWatchOutcomes(dir), +}) { const slice = path.resolve(sliceDir); const target = path.resolve(targetDir); const refuse = (/** @type {string} */ message) => { @@ -378,8 +431,19 @@ export async function relayoutSlice({ sliceDir, targetDir, noMedia = false }) { } catch { return refuse(`no readable run-state/watch.json under ${slice}`); } - if (state.status !== "complete") { - return refuse(`slice status is "${state.status}", not "complete"; close the slice first`); + const outcomesPass = () => { + try { + return verifyOutcomes(slice) === 0; + } catch (error) { + writeStderr(`relayout: outcome checks threw: ${/** @type {Error} */ (error).message}`); + return false; + } + }; + if (state.status !== "complete" && !outcomesPass()) { + return refuse( + `slice status is "${state.status}" and its outcome checks fail; ` + + "re-lay out a closed slice, or one whose check-watch-outcomes.js passes", + ); } const temp = resolveTempSession(state.tempSession ?? {}); @@ -394,7 +458,13 @@ export async function relayoutSlice({ sliceDir, targetDir, noMedia = false }) { /** @type {{ from: string, to: string, sourceFile?: string }[]} */ const plan = []; - for (const file of listFiles(slice)) { + const sliceFiles = listFiles(slice); + const sliceEntries = new Set( + sliceFiles.flatMap((file) => + file.split("/").map((_part, index, parts) => parts.slice(0, index + 1).join("/")), + ), + ); + for (const file of sliceFiles) { const to = corpusPathForSliceFile(file); if (to) plan.push({ from: path.join(slice, file), to, sourceFile: file }); } @@ -430,6 +500,7 @@ export async function relayoutSlice({ sliceDir, targetDir, noMedia = false }) { dirs: deriveDirectoryMap(files), tempFrames, sliceName: path.basename(slice), + sliceEntries, }; let rewrites = 0; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 14f61d697a3..6ee0a6f3178 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -89,7 +89,10 @@ function makeFixture({ status = "complete", withMedia = true, extraFiles = {} } "| `frames/title-slide.png` | `scene_0001.png` |\n\nTiers: [manifest](key-frames-manifest.md)\n", "key-frames/key-frames-manifest.md": "| title-slide.png |\n", "key-frames/key-frame-quality-audit.md": "| title-slide.png | yes |\n", + "key-frames/frame-triage-log.md": "| sheet_001 | promote |\n", "key-frames/selection.json": "{}", + "source/companion-sources.md": "Companion: [digest](companion-digest/README.md).\n", + "source/companion-digest/README.md": "Brief: `source/companion-sources.md`.\n", "research/claim-inventory.md": "Transcript: `source/transcript.txt`.\n", "research/findings/topic.md": "Quote at `SLICE/source/transcript.txt:3`; see [agenda](../claim-inventory.md#sessions).\n", @@ -97,6 +100,8 @@ function makeFixture({ status = "complete", withMedia = true, extraFiles = {} } "Evidence: `source/transcript.txt` and ![slide](../key-frames/frames/title-slide.png).", "Detail: `research/findings/topic.md`, the hub [README](README.md), the [journey](../README.md).", "Target file: `plugins/knowledge/skills/video-digest/SKILL.md`.", + "Run state: `run-state/watch.json`, [selection](../key-frames/selection.json).", + "Upstream: `research/deepening/html-report.md:92` in mattpocock/skills.", "", ].join("\n"), "recommendations/README.md": "# Hub\n", @@ -129,6 +134,9 @@ describe("relayoutSlice", () => { "frames/visual-frames.md", "frames/key-frames-manifest.md", "frames/key-frame-quality-audit.md", + "frames/frame-triage-log.md", + "metadata/companion-sources.md", + "analysis/companion-digest/README.md", `media/${VIDEO_ID}.mp4`, "analysis/RESEARCH.md", "analysis/research/claim-inventory.md", @@ -163,9 +171,17 @@ describe("relayoutSlice", () => { "Evidence: `../../transcript/transcript.txt` and ![slide](../../frames/key/title-slide.png).", "Detail: `../research/findings/topic.md`, the hub [README](README.md), the [journey](../../README.md).", "Target file: `plugins/knowledge/skills/video-digest/SKILL.md`.", + "Run state: run-state/watch.json (not retained in this copy), selection (key-frames/selection.json, not retained in this copy).", + "Upstream: `research/deepening/html-report.md:92` in mattpocock/skills.", "", ].join("\n"), ); + expect(read(targetDir, "metadata/companion-sources.md")).toBe( + "Companion: [digest](../analysis/companion-digest/README.md).\n", + ); + expect(read(targetDir, "analysis/companion-digest/README.md")).toBe( + "Brief: `../../metadata/companion-sources.md`.\n", + ); expect(read(targetDir, "frames/visual-frames.md")).toBe( "| `key/title-slide.png` | `all/scene_0001.png` |\n\nTiers: [manifest](key-frames-manifest.md)\n", ); @@ -190,20 +206,48 @@ describe("relayoutSlice", () => { expect(read(targetDir, "README.md")).toBe("# Curated\n"); }); - it("fails the link check on a slice path the corpus layout does not keep", async () => { + it("fails the link check on a missing file under a slice directory", async () => { const { sliceDir, targetDir } = makeFixture({ - extraFiles: { "recommendations/questions.md": "Plan: `key-frames/selection.json`.\n" }, + extraFiles: { "recommendations/questions.md": "Slide: `key-frames/frames/deleted.png`.\n" }, }); const result = await relayoutSlice({ sliceDir, targetDir }); expect(result.exitCode).toBe(1); expect(result.unresolved).toEqual([ - { file: "analysis/recommendations/questions.md", ref: "key-frames/selection.json" }, + { file: "analysis/recommendations/questions.md", ref: "../../frames/key/deleted.png" }, ]); }); - it("refuses a slice that is not closed", async () => { + it("re-lays out an unclosed slice, media included, when its outcome checks pass", async () => { + const { sliceDir, targetDir } = makeFixture({ status: "synthesizing" }); + /** @type {string[]} */ + const verified = []; + + const result = await relayoutSlice({ + sliceDir, + targetDir, + verifyOutcomes: (dir) => { + verified.push(dir); + return 0; + }, + }); + + expect(result.exitCode).toBe(0); + expect(verified).toEqual([path.resolve(sliceDir)]); + expect(fs.existsSync(path.join(targetDir, `media/${VIDEO_ID}.mp4`))).toBe(true); + }); + + it("refuses an unclosed slice whose outcome checks fail", async () => { + const { sliceDir, targetDir } = makeFixture({ status: "synthesizing" }); + + const result = await relayoutSlice({ sliceDir, targetDir, verifyOutcomes: () => 1 }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("refuses an unclosed slice that fails the real outcome checks", async () => { const { sliceDir, targetDir } = makeFixture({ status: "synthesizing" }); const result = await relayoutSlice({ sliceDir, targetDir }); @@ -212,6 +256,14 @@ describe("relayoutSlice", () => { expect(fs.existsSync(targetDir)).toBe(false); }); + it("skips the outcome checks for a closed slice", async () => { + const { sliceDir, targetDir } = makeFixture(); + + const result = await relayoutSlice({ sliceDir, targetDir, verifyOutcomes: () => 1 }); + + expect(result.exitCode).toBe(0); + }); + it("refuses when the temp session's media is gone, unless --no-media", async () => { const { sliceDir, targetDir } = makeFixture({ withMedia: false }); From d5a7de5aae1973dd15f67fd7bdb6dadbf40dc6a8 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:53:21 -0400 Subject: [PATCH 04/73] fix(knowledge): tighten caption boundary repeats; stage relayout output atomically - A paragraph-boundary repeat now needs 3 or more matching words, or 2 when the opening cue carries them over from the previous cue (time overlap or a verbatim rolling-caption carry), so a phrase the speaker genuinely repeats stays. - relayout-slice.js --no-media takes nothing from the temp session. - The layout is built in a sibling staging directory, link-checked there, and moved into place only on success; an existing target needs --replace. Any failure removes staging and leaves the target as it was. - relayout-slice.js is mode 100755 like its shebang siblings. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 21 ++- .../extraction/transcript/write-transcript.js | 73 ++++++-- .../transcript/write-transcript.test.js | 60 ++++++- .../extraction/watch/relayout-slice.js | 157 ++++++++++++------ .../extraction/watch/relayout-slice.test.js | 56 ++++++- 5 files changed, 280 insertions(+), 87 deletions(-) mode change 100644 => 100755 plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index daca1e01066..0d359f4fd94 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -59,7 +59,7 @@ material at a differently-shaped path (for example `sources///`) lan `library_dir` as written, then re-lays out the slice with `relayout-slice.js`: ```bash -node "/extraction/run.mjs" --data-dir "" watch/relayout-slice.js "" "" [--no-media] +node "/extraction/run.mjs" --data-dir "" watch/relayout-slice.js "" "" [--no-media] [--replace] ``` The media, every extracted frame, the served caption tracks and the info JSON come from the temp @@ -71,8 +71,10 @@ session `watch.json` records, which `close` removes. To keep them, run in this o The script runs on a closed slice, or on an unclosed one whose outcome checks pass (it runs them itself and refuses when they fail). It refuses any slice whose temp media is gone unless -`--no-media` is passed, which re-lays out the slice files alone; after a normal close that is the -only way it runs. +`--no-media` is passed, which takes nothing from the temp session (no media, frames, caption +tracks or info JSON) even when it still exists; after a normal close that is the only way it runs. +It refuses an existing target unless `--replace` is passed, which replaces the target and keeps +its `README.md`. It writes the knowledge-corpus layout: `transcript/`, `metadata/` (with a trimmed `info.json` and the companion sources brief), `frames/all/`, `frames/key/` beside the frame logs in @@ -81,12 +83,13 @@ companion digest), and a provenance `README.md` when the target has none. The sc comment is the authoritative mapping. Each slice path the copied markdown names (links and backticked paths) is rewritten through a path map relative to the file's new location, and one naming a pipeline file the layout does not keep (such as `run-state/watch.json`) becomes plain -text saying it is not retained. It then link-checks the target and exits 1 listing every -slice-internal path that still does not resolve. Not checked: paths under `plugins/`, `docs/`, -`templates/` and `.claude/`, `AGENTS.md`, paths that leave the target, and backticked paths whose -first segment is neither a top-level directory of the target nor an entry beside the naming file -(paths into another repository). Fix what it lists by hand and re-run; it overwrites the copied -files. +text saying it is not retained. It builds the layout in a sibling staging directory and +link-checks it there; it exits 1 listing every slice-internal path that still does not resolve. +Not checked: paths under `plugins/`, `docs/`, `templates/` and `.claude/`, `AGENTS.md`, paths +that leave the target, and backticked paths whose first segment is neither a top-level directory +of the target nor an entry beside the naming file (paths into another repository). Only a passing +check moves the staging directory into place; on any failure the staging directory is removed and +the target is left as it was. Fix what it lists in the slice and re-run. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index bb8f35b88e8..8b50d317f3f 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -11,7 +11,7 @@ import { cleanManualCaptions, stripCaptionHtmlEntities, } from "@melodic/video-digestion/transcript/manual-caption-clean"; -import { formatTranscript } from "@melodic/video-digestion/transcript/vtt-parser"; +import { formatTimestamp, formatTranscript } from "@melodic/video-digestion/transcript/vtt-parser"; import { primaryEntry } from "../adapters/adapter-contract.js"; import { LANES, lanePath } from "../lib/slice-lanes.js"; @@ -34,51 +34,88 @@ function countParagraphs(transcript) { return transcript ? transcript.split("\n\n").length : 0; } -const BOUNDARY_REPEAT_MIN_WORDS = 2; +const BOUNDARY_REPEAT_MIN_WORDS = 3; +const BOUNDARY_CARRY_OVER_WORDS = 2; const BOUNDARY_REPEAT_MAX_WORDS = 12; const PARAGRAPH_STAMP = /^(\[[^\]]*\]\s*)([\s\S]*)$/; +/** @param {string} text */ +function words(text) { + return text.split(/\s+/).filter(Boolean); +} + /** @param {string} word */ function comparableWord(word) { return word.toLowerCase().replace(/[^\p{L}\p{N}']/gu, ""); } -/** @param {string[]} words */ -function comparableRun(words) { - return words.map(comparableWord).join(" "); +/** @param {string[]} run */ +function comparableRun(run) { + return run.map(comparableWord).join(" "); +} + +/** + * True when a cue opening the paragraph stamped `stamp` carries `run` over + * from the cue before it: the two cues overlap in time, or the opening cue + * starts with the previous cue's closing words verbatim (a rolling caption). + * + * @param {string} stamp - the paragraph's `[M:SS]` + * @param {string[]} run - the paragraph's leading words + * @param {readonly { startSec: number, endSec: number, text: string }[]} cues + */ +function carriesOverFromPreviousCue(stamp, run, cues) { + const head = run.join(" "); + for (let index = 1; index < cues.length; index++) { + const cue = cues[index]; + if (`[${formatTimestamp(cue.startSec)}]` !== stamp) continue; + const previous = cues[index - 1]; + const cueWords = words(cue.text); + if (cueWords.slice(0, run.length).join(" ") !== head) continue; + if (previous.endSec > cue.startSec) return true; + if (words(previous.text).slice(-run.length).join(" ") === head) return true; + } + return false; } /** * Drop the words a paragraph opens with when they repeat the previous - * paragraph's closing words: the longest run of 2 to 12 words, compared - * case- and punctuation-insensitively. A rolling auto caption carries its - * tail into the next cue, and the vendor formatter's case-only overlap check - * misses a repeat that differs in punctuation. A paragraph always keeps at - * least one word. + * paragraph's closing words, the longest run first: 3 to 12 words compared + * case- and punctuation-insensitively, or 2 words when the opening cue carries + * them over from the cue before it ({@link carriesOverFromPreviousCue}). A + * rolling auto caption carries its tail into the next cue, and the vendor + * formatter only drops overlaps of 3 or more words it matches ignoring case + * alone. A two-word repeat the speaker actually said ("very good." / "Very + * good question") stays. A paragraph always keeps at least one word. * * @param {string} transcript - paragraphs joined by a blank line, each `[M:SS] text` + * @param {readonly { startSec: number, endSec: number, text: string }[]} cues - the + * cues the transcript was formatted from * @returns {string} */ -export function dropParagraphBoundaryRepeats(transcript) { +export function dropParagraphBoundaryRepeats(transcript, cues) { const paragraphs = transcript.split("\n\n"); for (let index = 1; index < paragraphs.length; index++) { const current = PARAGRAPH_STAMP.exec(paragraphs[index]); const previous = PARAGRAPH_STAMP.exec(paragraphs[index - 1]); if (!current || !previous) continue; - const previousWords = previous[2].split(/\s+/).filter(Boolean); - const currentWords = current[2].split(/\s+/).filter(Boolean); + const previousWords = words(previous[2]); + const currentWords = words(current[2]); const longest = Math.min( BOUNDARY_REPEAT_MAX_WORDS, previousWords.length, currentWords.length - 1, ); - for (let length = longest; length >= BOUNDARY_REPEAT_MIN_WORDS; length--) { + for (let length = longest; length >= BOUNDARY_CARRY_OVER_WORDS; length--) { + const run = currentWords.slice(0, length); + if (comparableRun(previousWords.slice(-length)) !== comparableRun(run)) continue; if ( - comparableRun(previousWords.slice(-length)) === comparableRun(currentWords.slice(0, length)) + length < BOUNDARY_REPEAT_MIN_WORDS && + !carriesOverFromPreviousCue(current[1].trim(), run, cues) ) { - paragraphs[index] = current[1] + currentWords.slice(length).join(" "); - break; + continue; } + paragraphs[index] = current[1] + currentWords.slice(length).join(" "); + break; } } return paragraphs.join("\n\n"); @@ -117,7 +154,7 @@ export function buildTranscriptText(vttText, isAutoCaption, { repairLexicon = nu } const formatted = formatTranscript(cues); - const transcript = isAutoCaption ? dropParagraphBoundaryRepeats(formatted) : formatted; + const transcript = isAutoCaption ? dropParagraphBoundaryRepeats(formatted, cues) : formatted; return { transcript, diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index b6384f6295d..f2c77ccca51 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -41,9 +41,9 @@ describe("buildTranscriptText", () => { expect(result.transcript).toBe("[0:04] [music] >> Hi everybody."); }); - it("drops auto-caption words a paragraph repeats from the previous paragraph's tail", () => { - // The rolling caption carries "benchmarks, too." into the next cue, which - // opens a new paragraph after the sentence end (#6992). + it("drops two words a rolling caption carries into the next paragraph's cue", () => { + // The 51Eb4EtGqrI shape (#6992): the cue opening the new paragraph starts + // with the previous cue's closing words verbatim. const autoVtt = `WEBVTT 00:00:01.000 --> 00:00:03.000 @@ -52,6 +52,24 @@ we ran the 00:00:03.000 --> 00:00:05.000 numbers on benchmarks, too. +00:00:05.000 --> 00:00:07.000 +benchmarks, too. And then we moved on. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe( + "[0:01] we ran the numbers on benchmarks, too.\n\n[0:05] And then we moved on.", + ); + }); + + it("drops two repeated words when the opening cue overlaps the previous cue in time", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +we ran the + +00:00:03.000 --> 00:00:05.500 +numbers on benchmarks, too. + 00:00:05.000 --> 00:00:07.000 Benchmarks too and then we moved on. `; @@ -61,6 +79,42 @@ Benchmarks too and then we moved on. ); }); + it("drops a repeat of three or more words compared case- and punctuation-insensitively", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +we ran the + +00:00:03.000 --> 00:00:05.000 +numbers on benchmarks, too. + +00:00:05.000 --> 00:00:07.000 +On benchmarks too and then we moved on. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe( + "[0:01] we ran the numbers on benchmarks, too.\n\n[0:05] and then we moved on.", + ); + }); + + it("keeps two words the speaker genuinely repeats across a paragraph boundary", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +that answer was + +00:00:03.000 --> 00:00:05.000 +very good. + +00:00:05.000 --> 00:00:07.000 +Very good question, thanks. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe( + "[0:01] that answer was very good.\n\n[0:05] Very good question, thanks.", + ); + }); + it("keeps a single repeated word at a paragraph boundary", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js old mode 100644 new mode 100755 index 2cd0f296247..4f8f4a540fa --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -21,9 +21,14 @@ * layout does not keep becomes plain text naming it as not retained; anything * else is left alone. The link check then reports what still does not resolve. * - * Usage: node watch/relayout-slice.js [--no-media] - * Exit: 0 copied and every slice-internal path resolves; 1 refused or - * unresolved paths (listed on stderr); 2 usage. + * The layout is built in a sibling staging directory and moved into place only + * when the link check passes; on any failure the target is left untouched. An + * existing target is replaced only with `--replace`, which keeps its README. + * `--no-media` copies nothing from the temp session. + * + * Usage: node watch/relayout-slice.js [--no-media] [--replace] + * Exit: 0 written and every slice-internal path resolves; 1 refused, failed, + * or unresolved paths (listed on stderr, nothing written); 2 usage. */ import fs from "node:fs"; @@ -407,16 +412,21 @@ function provenanceReadme({ title, sourceUrl, videoId, acquiredAt }, presentTopD * sliceDir: string, * targetDir: string, * noMedia?: boolean, + * replace?: boolean, * verifyOutcomes?: (sliceDir: string) => number, - * }} options - `verifyOutcomes` gates a slice that is not closed yet; it - * returns 0 when the slice's outcome checks pass + * copyFile?: (from: string, to: string) => void, + * }} options - `noMedia` copies nothing from the temp session; `replace` + * allows an existing target, keeping its `README.md`; `verifyOutcomes` gates + * a slice that is not closed yet and returns 0 when its outcome checks pass * @returns {Promise} */ export async function relayoutSlice({ sliceDir, targetDir, noMedia = false, + replace = false, verifyOutcomes = (dir) => runCheckWatchOutcomes(dir), + copyFile = fs.copyFileSync, }) { const slice = path.resolve(sliceDir); const target = path.resolve(targetDir); @@ -446,7 +456,12 @@ export async function relayoutSlice({ ); } - const temp = resolveTempSession(state.tempSession ?? {}); + if (fs.existsSync(target) && !replace) { + return refuse(`${target} exists; pass --replace to replace it`); + } + + // `--no-media` takes nothing from the temp session, even when it still exists. + const temp = resolveTempSession(noMedia ? {} : (state.tempSession ?? {})); const workFiles = temp.workDir ? await listWorkDirFiles(temp.workDir) : []; const media = resolveMediaArtifacts(workFiles, state.videoId); if (!media.videoPath && !noMedia) { @@ -480,7 +495,7 @@ export async function relayoutSlice({ tempFrames.set(name, `frames/all/${name}`); } } - if (media.videoPath && !noMedia) { + if (media.videoPath) { plan.push({ from: media.videoPath, to: `media/${state.videoId}${path.extname(media.videoPath)}`, @@ -503,56 +518,89 @@ export async function relayoutSlice({ sliceEntries, }; + // Build the whole layout in a sibling staging directory and move it into + // place only after the link check passes, so a failure never leaves a + // partial target. + fs.mkdirSync(path.dirname(target), { recursive: true }); + const staging = fs.mkdtempSync( + path.join(path.dirname(target), `.${path.basename(target)}.relayout-`), + ); + const discard = (/** @type {string} */ dir) => fs.rmSync(dir, { recursive: true, force: true }); let rewrites = 0; - const write = (/** @type {string} */ to, /** @type {string|Buffer} */ body) => { - const dest = path.join(target, to); - fs.mkdirSync(path.dirname(dest), { recursive: true }); - fs.writeFileSync(dest, body); - }; - for (const entry of plan) { - if (entry.sourceFile?.endsWith(".md")) { - const result = rewriteSliceReferences( - fs.readFileSync(entry.from, "utf8"), - entry.sourceFile, - entry.to, - map, - ); - rewrites += result.rewrites; - write(entry.to, result.text); + let copied = 0; + /** @type {{ file: string, ref: string }[]} */ + let unresolved; + try { + const write = (/** @type {string} */ to, /** @type {string} */ body) => { + const dest = path.join(staging, to); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.writeFileSync(dest, body); + copied += 1; + }; + for (const entry of plan) { + if (entry.sourceFile?.endsWith(".md")) { + const result = rewriteSliceReferences( + fs.readFileSync(entry.from, "utf8"), + entry.sourceFile, + entry.to, + map, + ); + rewrites += result.rewrites; + write(entry.to, result.text); + } else { + fs.mkdirSync(path.dirname(path.join(staging, entry.to)), { recursive: true }); + copyFile(entry.from, path.join(staging, entry.to)); + copied += 1; + } + } + if (media.metadataPath) { + const info = JSON.parse(fs.readFileSync(media.metadataPath, "utf8")); + write("metadata/info.json", `${JSON.stringify(trimInfoJson(info), null, 2)}\n`); + } + + const existingReadme = path.join(target, "README.md"); + if (fs.existsSync(existingReadme)) { + write("README.md", fs.readFileSync(existingReadme, "utf8")); } else { - fs.mkdirSync(path.dirname(path.join(target, entry.to)), { recursive: true }); - fs.copyFileSync(entry.from, path.join(target, entry.to)); + const present = new Set( + CONTENTS_LINES.map(([dir]) => dir).filter((dir) => fs.existsSync(path.join(staging, dir))), + ); + write( + "README.md", + provenanceReadme( + { + title: state.title, + sourceUrl: state.sourceUrl, + videoId: state.videoId, + acquiredAt: state.tempSession?.acquiredAt, + }, + present, + ), + ); } + + unresolved = checkLayoutLinks(staging); + } catch (error) { + discard(staging); + return refuse(`copy failed, target left untouched: ${/** @type {Error} */ (error).message}`); } - let copied = plan.length; - if (media.metadataPath) { - const info = JSON.parse(fs.readFileSync(media.metadataPath, "utf8")); - write("metadata/info.json", `${JSON.stringify(trimInfoJson(info), null, 2)}\n`); - copied += 1; + if (unresolved.length > 0) { + discard(staging); + return { exitCode: 1, copied: 0, rewrites, unresolved }; } - const readmePath = path.join(target, "README.md"); - if (!fs.existsSync(readmePath)) { - const present = new Set( - CONTENTS_LINES.map(([dir]) => dir).filter((dir) => fs.existsSync(path.join(target, dir))), - ); - write( - "README.md", - provenanceReadme( - { - title: state.title, - sourceUrl: state.sourceUrl, - videoId: state.videoId, - acquiredAt: state.tempSession?.acquiredAt, - }, - present, - ), - ); - copied += 1; + const backup = `${staging}.previous`; + const hadTarget = fs.existsSync(target); + try { + if (hadTarget) fs.renameSync(target, backup); + fs.renameSync(staging, target); + } catch (error) { + if (hadTarget && !fs.existsSync(target) && fs.existsSync(backup)) fs.renameSync(backup, target); + discard(staging); + return refuse(`could not move the layout into place: ${/** @type {Error} */ (error).message}`); } - - const unresolved = checkLayoutLinks(target); - return { exitCode: unresolved.length > 0 ? 1 : 0, copied, rewrites, unresolved }; + if (hadTarget) discard(backup); + return { exitCode: 0, copied, rewrites, unresolved }; } /** @@ -561,13 +609,16 @@ export async function relayoutSlice({ */ export async function runRelayoutCli(args) { const noMedia = args.includes("--no-media"); - const positional = args.filter((arg) => arg !== "--no-media"); + const replace = args.includes("--replace"); + const positional = args.filter((arg) => arg !== "--no-media" && arg !== "--replace"); if (positional.length !== 2 || positional.some((arg) => arg.startsWith("--"))) { - writeStderr("Usage: node watch/relayout-slice.js [--no-media]"); + writeStderr( + "Usage: node watch/relayout-slice.js [--no-media] [--replace]", + ); return 2; } const [sliceDir, targetDir] = positional; - const result = await relayoutSlice({ sliceDir, targetDir, noMedia }); + const result = await relayoutSlice({ sliceDir, targetDir, noMedia, replace }); if (result.copied > 0) { writeStdout( `relayout: ${result.copied} files written to ${path.resolve(targetDir)}, ` + diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 6ee0a6f3178..20428105d13 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -196,14 +196,61 @@ describe("relayoutSlice", () => { ); }); - it("keeps a README the target already has", async () => { + it("replaces an existing target with --replace, keeping its README", async () => { const { sliceDir, targetDir } = makeFixture(); - fs.mkdirSync(targetDir, { recursive: true }); - fs.writeFileSync(path.join(targetDir, "README.md"), "# Curated\n"); + writeTree(targetDir, { "README.md": "# Curated\n", "stale.md": "old\n" }); - await relayoutSlice({ sliceDir, targetDir }); + const result = await relayoutSlice({ sliceDir, targetDir, replace: true }); + expect(result.exitCode).toBe(0); expect(read(targetDir, "README.md")).toBe("# Curated\n"); + expect(fs.existsSync(path.join(targetDir, "stale.md"))).toBe(false); + expect(fs.readdirSync(path.dirname(targetDir))).toEqual(["out"]); + }); + + it("refuses an existing target without --replace and leaves it untouched", async () => { + const { sliceDir, targetDir } = makeFixture(); + writeTree(targetDir, { "stale.md": "old\n" }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(targetDir)).toEqual(["stale.md"]); + }); + + it("leaves the target untouched and no staging behind when a copy fails midway", async () => { + const { sliceDir, targetDir } = makeFixture(); + writeTree(targetDir, { "README.md": "# Curated\n", "stale.md": "old\n" }); + let calls = 0; + + const result = await relayoutSlice({ + sliceDir, + targetDir, + replace: true, + copyFile: (from, to) => { + calls += 1; + if (calls === 3) throw new Error("disk full"); + fs.copyFileSync(from, to); + }, + }); + + expect(result.exitCode).toBe(1); + expect(calls).toBe(3); + expect(fs.readdirSync(targetDir).sort()).toEqual(["README.md", "stale.md"]); + expect(read(targetDir, "stale.md")).toBe("old\n"); + expect(fs.readdirSync(path.dirname(targetDir))).toEqual(["out"]); + }); + + it("takes nothing from a temp session that still exists under --no-media", async () => { + const { sliceDir, targetDir } = makeFixture(); + + const result = await relayoutSlice({ sliceDir, targetDir, noMedia: true }); + + expect(result.exitCode).toBe(0); + for (const rel of ["media", "frames/all", "transcript/en.vtt", "metadata/info.json"]) { + expect(fs.existsSync(path.join(targetDir, rel)), rel).toBe(false); + } + expect(read(targetDir, "transcript/transcript.txt")).toBe("[0:01] hello\n"); }); it("fails the link check on a missing file under a slice directory", async () => { @@ -217,6 +264,7 @@ describe("relayoutSlice", () => { expect(result.unresolved).toEqual([ { file: "analysis/recommendations/questions.md", ref: "../../frames/key/deleted.png" }, ]); + expect(fs.readdirSync(path.dirname(targetDir))).toEqual([]); }); it("re-lays out an unclosed slice, media included, when its outcome checks pass", async () => { From da7392a150f3fb0a0c7fbecff7e0ce158c1d196a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:53:53 -0400 Subject: [PATCH 05/73] docs(knowledge): describe the tightened boundary rule and staged relayout in the changelog fragment Co-Authored-By: Claude Opus 5.5 --- .../fix-video-digest-relayout-and-caption-dedupe-59495b96.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md index fd79ef46b36..7da2a67ed26 100644 --- a/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md +++ b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md @@ -4,8 +4,8 @@ bump: minor ### Added -- video-digest: `watch/relayout-slice.js` copies a slice into the knowledge-corpus layout (`transcript/`, `metadata/`, `frames/all/`, `frames/key/`, `media/`, `analysis/`, provenance `README.md`), rewrites the slice paths its markdown names through a path map (a pipeline file the layout does not keep becomes plain text saying so), and exits 1 listing every slice-internal path that still does not resolve. It runs on a closed slice, or before `close` on one whose outcome checks pass so the temp media is still there, and refuses a slice whose temp media is gone unless `--no-media` ([#6902](https://github.com/melodic-software/claude-code-plugins/issues/6902)). +- video-digest: `watch/relayout-slice.js` copies a slice into the knowledge-corpus layout (`transcript/`, `metadata/`, `frames/all/`, `frames/key/`, `media/`, `analysis/`, provenance `README.md`), rewrites the slice paths its markdown names through a path map (a pipeline file the layout does not keep becomes plain text saying so), and exits 1 listing every slice-internal path that still does not resolve. It runs on a closed slice, or before `close` on one whose outcome checks pass so the temp media is still there, and refuses a slice whose temp media is gone unless `--no-media`, which takes nothing from the temp session. It builds the layout in a staging directory and moves it into place only when the link check passes; an existing target needs `--replace` ([#6902](https://github.com/melodic-software/claude-code-plugins/issues/6902)). ### Fixed -- video-digest: an auto-caption transcript paragraph no longer repeats the previous paragraph's closing words (2 to 12 words, compared case- and punctuation-insensitively) that a rolling caption carried over ([#6992](https://github.com/melodic-software/claude-code-plugins/issues/6992)). +- video-digest: an auto-caption transcript paragraph no longer repeats the previous paragraph's closing words that a rolling caption carried over: 3 to 12 words compared case- and punctuation-insensitively, or 2 when the opening cue carries them from the cue before it, so a phrase the speaker genuinely repeats stays ([#6992](https://github.com/melodic-software/claude-code-plugins/issues/6992)). From 7cfed3c4f774a3308c2c8347e6a2658bea672662 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:01:50 -0400 Subject: [PATCH 06/73] fix(knowledge): guard relayout against slice overlap, interrupted swaps and partial temp sessions - Refuse a target that is the slice, contains it, or sits inside it, comparing real paths before any staging, in every mode. - The --replace swap moves the old target to .relayout-backup and deletes it only once the new layout is in place; a later run restores a backup whose target is missing and refuses when both exist. - Without --no-media, refuse a temp session missing the video, the frames or (for a captioned watch) the caption tracks, naming what is missing. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 9 ++- .../extraction/watch/relayout-slice.js | 71 +++++++++++++++++-- .../extraction/watch/relayout-slice.test.js | 65 ++++++++++++++++- 3 files changed, 136 insertions(+), 9 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 0d359f4fd94..2947eb1ebe5 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -73,8 +73,13 @@ The script runs on a closed slice, or on an unclosed one whose outcome checks pa itself and refuses when they fail). It refuses any slice whose temp media is gone unless `--no-media` is passed, which takes nothing from the temp session (no media, frames, caption tracks or info JSON) even when it still exists; after a normal close that is the only way it runs. -It refuses an existing target unless `--replace` is passed, which replaces the target and keeps -its `README.md`. +Without `--no-media` it also refuses a partly removed temp session, naming the missing video, +frames or caption tracks. It refuses a target that is the slice, contains it, or sits inside it, +and an existing target unless `--replace` is passed, which replaces the target and keeps its +`README.md`. The replaced target is moved to `.relayout-backup` and deleted only once the +new layout is in place. If a run is interrupted mid-swap, the next run restores the target from +that backup when the target is missing (and says so), and refuses, naming the backup, when both +exist; remove whichever one you do not want to keep. It writes the knowledge-corpus layout: `transcript/`, `metadata/` (with a trimmed `info.json` and the companion sources brief), `frames/all/`, `frames/key/` beside the frame logs in diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 4f8f4a540fa..c6e3b63831c 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -399,6 +399,32 @@ function provenanceReadme({ title, sourceUrl, videoId, acquiredAt }, presentTopD ].join("\n"); } +/** + * Real path of `p`, resolving symlinks through its nearest existing ancestor + * when `p` itself does not exist yet. + * + * @param {string} p - absolute path + * @returns {string} + */ +function realPathOf(p) { + let existing = p; + const rest = []; + while (!fs.existsSync(existing) && path.dirname(existing) !== existing) { + rest.unshift(path.basename(existing)); + existing = path.dirname(existing); + } + return path.join(fs.realpathSync.native(existing), ...rest); +} + +/** + * @param {string} inner + * @param {string} outer + */ +function isSameOrInside(inner, outer) { + const rel = path.relative(outer, inner); + return rel === "" || (rel !== ".." && !rel.startsWith(`..${path.sep}`) && !path.isAbsolute(rel)); +} + /** * @typedef {Object} RelayoutResult * @property {number} exitCode @@ -435,6 +461,26 @@ export async function relayoutSlice({ return { exitCode: 1, copied: 0, rewrites: 0, unresolved: [] }; }; + // A target that is, holds, or sits inside the slice would copy the slice + // into itself, and `--replace` would move the slice away and delete it. + const realSlice = realPathOf(slice); + const realTarget = realPathOf(target); + if (isSameOrInside(realTarget, realSlice) || isSameOrInside(realSlice, realTarget)) { + return refuse(`target ${target} overlaps the slice ${slice}; pick a target outside it`); + } + + // An interrupted `--replace` swap leaves the previous target at this name. + const backup = `${target}.relayout-backup`; + if (fs.existsSync(backup)) { + if (fs.existsSync(target)) { + return refuse( + `a leftover backup ${backup} sits beside ${target}; keep one of them and remove the other`, + ); + } + fs.renameSync(backup, target); + writeStderr(`relayout: restored ${target} from the backup an interrupted run left`); + } + let state; try { state = JSON.parse(fs.readFileSync(watchStatePath(slice), "utf8")); @@ -464,11 +510,23 @@ export async function relayoutSlice({ const temp = resolveTempSession(noMedia ? {} : (state.tempSession ?? {})); const workFiles = temp.workDir ? await listWorkDirFiles(temp.workDir) : []; const media = resolveMediaArtifacts(workFiles, state.videoId); - if (!media.videoPath && !noMedia) { - return refuse( - `the temp session's media is gone (${temp.workDir ?? "no tempSession.workDir"}); ` + - "pass --no-media to re-lay out without media/, frames/all/, the caption tracks and info.json", - ); + if (!noMedia) { + // A partly removed temp session would yield a layout silently missing a part. + const frameNames = temp.framesDir ? listFiles(temp.framesDir) : []; + const captioned = Boolean(state.phases?.acquire?.metrics?.captionRung); + const missing = [ + ...(media.videoPath ? [] : [`the video (${temp.workDir ?? "no tempSession.workDir"})`]), + ...(frameNames.length > 0 + ? [] + : [`the frames (${temp.framesDir ?? "no tempSession.framesDir"})`]), + ...(captioned && media.captionPaths.length === 0 ? ["the caption tracks"] : []), + ]; + if (missing.length > 0) { + return refuse( + `the temp session is missing ${missing.join(", ")}; ` + + "pass --no-media to re-lay out without media/, frames/all/, the caption tracks and info.json", + ); + } } /** @type {{ from: string, to: string, sourceFile?: string }[]} */ @@ -589,7 +647,8 @@ export async function relayoutSlice({ return { exitCode: 1, copied: 0, rewrites, unresolved }; } - const backup = `${staging}.previous`; + // The backup keeps a fixed name so a run killed between the two renames is + // recovered at the next start (see the backup check above). const hadTarget = fs.existsSync(target); try { if (hadTarget) fs.renameSync(target, backup); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 20428105d13..bc827a7c281 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -78,6 +78,7 @@ function makeFixture({ status = "complete", withMedia = true, extraFiles = {} } sourceUrl: `https://www.youtube.com/watch?v=${VIDEO_ID}`, title: "Sample Talk", status, + phases: { acquire: { metrics: { captionRung: "auto-en" } } }, tempSession: { workDir, framesDir, acquiredAt: "2026-10-10T12:00:00.000Z" }, }), "README.md": "# Sample Talk journey\n", @@ -107,7 +108,7 @@ function makeFixture({ status = "complete", withMedia = true, extraFiles = {} } "recommendations/README.md": "# Hub\n", ...extraFiles, }); - return { sliceDir, targetDir: path.join(makeDir("relayout-target-"), "out") }; + return { sliceDir, workDir, framesDir, targetDir: path.join(makeDir("relayout-target-"), "out") }; } afterEach(() => { @@ -324,6 +325,68 @@ describe("relayoutSlice", () => { expect(fs.existsSync(path.join(targetDir, "media"))).toBe(false); expect(fs.existsSync(path.join(targetDir, "analysis/recommendations/menu.md"))).toBe(true); }); + + it.each([ + ["the slice itself", (/** @type {string} */ slice) => slice], + ["an ancestor of the slice", (/** @type {string} */ slice) => path.dirname(slice)], + ["a directory inside the slice", (/** @type {string} */ slice) => path.join(slice, "out")], + ])("refuses a target that is %s, even with --replace", async (_label, toTarget) => { + const { sliceDir } = makeFixture(); + const targetDir = toTarget(sliceDir); + const before = fs.readdirSync(sliceDir).sort(); + + const result = await relayoutSlice({ sliceDir, targetDir, replace: true }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(sliceDir).sort()).toEqual(before); + expect(fs.existsSync(path.join(sliceDir, "out"))).toBe(false); + }); + + it("restores the target from the backup an interrupted --replace left", async () => { + const { sliceDir, targetDir } = makeFixture(); + // Killed between the two renames: the old target sits at the backup name. + writeTree(`${targetDir}.relayout-backup`, { "README.md": "# Curated\n" }); + + const result = await relayoutSlice({ sliceDir, targetDir, replace: true }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "README.md")).toBe("# Curated\n"); + expect(fs.readdirSync(path.dirname(targetDir))).toEqual(["out"]); + }); + + it("refuses when both the target and a leftover backup exist", async () => { + const { sliceDir, targetDir } = makeFixture(); + writeTree(targetDir, { "README.md": "# New\n" }); + writeTree(`${targetDir}.relayout-backup`, { "README.md": "# Old\n" }); + + const result = await relayoutSlice({ sliceDir, targetDir, replace: true }); + + expect(result.exitCode).toBe(1); + expect(read(targetDir, "README.md")).toBe("# New\n"); + expect(read(`${targetDir}.relayout-backup`, "README.md")).toBe("# Old\n"); + }); + + it("refuses a temp session whose frames are gone though the video survives", async () => { + const { sliceDir, targetDir, framesDir } = makeFixture(); + fs.rmSync(framesDir, { recursive: true, force: true }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("refuses a captioned temp session whose caption tracks are gone", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + for (const name of fs.readdirSync(workDir).filter((file) => file.endsWith(".vtt"))) { + fs.rmSync(path.join(workDir, name)); + } + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); }); describe("checkLayoutLinks", () => { From f3413fc8f37102632e783c23b4471ea726950005 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:13:59 -0400 Subject: [PATCH 07/73] fix(knowledge): refuse ambiguous or info-less temp sessions in video-digest relayout The watch does not record which work-dir video is the primary entry's, so media mode now refuses a temp session holding several videos (an X post with more than one) and names them instead of taking the first file. The single video's id selects its info JSON and caption tracks, and a missing info JSON is now a refusal like the other temp artifacts. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 3 +- .../extraction/watch/relayout-slice.js | 26 +++++++++++++-- .../extraction/watch/relayout-slice.test.js | 32 ++++++++++++++++++- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 2947eb1ebe5..9a8ef2c21d1 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -74,7 +74,8 @@ itself and refuses when they fail). It refuses any slice whose temp media is gon `--no-media` is passed, which takes nothing from the temp session (no media, frames, caption tracks or info JSON) even when it still exists; after a normal close that is the only way it runs. Without `--no-media` it also refuses a partly removed temp session, naming the missing video, -frames or caption tracks. It refuses a target that is the slice, contains it, or sits inside it, +info JSON, frames or caption tracks, and a temp session holding several videos (an X post with +more than one), naming them, since the watch does not record which one is primary. It refuses a target that is the slice, contains it, or sits inside it, and an existing target unless `--replace` is passed, which replaces the target and keeps its `README.md`. The replaced target is moved to `.relayout-backup` and deleted only once the new layout is in place. If a run is interrupted mid-swap, the next run restores the target from diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index c6e3b63831c..6e794b0ec77 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -37,7 +37,7 @@ import path from "node:path"; import { isMainModule } from "@melodic/video-digestion/shared/main-module"; import { writeStderr, writeStdout } from "@melodic/video-digestion/shared/terminal"; -import { listWorkDirFiles, resolveMediaArtifacts } from "../acquisition/acquire.js"; +import { listWorkDirFiles } from "../acquisition/acquire.js"; import { runCheckWatchOutcomes } from "../evals/check-watch-outcomes.js"; import { LANES } from "../lib/slice-lanes.js"; import { resolveTempSession } from "../lib/temp-session-paths.js"; @@ -82,6 +82,7 @@ const SOURCE_METADATA = new Set([ ]); const TRANSCRIPT_TEXT = /^transcript(?:-\d+)?\.txt$/; +const VIDEO_FILE = /\.(?:mp4|mkv|webm)$/; const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$\s]/; const LINE_SUFFIX = /:\d+(?:[-,]\d+)*$/; @@ -509,13 +510,32 @@ export async function relayoutSlice({ // `--no-media` takes nothing from the temp session, even when it still exists. const temp = resolveTempSession(noMedia ? {} : (state.tempSession ?? {})); const workFiles = temp.workDir ? await listWorkDirFiles(temp.workDir) : []; - const media = resolveMediaArtifacts(workFiles, state.videoId); + // The watch does not record which work file is the primary entry's, so only + // a work dir holding exactly one video is unambiguous (an X post with + // several videos is not). That video's id, the `%(id)s` of yt-dlp's output + // template, picks its info JSON and caption tracks. + const videos = workFiles.filter((file) => VIDEO_FILE.test(file)); + if (!noMedia && videos.length > 1) { + return refuse( + `the temp session holds several videos (${videos.map((file) => path.basename(file)).join(", ")}) ` + + "and the watch did not record which is primary; pass --no-media", + ); + } + const mediaId = videos.length === 1 ? path.basename(videos[0], path.extname(videos[0])) : null; + const ofMedia = (/** @type {string} */ file) => + mediaId !== null && path.basename(file).startsWith(`${mediaId}.`); + const media = { + videoPath: videos[0] ?? "", + captionPaths: workFiles.filter((file) => ofMedia(file) && file.endsWith(".vtt")), + metadataPath: workFiles.find((file) => ofMedia(file) && file.endsWith(".info.json")) ?? "", + }; if (!noMedia) { // A partly removed temp session would yield a layout silently missing a part. const frameNames = temp.framesDir ? listFiles(temp.framesDir) : []; const captioned = Boolean(state.phases?.acquire?.metrics?.captionRung); const missing = [ ...(media.videoPath ? [] : [`the video (${temp.workDir ?? "no tempSession.workDir"})`]), + ...(media.videoPath && !media.metadataPath ? [`the info JSON (${mediaId}.info.json)`] : []), ...(frameNames.length > 0 ? [] : [`the frames (${temp.framesDir ?? "no tempSession.framesDir"})`]), @@ -542,7 +562,7 @@ export async function relayoutSlice({ if (to) plan.push({ from: path.join(slice, file), to, sourceFile: file }); } for (const caption of media.captionPaths) { - const name = path.basename(caption).replace(`${state.videoId}.`, ""); + const name = path.basename(caption).replace(`${mediaId}.`, ""); plan.push({ from: caption, to: `transcript/${name}` }); } /** @type {Map} */ diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index bc827a7c281..6efc8a3defc 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -2,7 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { checkLayoutLinks, relayoutSlice } from "./relayout-slice.js"; @@ -376,6 +376,36 @@ describe("relayoutSlice", () => { expect(fs.existsSync(targetDir)).toBe(false); }); + it("refuses a temp session holding several videos, since the primary is not recorded", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + // An X post with two videos: yt-dlp writes each as `.`. + writeTree(workDir, { + "1800000000000000002.mp4": "video-2", + "1800000000000000002.info.json": "{}", + "1800000000000000002.en.vtt": "WEBVTT\n", + }); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + const result = await relayoutSlice({ sliceDir, targetDir }); + const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); + stderr.mockRestore(); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + expect(message).toContain("1800000000000000002.mp4"); + expect(message).toContain(`${VIDEO_ID}.mp4`); + }); + + it("refuses a temp session whose info JSON is gone though the video survives", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + fs.rmSync(path.join(workDir, `${VIDEO_ID}.info.json`)); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + it("refuses a captioned temp session whose caption tracks are gone", async () => { const { sliceDir, targetDir, workDir } = makeFixture(); for (const name of fs.readdirSync(workDir).filter((file) => file.endsWith(".vtt"))) { From aee123c39d80a8c53546c150501405324532f5db Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:22:15 -0400 Subject: [PATCH 08/73] fix(knowledge): close the remaining overlap and partial-copy cases in video-digest relayout - Refuse when the slice is, holds, or sits inside any path the run writes: the target, .relayout-backup, or a staging dir beside the target, checked on real paths before backup recovery. - Validate frames individually: frame-times.json, at least one frame image, and every frame the sidecar lists. Refuse an empty video. - Refuse symlinks in the slice or the temp session (a copy would follow them), recorded temp dirs outside the OS temp dir or overlapping the slice or target, and report an unwritable target parent as a refusal instead of throwing. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 14 +- .../extraction/watch/relayout-slice.js | 118 +++++++++++++++-- .../extraction/watch/relayout-slice.test.js | 121 +++++++++++++++++- 3 files changed, 233 insertions(+), 20 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 9a8ef2c21d1..5064911eb17 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -73,11 +73,15 @@ The script runs on a closed slice, or on an unclosed one whose outcome checks pa itself and refuses when they fail). It refuses any slice whose temp media is gone unless `--no-media` is passed, which takes nothing from the temp session (no media, frames, caption tracks or info JSON) even when it still exists; after a normal close that is the only way it runs. -Without `--no-media` it also refuses a partly removed temp session, naming the missing video, -info JSON, frames or caption tracks, and a temp session holding several videos (an X post with -more than one), naming them, since the watch does not record which one is primary. It refuses a target that is the slice, contains it, or sits inside it, -and an existing target unless `--replace` is passed, which replaces the target and keeps its -`README.md`. The replaced target is moved to `.relayout-backup` and deleted only once the +Without `--no-media` it also refuses a partly removed temp session, naming the missing piece: the +video (or an empty one), the info JSON, `frame-times.json`, any frame image, a frame +`frame-times.json` lists, or the caption tracks. It refuses a temp session holding several videos +(an X post with more than one), naming them, since the watch does not record which one is +primary; a recorded temp dir outside the OS temp dir or overlapping the slice or target; and a +symlink in the slice or the temp session, since a copy would follow it. It refuses when the slice +is, holds, or sits inside a path the run writes: the target, `.relayout-backup`, or a +`..relayout-*` staging dir beside the target. It refuses an existing target unless +`--replace` is passed, which replaces the target and keeps its `README.md`. The replaced target is moved to `.relayout-backup` and deleted only once the new layout is in place. If a run is interrupted mid-swap, the next run restores the target from that backup when the target is missing (and says so), and refuses, naming the backup, when both exist; remove whichever one you do not want to keep. diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 6e794b0ec77..c8c98075e53 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -32,6 +32,7 @@ */ import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { isMainModule } from "@melodic/video-digestion/shared/main-module"; @@ -83,6 +84,7 @@ const SOURCE_METADATA = new Set([ const TRANSCRIPT_TEXT = /^transcript(?:-\d+)?\.txt$/; const VIDEO_FILE = /\.(?:mp4|mkv|webm)$/; +const FRAME_IMAGE = /\.(?:png|jpe?g)$/i; const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$\s]/; const LINE_SUFFIX = /:\d+(?:[-,]\d+)*$/; @@ -400,6 +402,53 @@ function provenanceReadme({ title, sourceUrl, videoId, acquiredAt }, presentTopD ].join("\n"); } +/** + * What the frames dir lacks for a complete `frames/all/`: `frame-times.json` + * (an object keyed by frame file), at least one frame image, and every frame + * the sidecar lists. Empty when nothing is missing. + * + * @param {string|undefined} framesDir + * @returns {string[]} + */ +function missingFrames(framesDir) { + if (!framesDir || !fs.existsSync(framesDir)) { + return [`the frames (${framesDir ?? "no tempSession.framesDir"})`]; + } + const names = new Set(fs.readdirSync(framesDir)); + const missing = []; + /** @type {unknown} */ + let times = null; + try { + times = JSON.parse(fs.readFileSync(path.join(framesDir, "frame-times.json"), "utf8")); + } catch { + missing.push(`a readable frame-times.json (${framesDir})`); + } + if (![...names].some((name) => FRAME_IMAGE.test(name))) { + missing.push(`any frame image (${framesDir})`); + } + if (times && typeof times === "object") { + const listed = Object.keys(times).filter((name) => !names.has(name)); + if (listed.length > 0) missing.push(`the frames frame-times.json lists: ${listed.join(", ")}`); + } + return missing; +} + +/** + * posix paths, relative to `dir`, of every symlink under it. + * + * @param {string} dir + * @param {string} [prefix] + * @returns {string[]} + */ +function findSymlinks(dir, prefix = "") { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isSymbolicLink()) return [rel]; + return entry.isDirectory() ? findSymlinks(path.join(dir, entry.name), rel) : []; + }); +} + /** * Real path of `p`, resolving symlinks through its nearest existing ancestor * when `p` itself does not exist yet. @@ -462,16 +511,35 @@ export async function relayoutSlice({ return { exitCode: 1, copied: 0, rewrites: 0, unresolved: [] }; }; - // A target that is, holds, or sits inside the slice would copy the slice - // into itself, and `--replace` would move the slice away and delete it. + // Every path this run creates, renames or deletes: the target, its backup + // and its staging dirs (`..relayout-*` beside the target). One that + // is, holds, or sits inside the slice would copy the slice into itself or + // move it away and delete it. Real paths, so a symlink cannot hide overlap. + const backup = `${target}.relayout-backup`; const realSlice = realPathOf(slice); - const realTarget = realPathOf(target); - if (isSameOrInside(realTarget, realSlice) || isSameOrInside(realSlice, realTarget)) { - return refuse(`target ${target} overlaps the slice ${slice}; pick a target outside it`); + const stagingPrefix = `.${path.basename(target)}.relayout-`; + const overlapsSlice = (/** @type {string} */ p) => { + const real = realPathOf(p); + return isSameOrInside(real, realSlice) || isSameOrInside(realSlice, real); + }; + const sliceInStaging = path + .relative(realPathOf(path.dirname(target)), realSlice) + .split(path.sep)[0] + .startsWith(stagingPrefix); + for (const written of [target, backup]) { + if (overlapsSlice(written)) { + return refuse(`${written} overlaps the slice ${slice}; pick a target outside it`); + } + } + if (sliceInStaging) { + return refuse(`the slice ${slice} sits in a relayout staging dir of ${target}`); + } + const symlinks = findSymlinks(slice); + if (symlinks.length > 0) { + return refuse(`the slice holds symlinks, which a copy would follow: ${symlinks.join(", ")}`); } // An interrupted `--replace` swap leaves the previous target at this name. - const backup = `${target}.relayout-backup`; if (fs.existsSync(backup)) { if (fs.existsSync(target)) { return refuse( @@ -509,6 +577,22 @@ export async function relayoutSlice({ // `--no-media` takes nothing from the temp session, even when it still exists. const temp = resolveTempSession(noMedia ? {} : (state.tempSession ?? {})); + // watch.json names the temp dirs, and the watch only ever makes them in the + // OS temp dir; anything else (a hand-edited path, the slice, the target) is + // not a temp session to read from. + const realTmp = fs.realpathSync.native(os.tmpdir()); + for (const dir of [temp.workDir, temp.framesDir]) { + if (!dir) continue; + if (!isSameOrInside(realPathOf(dir), realTmp) || realPathOf(dir) === realTmp) { + return refuse(`temp session dir ${dir} is not inside the OS temp dir`); + } + if ( + overlapsSlice(dir) || + [target, backup].some((p) => isSameOrInside(realPathOf(p), realPathOf(dir))) + ) { + return refuse(`temp session dir ${dir} overlaps the slice or the target`); + } + } const workFiles = temp.workDir ? await listWorkDirFiles(temp.workDir) : []; // The watch does not record which work file is the primary entry's, so only // a work dir holding exactly one video is unambiguous (an X post with @@ -531,14 +615,17 @@ export async function relayoutSlice({ }; if (!noMedia) { // A partly removed temp session would yield a layout silently missing a part. - const frameNames = temp.framesDir ? listFiles(temp.framesDir) : []; const captioned = Boolean(state.phases?.acquire?.metrics?.captionRung); const missing = [ ...(media.videoPath ? [] : [`the video (${temp.workDir ?? "no tempSession.workDir"})`]), + ...(media.videoPath && fs.statSync(media.videoPath).size === 0 + ? [`a non-empty video (${path.basename(media.videoPath)} is empty)`] + : []), ...(media.videoPath && !media.metadataPath ? [`the info JSON (${mediaId}.info.json)`] : []), - ...(frameNames.length > 0 - ? [] - : [`the frames (${temp.framesDir ?? "no tempSession.framesDir"})`]), + ...missingFrames(temp.framesDir), + ...[temp.workDir, temp.framesDir] + .flatMap((dir) => (dir ? findSymlinks(dir).map((link) => path.join(dir, link)) : [])) + .map((link) => `regular files in place of symlinks (${link})`), ...(captioned && media.captionPaths.length === 0 ? ["the caption tracks"] : []), ]; if (missing.length > 0) { @@ -599,10 +686,13 @@ export async function relayoutSlice({ // Build the whole layout in a sibling staging directory and move it into // place only after the link check passes, so a failure never leaves a // partial target. - fs.mkdirSync(path.dirname(target), { recursive: true }); - const staging = fs.mkdtempSync( - path.join(path.dirname(target), `.${path.basename(target)}.relayout-`), - ); + let staging; + try { + fs.mkdirSync(path.dirname(target), { recursive: true }); + staging = fs.mkdtempSync(path.join(path.dirname(target), stagingPrefix)); + } catch (error) { + return refuse(`cannot stage beside ${target}: ${/** @type {Error} */ (error).message}`); + } const discard = (/** @type {string} */ dir) => fs.rmSync(dir, { recursive: true, force: true }); let rewrites = 0; let copied = 0; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 6efc8a3defc..f0d0425f94a 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -65,7 +65,7 @@ function makeFixture({ status = "complete", withMedia = true, extraFiles = {} } writeTree(framesDir, { "scene_0001.png": "png", "anchor_00012000_0001.png": "png", - "frame-times.json": "{}", + "frame-times.json": JSON.stringify({ "scene_0001.png": { timestampSec: 1 } }), }); } else { fs.rmSync(workDir, { recursive: true, force: true }); @@ -406,6 +406,125 @@ describe("relayoutSlice", () => { expect(fs.existsSync(targetDir)).toBe(false); }); + it.each([ + [ + "is the backup path", + (/** @type {string} */ parent) => path.join(parent, "out.relayout-backup"), + ], + [ + "sits inside the backup path", + (/** @type {string} */ parent) => path.join(parent, "out.relayout-backup", "slice"), + ], + [ + "sits inside a staging dir", + (/** @type {string} */ parent) => path.join(parent, ".out.relayout-abc123", "slice"), + ], + ])("refuses when the slice %s of the target", async (_label, toSlice) => { + const { sliceDir: original } = makeFixture(); + const parent = makeDir("relayout-overlap-"); + const sliceDir = toSlice(parent); + fs.mkdirSync(path.dirname(sliceDir), { recursive: true }); + fs.renameSync(original, sliceDir); + const before = fs.readdirSync(sliceDir).sort(); + + const result = await relayoutSlice({ + sliceDir, + targetDir: path.join(parent, "out"), + replace: true, + }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(sliceDir).sort()).toEqual(before); + expect(fs.existsSync(path.join(parent, "out"))).toBe(false); + }); + + it("refuses a slice holding a symlink, which a copy would follow", async () => { + const { sliceDir, targetDir } = makeFixture(); + const outside = path.join(makeDir("relayout-secret-"), "secret.md"); + fs.writeFileSync(outside, "secret\n"); + fs.symlinkSync(outside, path.join(sliceDir, "research", "leak.md")); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it.each([ + ["outside the OS temp dir", () => path.parse(os.tmpdir()).root], + ["the slice itself", (/** @type {string} */ slice) => slice], + ])("refuses a recorded temp dir %s", async (_label, toWorkDir) => { + const { sliceDir, targetDir } = makeFixture(); + const statePath = path.join(sliceDir, "run-state", "watch.json"); + const state = JSON.parse(read(sliceDir, "run-state/watch.json")); + state.tempSession.workDir = toWorkDir(sliceDir); + fs.writeFileSync(statePath, JSON.stringify(state)); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it.each([ + [ + "frame-times.json is gone", + (/** @type {string} */ dir) => fs.rmSync(path.join(dir, "frame-times.json")), + ], + [ + "a frame frame-times.json lists is gone", + (/** @type {string} */ dir) => fs.rmSync(path.join(dir, "scene_0001.png")), + ], + [ + "only the sidecar is left", + (/** @type {string} */ dir) => { + for (const name of fs.readdirSync(dir).filter((file) => file.endsWith(".png"))) { + fs.rmSync(path.join(dir, name)); + } + }, + ], + ])("refuses a temp session where %s", async (_label, damage) => { + const { sliceDir, targetDir, framesDir } = makeFixture(); + damage(framesDir); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("refuses a temp session holding a symlink, which a copy would follow", async () => { + const { sliceDir, targetDir, framesDir } = makeFixture(); + const outside = path.join(makeDir("relayout-secret-"), "secret.png"); + fs.writeFileSync(outside, "secret"); + fs.symlinkSync(outside, path.join(framesDir, "scene_0002.png")); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("refuses, without throwing, a target whose parent is a file", async () => { + const { sliceDir } = makeFixture(); + const blocker = path.join(makeDir("relayout-blocker-"), "file"); + fs.writeFileSync(blocker, ""); + + const result = await relayoutSlice({ sliceDir, targetDir: path.join(blocker, "out") }); + + expect(result.exitCode).toBe(1); + }); + + it("refuses an empty video file", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + fs.writeFileSync(path.join(workDir, `${VIDEO_ID}.mp4`), ""); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + it("refuses a captioned temp session whose caption tracks are gone", async () => { const { sliceDir, targetDir, workDir } = makeFixture(); for (const name of fs.readdirSync(workDir).filter((file) => file.endsWith(".vtt"))) { From e5f1b7a7c8fe437580dfae820901ede3346b04a3 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:38:58 -0400 Subject: [PATCH 09/73] fix(knowledge): cover decks, bare frame names, >> tokens and install races in video-digest - Boundary dedupe counts lexical words only, so `>> Very good.` / `>> Very good question` is a 2-word repeat and is kept. - Relayout copies source/decks/ and source/attachments/ to metadata/, and vision-plan.md and visual-gaps.md to frames/. - Under --no-media a bare extracted-frame name becomes not-retained text; the link check now flags a bare frame name that does not resolve. - A reference to a directory split across the layout (source/) names where its files went. - Without --replace the install is a single rename, so a target that appears during the run is refused instead of being replaced. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 32 ++++--- .../extraction/transcript/write-transcript.js | 24 ++++- .../transcript/write-transcript.test.js | 18 ++++ .../extraction/watch/relayout-slice.js | 62 ++++++++++--- .../extraction/watch/relayout-slice.test.js | 87 +++++++++++++++++++ 5 files changed, 195 insertions(+), 28 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 5064911eb17..0d969e9d591 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -86,20 +86,24 @@ new layout is in place. If a run is interrupted mid-swap, the next run restores that backup when the target is missing (and says so), and refuses, naming the backup, when both exist; remove whichever one you do not want to keep. -It writes the knowledge-corpus layout: `transcript/`, `metadata/` (with a trimmed `info.json` -and the companion sources brief), `frames/all/`, `frames/key/` beside the frame logs in -`frames/`, `media/.`, `analysis/` (`RESEARCH.md`, `research/`, `recommendations/`, the -companion digest), and a provenance `README.md` when the target has none. The script's header -comment is the authoritative mapping. Each slice path the copied markdown names (links and -backticked paths) is rewritten through a path map relative to the file's new location, and one -naming a pipeline file the layout does not keep (such as `run-state/watch.json`) becomes plain -text saying it is not retained. It builds the layout in a sibling staging directory and -link-checks it there; it exits 1 listing every slice-internal path that still does not resolve. -Not checked: paths under `plugins/`, `docs/`, `templates/` and `.claude/`, `AGENTS.md`, paths -that leave the target, and backticked paths whose first segment is neither a top-level directory -of the target nor an entry beside the naming file (paths into another repository). Only a passing -check moves the staging directory into place; on any failure the staging directory is removed and -the target is left as it was. Fix what it lists in the slice and re-run. +It writes the knowledge-corpus layout: `transcript/`, `metadata/` (with a trimmed `info.json`, +the companion sources brief, and the fetched `decks/` and `attachments/`), `frames/all/`, +`frames/key/` beside the frame and vision logs in `frames/`, `media/.`, `analysis/` +(`RESEARCH.md`, `research/`, `recommendations/`, the companion digest), and a provenance +`README.md` when the target has none. The script's header comment is the authoritative mapping. +Each slice path the copied markdown names (links and backticked paths) is rewritten through a +path map relative to the file's new location. One naming a pipeline file the layout does not keep +(such as `run-state/watch.json`, or an extracted frame's bare name under `--no-media`) becomes +plain text saying it is not retained, and one naming a directory whose files went to several +places (`source/`) becomes plain text naming where they went. It builds the layout in a sibling +staging directory and link-checks it there; it exits 1 listing every slice-internal path that +still does not resolve, including a bare extracted-frame name. Not checked: paths under +`plugins/`, `docs/`, `templates/` and `.claude/`, `AGENTS.md`, paths that leave the target, and +backticked paths whose first segment is neither a top-level directory of the target nor an entry +beside the naming file (paths into another repository). Only a passing check moves the staging +directory into place; on any failure the staging directory is removed and the target is left as +it was. Without `--replace` the move is a single rename that fails, rather than replacing +anything, when a target appeared during the run. Fix what it lists in the slice and re-run. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index 8b50d317f3f..c20008b12ae 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -40,7 +40,7 @@ const BOUNDARY_REPEAT_MAX_WORDS = 12; const PARAGRAPH_STAMP = /^(\[[^\]]*\]\s*)([\s\S]*)$/; /** @param {string} text */ -function words(text) { +function tokens(text) { return text.split(/\s+/).filter(Boolean); } @@ -49,6 +49,16 @@ function comparableWord(word) { return word.toLowerCase().replace(/[^\p{L}\p{N}']/gu, ""); } +/** + * The lexical words of `text`: punctuation-only tokens such as `>>` are not + * words, so they never count toward a run's length. + * + * @param {string} text + */ +function words(text) { + return tokens(text).filter((token) => comparableWord(token) !== ""); +} + /** @param {string[]} run */ function comparableRun(run) { return run.map(comparableWord).join(" "); @@ -99,7 +109,9 @@ export function dropParagraphBoundaryRepeats(transcript, cues) { const previous = PARAGRAPH_STAMP.exec(paragraphs[index - 1]); if (!current || !previous) continue; const previousWords = words(previous[2]); - const currentWords = words(current[2]); + const currentTokens = tokens(current[2]); + const wordAt = currentTokens.flatMap((token, at) => (comparableWord(token) ? [at] : [])); + const currentWords = wordAt.map((at) => currentTokens[at]); const longest = Math.min( BOUNDARY_REPEAT_MAX_WORDS, previousWords.length, @@ -114,7 +126,13 @@ export function dropParagraphBoundaryRepeats(transcript, cues) { ) { continue; } - paragraphs[index] = current[1] + currentWords.slice(length).join(" "); + // Keep punctuation-only tokens before the dropped words (a `>>` speaker + // marker); drop any between them. + const kept = [ + ...currentTokens.slice(0, wordAt[0]), + ...currentTokens.slice(wordAt[length - 1] + 1), + ]; + paragraphs[index] = current[1] + kept.join(" "); break; } } diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index f2c77ccca51..8ea1998323a 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -97,6 +97,24 @@ On benchmarks too and then we moved on. ); }); + it("does not count a punctuation-only token like >> as a repeated word", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +that answer was + +00:00:03.000 --> 00:00:05.000 +>> Very good. + +00:00:05.000 --> 00:00:07.000 +>> Very good question, thanks. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe( + "[0:01] that answer was >> Very good.\n\n[0:05] >> Very good question, thanks.", + ); + }); + it("keeps two words the speaker genuinely repeats across a paragraph boundary", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index c8c98075e53..ef25b8ab637 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -69,11 +69,21 @@ export const ALLOWED_MISS_PREFIXES = Object.freeze([ "AGENTS.md", ]); +/** Source-lane folders the layout keeps, and where each one goes. */ +const SOURCE_FOLDERS = new Map([ + ["companion-digest", "analysis/companion-digest"], + ["decks", "metadata/decks"], + ["attachments", "metadata/attachments"], +]); +/** The pipeline's extracted frame names: scene, interval and anchor frames. */ +const FRAME_NAME = /^(?:(?:scene|interval)_\d{4}|anchor_\d+_\d{4})\.png$/; const KEY_FRAME_LOGS = new Set([ "key-frames-manifest.md", "visual-frames.md", "key-frame-quality-audit.md", "frame-triage-log.md", + "vision-plan.md", + "visual-gaps.md", ]); const SOURCE_METADATA = new Set([ "harvested-links.json", @@ -106,8 +116,8 @@ export function corpusPathForSliceFile(sliceRel) { const name = rest.join("/"); if (sliceRel === "RESEARCH.md") return "analysis/RESEARCH.md"; if (lane === LANES.research || lane === LANES.recommendations) return `analysis/${sliceRel}`; - if (lane === LANES.source && rest[0] === "companion-digest" && rest.length > 1) { - return `analysis/${name}`; + if (lane === LANES.source && SOURCE_FOLDERS.has(rest[0]) && rest.length > 1) { + return `${SOURCE_FOLDERS.get(rest[0])}/${rest.slice(1).join("/")}`; } if (lane === LANES.source && rest.length === 1) { if (TRANSCRIPT_TEXT.test(name)) return `transcript/${name}`; @@ -166,11 +176,12 @@ function deriveDirectoryMap(fileMap) { * @property {Map} files - slice-relative file → target-relative file * @property {Map} dirs - slice-relative directory → target-relative directory * @property {Map} tempFrames - temp frame file name → target-relative file + * @property {boolean} framesDropped - the layout keeps no extracted frames (`--no-media`) * @property {string} sliceName - the slice directory's own name * @property {Set} sliceEntries - every slice-relative file and directory */ -/** @typedef {{ target: string, isDir: boolean } | { retired: string }} Lookup */ +/** @typedef {{ target: string, isDir: boolean } | { retired: string, note: string }} Lookup */ /** * Split a reference into its path and the suffix to carry over unchanged @@ -233,7 +244,22 @@ function lookUpReference(refPath, sourceFile, map) { if (found) return found; } const retired = keys.find((key) => key !== null && map.sliceEntries.has(key)); - if (retired) return { retired }; + if (retired) { + // A directory whose kept files went to several places (`source/`) has no + // single new location: name where they went instead. + const splitInto = [ + ...new Set( + [...map.files] + .filter(([from]) => from.startsWith(`${retired}/`)) + .map(([, to]) => `${to.split("/")[0]}/`), + ), + ].sort(); + return { + retired, + note: + splitInto.length > 0 ? `split across ${splitInto.join(", ")} in this copy` : NOT_RETAINED, + }; + } // An unknown file in a copied directory moves with the directory, so the // link check flags it at its new location. for (const key of bare.includes("/") ? keys : []) { @@ -242,8 +268,11 @@ function lookUpReference(refPath, sourceFile, map) { return { target: `${dir}/${path.posix.basename(key)}`, isDir: false }; } } - const frame = bare.includes("/") ? undefined : map.tempFrames.get(bare); - return frame ? { target: frame, isDir: false } : null; + if (bare.includes("/")) return null; + const frame = map.tempFrames.get(bare); + if (frame) return { target: frame, isDir: false }; + // Without the temp frames (`--no-media`) a frame name names nothing kept. + return map.framesDropped && FRAME_NAME.test(bare) ? { retired: bare, note: NOT_RETAINED } : null; } const NOT_RETAINED = "not retained in this copy"; @@ -278,7 +307,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const found = lookUpReference(splitReference(unwrap(target)).refPath, sourceFile, map); if (!found || !("retired" in found)) return match; rewrites += 1; - return `${label} (${found.retired}, ${NOT_RETAINED})`; + return `${label} (${found.retired}, ${found.note})`; }) .replace(MARKDOWN_LINK, (_match, open, target) => target.startsWith("<") ? `${open}<${rewrite(unwrap(target))}>` : `${open}${rewrite(target)}`, @@ -288,7 +317,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const found = lookUpReference(splitReference(span).refPath, sourceFile, map); if (found && "retired" in found) { rewrites += 1; - return `${found.retired} (${NOT_RETAINED})`; + return `${found.retired} (${found.note})`; } return `\`${rewrite(span)}\``; }); @@ -300,14 +329,16 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { * explicit `./` or `../` path, or one whose first segment is a top-level * entry of the re-laid-out slice or an entry beside the naming file. Any * other path is one the path map did not know under no slice directory, a - * path into another repository. + * path into another repository. A bare name reads as a slice path only when + * it is an extracted frame's name, which the rewrite turns into a path or + * not-retained text. * * @param {string} refPath * @param {string} fileDir - absolute directory of the naming file * @param {Set} topLevel - top-level entries of the re-laid-out slice */ function looksSliceInternal(refPath, fileDir, topLevel) { - if (!refPath.includes("/")) return false; + if (!refPath.includes("/")) return FRAME_NAME.test(refPath); if (/^(?:\.\.?|SLICE)\//.test(refPath)) return true; const first = refPath.split("/")[0]; return topLevel.has(first) || fs.existsSync(path.join(fileDir, first)); @@ -679,6 +710,7 @@ export async function relayoutSlice({ files, dirs: deriveDirectoryMap(files), tempFrames, + framesDropped: noMedia, sliceName: path.basename(slice), sliceEntries, }; @@ -759,13 +791,21 @@ export async function relayoutSlice({ // The backup keeps a fixed name so a run killed between the two renames is // recovered at the next start (see the backup check above). - const hadTarget = fs.existsSync(target); + // Without --replace the install is a single rename onto the target path, + // never a backup-and-swap: a target that appeared after the early check + // (another run, a person) makes the rename fail (ENOTEMPTY or EEXIST on + // POSIX, EPERM on Windows) instead of being replaced. POSIX lets the rename + // replace an empty directory, which holds nothing to lose. + const hadTarget = replace && fs.existsSync(target); try { if (hadTarget) fs.renameSync(target, backup); fs.renameSync(staging, target); } catch (error) { if (hadTarget && !fs.existsSync(target) && fs.existsSync(backup)) fs.renameSync(backup, target); discard(staging); + if (!replace && fs.existsSync(target)) { + return refuse(`${target} appeared during the run; pass --replace to replace it`); + } return refuse(`could not move the layout into place: ${/** @type {Error} */ (error).message}`); } if (hadTarget) discard(backup); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index f0d0425f94a..d9d96b72b48 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -252,6 +252,80 @@ describe("relayoutSlice", () => { expect(fs.existsSync(path.join(targetDir, rel)), rel).toBe(false); } expect(read(targetDir, "transcript/transcript.txt")).toBe("[0:01] hello\n"); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "| `key/title-slide.png` | scene_0001.png (not retained in this copy) |\n\nTiers: [manifest](key-frames-manifest.md)\n", + ); + }); + + it("copies decks, attachments and the vision logs, rewriting their references", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/session-1/slides.pdf": "pdf", + "source/attachments/pdf/notes.pdf": "pdf", + "source/deck-inventory.md": "Fetched: `decks/session-1/slides.pdf`.\n", + "research/sources.md": "Notes: [pdf](../source/attachments/pdf/notes.pdf).\n", + "key-frames/vision-plan.md": "# Vision plan\n", + "key-frames/visual-gaps.md": "# Visual gaps\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "metadata/decks/session-1/slides.pdf")).toBe("pdf"); + expect(read(targetDir, "metadata/attachments/pdf/notes.pdf")).toBe("pdf"); + expect(read(targetDir, "metadata/deck-inventory.md")).toBe( + "Fetched: `decks/session-1/slides.pdf`.\n", + ); + expect(read(targetDir, "analysis/research/sources.md")).toBe( + "Notes: [pdf](../../metadata/attachments/pdf/notes.pdf).\n", + ); + expect(read(targetDir, "frames/vision-plan.md")).toBe("# Vision plan\n"); + expect(read(targetDir, "frames/visual-gaps.md")).toBe("# Visual gaps\n"); + }); + + it("names where a split directory's files went instead of a single new path", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "recommendations/questions.md": "Inputs: `source/`.\n" }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "analysis/recommendations/questions.md")).toBe( + "Inputs: source (split across analysis/, metadata/, transcript/ in this copy).\n", + ); + }); + + it("fails the link check on a frame name the temp session does not hold", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": "Rejected: `scene_0099.png`.\n" }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([{ file: "frames/visual-frames.md", ref: "scene_0099.png" }]); + }); + + it("refuses, without --replace, a target that appears after the early check", async () => { + const { sliceDir, targetDir } = makeFixture(); + let calls = 0; + + const result = await relayoutSlice({ + sliceDir, + targetDir, + copyFile: (from, to) => { + calls += 1; + // Another run installs its layout while this one is staging. + if (calls === 1) writeTree(targetDir, { "theirs.md": "theirs\n" }); + fs.copyFileSync(from, to); + }, + }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(targetDir)).toEqual(["theirs.md"]); + expect(fs.readdirSync(path.dirname(targetDir))).toEqual(["out"]); }); it("fails the link check on a missing file under a slice directory", async () => { @@ -557,4 +631,17 @@ describe("checkLayoutLinks", () => { expect(checkLayoutLinks(root)).toEqual([]); }); + + it("flags a bare extracted-frame name but not other bare file names", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "frames/all/scene_0001.png": "png", + "frames/visual-frames.md": + "Kept: `all/scene_0001.png`. Dangling: `anchor_00012000_0003.png`. Prose: `SKILL.md`, `README.md`.\n", + }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "frames/visual-frames.md", ref: "anchor_00012000_0003.png" }, + ]); + }); }); From 78c4c45547d36e33cc3043c430977c2f36dcb94f Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:47:24 -0400 Subject: [PATCH 10/73] fix(knowledge): check video-digest frames against frame-times.json both ways Every scene_ and interval_ frame in the temp frames dir must have a frame-times.json entry, as well as every listed entry existing. Anchor frames carry their time in their file name and are never listed, which matches what scene detection and anchor extraction write. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 3 ++- .../extraction/watch/relayout-slice.js | 15 +++++++++++++-- .../extraction/watch/relayout-slice.test.js | 10 +++++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 0d969e9d591..b3369b39abb 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -75,7 +75,8 @@ itself and refuses when they fail). It refuses any slice whose temp media is gon tracks or info JSON) even when it still exists; after a normal close that is the only way it runs. Without `--no-media` it also refuses a partly removed temp session, naming the missing piece: the video (or an empty one), the info JSON, `frame-times.json`, any frame image, a frame -`frame-times.json` lists, or the caption tracks. It refuses a temp session holding several videos +`frame-times.json` lists, a `frame-times.json` entry for a scene or interval frame on disk +(anchor frames carry their time in their name), or the caption tracks. It refuses a temp session holding several videos (an X post with more than one), naming them, since the watch does not record which one is primary; a recorded temp dir outside the OS temp dir or overlapping the slice or target; and a symlink in the slice or the temp session, since a copy would follow it. It refuses when the slice diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index ef25b8ab637..082337b780f 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -77,6 +77,8 @@ const SOURCE_FOLDERS = new Map([ ]); /** The pipeline's extracted frame names: scene, interval and anchor frames. */ const FRAME_NAME = /^(?:(?:scene|interval)_\d{4}|anchor_\d+_\d{4})\.png$/; +/** Frames scene detection writes and records in `frame-times.json`. */ +const TIMED_FRAME = /^(?:scene|interval)_\d{4}\.png$/; const KEY_FRAME_LOGS = new Set([ "key-frames-manifest.md", "visual-frames.md", @@ -458,8 +460,17 @@ function missingFrames(framesDir) { missing.push(`any frame image (${framesDir})`); } if (times && typeof times === "object") { - const listed = Object.keys(times).filter((name) => !names.has(name)); - if (listed.length > 0) missing.push(`the frames frame-times.json lists: ${listed.join(", ")}`); + const listed = Object.keys(times); + const gone = listed.filter((name) => !names.has(name)); + if (gone.length > 0) missing.push(`the frames frame-times.json lists: ${gone.join(", ")}`); + // Scene detection records every scene and interval frame it writes; an + // anchor frame carries its time in its own name and is never listed. + const untimed = [...names].filter( + (name) => TIMED_FRAME.test(name) && !Object.hasOwn(times, name), + ); + if (untimed.length > 0) { + missing.push(`frame-times.json entries for: ${untimed.sort().join(", ")}`); + } } return missing; } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index d9d96b72b48..20b44421886 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -549,6 +549,14 @@ describe("relayoutSlice", () => { "a frame frame-times.json lists is gone", (/** @type {string} */ dir) => fs.rmSync(path.join(dir, "scene_0001.png")), ], + [ + "a scene frame has no frame-times.json entry", + (/** @type {string} */ dir) => fs.writeFileSync(path.join(dir, "scene_0002.png"), "png"), + ], + [ + "an interval frame has no frame-times.json entry", + (/** @type {string} */ dir) => fs.writeFileSync(path.join(dir, "interval_0001.png"), "png"), + ], [ "only the sidecar is left", (/** @type {string} */ dir) => { @@ -571,7 +579,7 @@ describe("relayoutSlice", () => { const { sliceDir, targetDir, framesDir } = makeFixture(); const outside = path.join(makeDir("relayout-secret-"), "secret.png"); fs.writeFileSync(outside, "secret"); - fs.symlinkSync(outside, path.join(framesDir, "scene_0002.png")); + fs.symlinkSync(outside, path.join(framesDir, "anchor_00030000_0002.png")); const result = await relayoutSlice({ sliceDir, targetDir }); From 42945b9ed85dfd74c6f7c7ca9a3e76d9520969c5 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:55:30 -0400 Subject: [PATCH 11/73] fix(knowledge): bind caption carry-over to the exact opener; require captions after recovery - A 2-word boundary repeat is dropped only when exactly one cue starts in the paragraph's second, so the opening cue is known; otherwise the words stay. - Relayout requires caption tracks unless the transcript phase records ASR, a skip or a degradation, which also covers a bootstrap-recovered watch (recovery needs a caption track but records no caption rung). Co-Authored-By: Claude Opus 5.5 --- .../extraction/transcript/write-transcript.js | 26 +++++++----- .../transcript/write-transcript.test.js | 21 ++++++++++ .../extraction/watch/relayout-slice.js | 11 ++++- .../extraction/watch/relayout-slice.test.js | 41 ++++++++++++++++++- 4 files changed, 86 insertions(+), 13 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index c20008b12ae..100e785e4b2 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -65,26 +65,30 @@ function comparableRun(run) { } /** - * True when a cue opening the paragraph stamped `stamp` carries `run` over + * True when the cue opening the paragraph stamped `stamp` carries `run` over * from the cue before it: the two cues overlap in time, or the opening cue * starts with the previous cue's closing words verbatim (a rolling caption). + * False whenever the opening cue cannot be identified exactly. * * @param {string} stamp - the paragraph's `[M:SS]` * @param {string[]} run - the paragraph's leading words * @param {readonly { startSec: number, endSec: number, text: string }[]} cues */ function carriesOverFromPreviousCue(stamp, run, cues) { + // The paragraph stamp floors the opening cue's start to the second, so it + // identifies the opener only when exactly one cue starts in that second. + // With several, the evidence is ambiguous and the words stay. + const openers = cues.flatMap((cue, at) => + `[${formatTimestamp(cue.startSec)}]` === stamp ? [at] : [], + ); + if (openers.length !== 1 || openers[0] === 0) return false; + const cue = cues[openers[0]]; + const previous = cues[openers[0] - 1]; const head = run.join(" "); - for (let index = 1; index < cues.length; index++) { - const cue = cues[index]; - if (`[${formatTimestamp(cue.startSec)}]` !== stamp) continue; - const previous = cues[index - 1]; - const cueWords = words(cue.text); - if (cueWords.slice(0, run.length).join(" ") !== head) continue; - if (previous.endSec > cue.startSec) return true; - if (words(previous.text).slice(-run.length).join(" ") === head) return true; - } - return false; + if (words(cue.text).slice(0, run.length).join(" ") !== head) return false; + return ( + previous.endSec > cue.startSec || words(previous.text).slice(-run.length).join(" ") === head + ); } /** diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index 8ea1998323a..24fb8d21a41 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -97,6 +97,27 @@ On benchmarks too and then we moved on. ); }); + it("keeps two repeated words when a later cue in the opener's second carries them", () => { + // The opener (5.0s) does not carry "Very good" over; a later cue in the + // same second (5.4s) overlaps the opener and starts with those words. + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +that answer was + +00:00:03.000 --> 00:00:05.000 +very good. + +00:00:05.000 --> 00:00:05.600 +Very good question + +00:00:05.400 --> 00:00:07.000 +Very good indeed, thanks. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript.split("\n\n")[1]).toMatch(/^\[0:05\] Very good question/); + }); + it("does not count a punctuation-only token like >> as a repeated word", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 082337b780f..02e8cb03377 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -657,7 +657,16 @@ export async function relayoutSlice({ }; if (!noMedia) { // A partly removed temp session would yield a layout silently missing a part. - const captioned = Boolean(state.phases?.acquire?.metrics?.captionRung); + // Captions are required unless the transcript phase records that it did + // not come from captions (ASR, skipped, or degraded). A recovered watch + // records only `recovered`, and recovery itself needs a caption track, so + // it requires them too. + const transcriptMetrics = state.phases?.transcript?.metrics ?? {}; + const captioned = !( + transcriptMetrics.transcriptStrategy === "asr" || + transcriptMetrics.skipped || + transcriptMetrics.transcriptDegradation + ); const missing = [ ...(media.videoPath ? [] : [`the video (${temp.workDir ?? "no tempSession.workDir"})`]), ...(media.videoPath && fs.statSync(media.videoPath).size === 0 diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 20b44421886..dfb320139d1 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -78,7 +78,7 @@ function makeFixture({ status = "complete", withMedia = true, extraFiles = {} } sourceUrl: `https://www.youtube.com/watch?v=${VIDEO_ID}`, title: "Sample Talk", status, - phases: { acquire: { metrics: { captionRung: "auto-en" } } }, + phases: { transcript: { metrics: { transcriptStrategy: "captions" } } }, tempSession: { workDir, framesDir, acquiredAt: "2026-10-10T12:00:00.000Z" }, }), "README.md": "# Sample Talk journey\n", @@ -618,6 +618,45 @@ describe("relayoutSlice", () => { expect(result.exitCode).toBe(1); expect(fs.existsSync(targetDir)).toBe(false); }); + + /** + * @param {string} sliceDir + * @param {Record} transcriptMetrics + * @param {string} workDir + */ + function withTranscriptMetricsAndNoCaptions(sliceDir, transcriptMetrics, workDir) { + const statePath = path.join(sliceDir, "run-state", "watch.json"); + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + // What recover-watch-bootstrap.js writes: acquire and transcript carry only `recovered`. + state.phases = { + acquire: { metrics: { videoDownloaded: true, recovered: true } }, + transcript: { metrics: transcriptMetrics }, + }; + fs.writeFileSync(statePath, JSON.stringify(state)); + for (const name of fs.readdirSync(workDir).filter((file) => file.endsWith(".vtt"))) { + fs.rmSync(path.join(workDir, name)); + } + } + + it("refuses a bootstrap-recovered session whose caption tracks are gone", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + withTranscriptMetricsAndNoCaptions(sliceDir, { recovered: true }, workDir); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("needs no caption tracks when the transcript came from ASR", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + withTranscriptMetricsAndNoCaptions(sliceDir, { transcriptStrategy: "asr" }, workDir); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(fs.existsSync(path.join(targetDir, "transcript/en.vtt"))).toBe(false); + }); }); describe("checkLayoutLinks", () => { From 0d7bf7e5f460fdd10edb8a2def2e751a68f3d4df Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:03:08 -0400 Subject: [PATCH 12/73] fix(knowledge): stop a caption provenance note from exempting video-digest relayout captions transcriptDegradation can carry the primary caption's provenanceNote even when that caption produced the transcript, so it no longer waives the caption-track requirement. Only transcriptStrategy "asr" or a skipped transcript phase (no transcript-bearing entries) does. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 14 ++++----- .../extraction/watch/relayout-slice.test.js | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 02e8cb03377..1a2a1d364a5 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -657,15 +657,15 @@ export async function relayoutSlice({ }; if (!noMedia) { // A partly removed temp session would yield a layout silently missing a part. - // Captions are required unless the transcript phase records that it did - // not come from captions (ASR, skipped, or degraded). A recovered watch - // records only `recovered`, and recovery itself needs a caption track, so - // it requires them too. + // Captions are required unless the transcript phase records that no + // caption fed the transcript: run-watch.js writes `transcriptStrategy: + // "asr"`, or `skipped: true` when no entry produced one. A + // `transcriptDegradation` can be a caption's provenance note, so it is not + // that evidence. A recovered watch records only `recovered`, and recovery + // itself needs a caption track, so it requires them too. const transcriptMetrics = state.phases?.transcript?.metrics ?? {}; const captioned = !( - transcriptMetrics.transcriptStrategy === "asr" || - transcriptMetrics.skipped || - transcriptMetrics.transcriptDegradation + transcriptMetrics.transcriptStrategy === "asr" || transcriptMetrics.skipped === true ); const missing = [ ...(media.videoPath ? [] : [`the video (${temp.workDir ?? "no tempSession.workDir"})`]), diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index dfb320139d1..c5faecaadf5 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -648,6 +648,35 @@ describe("relayoutSlice", () => { expect(fs.existsSync(targetDir)).toBe(false); }); + it("still requires caption tracks when a caption transcript records a degradation", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + // write-transcript.js sets transcriptDegradation to the caption's + // provenanceNote even when that caption produced the transcript. + withTranscriptMetricsAndNoCaptions( + sliceDir, + { transcriptStrategy: "captions", transcriptDegradation: "auto captions only" }, + workDir, + ); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("needs no caption tracks when no entry produced a transcript", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + withTranscriptMetricsAndNoCaptions( + sliceDir, + { skipped: true, reason: "no transcript-bearing entries" }, + workDir, + ); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + }); + it("needs no caption tracks when the transcript came from ASR", async () => { const { sliceDir, targetDir, workDir } = makeFixture(); withTranscriptMetricsAndNoCaptions(sliceDir, { transcriptStrategy: "asr" }, workDir); From b0581ed6fba218f18b6fd4de012682e4c22022f2 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:11:37 -0400 Subject: [PATCH 13/73] fix(knowledge): check relayout targets against recorded temp dirs in every mode --no-media swapped the temp session for {} before the overlap check, so a target under a recorded workDir, framesDir or contactSheetsDir passed and a later close would delete it. The target, its backup and the staging dir are now checked against every recorded temp dir, existing or not, in every mode; --no-media only skips reading temp artifacts. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 10 +++-- .../extraction/watch/relayout-slice.js | 27 ++++++++++--- .../extraction/watch/relayout-slice.test.js | 38 ++++++++++++++++++- 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index b3369b39abb..d054d0d356d 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -78,10 +78,12 @@ video (or an empty one), the info JSON, `frame-times.json`, any frame image, a f `frame-times.json` lists, a `frame-times.json` entry for a scene or interval frame on disk (anchor frames carry their time in their name), or the caption tracks. It refuses a temp session holding several videos (an X post with more than one), naming them, since the watch does not record which one is -primary; a recorded temp dir outside the OS temp dir or overlapping the slice or target; and a -symlink in the slice or the temp session, since a copy would follow it. It refuses when the slice -is, holds, or sits inside a path the run writes: the target, `.relayout-backup`, or a -`..relayout-*` staging dir beside the target. It refuses an existing target unless +primary; a recorded temp dir outside the OS temp dir or overlapping the slice; and a symlink in +the slice or the temp session, since a copy would follow it. It refuses when the slice is, holds, +or sits inside a path the run writes: the target, `.relayout-backup`, or a +`..relayout-*` staging dir beside the target. In every mode, `--no-media` included, +it refuses when one of those paths is, holds, or sits inside a temp dir `watch.json` records, +existing or not, since `close` deletes those. It refuses an existing target unless `--replace` is passed, which replaces the target and keeps its `README.md`. The replaced target is moved to `.relayout-backup` and deleted only once the new layout is in place. If a run is interrupted mid-swap, the next run restores the target from that backup when the target is missing (and says so), and refuses, naming the backup, when both diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 1a2a1d364a5..c9acd1b1920 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -618,7 +618,25 @@ export async function relayoutSlice({ } // `--no-media` takes nothing from the temp session, even when it still exists. - const temp = resolveTempSession(noMedia ? {} : (state.tempSession ?? {})); + // `close` deletes every temp dir watch.json records, so in every mode, + // `--no-media` included and whether or not the dir still exists, nothing + // this run writes may sit in, hold, or be one of them. + const recorded = resolveTempSession(state.tempSession ?? {}); + for (const dir of [recorded.workDir, recorded.framesDir, recorded.contactSheetsDir]) { + if (!dir) continue; + const realDir = realPathOf(dir); + const clashes = + [target, backup].some((p) => { + const real = realPathOf(p); + return isSameOrInside(real, realDir) || isSameOrInside(realDir, real); + }) || isSameOrInside(realPathOf(path.dirname(target)), realDir); + if (clashes) { + return refuse(`target ${target} overlaps the recorded temp session dir ${dir}`); + } + } + + // `--no-media` reads nothing from the temp session. + const temp = noMedia ? {} : recorded; // watch.json names the temp dirs, and the watch only ever makes them in the // OS temp dir; anything else (a hand-edited path, the slice, the target) is // not a temp session to read from. @@ -628,11 +646,8 @@ export async function relayoutSlice({ if (!isSameOrInside(realPathOf(dir), realTmp) || realPathOf(dir) === realTmp) { return refuse(`temp session dir ${dir} is not inside the OS temp dir`); } - if ( - overlapsSlice(dir) || - [target, backup].some((p) => isSameOrInside(realPathOf(p), realPathOf(dir))) - ) { - return refuse(`temp session dir ${dir} overlaps the slice or the target`); + if (overlapsSlice(dir)) { + return refuse(`temp session dir ${dir} overlaps the slice`); } } const workFiles = temp.workDir ? await listWorkDirFiles(temp.workDir) : []; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index c5faecaadf5..c028c0a2fd2 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -525,7 +525,7 @@ describe("relayoutSlice", () => { }); it.each([ - ["outside the OS temp dir", () => path.parse(os.tmpdir()).root], + ["outside the OS temp dir", () => path.join(path.parse(os.tmpdir()).root, "relayout-not-tmp")], ["the slice itself", (/** @type {string} */ slice) => slice], ])("refuses a recorded temp dir %s", async (_label, toWorkDir) => { const { sliceDir, targetDir } = makeFixture(); @@ -597,6 +597,42 @@ describe("relayoutSlice", () => { expect(result.exitCode).toBe(1); }); + it.each([ + [ + "inside the recorded workDir", + (/** @type {Record} */ dirs) => path.join(dirs.workDir, "out"), + ], + [ + "the recorded framesDir itself", + (/** @type {Record} */ dirs) => dirs.framesDir, + ], + [ + "holding a recorded contactSheetsDir", + (/** @type {Record} */ dirs) => path.dirname(dirs.contactSheetsDir), + ], + [ + "inside a recorded contactSheetsDir that no longer exists", + (/** @type {Record} */ dirs) => path.join(dirs.contactSheetsDir, "out"), + ], + ])("refuses, even under --no-media, a target %s", async (_label, toTarget) => { + const { sliceDir, workDir, framesDir } = makeFixture(); + // `close` already removed this one; its parent holds nothing else. + const contactSheetsDir = path.join(makeDir("relayout-holder-"), "sheets"); + const statePath = path.join(sliceDir, "run-state", "watch.json"); + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + state.tempSession.contactSheetsDir = contactSheetsDir; + fs.writeFileSync(statePath, JSON.stringify(state)); + const targetDir = toTarget({ workDir, framesDir, contactSheetsDir }); + const before = fs.readdirSync(framesDir).sort(); + + const result = await relayoutSlice({ sliceDir, targetDir, noMedia: true, replace: true }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(framesDir).sort()).toEqual(before); + expect(fs.existsSync(path.join(workDir, "out"))).toBe(false); + expect(fs.existsSync(contactSheetsDir)).toBe(false); + }); + it("refuses an empty video file", async () => { const { sliceDir, targetDir, workDir } = makeFixture(); fs.writeFileSync(path.join(workDir, `${VIDEO_ID}.mp4`), ""); From 1e84ce746278c8f8ba8f8bfd0f531fbc1e96d5a3 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:21:25 -0400 Subject: [PATCH 14/73] fix(knowledge): tighten relayout lane checks, frame-times shape and no-clobber install - The link check counts a path under an original slice lane as the slice's when its parent directory existed in the slice, so a typo such as source/missing.txt is flagged while another repo's research/... path stays external. Prose naming a lane folder the slice never produced becomes "(none in this slice)" text. - frame-times.json must parse to a JSON object; null, arrays and scalars are refused. - Without --replace, POSIX installs claim the target with an exclusive mkdir and rename onto that claim, so even an empty directory created during the run is never replaced. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 77 +++++++++++++++---- .../extraction/watch/relayout-slice.test.js | 67 ++++++++++++++++ 2 files changed, 131 insertions(+), 13 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index c9acd1b1920..93d64bf2036 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -75,6 +75,12 @@ const SOURCE_FOLDERS = new Map([ ["decks", "metadata/decks"], ["attachments", "metadata/attachments"], ]); +/** Lane folders the output contract names, present in a slice only when produced. */ +const KNOWN_LANE_FOLDERS = new Set([ + ...[...SOURCE_FOLDERS.keys()].map((folder) => `${LANES.source}/${folder}`), + `${LANES.research}/findings`, + `${LANES.keyFrames}/frames`, +]); /** The pipeline's extracted frame names: scene, interval and anchor frames. */ const FRAME_NAME = /^(?:(?:scene|interval)_\d{4}|anchor_\d+_\d{4})\.png$/; /** Frames scene detection writes and records in `frame-times.json`. */ @@ -97,6 +103,8 @@ const TRANSCRIPT_TEXT = /^transcript(?:-\d+)?\.txt$/; const VIDEO_FILE = /\.(?:mp4|mkv|webm)$/; const FRAME_IMAGE = /\.(?:png|jpe?g)$/i; +/** The slice lane directories, plus the `SLICE/` root alias. */ +const SLICE_LANE_NAMES = new Set([...Object.values(LANES), "SLICE"]); const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$\s]/; const LINE_SUFFIX = /:\d+(?:[-,]\d+)*$/; @@ -245,6 +253,12 @@ function lookUpReference(refPath, sourceFile, map) { const found = lookup(key); if (found) return found; } + // A lane folder the layout knows but this slice never produced (prose such + // as "no deck in `source/decks/`") names nothing to point at. + const absentFolder = keys.find( + (key) => key !== null && KNOWN_LANE_FOLDERS.has(key) && !map.sliceEntries.has(key), + ); + if (absentFolder) return { retired: absentFolder, note: "none in this slice" }; const retired = keys.find((key) => key !== null && map.sliceEntries.has(key)); if (retired) { // A directory whose kept files went to several places (`source/`) has no @@ -338,12 +352,21 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { * @param {string} refPath * @param {string} fileDir - absolute directory of the naming file * @param {Set} topLevel - top-level entries of the re-laid-out slice + * @param {Set} [sliceDirs] - posix directories of the source slice */ -function looksSliceInternal(refPath, fileDir, topLevel) { +function looksSliceInternal(refPath, fileDir, topLevel, sliceDirs) { if (!refPath.includes("/")) return FRAME_NAME.test(refPath); if (/^(?:\.\.?|SLICE)\//.test(refPath)) return true; const first = refPath.split("/")[0]; - return topLevel.has(first) || fs.existsSync(path.join(fileDir, first)); + if (topLevel.has(first) || fs.existsSync(path.join(fileDir, first))) return true; + // A path left under an original slice lane (`source/missing.txt`) is a + // slice path the rewrite could not place. With the slice's directories + // known, it counts when its parent directory existed in the slice, so a + // path into another repository that shares a lane name + // (`research/deepening/x.md`) stays external; without them, any lane name + // counts. + if (!SLICE_LANE_NAMES.has(first)) return false; + return sliceDirs === undefined || sliceDirs.has(path.posix.dirname(refPath.replace(/\/+$/, ""))); } /** @@ -353,9 +376,11 @@ function looksSliceInternal(refPath, fileDir, topLevel) { * slice (another repository) and the allowed-miss prefixes are skipped. * * @param {string} rootDir + * @param {{ sliceDirs?: Set }} [options] - `sliceDirs`, the source + * slice's directories, narrows which lane-named paths count as the slice's * @returns {{ file: string, ref: string }[]} unresolved references */ -export function checkLayoutLinks(rootDir) { +export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { const root = path.resolve(rootDir); const topLevel = new Set(fs.existsSync(root) ? fs.readdirSync(root) : []); /** @type {{ file: string, ref: string }[]} */ @@ -371,7 +396,7 @@ export function checkLayoutLinks(rootDir) { if (!refPath || URL_SCHEME.test(refPath) || PLACEHOLDER.test(refPath)) return; if (refPath.startsWith("/") || refPath.startsWith("~")) return; if (ALLOWED_MISS_PREFIXES.some((prefix) => refPath.startsWith(prefix))) return; - if (!isLink && !looksSliceInternal(refPath, fileDir, topLevel)) return; + if (!isLink && !looksSliceInternal(refPath, fileDir, topLevel, sliceDirs)) return; const resolved = path.resolve(fileDir, refPath); const rel = path.relative(root, resolved); if (rel.startsWith("..") || path.isAbsolute(rel)) return; @@ -451,22 +476,31 @@ function missingFrames(framesDir) { const missing = []; /** @type {unknown} */ let times = null; + let readable = true; try { times = JSON.parse(fs.readFileSync(path.join(framesDir, "frame-times.json"), "utf8")); } catch { + readable = false; missing.push(`a readable frame-times.json (${framesDir})`); } + // Scene detection writes an object keyed by frame file; null, an array or + // a scalar is not one, and `Object.keys` would read it as an empty listing. + const isTimesObject = times !== null && typeof times === "object" && !Array.isArray(times); + if (readable && !isTimesObject) { + missing.push(`a frame-times.json holding a JSON object (${framesDir})`); + } if (![...names].some((name) => FRAME_IMAGE.test(name))) { missing.push(`any frame image (${framesDir})`); } - if (times && typeof times === "object") { - const listed = Object.keys(times); + if (isTimesObject) { + const listing = /** @type {Record} */ (times); + const listed = Object.keys(listing); const gone = listed.filter((name) => !names.has(name)); if (gone.length > 0) missing.push(`the frames frame-times.json lists: ${gone.join(", ")}`); // Scene detection records every scene and interval frame it writes; an // anchor frame carries its time in its own name and is never listed. const untimed = [...names].filter( - (name) => TIMED_FRAME.test(name) && !Object.hasOwn(times, name), + (name) => TIMED_FRAME.test(name) && !Object.hasOwn(listing, name), ); if (untimed.length > 0) { missing.push(`frame-times.json entries for: ${untimed.sort().join(", ")}`); @@ -814,7 +848,9 @@ export async function relayoutSlice({ ); } - unresolved = checkLayoutLinks(staging); + unresolved = checkLayoutLinks(staging, { + sliceDirs: new Set([...sliceEntries].filter((entry) => !sliceFiles.includes(entry))), + }); } catch (error) { discard(staging); return refuse(`copy failed, target left untouched: ${/** @type {Error} */ (error).message}`); @@ -826,17 +862,32 @@ export async function relayoutSlice({ // The backup keeps a fixed name so a run killed between the two renames is // recovered at the next start (see the backup check above). - // Without --replace the install is a single rename onto the target path, - // never a backup-and-swap: a target that appeared after the early check - // (another run, a person) makes the rename fail (ENOTEMPTY or EEXIST on - // POSIX, EPERM on Windows) instead of being replaced. POSIX lets the rename - // replace an empty directory, which holds nothing to lose. + // Without --replace the install never replaces anything that appeared after + // the early check (another run, a person), even an empty directory. POSIX + // rename replaces an empty directory, so the run first claims the target + // with an exclusive mkdir (EEXIST when anything is there) and renames onto + // its own empty claim; anything written into the claim meanwhile makes the + // rename fail with ENOTEMPTY. Windows rename never replaces an existing + // directory, so there the rename alone is the claim. const hadTarget = replace && fs.existsSync(target); + const claim = !replace && process.platform !== "win32"; + let claimed = false; try { if (hadTarget) fs.renameSync(target, backup); + if (claim) { + fs.mkdirSync(target); + claimed = true; + } fs.renameSync(staging, target); } catch (error) { if (hadTarget && !fs.existsSync(target) && fs.existsSync(backup)) fs.renameSync(backup, target); + if (claimed) { + try { + fs.rmdirSync(target); + } catch { + // Not empty: someone else's files now sit in the claim; leave them. + } + } discard(staging); if (!replace && fs.existsSync(target)) { return refuse(`${target} appeared during the run; pass --replace to replace it`); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index c028c0a2fd2..c14cb5e8757 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -308,6 +308,26 @@ describe("relayoutSlice", () => { expect(result.unresolved).toEqual([{ file: "frames/visual-frames.md", ref: "scene_0099.png" }]); }); + it("refuses, without --replace, an empty target directory created during the run", async () => { + const { sliceDir, targetDir } = makeFixture(); + let calls = 0; + + const result = await relayoutSlice({ + sliceDir, + targetDir, + copyFile: (from, to) => { + calls += 1; + // Created late and still empty: a plain POSIX rename would replace it. + if (calls === 1) fs.mkdirSync(targetDir); + fs.copyFileSync(from, to); + }, + }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(targetDir)).toEqual([]); + expect(fs.readdirSync(path.dirname(targetDir))).toEqual(["out"]); + }); + it("refuses, without --replace, a target that appears after the early check", async () => { const { sliceDir, targetDir } = makeFixture(); let calls = 0; @@ -557,6 +577,15 @@ describe("relayoutSlice", () => { "an interval frame has no frame-times.json entry", (/** @type {string} */ dir) => fs.writeFileSync(path.join(dir, "interval_0001.png"), "png"), ], + [ + "frame-times.json holds null", + (/** @type {string} */ dir) => fs.writeFileSync(path.join(dir, "frame-times.json"), "null"), + ], + [ + "frame-times.json holds an array", + (/** @type {string} */ dir) => + fs.writeFileSync(path.join(dir, "frame-times.json"), '["scene_0001.png"]'), + ], [ "only the sidecar is left", (/** @type {string} */ dir) => { @@ -744,6 +773,44 @@ describe("checkLayoutLinks", () => { expect(checkLayoutLinks(root)).toEqual([]); }); + it("flags an unresolved path under a slice lane whose directory the slice had", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "recommendations/questions.md": + "Typo: `source/missing.txt`. Upstream: `research/deepening/html-report.md:92`.\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/recommendations/questions.md", ref: "source/missing.txt" }, + ]); + }); + + it("says so when prose names a lane folder this slice never produced", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/vision-plan.md": "No deck in the `source/decks/` lane.\n" }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/vision-plan.md")).toBe( + "No deck in the source/decks (none in this slice) lane.\n", + ); + }); + + it("treats any lane-named path as the slice's when no slice directories are given", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { "analysis/RESEARCH.md": "See `run-state/watch.json`.\n" }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "analysis/RESEARCH.md", ref: "run-state/watch.json" }, + ]); + }); + it("flags a bare extracted-frame name but not other bare file names", () => { const root = makeDir("relayout-check-"); writeTree(root, { From 6f642bbd6eef6fdc772c0dfe96229cb90f282698 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:29:17 -0400 Subject: [PATCH 15/73] fix(knowledge): close the input-shape and reference-form gaps an adversarial sweep found - Refuse a watch.json that is not an object, whose videoId is not a plain id (it named media/., so "../../x" wrote outside the target), that lacks a title or sourceUrl, or whose temp dirs are not strings, instead of writing "undefined" or throwing. - Drop yt-dlp's local download paths (_filename, filename, __files_to_move) from the trimmed info.json. - Rewrite and check reference-style definitions and HTML src/href, decode percent-encoded paths, accept spaces in angle-bracket links, normalize backslash separators, and keep links that leave the slice pointing at the same file from the deeper layout. - Judge lane-named backticked paths by the slice's directories before the beside-the-file shortcut, so another repo's research/... path in RESEARCH.md stays external. - Retire references to the slice's journey README instead of pointing them at the provenance README, whose headings differ. - Refuse empty caption tracks and empty frame images in media mode. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 60 ++++--- .../extraction/watch/relayout-slice.js | 149 ++++++++++++++---- .../extraction/watch/relayout-slice.test.js | 122 +++++++++++++- 3 files changed, 276 insertions(+), 55 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index d054d0d356d..7b13545171a 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -73,18 +73,22 @@ The script runs on a closed slice, or on an unclosed one whose outcome checks pa itself and refuses when they fail). It refuses any slice whose temp media is gone unless `--no-media` is passed, which takes nothing from the temp session (no media, frames, caption tracks or info JSON) even when it still exists; after a normal close that is the only way it runs. -Without `--no-media` it also refuses a partly removed temp session, naming the missing piece: the -video (or an empty one), the info JSON, `frame-times.json`, any frame image, a frame +It refuses a `watch.json` that is not an object, whose `videoId` is not a plain id (it names +`media/.`), that lacks a `title` or `sourceUrl`, or whose temp session dirs are not +strings. Without `--no-media` it also refuses a partly removed temp session, naming the missing +piece: the video, an empty video, caption track or frame image, the info JSON, +`frame-times.json` (or one that is not a JSON object), any frame image, a frame `frame-times.json` lists, a `frame-times.json` entry for a scene or interval frame on disk -(anchor frames carry their time in their name), or the caption tracks. It refuses a temp session holding several videos -(an X post with more than one), naming them, since the watch does not record which one is -primary; a recorded temp dir outside the OS temp dir or overlapping the slice; and a symlink in -the slice or the temp session, since a copy would follow it. It refuses when the slice is, holds, -or sits inside a path the run writes: the target, `.relayout-backup`, or a -`..relayout-*` staging dir beside the target. In every mode, `--no-media` included, -it refuses when one of those paths is, holds, or sits inside a temp dir `watch.json` records, -existing or not, since `close` deletes those. It refuses an existing target unless -`--replace` is passed, which replaces the target and keeps its `README.md`. The replaced target is moved to `.relayout-backup` and deleted only once the +(anchor frames carry their time in their name), or the caption tracks. It refuses a temp session +holding several videos (an X post with more than one), naming them, since the watch does not +record which one is primary; a recorded temp dir outside the OS temp dir or overlapping the +slice; and a symlink in the slice or the temp session, since a copy would follow it. It refuses +when the slice is, holds, or sits inside a path the run writes: the target, +`.relayout-backup`, or a `..relayout-*` staging dir beside the target. In +every mode, `--no-media` included, it refuses when one of those paths is, holds, or sits inside a +temp dir `watch.json` records, existing or not, since `close` deletes those. It refuses an +existing target unless `--replace` is passed, which replaces the target and keeps its +`README.md`. The replaced target is moved to `.relayout-backup` and deleted only once the new layout is in place. If a run is interrupted mid-swap, the next run restores the target from that backup when the target is missing (and says so), and refuses, naming the backup, when both exist; remove whichever one you do not want to keep. @@ -93,20 +97,26 @@ It writes the knowledge-corpus layout: `transcript/`, `metadata/` (with a trimme the companion sources brief, and the fetched `decks/` and `attachments/`), `frames/all/`, `frames/key/` beside the frame and vision logs in `frames/`, `media/.`, `analysis/` (`RESEARCH.md`, `research/`, `recommendations/`, the companion digest), and a provenance -`README.md` when the target has none. The script's header comment is the authoritative mapping. -Each slice path the copied markdown names (links and backticked paths) is rewritten through a -path map relative to the file's new location. One naming a pipeline file the layout does not keep -(such as `run-state/watch.json`, or an extracted frame's bare name under `--no-media`) becomes -plain text saying it is not retained, and one naming a directory whose files went to several -places (`source/`) becomes plain text naming where they went. It builds the layout in a sibling -staging directory and link-checks it there; it exits 1 listing every slice-internal path that -still does not resolve, including a bare extracted-frame name. Not checked: paths under -`plugins/`, `docs/`, `templates/` and `.claude/`, `AGENTS.md`, paths that leave the target, and -backticked paths whose first segment is neither a top-level directory of the target nor an entry -beside the naming file (paths into another repository). Only a passing check moves the staging -directory into place; on any failure the staging directory is removed and the target is left as -it was. Without `--replace` the move is a single rename that fails, rather than replacing -anything, when a target appeared during the run. Fix what it lists in the slice and re-run. +`README.md` when the target has none; the trimmed `info.json` also drops the local download +paths yt-dlp records. The script's header comment is the authoritative mapping. Each slice path +the copied markdown names (inline and reference-style links, HTML `src`/`href`, and backticked +paths, percent-encoded or with `\` separators) is rewritten through a path map relative to the +file's new location, and a relative link that leaves the slice keeps pointing at the same file. +One naming a pipeline file the layout does not keep (such as `run-state/watch.json`, the slice's +own journey `README.md`, or an extracted frame's bare name under `--no-media`) becomes plain text +saying it is not retained; one naming a directory whose files went to several places (`source/`) +names where they went; one naming a lane folder the slice never produced (`source/decks/`) says +so. It builds the layout in a sibling staging directory and link-checks it there; it exits 1 +listing every slice-internal path that still does not resolve, including a bare extracted-frame +name and a path under an original lane whose parent directory the slice had (`source/missing.txt`). +Not checked: paths under `plugins/`, `docs/`, `templates/` and `.claude/`, `AGENTS.md`, paths +that leave the target, and backticked paths that are neither under such a lane directory nor +under a top-level directory of the target or an entry beside the naming file (paths into another +repository, such as `research/deepening/...` from an upstream repo). Only a passing check moves +the staging directory into place; on any failure the staging directory is removed and the target +is left as it was. Without `--replace` the install never replaces anything that appeared during +the run, even an empty directory: on POSIX it claims the target with an exclusive mkdir before +renaming onto it. Fix what it lists in the slice and re-run. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 93d64bf2036..dc5a66e4307 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -44,7 +44,7 @@ import { LANES } from "../lib/slice-lanes.js"; import { resolveTempSession } from "../lib/temp-session-paths.js"; import { watchStatePath } from "./watch-state.js"; -/** info.json keys the corpus copy drops: format, caption-URL, heatmap and request lists. */ +/** info.json keys the corpus copy drops: format, caption-URL, heatmap and request lists, and the local download paths yt-dlp records. */ export const INFO_JSON_DROPPED_KEYS = Object.freeze([ "formats", "requested_formats", @@ -55,6 +55,9 @@ export const INFO_JSON_DROPPED_KEYS = Object.freeze([ "http_headers", "requested_subtitles", "_format_sort_fields", + "_filename", + "filename", + "__files_to_move", "requested_downloads", ]); @@ -106,11 +109,15 @@ const FRAME_IMAGE = /\.(?:png|jpe?g)$/i; /** The slice lane directories, plus the `SLICE/` root alias. */ const SLICE_LANE_NAMES = new Set([...Object.values(LANES), "SLICE"]); const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; -const PLACEHOLDER = /[<>{}*$\s]/; +const PLACEHOLDER = /[<>{}*$]/; const LINE_SUFFIX = /:\d+(?:[-,]\d+)*$/; const MARKDOWN_LINK = /(\]\()(<[^>]+>|[^)\s]+)/g; const FULL_LINK = /!?\[([^\]\n]*)\]\((<[^>]+>|[^)\s]+)[^)\n]*\)/g; const BACKTICK_SPAN = /`([^`\s]+)`/g; +/** `[label]: path` reference-style link definitions. */ +const REFERENCE_DEFINITION = /^( {0,3}\[[^\]\n]+\]:[ \t]*)(<[^>\n]+>|\S+)/gm; +/** `src="…"` and `href="…"` in inline HTML. */ +const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)=")([^"\n]+)"/g; /** A backticked span names a path when it holds a separator or ends in an extension. */ const PATH_LIKE = /\/|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)?(?:#.*)?$/; @@ -189,16 +196,18 @@ function deriveDirectoryMap(fileMap) { * @property {boolean} framesDropped - the layout keeps no extracted frames (`--no-media`) * @property {string} sliceName - the slice directory's own name * @property {Set} sliceEntries - every slice-relative file and directory + * @property {string} sliceAbs - absolute slice directory + * @property {string} targetAbs - absolute final target directory */ -/** @typedef {{ target: string, isDir: boolean } | { retired: string, note: string }} Lookup */ +/** @typedef {{ target: string, isDir: boolean } | { retired: string, note: string } | { outside: string }} Lookup */ /** * Split a reference into its path and the suffix to carry over unchanged * (`#anchor`, `?query`, `:line`). * * @param {string} ref - * @returns {{ refPath: string, suffix: string }} + * @returns {{ refPath: string, suffix: string, encoded: boolean }} */ function splitReference(ref) { const hash = ref.search(/[#?]/); @@ -209,7 +218,20 @@ function splitReference(ref) { suffix = line[0] + suffix; refPath = refPath.slice(0, line.index); } - return { refPath, suffix }; + // Compare paths the way the filesystem sees them: percent-decoded, with + // Windows separators as `/`. A rewrite re-encodes when the original was. + let encoded = false; + if (refPath.includes("%")) { + try { + const decoded = decodeURI(refPath); + encoded = decoded !== refPath; + refPath = decoded; + } catch { + // Not valid percent-encoding: keep it as written. + } + } + if (!URL_SCHEME.test(refPath)) refPath = refPath.replace(/\\/g, "/"); + return { refPath, suffix, encoded }; } /** @@ -244,10 +266,13 @@ function lookUpReference(refPath, sourceFile, map) { } else if (bare.includes(marker)) { keys = [path.posix.normalize(bare.slice(bare.indexOf(marker) + marker.length))]; } else { - keys = [ - path.posix.normalize(path.posix.join(path.posix.dirname(sourceFile), bare)), - bare.startsWith(".") ? null : path.posix.normalize(bare), - ]; + const fromFile = path.posix.normalize(path.posix.join(path.posix.dirname(sourceFile), bare)); + // A relative path that leaves the slice (a sibling slice, the repo) keeps + // pointing at the same place from the file's new, deeper location. + if (fromFile === ".." || fromFile.startsWith("../")) { + return { outside: path.resolve(map.sliceAbs, ...fromFile.split("/")) }; + } + keys = [fromFile, bare.startsWith(".") ? null : path.posix.normalize(bare)]; } for (const key of keys) { const found = lookup(key); @@ -309,12 +334,20 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const unwrap = (/** @type {string} */ target) => target.startsWith("<") ? target.slice(1, -1) : target; const rewrite = (/** @type {string} */ ref) => { - const { refPath, suffix } = splitReference(ref); + const { refPath, suffix, encoded } = splitReference(ref); const found = lookUpReference(refPath, sourceFile, map); if (!found || "retired" in found) return ref; - const relative = path.posix.relative(path.posix.dirname(targetFile), found.target) || "."; - const trailing = found.isDir && refPath.endsWith("/") ? "/" : ""; - const next = `${relative}${trailing}${suffix}`; + let relative; + let trailing = ""; + if ("outside" in found) { + const fileDir = path.dirname(path.join(map.targetAbs, targetFile)); + relative = path.relative(fileDir, found.outside).split(path.sep).join("/"); + trailing = refPath.endsWith("/") ? "/" : ""; + } else { + relative = path.posix.relative(path.posix.dirname(targetFile), found.target) || "."; + trailing = found.isDir && refPath.endsWith("/") ? "/" : ""; + } + const next = `${encoded ? encodeURI(relative) : relative}${trailing}${suffix}`; if (next !== ref) rewrites += 1; return next; }; @@ -328,6 +361,13 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { .replace(MARKDOWN_LINK, (_match, open, target) => target.startsWith("<") ? `${open}<${rewrite(unwrap(target))}>` : `${open}${rewrite(target)}`, ) + // Reference-style definitions and HTML src/href attributes take the same + // path rewrite. One naming a file the layout drops is left for the link + // check to flag: a definition or attribute has no prose to rewrite into. + .replace(REFERENCE_DEFINITION, (_match, open, target) => + target.startsWith("<") ? `${open}<${rewrite(unwrap(target))}>` : `${open}${rewrite(target)}`, + ) + .replace(HTML_PATH_ATTRIBUTE, (_match, open, target) => `${open}${rewrite(target)}"`) .replace(BACKTICK_SPAN, (match, span) => { if (!PATH_LIKE.test(span)) return match; const found = lookUpReference(splitReference(span).refPath, sourceFile, map); @@ -358,15 +398,17 @@ function looksSliceInternal(refPath, fileDir, topLevel, sliceDirs) { if (!refPath.includes("/")) return FRAME_NAME.test(refPath); if (/^(?:\.\.?|SLICE)\//.test(refPath)) return true; const first = refPath.split("/")[0]; - if (topLevel.has(first) || fs.existsSync(path.join(fileDir, first))) return true; - // A path left under an original slice lane (`source/missing.txt`) is a - // slice path the rewrite could not place. With the slice's directories - // known, it counts when its parent directory existed in the slice, so a - // path into another repository that shares a lane name - // (`research/deepening/x.md`) stays external; without them, any lane name - // counts. - if (!SLICE_LANE_NAMES.has(first)) return false; - return sliceDirs === undefined || sliceDirs.has(path.posix.dirname(refPath.replace(/\/+$/, ""))); + // A path under an original slice lane name (`source/missing.txt`) is the + // slice's when its parent directory existed in the slice, so a path into + // another repository that shares a lane name (`research/deepening/x.md`) + // stays external even beside `analysis/research/`. Without the slice's + // directories, any lane name counts. + if (SLICE_LANE_NAMES.has(first)) { + return ( + sliceDirs === undefined || sliceDirs.has(path.posix.dirname(refPath.replace(/\/+$/, ""))) + ); + } + return topLevel.has(first) || fs.existsSync(path.join(fileDir, first)); } /** @@ -402,8 +444,10 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { if (rel.startsWith("..") || path.isAbsolute(rel)) return; if (!fs.existsSync(resolved)) unresolved.push({ file, ref }); }; - for (const match of text.matchAll(MARKDOWN_LINK)) { - check(match[2].replace(/^<|>$/g, ""), true); + for (const pattern of [MARKDOWN_LINK, REFERENCE_DEFINITION, HTML_PATH_ATTRIBUTE]) { + for (const match of text.matchAll(pattern)) { + check(match[2].replace(/^<|>$/g, ""), true); + } } for (const match of text.matchAll(BACKTICK_SPAN)) { check(match[1], false); @@ -460,6 +504,40 @@ function provenanceReadme({ title, sourceUrl, videoId, acquiredAt }, presentTopD ].join("\n"); } +/** A video id as yt-dlp's `%(id)s` writes it: no separator, no `..`. */ +const VIDEO_ID = /^[A-Za-z0-9_-]+$/; + +/** + * What is wrong with a parsed watch.json's shape for this script, or null. + * `videoId` names `media/.`, so it must not hold a path; the title + * and source URL fill the provenance README; the temp session dirs are paths. + * + * @param {unknown} state + * @returns {string|null} + */ +function watchStateShapeProblem(state) { + if (!state || typeof state !== "object" || Array.isArray(state)) return "is not a JSON object"; + const record = /** @type {Record} */ (state); + if (typeof record.videoId !== "string" || !VIDEO_ID.test(record.videoId)) { + return `videoId ${JSON.stringify(record.videoId)} is not a plain video id`; + } + for (const field of ["title", "sourceUrl"]) { + if (typeof record[field] !== "string" || record[field] === "") { + return `has no ${field} string`; + } + } + const temp = record.tempSession; + if (temp === undefined || temp === null) return null; + if (typeof temp !== "object" || Array.isArray(temp)) return "tempSession is not an object"; + const session = /** @type {Record} */ (temp); + for (const field of ["workDir", "framesDir", "contactSheetsDir", "acquiredAt"]) { + if (session[field] !== undefined && typeof session[field] !== "string") { + return `tempSession.${field} is not a string`; + } + } + return null; +} + /** * What the frames dir lacks for a complete `frames/all/`: `frame-times.json` * (an object keyed by frame file), at least one frame image, and every frame @@ -489,6 +567,11 @@ function missingFrames(framesDir) { if (readable && !isTimesObject) { missing.push(`a frame-times.json holding a JSON object (${framesDir})`); } + const emptyImages = [...names].filter( + (name) => FRAME_IMAGE.test(name) && fs.statSync(path.join(framesDir, name)).size === 0, + ); + if (emptyImages.length > 0) + missing.push(`non-empty frames (empty: ${emptyImages.sort().join(", ")})`); if (![...names].some((name) => FRAME_IMAGE.test(name))) { missing.push(`any frame image (${framesDir})`); } @@ -632,6 +715,8 @@ export async function relayoutSlice({ } catch { return refuse(`no readable run-state/watch.json under ${slice}`); } + const shapeProblem = watchStateShapeProblem(state); + if (shapeProblem) return refuse(`run-state/watch.json ${shapeProblem}`); const outcomesPass = () => { try { return verifyOutcomes(slice) === 0; @@ -727,6 +812,9 @@ export async function relayoutSlice({ .flatMap((dir) => (dir ? findSymlinks(dir).map((link) => path.join(dir, link)) : [])) .map((link) => `regular files in place of symlinks (${link})`), ...(captioned && media.captionPaths.length === 0 ? ["the caption tracks"] : []), + ...media.captionPaths + .filter((caption) => fs.statSync(caption).size === 0) + .map((caption) => `a non-empty caption track (${path.basename(caption)} is empty)`), ]; if (missing.length > 0) { return refuse( @@ -767,13 +855,14 @@ export async function relayoutSlice({ }); } - // A reference to the slice README lands on the target README, provenance or existing. - const files = new Map([ - ["README.md", "README.md"], - ...plan.flatMap((entry) => + // The slice's own README (its journey) is not copied: the target's README is + // a provenance page with other headings, so a reference to the slice README + // becomes not-retained text rather than a link to a different page. + const files = new Map( + plan.flatMap((entry) => entry.sourceFile ? [/** @type {[string, string]} */ ([entry.sourceFile, entry.to])] : [], ), - ]); + ); /** @type {PathMap} */ const map = { files, @@ -782,6 +871,8 @@ export async function relayoutSlice({ framesDropped: noMedia, sliceName: path.basename(slice), sliceEntries, + sliceAbs: slice, + targetAbs: target, }; // Build the whole layout in a sibling staging directory and move it into diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index c14cb5e8757..1cee306bc32 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -59,6 +59,9 @@ function makeFixture({ status = "complete", withMedia = true, extraFiles = {} } requested_subtitles: {}, _format_sort_fields: [], requested_downloads: [{}], + _filename: "/tmp/video-extraction-x/abc123XYZ_0.mp4", + filename: "/tmp/video-extraction-x/abc123XYZ_0.mp4", + __files_to_move: {}, comments: Array.from({ length: 25 }, (_, index) => ({ id: `c${index}` })), }), }); @@ -170,7 +173,7 @@ describe("relayoutSlice", () => { expect(read(targetDir, "analysis/recommendations/menu.md")).toBe( [ "Evidence: `../../transcript/transcript.txt` and ![slide](../../frames/key/title-slide.png).", - "Detail: `../research/findings/topic.md`, the hub [README](README.md), the [journey](../../README.md).", + "Detail: `../research/findings/topic.md`, the hub [README](README.md), the journey (README.md, not retained in this copy).", "Target file: `plugins/knowledge/skills/video-digest/SKILL.md`.", "Run state: run-state/watch.json (not retained in this copy), selection (key-frames/selection.json, not retained in this copy).", "Upstream: `research/deepening/html-report.md:92` in mattpocock/skills.", @@ -753,6 +756,123 @@ describe("relayoutSlice", () => { }); }); +describe("relayoutSlice input shapes and reference forms (adversarial sweep)", () => { + /** @param {string} sliceDir @param {(state: Record) => unknown} edit */ + function editState(sliceDir, edit) { + const statePath = path.join(sliceDir, "run-state", "watch.json"); + const next = edit(JSON.parse(fs.readFileSync(statePath, "utf8"))); + fs.writeFileSync(statePath, JSON.stringify(next)); + } + + it.each([ + ["a videoId that is a path", (/** @type {any} */ s) => ({ ...s, videoId: "../../escaped" })], + ["no title", (/** @type {any} */ s) => ({ ...s, title: undefined })], + ["no sourceUrl", (/** @type {any} */ s) => ({ ...s, sourceUrl: "" })], + ["null instead of an object", () => null], + [ + "a temp dir that is not a string", + (/** @type {any} */ s) => ({ ...s, tempSession: { ...s.tempSession, workDir: 123 } }), + ], + ])("refuses, without throwing, a watch.json with %s", async (_label, edit) => { + const { sliceDir, targetDir } = makeFixture(); + editState(sliceDir, edit); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(path.dirname(targetDir))).toEqual([]); + }); + + it("rewrites reference-style definitions, HTML src, encoded, spaced and backslash paths", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/findings/my notes.md": "notes\n", + "key-frames/visual-frames.md": [ + '[defined][1] and ', + "[spaced](<../research/findings/my notes.md>) [encoded](../research/findings/my%20notes.md)", + "Windows: `frames\\title-slide.png`.", + "", + "[1]: ../source/transcript.txt", + "", + ].join("\n"), + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + [ + '[defined][1] and ', + "[spaced](<../analysis/research/findings/my notes.md>) [encoded](../analysis/research/findings/my%20notes.md)", + "Windows: `key/title-slide.png`.", + "", + "[1]: ../transcript/transcript.txt", + "", + ].join("\n"), + ); + }); + + it("flags a reference-style definition to a path that does not exist", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "research/claim-inventory.md": "[gone][g]\n\n[g]: findings/gone.md\n" }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/research/claim-inventory.md", ref: "findings/gone.md" }, + ]); + }); + + it("keeps a link that leaves the slice pointing at the same file from its deeper location", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "research/claim-inventory.md": "[other](../../other-slice/RESEARCH.md)\n" }, + }); + const sibling = path.join(path.dirname(sliceDir), "other-slice", "RESEARCH.md"); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + const link = /\]\(([^)]+)\)/.exec(read(targetDir, "analysis/research/claim-inventory.md"))?.[1]; + expect(path.resolve(targetDir, "analysis/research", String(link))).toBe(sibling); + }); + + it("leaves another repo's lane-named path in RESEARCH.md alone", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "RESEARCH.md": "Upstream: `research/deepening/html-report.md:92` in mattpocock/skills.\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + }); + + it.each([ + [ + "an empty caption track", + (/** @type {Record} */ d) => + fs.writeFileSync(path.join(d.workDir, `${VIDEO_ID}.en.vtt`), ""), + ], + [ + "an empty frame image", + (/** @type {Record} */ d) => + fs.writeFileSync(path.join(d.framesDir, "anchor_00012000_0001.png"), ""), + ], + ])("refuses a temp session with %s", async (_label, damage) => { + const { sliceDir, targetDir, workDir, framesDir } = makeFixture(); + damage({ workDir, framesDir }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); +}); + describe("checkLayoutLinks", () => { it("stays quiet on URLs, placeholders, other-repository paths and resolvable paths", () => { const root = makeDir("relayout-check-"); From c24691fc62ba56c8cf5ed0ef8251282a48d1f1b2 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:43:25 -0400 Subject: [PATCH 16/73] fix(knowledge): handle any HTML attribute quoting and shape-check info JSON in relayout - src/href are rewritten and link-checked whether double-, single- or unquoted. - Media mode refuses, before staging, an info JSON that does not parse to a plain object (null, an array, corrupt JSON). Every JSON.parse in the script is now shape-checked: watch.json, frame-times.json and the info JSON. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 8 ++-- .../extraction/watch/relayout-slice.js | 41 ++++++++++++++++--- .../extraction/watch/relayout-slice.test.js | 39 ++++++++++++++++++ 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 7b13545171a..a2c3ea26f13 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -76,8 +76,8 @@ tracks or info JSON) even when it still exists; after a normal close that is the It refuses a `watch.json` that is not an object, whose `videoId` is not a plain id (it names `media/.`), that lacks a `title` or `sourceUrl`, or whose temp session dirs are not strings. Without `--no-media` it also refuses a partly removed temp session, naming the missing -piece: the video, an empty video, caption track or frame image, the info JSON, -`frame-times.json` (or one that is not a JSON object), any frame image, a frame +piece: the video, an empty video, caption track or frame image, the info JSON or +`frame-times.json` (or either one not holding a JSON object), any frame image, a frame `frame-times.json` lists, a `frame-times.json` entry for a scene or interval frame on disk (anchor frames carry their time in their name), or the caption tracks. It refuses a temp session holding several videos (an X post with more than one), naming them, since the watch does not @@ -99,8 +99,8 @@ the companion sources brief, and the fetched `decks/` and `attachments/`), `fram (`RESEARCH.md`, `research/`, `recommendations/`, the companion digest), and a provenance `README.md` when the target has none; the trimmed `info.json` also drops the local download paths yt-dlp records. The script's header comment is the authoritative mapping. Each slice path -the copied markdown names (inline and reference-style links, HTML `src`/`href`, and backticked -paths, percent-encoded or with `\` separators) is rewritten through a path map relative to the +the copied markdown names (inline and reference-style links, HTML `src`/`href` in any quoting, +and backticked paths, percent-encoded or with `\` separators) is rewritten through a path map relative to the file's new location, and a relative link that leaves the slice keeps pointing at the same file. One naming a pipeline file the layout does not keep (such as `run-state/watch.json`, the slice's own journey `README.md`, or an extracted frame's bare name under `--no-media`) becomes plain text diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index dc5a66e4307..e4124db7864 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -116,8 +116,8 @@ const FULL_LINK = /!?\[([^\]\n]*)\]\((<[^>]+>|[^)\s]+)[^)\n]*\)/g; const BACKTICK_SPAN = /`([^`\s]+)`/g; /** `[label]: path` reference-style link definitions. */ const REFERENCE_DEFINITION = /^( {0,3}\[[^\]\n]+\]:[ \t]*)(<[^>\n]+>|\S+)/gm; -/** `src="…"` and `href="…"` in inline HTML. */ -const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)=")([^"\n]+)"/g; +/** `src` and `href` in inline HTML, double-, single- or unquoted. */ +const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/g; /** A backticked span names a path when it holds a separator or ends in an extension. */ const PATH_LIKE = /\/|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)?(?:#.*)?$/; @@ -367,7 +367,10 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { .replace(REFERENCE_DEFINITION, (_match, open, target) => target.startsWith("<") ? `${open}<${rewrite(unwrap(target))}>` : `${open}${rewrite(target)}`, ) - .replace(HTML_PATH_ATTRIBUTE, (_match, open, target) => `${open}${rewrite(target)}"`) + .replace(HTML_PATH_ATTRIBUTE, (_match, open, doubled, single, bare) => { + const quote = doubled !== undefined ? '"' : single !== undefined ? "'" : ""; + return `${open}${quote}${rewrite(doubled ?? single ?? bare)}${quote}`; + }) .replace(BACKTICK_SPAN, (match, span) => { if (!PATH_LIKE.test(span)) return match; const found = lookUpReference(splitReference(span).refPath, sourceFile, map); @@ -444,11 +447,14 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { if (rel.startsWith("..") || path.isAbsolute(rel)) return; if (!fs.existsSync(resolved)) unresolved.push({ file, ref }); }; - for (const pattern of [MARKDOWN_LINK, REFERENCE_DEFINITION, HTML_PATH_ATTRIBUTE]) { + for (const pattern of [MARKDOWN_LINK, REFERENCE_DEFINITION]) { for (const match of text.matchAll(pattern)) { check(match[2].replace(/^<|>$/g, ""), true); } } + for (const match of text.matchAll(HTML_PATH_ATTRIBUTE)) { + check(match[2] ?? match[3] ?? match[4], true); + } for (const match of text.matchAll(BACKTICK_SPAN)) { check(match[1], false); } @@ -504,6 +510,26 @@ function provenanceReadme({ title, sourceUrl, videoId, acquiredAt }, presentTopD ].join("\n"); } +/** + * The JSON object in `file`, or null when it cannot be read, does not parse, + * or parses to anything but a plain object (null, an array, a scalar). + * + * @param {string} file + * @returns {Record|null} + */ +function readJsonObject(file) { + /** @type {unknown} */ + let parsed; + try { + parsed = JSON.parse(fs.readFileSync(file, "utf8")); + } catch { + return null; + } + return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) + ? /** @type {Record} */ (parsed) + : null; +} + /** A video id as yt-dlp's `%(id)s` writes it: no separator, no `..`. */ const VIDEO_ID = /^[A-Za-z0-9_-]+$/; @@ -807,6 +833,9 @@ export async function relayoutSlice({ ? [`a non-empty video (${path.basename(media.videoPath)} is empty)`] : []), ...(media.videoPath && !media.metadataPath ? [`the info JSON (${mediaId}.info.json)`] : []), + ...(media.metadataPath && !readJsonObject(media.metadataPath) + ? [`an info JSON holding a JSON object (${path.basename(media.metadataPath)})`] + : []), ...missingFrames(temp.framesDir), ...[temp.workDir, temp.framesDir] .flatMap((dir) => (dir ? findSymlinks(dir).map((link) => path.join(dir, link)) : [])) @@ -914,7 +943,9 @@ export async function relayoutSlice({ } } if (media.metadataPath) { - const info = JSON.parse(fs.readFileSync(media.metadataPath, "utf8")); + // Checked to be an object before staging; a change since then is a failure. + const info = readJsonObject(media.metadataPath); + if (!info) throw new Error(`${media.metadataPath} no longer holds a JSON object`); write("metadata/info.json", `${JSON.stringify(trimInfoJson(info), null, 2)}\n`); } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 1cee306bc32..20bbc70d944 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -813,6 +813,45 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ); }); + it("rewrites single-quoted and unquoted HTML src and href", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "key-frames/visual-frames.md": + " t\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + " t\n", + ); + }); + + it("flags a single-quoted HTML src that does not resolve", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { "frames/visual-frames.md": "\n" }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "frames/visual-frames.md", ref: "key/gone.png" }, + ]); + }); + + it.each([ + ["null", "null"], + ["an array", "[]"], + ["corrupt JSON", "{"], + ])("refuses, before staging, an info JSON holding %s", async (_label, body) => { + const { sliceDir, targetDir, workDir } = makeFixture(); + fs.writeFileSync(path.join(workDir, `${VIDEO_ID}.info.json`), body); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(path.dirname(targetDir))).toEqual([]); + }); + it("flags a reference-style definition to a path that does not exist", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "research/claim-inventory.md": "[gone][g]\n\n[g]: findings/gone.md\n" }, From c65c7c7f931c569a7175904c2d2879730a87f9e1 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:53:47 -0400 Subject: [PATCH 17/73] fix(knowledge): scan link destinations with parentheses; make the relayout claim recoverable - Inline link destinations are read by a small scanner (balanced and backslash-escaped parentheses, angle brackets, optional titles), in the rewrite and the link check, so slides_(final).pdf is one destination. - Without --replace on POSIX the run claims the target before staging with an exclusive mkdir plus a .relayout-claim marker (pid, time), releases it on failure, and removes it as part of the install. A target holding only that marker from a dead run is removed with a notice; a live pid's claim, or a marker beside other files, is refused. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 16 +- .../extraction/watch/relayout-slice.js | 259 +++++++++++++++--- .../extraction/watch/relayout-slice.test.js | 121 +++++++- 3 files changed, 341 insertions(+), 55 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index a2c3ea26f13..950a756ba7e 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -99,9 +99,10 @@ the companion sources brief, and the fetched `decks/` and `attachments/`), `fram (`RESEARCH.md`, `research/`, `recommendations/`, the companion digest), and a provenance `README.md` when the target has none; the trimmed `info.json` also drops the local download paths yt-dlp records. The script's header comment is the authoritative mapping. Each slice path -the copied markdown names (inline and reference-style links, HTML `src`/`href` in any quoting, -and backticked paths, percent-encoded or with `\` separators) is rewritten through a path map relative to the -file's new location, and a relative link that leaves the slice keeps pointing at the same file. +the copied markdown names is rewritten through a path map relative to the file's new location: +inline links (destinations with balanced or escaped parentheses too), reference-style links, +HTML `src`/`href` in any quoting, and backticked paths, percent-encoded or with `\` separators. +A relative link that leaves the slice keeps pointing at the same file. One naming a pipeline file the layout does not keep (such as `run-state/watch.json`, the slice's own journey `README.md`, or an extracted frame's bare name under `--no-media`) becomes plain text saying it is not retained; one naming a directory whose files went to several places (`source/`) @@ -115,8 +116,13 @@ under a top-level directory of the target or an entry beside the naming file (pa repository, such as `research/deepening/...` from an upstream repo). Only a passing check moves the staging directory into place; on any failure the staging directory is removed and the target is left as it was. Without `--replace` the install never replaces anything that appeared during -the run, even an empty directory: on POSIX it claims the target with an exclusive mkdir before -renaming onto it. Fix what it lists in the slice and re-run. +the run, even an empty directory. On POSIX it claims the target before staging with an exclusive +mkdir and a `.relayout-claim` marker holding its pid, then removes the marker and renames the +staged layout onto the claim, so the installed target never carries it; a failed run releases +its claim. A run killed while holding the claim leaves a directory holding only that marker: the +next run removes it with a notice and proceeds, unless the marker's pid is still running, which +it refuses. A target holding the marker beside anything else stays refused. Fix what it lists in +the slice and re-run. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index e4124db7864..be99d8fb87c 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -111,8 +111,6 @@ const SLICE_LANE_NAMES = new Set([...Object.values(LANES), "SLICE"]); const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$]/; const LINE_SUFFIX = /:\d+(?:[-,]\d+)*$/; -const MARKDOWN_LINK = /(\]\()(<[^>]+>|[^)\s]+)/g; -const FULL_LINK = /!?\[([^\]\n]*)\]\((<[^>]+>|[^)\s]+)[^)\n]*\)/g; const BACKTICK_SPAN = /`([^`\s]+)`/g; /** `[label]: path` reference-style link definitions. */ const REFERENCE_DEFINITION = /^( {0,3}\[[^\]\n]+\]:[ \t]*)(<[^>\n]+>|\S+)/gm; @@ -318,6 +316,109 @@ function lookUpReference(refPath, sourceFile, map) { const NOT_RETAINED = "not retained in this copy"; +/** + * @typedef {Object} InlineLink + * @property {string} dest - the destination, angle brackets removed and `\(` `\)` unescaped + * @property {number} destStart - offset of the destination as written (its `<` when angled) + * @property {number} destEnd - offset just past it + * @property {boolean} angle - written as `<...>` + * @property {boolean} escaped - held backslash-escaped parentheses + * @property {string} label - the link text, `!` and brackets removed + * @property {number} labelStart - offset of the link's `[` (or its `!`), -1 when unmatched + * @property {number} linkStart - offset where the whole link starts + * @property {number} linkEnd - offset just past the link's closing `)` + */ + +/** + * Every inline link in `text`, scanned the way CommonMark reads a + * destination: `<...>`, or a run with balanced or backslash-escaped + * parentheses up to whitespace or the unbalanced `)`. A regex stops at the + * first `)`, which breaks `slides_(final).pdf`. + * + * @param {string} text + * @returns {InlineLink[]} + */ +export function inlineLinks(text) { + /** @type {InlineLink[]} */ + const links = []; + let from = 0; + for (;;) { + const open = text.indexOf("](", from); + if (open === -1) return links; + from = open + 2; + let at = open + 2; + while (text[at] === " " || text[at] === "\t") at += 1; + const destStart = at; + let raw; + let angle = false; + if (text[at] === "<") { + const close = text.indexOf(">", at); + if (close === -1 || text.slice(at, close).includes("\n")) continue; + angle = true; + raw = text.slice(at + 1, close); + at = close + 1; + } else { + let depth = 0; + while (at < text.length) { + const ch = text[at]; + if (ch === "\\" && (text[at + 1] === "(" || text[at + 1] === ")")) { + at += 2; + continue; + } + if (/\s/.test(ch)) break; + if (ch === "(") depth += 1; + else if (ch === ")") { + if (depth === 0) break; + depth -= 1; + } + at += 1; + } + if (at === destStart || depth !== 0) continue; + raw = text.slice(destStart, at); + } + const destEnd = at; + // An optional title, then the closing `)`. + while (text[at] === " " || text[at] === "\t") at += 1; + const quote = text[at]; + if (quote === '"' || quote === "'") { + const closeQuote = text.indexOf(quote, at + 1); + if (closeQuote === -1) continue; + at = closeQuote + 1; + while (text[at] === " " || text[at] === "\t") at += 1; + } + if (text[at] !== ")") continue; + const linkEnd = at + 1; + // The `[` that opens this link's text, matching nested brackets. + let labelStart = -1; + let depth = 0; + for (let back = open - 1; back >= 0; back -= 1) { + if (text[back - 1] === "\\") continue; + if (text[back] === "]") depth += 1; + else if (text[back] === "[") { + if (depth === 0) { + labelStart = back; + break; + } + depth -= 1; + } else if (text[back] === "\n" && text[back - 1] === "\n") break; + } + const label = labelStart === -1 ? "" : text.slice(labelStart + 1, open); + const linkStart = labelStart > 0 && text[labelStart - 1] === "!" ? labelStart - 1 : labelStart; + links.push({ + dest: raw.replace(/\\([()])/g, "$1"), + destStart, + destEnd, + angle, + escaped: /\\[()]/.test(raw), + label, + labelStart: linkStart, + linkStart, + linkEnd, + }); + from = linkEnd; + } +} + /** * Rewrite every markdown link and backticked path in `text` that names a * slice path: a copied one relative to the file's target location, one the @@ -351,16 +452,43 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { if (next !== ref) rewrites += 1; return next; }; - const rewritten = text - .replace(FULL_LINK, (match, label, target) => { - const found = lookUpReference(splitReference(unwrap(target)).refPath, sourceFile, map); - if (!found || !("retired" in found)) return match; + // Inline links: a link naming a file the layout drops becomes its label plus + // not-retained text; any other destination takes the path rewrite. Edits + // apply back to front; one overlapping an earlier-starting edit (an image + // inside a link that is itself replaced) is dropped. + /** @type {{ from: number, to: number, text: string }[]} */ + const edits = []; + for (const link of inlineLinks(text)) { + const found = lookUpReference(splitReference(link.dest).refPath, sourceFile, map); + if (found && "retired" in found && link.labelStart !== -1) { rewrites += 1; - return `${label} (${found.retired}, ${found.note})`; - }) - .replace(MARKDOWN_LINK, (_match, open, target) => - target.startsWith("<") ? `${open}<${rewrite(unwrap(target))}>` : `${open}${rewrite(target)}`, - ) + edits.push({ + from: link.linkStart, + to: link.linkEnd, + text: `${link.label} (${found.retired}, ${found.note})`, + }); + continue; + } + const next = rewrite(link.dest); + if (next === link.dest) continue; + const written = link.escaped ? next.replace(/[()]/g, "\\$&") : next; + edits.push({ + from: link.destStart, + to: link.destEnd, + text: link.angle ? `<${written}>` : written, + }); + } + edits.sort((a, b) => a.from - b.from); + /** @type {typeof edits} */ + const kept = []; + for (const edit of edits) { + if (kept.length === 0 || edit.from >= kept[kept.length - 1].to) kept.push(edit); + } + let linked = text; + for (const edit of kept.reverse()) { + linked = linked.slice(0, edit.from) + edit.text + linked.slice(edit.to); + } + const rewritten = linked // Reference-style definitions and HTML src/href attributes take the same // path rewrite. One naming a file the layout drops is left for the link // check to flag: a definition or attribute has no prose to rewrite into. @@ -447,10 +575,9 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { if (rel.startsWith("..") || path.isAbsolute(rel)) return; if (!fs.existsSync(resolved)) unresolved.push({ file, ref }); }; - for (const pattern of [MARKDOWN_LINK, REFERENCE_DEFINITION]) { - for (const match of text.matchAll(pattern)) { - check(match[2].replace(/^<|>$/g, ""), true); - } + for (const link of inlineLinks(text)) check(link.dest, true); + for (const match of text.matchAll(REFERENCE_DEFINITION)) { + check(match[2].replace(/^<|>$/g, ""), true); } for (const match of text.matchAll(HTML_PATH_ATTRIBUTE)) { check(match[2] ?? match[3] ?? match[4], true); @@ -530,6 +657,40 @@ function readJsonObject(file) { : null; } +/** The file a run writes into the target directory it has claimed. */ +export const CLAIM_MARKER = ".relayout-claim"; + +/** + * Whether `target` is a claim left by a relayout: `"none"` when it is not a + * directory holding only the claim marker, `"live"` when the marker names a + * process still running on this host, `"stale"` otherwise. + * + * @param {string} target + * @returns {"none"|"live"|"stale"} + */ +function staleClaimProblem(target) { + let entries; + try { + if (!fs.lstatSync(target).isDirectory()) return "none"; + entries = fs.readdirSync(target); + } catch { + return "none"; + } + if (entries.length !== 1 || entries[0] !== CLAIM_MARKER) return "none"; + const claim = readJsonObject(path.join(target, CLAIM_MARKER)); + const pid = claim?.pid; + if (typeof pid === "number" && Number.isInteger(pid) && pid > 0 && pid !== process.pid) { + try { + process.kill(pid, 0); + return "live"; + } catch (error) { + // EPERM: the process exists but belongs to someone else. + if (/** @type {NodeJS.ErrnoException} */ (error).code === "EPERM") return "live"; + } + } + return "stale"; +} + /** A video id as yt-dlp's `%(id)s` writes it: no separator, no `..`. */ const VIDEO_ID = /^[A-Za-z0-9_-]+$/; @@ -735,6 +896,19 @@ export async function relayoutSlice({ writeStderr(`relayout: restored ${target} from the backup an interrupted run left`); } + // A run interrupted while it held the target's claim leaves a directory + // holding only the claim marker. Unless that run is still alive, the claim + // is stale: remove it and go on. Anything else in the target stays refused. + const stale = staleClaimProblem(target); + if (stale === "live") { + return refuse(`${target} is claimed by a relayout that is still running`); + } + if (stale === "stale") { + fs.rmSync(path.join(target, CLAIM_MARKER), { force: true }); + fs.rmdirSync(target); + writeStderr(`relayout: removed a stale claim on ${target} an interrupted run left`); + } + let state; try { state = JSON.parse(fs.readFileSync(watchStatePath(slice), "utf8")); @@ -907,11 +1081,42 @@ export async function relayoutSlice({ // Build the whole layout in a sibling staging directory and move it into // place only after the link check passes, so a failure never leaves a // partial target. + // Without --replace on POSIX, claim the target before staging: an exclusive + // mkdir (EEXIST when anything is there, an empty directory included), then a + // marker naming this run, so an interrupted run's claim is recognizable and + // recoverable at the next start. POSIX rename replaces an empty directory, + // so the install removes the marker and renames onto the claim; anything + // someone else writes into the claim makes that rename fail. Windows rename + // never replaces an existing directory, so there the rename is the claim. + const claim = !replace && process.platform !== "win32"; + let claimed = false; + const releaseClaim = () => { + if (!claimed) return; + fs.rmSync(path.join(target, CLAIM_MARKER), { force: true }); + try { + fs.rmdirSync(target); + } catch { + // Not empty: someone else's files now sit in the claim; leave them. + } + }; let staging; try { fs.mkdirSync(path.dirname(target), { recursive: true }); + if (claim) { + fs.mkdirSync(target); + claimed = true; + fs.writeFileSync( + path.join(target, CLAIM_MARKER), + `${JSON.stringify({ pid: process.pid, claimedAt: new Date().toISOString() })}\n`, + ); + } staging = fs.mkdtempSync(path.join(path.dirname(target), stagingPrefix)); } catch (error) { + const code = /** @type {NodeJS.ErrnoException} */ (error).code; + if (claim && !claimed && code === "EEXIST") { + return refuse(`${target} appeared during the run; pass --replace to replace it`); + } + releaseClaim(); return refuse(`cannot stage beside ${target}: ${/** @type {Error} */ (error).message}`); } const discard = (/** @type {string} */ dir) => fs.rmSync(dir, { recursive: true, force: true }); @@ -975,41 +1180,25 @@ export async function relayoutSlice({ }); } catch (error) { discard(staging); + releaseClaim(); return refuse(`copy failed, target left untouched: ${/** @type {Error} */ (error).message}`); } if (unresolved.length > 0) { discard(staging); + releaseClaim(); return { exitCode: 1, copied: 0, rewrites, unresolved }; } // The backup keeps a fixed name so a run killed between the two renames is // recovered at the next start (see the backup check above). - // Without --replace the install never replaces anything that appeared after - // the early check (another run, a person), even an empty directory. POSIX - // rename replaces an empty directory, so the run first claims the target - // with an exclusive mkdir (EEXIST when anything is there) and renames onto - // its own empty claim; anything written into the claim meanwhile makes the - // rename fail with ENOTEMPTY. Windows rename never replaces an existing - // directory, so there the rename alone is the claim. const hadTarget = replace && fs.existsSync(target); - const claim = !replace && process.platform !== "win32"; - let claimed = false; try { if (hadTarget) fs.renameSync(target, backup); - if (claim) { - fs.mkdirSync(target); - claimed = true; - } + if (claimed) fs.rmSync(path.join(target, CLAIM_MARKER)); fs.renameSync(staging, target); } catch (error) { if (hadTarget && !fs.existsSync(target) && fs.existsSync(backup)) fs.renameSync(backup, target); - if (claimed) { - try { - fs.rmdirSync(target); - } catch { - // Not empty: someone else's files now sit in the claim; leave them. - } - } + releaseClaim(); discard(staging); if (!replace && fs.existsSync(target)) { return refuse(`${target} appeared during the run; pass --replace to replace it`); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 20bbc70d944..5d0d18b8ce5 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -4,7 +4,7 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { checkLayoutLinks, relayoutSlice } from "./relayout-slice.js"; +import { CLAIM_MARKER, checkLayoutLinks, inlineLinks, relayoutSlice } from "./relayout-slice.js"; /** @type {string[]} */ const scratch = []; @@ -311,24 +311,73 @@ describe("relayoutSlice", () => { expect(result.unresolved).toEqual([{ file: "frames/visual-frames.md", ref: "scene_0099.png" }]); }); - it("refuses, without --replace, an empty target directory created during the run", async () => { + it.skipIf(process.platform === "win32")( + "holds the target with a claim while staging, so a late mkdir of it fails, and installs without the marker", + async () => { + const { sliceDir, targetDir } = makeFixture(); + /** @type {string[]} */ + let seenDuringCopy = []; + let lateMkdir = ""; + + const result = await relayoutSlice({ + sliceDir, + targetDir, + copyFile: (from, to) => { + if (seenDuringCopy.length === 0) { + seenDuringCopy = fs.readdirSync(targetDir); + try { + fs.mkdirSync(targetDir); + } catch (error) { + lateMkdir = /** @type {NodeJS.ErrnoException} */ (error).code ?? ""; + } + } + fs.copyFileSync(from, to); + }, + }); + + expect(seenDuringCopy).toEqual([CLAIM_MARKER]); + expect(lateMkdir).toBe("EEXIST"); + expect(result.exitCode).toBe(0); + expect(fs.existsSync(path.join(targetDir, CLAIM_MARKER))).toBe(false); + expect(fs.existsSync(path.join(targetDir, "README.md"))).toBe(true); + }, + ); + + it("removes a stale claim an interrupted run left, says so, and proceeds", async () => { const { sliceDir, targetDir } = makeFixture(); - let calls = 0; - - const result = await relayoutSlice({ - sliceDir, - targetDir, - copyFile: (from, to) => { - calls += 1; - // Created late and still empty: a plain POSIX rename would replace it. - if (calls === 1) fs.mkdirSync(targetDir); - fs.copyFileSync(from, to); - }, + // A pid far above any live one on a test host: the claiming run is gone. + writeTree(targetDir, { + [CLAIM_MARKER]: JSON.stringify({ pid: 2 ** 22 + 12345, claimedAt: "2026-10-10T00:00:00Z" }), }); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + const result = await relayoutSlice({ sliceDir, targetDir }); + const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); + stderr.mockRestore(); + + expect(result.exitCode).toBe(0); + expect(message).toContain("stale claim"); + expect(fs.existsSync(path.join(targetDir, CLAIM_MARKER))).toBe(false); + }); + + it("refuses a claim held by a process that is still running", async () => { + const { sliceDir, targetDir } = makeFixture(); + writeTree(targetDir, { [CLAIM_MARKER]: JSON.stringify({ pid: process.ppid }) }); + + const result = await relayoutSlice({ sliceDir, targetDir }); expect(result.exitCode).toBe(1); - expect(fs.readdirSync(targetDir)).toEqual([]); - expect(fs.readdirSync(path.dirname(targetDir))).toEqual(["out"]); + expect(fs.readdirSync(targetDir)).toEqual([CLAIM_MARKER]); + }); + + it("refuses a target holding the claim marker beside other files", async () => { + const { sliceDir, targetDir } = makeFixture(); + writeTree(targetDir, { [CLAIM_MARKER]: "{}", "theirs.md": "theirs\n" }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(targetDir).sort()).toEqual([CLAIM_MARKER, "theirs.md"]); }); it("refuses, without --replace, a target that appears after the early check", async () => { @@ -813,6 +862,48 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ); }); + it("rewrites link destinations holding balanced or escaped parentheses", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/session-1/slides_(final).pdf": "pdf", + "key-frames/visual-frames.md": [ + "[balanced](../source/decks/session-1/slides_(final).pdf)", + '[escaped](../source/decks/session-1/slides_\\(final\\).pdf "Deck")', + "", + ].join("\n"), + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + [ + "[balanced](../metadata/decks/session-1/slides_(final).pdf)", + '[escaped](../metadata/decks/session-1/slides_\\(final\\).pdf "Deck")', + "", + ].join("\n"), + ); + }); + + it("flags a missing destination with parentheses rather than a truncated one", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { "metadata/deck-inventory.md": "[deck](decks/slides_(v2).pdf)\n" }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "metadata/deck-inventory.md", ref: "decks/slides_(v2).pdf" }, + ]); + }); + + it("reads an image nested in a link as two links with the right labels", () => { + expect( + inlineLinks("[![alt](a.png)](b.md)").map(({ dest, label }) => ({ dest, label })), + ).toEqual([ + { dest: "a.png", label: "alt" }, + { dest: "b.md", label: "![alt](a.png)" }, + ]); + }); + it("rewrites single-quoted and unquoted HTML src and href", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From 41522737163d92f10a44281b18c8b8af453cd159 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:05:22 -0400 Subject: [PATCH 18/73] fix(knowledge): finish interrupted relayout installs; decode path segments; harden recovery - A staging dir that passes the link check gets a .relayout-complete marker, removed after the install. At start, an empty target (or one holding only a dead run's claim) beside exactly one completed staging dir has its install finished, with a notice; beside incomplete or several staging dirs it is refused, naming them. - Path segments are decoded with decodeURIComponent (malformed escapes and decoded separators stay as written), so %23 and %3F match copied files, and rewritten paths are re-encoded per segment. - HTML src/href attribute names match case-insensitively. - Backup restore, stale-claim removal, install rollback and post-install cleanup report a filesystem failure as a refusal instead of throwing. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 11 +- .../extraction/watch/relayout-slice.js | 142 +++++++++++++++--- .../extraction/watch/relayout-slice.test.js | 127 +++++++++++++++- 3 files changed, 253 insertions(+), 27 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 950a756ba7e..fea7e331a2d 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -101,7 +101,8 @@ the companion sources brief, and the fetched `decks/` and `attachments/`), `fram paths yt-dlp records. The script's header comment is the authoritative mapping. Each slice path the copied markdown names is rewritten through a path map relative to the file's new location: inline links (destinations with balanced or escaped parentheses too), reference-style links, -HTML `src`/`href` in any quoting, and backticked paths, percent-encoded or with `\` separators. +HTML `src`/`href` in any quoting or case, and backticked paths, with each segment +percent-decoded (and re-encoded on rewrite) and `\` separators read as `/`. A relative link that leaves the slice keeps pointing at the same file. One naming a pipeline file the layout does not keep (such as `run-state/watch.json`, the slice's own journey `README.md`, or an extracted frame's bare name under `--no-media`) becomes plain text @@ -121,8 +122,12 @@ mkdir and a `.relayout-claim` marker holding its pid, then removes the marker an staged layout onto the claim, so the installed target never carries it; a failed run releases its claim. A run killed while holding the claim leaves a directory holding only that marker: the next run removes it with a notice and proceeds, unless the marker's pid is still running, which -it refuses. A target holding the marker beside anything else stays refused. Fix what it lists in -the slice and re-run. +it refuses. A target holding the marker beside anything else stays refused. Once the link check +passes, the run writes `.relayout-complete` into its staging dir, and removes it after the +install. A run killed during the install leaves an empty target (or one holding only its claim +marker) beside that staging dir: the next run finishes the install from it with a notice and +stops, so re-run with `--replace` for a fresh copy. An empty target beside an incomplete staging +dir, or beside more than one, is refused, naming them. Fix what it lists in the slice and re-run. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index be99d8fb87c..d67948f8c7e 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -115,7 +115,7 @@ const BACKTICK_SPAN = /`([^`\s]+)`/g; /** `[label]: path` reference-style link definitions. */ const REFERENCE_DEFINITION = /^( {0,3}\[[^\]\n]+\]:[ \t]*)(<[^>\n]+>|\S+)/gm; /** `src` and `href` in inline HTML, double-, single- or unquoted. */ -const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/g; +const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/gi; /** A backticked span names a path when it holds a separator or ends in an extension. */ const PATH_LIKE = /\/|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)?(?:#.*)?$/; @@ -216,19 +216,27 @@ function splitReference(ref) { suffix = line[0] + suffix; refPath = refPath.slice(0, line.index); } - // Compare paths the way the filesystem sees them: percent-decoded, with - // Windows separators as `/`. A rewrite re-encodes when the original was. + // Compare paths the way the filesystem sees them: Windows separators as + // `/`, then each segment percent-decoded (`%23` is `#`, `%3F` is `?`). A + // malformed escape, or one decoding to a separator, stays as written. A + // rewrite re-encodes each segment when the original was encoded. + if (!URL_SCHEME.test(refPath)) refPath = refPath.replace(/\\/g, "/"); let encoded = false; if (refPath.includes("%")) { - try { - const decoded = decodeURI(refPath); - encoded = decoded !== refPath; - refPath = decoded; - } catch { - // Not valid percent-encoding: keep it as written. - } + const decoded = refPath + .split("/") + .map((segment) => { + try { + const plain = decodeURIComponent(segment); + return /[/\\]/.test(plain) ? segment : plain; + } catch { + return segment; + } + }) + .join("/"); + encoded = decoded !== refPath; + refPath = decoded; } - if (!URL_SCHEME.test(refPath)) refPath = refPath.replace(/\\/g, "/"); return { refPath, suffix, encoded }; } @@ -448,7 +456,8 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { relative = path.posix.relative(path.posix.dirname(targetFile), found.target) || "."; trailing = found.isDir && refPath.endsWith("/") ? "/" : ""; } - const next = `${encoded ? encodeURI(relative) : relative}${trailing}${suffix}`; + const written = encoded ? relative.split("/").map(encodeURIComponent).join("/") : relative; + const next = `${written}${trailing}${suffix}`; if (next !== ref) rewrites += 1; return next; }; @@ -659,6 +668,27 @@ function readJsonObject(file) { /** The file a run writes into the target directory it has claimed. */ export const CLAIM_MARKER = ".relayout-claim"; +/** The file a run writes into its staging dir once the link check passes. */ +export const COMPLETE_MARKER = ".relayout-complete"; + +/** @param {unknown} error */ +function errorMessage(error) { + return error instanceof Error ? error.message : String(error); +} + +/** + * The entries of `dir` when it is a real directory (not a symlink), else null. + * + * @param {string} dir + * @returns {string[]|null} + */ +function directoryEntries(dir) { + try { + return fs.lstatSync(dir).isDirectory() ? fs.readdirSync(dir) : null; + } catch { + return null; + } +} /** * Whether `target` is a claim left by a relayout: `"none"` when it is not a @@ -892,20 +922,64 @@ export async function relayoutSlice({ `a leftover backup ${backup} sits beside ${target}; keep one of them and remove the other`, ); } - fs.renameSync(backup, target); + try { + fs.renameSync(backup, target); + } catch (error) { + return refuse(`could not restore ${target} from ${backup}: ${errorMessage(error)}`); + } writeStderr(`relayout: restored ${target} from the backup an interrupted run left`); } - // A run interrupted while it held the target's claim leaves a directory - // holding only the claim marker. Unless that run is still alive, the claim - // is stale: remove it and go on. Anything else in the target stays refused. + // A run killed after its link check passed but before its staged layout was + // renamed into place leaves the target empty (or holding only its claim + // marker) beside a staging dir marked complete. Finish that install. Any + // other staging state beside an empty target is ambiguous: refuse. const stale = staleClaimProblem(target); if (stale === "live") { return refuse(`${target} is claimed by a relayout that is still running`); } + const targetEntries = directoryEntries(target); + const claimOnly = targetEntries?.length === 1 && targetEntries[0] === CLAIM_MARKER; + if (targetEntries && (targetEntries.length === 0 || claimOnly)) { + const stagings = (directoryEntries(path.dirname(target)) ?? []) + .filter((name) => name.startsWith(stagingPrefix)) + .map((name) => path.join(path.dirname(target), name)); + if (stagings.length > 0) { + const complete = stagings.filter((dir) => fs.existsSync(path.join(dir, COMPLETE_MARKER))); + if (stagings.length !== 1 || complete.length !== 1) { + return refuse( + `${target} is empty beside staging dirs an interrupted run left (${stagings.join(", ")}); ` + + "remove the ones you do not want and re-run", + ); + } + try { + fs.rmSync(path.join(target, CLAIM_MARKER), { force: true }); + fs.rmdirSync(target); + fs.renameSync(complete[0], target); + fs.rmSync(path.join(target, COMPLETE_MARKER), { force: true }); + } catch (error) { + return refuse( + `could not finish the interrupted install of ${target}: ${errorMessage(error)}`, + ); + } + writeStderr( + `relayout: finished the install an interrupted run left, from ${complete[0]}; ` + + "re-run with --replace for a fresh copy", + ); + return { exitCode: 0, copied: 0, rewrites: 0, unresolved: [] }; + } + } + + // A run interrupted while it held the target's claim, before staging + // finished, leaves a directory holding only the claim marker. Its run is not + // alive (checked above), so the claim is stale: remove it and go on. if (stale === "stale") { - fs.rmSync(path.join(target, CLAIM_MARKER), { force: true }); - fs.rmdirSync(target); + try { + fs.rmSync(path.join(target, CLAIM_MARKER), { force: true }); + fs.rmdirSync(target); + } catch (error) { + return refuse(`could not remove the stale claim on ${target}: ${errorMessage(error)}`); + } writeStderr(`relayout: removed a stale claim on ${target} an interrupted run left`); } @@ -1178,6 +1252,14 @@ export async function relayoutSlice({ unresolved = checkLayoutLinks(staging, { sliceDirs: new Set([...sliceEntries].filter((entry) => !sliceFiles.includes(entry))), }); + // Marks the staged layout as ready to install, so a run killed during the + // install can be finished by the next one. + if (unresolved.length === 0) { + fs.writeFileSync( + path.join(staging, COMPLETE_MARKER), + `${JSON.stringify({ pid: process.pid, completedAt: new Date().toISOString() })}\n`, + ); + } } catch (error) { discard(staging); releaseClaim(); @@ -1197,15 +1279,29 @@ export async function relayoutSlice({ if (claimed) fs.rmSync(path.join(target, CLAIM_MARKER)); fs.renameSync(staging, target); } catch (error) { - if (hadTarget && !fs.existsSync(target) && fs.existsSync(backup)) fs.renameSync(backup, target); - releaseClaim(); - discard(staging); + try { + if (hadTarget && !fs.existsSync(target) && fs.existsSync(backup)) { + fs.renameSync(backup, target); + } + releaseClaim(); + discard(staging); + } catch (cleanupError) { + return refuse( + `could not move the layout into place (${errorMessage(error)}), and cleanup failed ` + + `(${errorMessage(cleanupError)}); check ${target}, ${backup} and ${staging}`, + ); + } if (!replace && fs.existsSync(target)) { return refuse(`${target} appeared during the run; pass --replace to replace it`); } - return refuse(`could not move the layout into place: ${/** @type {Error} */ (error).message}`); + return refuse(`could not move the layout into place: ${errorMessage(error)}`); + } + try { + fs.rmSync(path.join(target, COMPLETE_MARKER), { force: true }); + if (hadTarget) discard(backup); + } catch (error) { + writeStderr(`relayout: installed ${target}, but cleanup failed: ${errorMessage(error)}`); } - if (hadTarget) discard(backup); return { exitCode: 0, copied, rewrites, unresolved }; } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 5d0d18b8ce5..42469b83cf4 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -4,7 +4,13 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { CLAIM_MARKER, checkLayoutLinks, inlineLinks, relayoutSlice } from "./relayout-slice.js"; +import { + CLAIM_MARKER, + COMPLETE_MARKER, + checkLayoutLinks, + inlineLinks, + relayoutSlice, +} from "./relayout-slice.js"; /** @type {string[]} */ const scratch = []; @@ -153,6 +159,8 @@ describe("relayoutSlice", () => { } expect(fs.existsSync(path.join(targetDir, "key-frames"))).toBe(false); expect(fs.existsSync(path.join(targetDir, "run-state"))).toBe(false); + expect(fs.existsSync(path.join(targetDir, COMPLETE_MARKER))).toBe(false); + expect(fs.existsSync(path.join(targetDir, CLAIM_MARKER))).toBe(false); }); it("trims info.json to the corpus copy", async () => { @@ -380,6 +388,89 @@ describe("relayoutSlice", () => { expect(fs.readdirSync(targetDir).sort()).toEqual([CLAIM_MARKER, "theirs.md"]); }); + /** + * The state a run killed during its install leaves: a staging dir beside + * the target, marked complete or not, and an empty target (or one holding + * only a dead run's claim marker). + * + * @param {string} targetDir + * @param {{ complete: boolean, claim?: boolean, name?: string }} options + */ + function interruptedInstall(targetDir, { complete, claim = false, name = "abc123" }) { + const staging = path.join(path.dirname(targetDir), `.out.relayout-${name}`); + writeTree(staging, { + "README.md": "# Staged\n", + ...(complete ? { [COMPLETE_MARKER]: "{}" } : {}), + }); + fs.mkdirSync(targetDir, { recursive: true }); + if (claim) { + fs.writeFileSync( + path.join(targetDir, CLAIM_MARKER), + JSON.stringify({ pid: 2 ** 22 + 12345 }), + ); + } + return staging; + } + + it.each([ + ["an empty target", false], + ["a target holding only a dead run's claim marker", true], + ])( + "finishes an install interrupted with %s beside a completed staging dir", + async (_label, claim) => { + const { sliceDir, targetDir } = makeFixture(); + const staging = interruptedInstall(targetDir, { complete: true, claim }); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + const result = await relayoutSlice({ sliceDir, targetDir }); + const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); + stderr.mockRestore(); + + expect(result.exitCode).toBe(0); + expect(message).toContain("finished the install an interrupted run left"); + expect(fs.readdirSync(targetDir)).toEqual(["README.md"]); + expect(read(targetDir, "README.md")).toBe("# Staged\n"); + expect(fs.existsSync(staging)).toBe(false); + }, + ); + + it.each([ + ["an incomplete staging dir", [{ complete: false }]], + [ + "two staging dirs", + [ + { complete: true, name: "aaa111" }, + { complete: false, name: "bbb222" }, + ], + ], + ])("refuses an empty target beside %s and changes nothing", async (_label, stagings) => { + const { sliceDir, targetDir } = makeFixture(); + const dirs = stagings.map((options) => interruptedInstall(targetDir, options)); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(targetDir)).toEqual([]); + for (const dir of dirs) expect(fs.existsSync(dir)).toBe(true); + }); + + it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "refuses, without throwing, when a stale claim cannot be removed", + async () => { + const { sliceDir, targetDir } = makeFixture(); + writeTree(targetDir, { [CLAIM_MARKER]: JSON.stringify({ pid: 2 ** 22 + 12345 }) }); + fs.chmodSync(targetDir, 0o555); + try { + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.readdirSync(targetDir)).toEqual([CLAIM_MARKER]); + } finally { + fs.chmodSync(targetDir, 0o755); + } + }, + ); + it("refuses, without --replace, a target that appears after the early check", async () => { const { sliceDir, targetDir } = makeFixture(); let calls = 0; @@ -920,6 +1011,40 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ); }); + it("rewrites percent-encoded #, ? and space in path segments, keeping the encoding", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/findings/notes#1.md": "n\n", + "research/findings/what?.md": "w\n", + "key-frames/visual-frames.md": + "[a](../research/findings/notes%231.md) [b](../research/findings/what%3F.md#top)\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "[a](../analysis/research/findings/notes%231.md) [b](../analysis/research/findings/what%3F.md#top)\n", + ); + }); + + it("rewrites and checks upper-case HTML attribute names", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": '\n' }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe('\n'); + const root = makeDir("relayout-check-"); + writeTree(root, { "frames/visual-frames.md": "x\n" }); + expect(checkLayoutLinks(root)).toEqual([ + { file: "frames/visual-frames.md", ref: "key/gone.png" }, + ]); + }); + it("flags a single-quoted HTML src that does not resolve", () => { const root = makeDir("relayout-check-"); writeTree(root, { "frames/visual-frames.md": "\n" }); From 7000a6ccc91f80a1cdfb573cc453569f4aafb9e2 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:15:31 -0400 Subject: [PATCH 19/73] refactor(knowledge): drop relayout claim and completion markers for a single-rename install The no-replace claim (exclusive mkdir plus .relayout-claim) and the .relayout-complete auto-finish recovery kept producing new edge cases, including one that could install another slice's staging dir. They are removed. Without --replace the run refuses an existing target at start and installs with one rename, refusing when a target with content appeared; an empty directory created meanwhile may be replaced on POSIX, which loses no data. Leftover staging dirs are only named in a warning for deletion by hand. The --replace backup and its startup restore stay, as do the refuse-instead-of-throw guards. The link scanner also accepts parenthesized titles. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 25 +-- .../extraction/watch/relayout-slice.js | 175 +++-------------- .../extraction/watch/relayout-slice.test.js | 181 +++++------------- 3 files changed, 84 insertions(+), 297 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index fea7e331a2d..b00daeab0f1 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -100,9 +100,9 @@ the companion sources brief, and the fetched `decks/` and `attachments/`), `fram `README.md` when the target has none; the trimmed `info.json` also drops the local download paths yt-dlp records. The script's header comment is the authoritative mapping. Each slice path the copied markdown names is rewritten through a path map relative to the file's new location: -inline links (destinations with balanced or escaped parentheses too), reference-style links, -HTML `src`/`href` in any quoting or case, and backticked paths, with each segment -percent-decoded (and re-encoded on rewrite) and `\` separators read as `/`. +inline links (balanced or escaped parentheses in the destination, quoted or parenthesized titles), +reference-style links, HTML `src`/`href` in any quoting or case, and backticked paths, with each +segment percent-decoded (and re-encoded on rewrite) and `\` separators read as `/`. A relative link that leaves the slice keeps pointing at the same file. One naming a pipeline file the layout does not keep (such as `run-state/watch.json`, the slice's own journey `README.md`, or an extracted frame's bare name under `--no-media`) becomes plain text @@ -116,18 +116,13 @@ that leave the target, and backticked paths that are neither under such a lane d under a top-level directory of the target or an entry beside the naming file (paths into another repository, such as `research/deepening/...` from an upstream repo). Only a passing check moves the staging directory into place; on any failure the staging directory is removed and the target -is left as it was. Without `--replace` the install never replaces anything that appeared during -the run, even an empty directory. On POSIX it claims the target before staging with an exclusive -mkdir and a `.relayout-claim` marker holding its pid, then removes the marker and renames the -staged layout onto the claim, so the installed target never carries it; a failed run releases -its claim. A run killed while holding the claim leaves a directory holding only that marker: the -next run removes it with a notice and proceeds, unless the marker's pid is still running, which -it refuses. A target holding the marker beside anything else stays refused. Once the link check -passes, the run writes `.relayout-complete` into its staging dir, and removes it after the -install. A run killed during the install leaves an empty target (or one holding only its claim -marker) beside that staging dir: the next run finishes the install from it with a notice and -stops, so re-run with `--replace` for a fresh copy. An empty target beside an incomplete staging -dir, or beside more than one, is refused, naming them. Fix what it lists in the slice and re-run. +is left as it was. Without `--replace` the install is a single rename of the staging dir onto the +target path, which fails, and the run refuses, when a target with content appeared during the +run. On POSIX that rename replaces an empty directory someone created meanwhile; that is +accepted, since an empty directory holds no data. A run killed mid-copy or mid-install leaves its +`..relayout-*` staging dir behind; the next run names any such dir in a warning for +deletion by hand and never installs or deletes one, since it may belong to another slice's run +or to a run still going. Fix what it lists in the slice and re-run. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index d67948f8c7e..bec25b23c9e 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -387,9 +387,10 @@ export function inlineLinks(text) { const destEnd = at; // An optional title, then the closing `)`. while (text[at] === " " || text[at] === "\t") at += 1; + // A title is "...", '...' or (...), set off from the destination by space. const quote = text[at]; - if (quote === '"' || quote === "'") { - const closeQuote = text.indexOf(quote, at + 1); + if (at > destEnd && (quote === '"' || quote === "'" || quote === "(")) { + const closeQuote = text.indexOf(quote === "(" ? ")" : quote, at + 1); if (closeQuote === -1) continue; at = closeQuote + 1; while (text[at] === " " || text[at] === "\t") at += 1; @@ -666,61 +667,11 @@ function readJsonObject(file) { : null; } -/** The file a run writes into the target directory it has claimed. */ -export const CLAIM_MARKER = ".relayout-claim"; -/** The file a run writes into its staging dir once the link check passes. */ -export const COMPLETE_MARKER = ".relayout-complete"; - /** @param {unknown} error */ function errorMessage(error) { return error instanceof Error ? error.message : String(error); } -/** - * The entries of `dir` when it is a real directory (not a symlink), else null. - * - * @param {string} dir - * @returns {string[]|null} - */ -function directoryEntries(dir) { - try { - return fs.lstatSync(dir).isDirectory() ? fs.readdirSync(dir) : null; - } catch { - return null; - } -} - -/** - * Whether `target` is a claim left by a relayout: `"none"` when it is not a - * directory holding only the claim marker, `"live"` when the marker names a - * process still running on this host, `"stale"` otherwise. - * - * @param {string} target - * @returns {"none"|"live"|"stale"} - */ -function staleClaimProblem(target) { - let entries; - try { - if (!fs.lstatSync(target).isDirectory()) return "none"; - entries = fs.readdirSync(target); - } catch { - return "none"; - } - if (entries.length !== 1 || entries[0] !== CLAIM_MARKER) return "none"; - const claim = readJsonObject(path.join(target, CLAIM_MARKER)); - const pid = claim?.pid; - if (typeof pid === "number" && Number.isInteger(pid) && pid > 0 && pid !== process.pid) { - try { - process.kill(pid, 0); - return "live"; - } catch (error) { - // EPERM: the process exists but belongs to someone else. - if (/** @type {NodeJS.ErrnoException} */ (error).code === "EPERM") return "live"; - } - } - return "stale"; -} - /** A video id as yt-dlp's `%(id)s` writes it: no separator, no `..`. */ const VIDEO_ID = /^[A-Za-z0-9_-]+$/; @@ -930,57 +881,23 @@ export async function relayoutSlice({ writeStderr(`relayout: restored ${target} from the backup an interrupted run left`); } - // A run killed after its link check passed but before its staged layout was - // renamed into place leaves the target empty (or holding only its claim - // marker) beside a staging dir marked complete. Finish that install. Any - // other staging state beside an empty target is ambiguous: refuse. - const stale = staleClaimProblem(target); - if (stale === "live") { - return refuse(`${target} is claimed by a relayout that is still running`); - } - const targetEntries = directoryEntries(target); - const claimOnly = targetEntries?.length === 1 && targetEntries[0] === CLAIM_MARKER; - if (targetEntries && (targetEntries.length === 0 || claimOnly)) { - const stagings = (directoryEntries(path.dirname(target)) ?? []) - .filter((name) => name.startsWith(stagingPrefix)) - .map((name) => path.join(path.dirname(target), name)); - if (stagings.length > 0) { - const complete = stagings.filter((dir) => fs.existsSync(path.join(dir, COMPLETE_MARKER))); - if (stagings.length !== 1 || complete.length !== 1) { - return refuse( - `${target} is empty beside staging dirs an interrupted run left (${stagings.join(", ")}); ` + - "remove the ones you do not want and re-run", - ); - } - try { - fs.rmSync(path.join(target, CLAIM_MARKER), { force: true }); - fs.rmdirSync(target); - fs.renameSync(complete[0], target); - fs.rmSync(path.join(target, COMPLETE_MARKER), { force: true }); - } catch (error) { - return refuse( - `could not finish the interrupted install of ${target}: ${errorMessage(error)}`, - ); - } - writeStderr( - `relayout: finished the install an interrupted run left, from ${complete[0]}; ` + - "re-run with --replace for a fresh copy", - ); - return { exitCode: 0, copied: 0, rewrites: 0, unresolved: [] }; - } + // A killed run leaves its staging dir behind. Name any for deletion by + // hand; never install or delete one: it may belong to another slice's run + // or to a run still in progress. + let siblings = []; + try { + siblings = fs.readdirSync(path.dirname(target)); + } catch { + // No parent yet: nothing left behind. } - - // A run interrupted while it held the target's claim, before staging - // finished, leaves a directory holding only the claim marker. Its run is not - // alive (checked above), so the claim is stale: remove it and go on. - if (stale === "stale") { - try { - fs.rmSync(path.join(target, CLAIM_MARKER), { force: true }); - fs.rmdirSync(target); - } catch (error) { - return refuse(`could not remove the stale claim on ${target}: ${errorMessage(error)}`); - } - writeStderr(`relayout: removed a stale claim on ${target} an interrupted run left`); + const leftovers = siblings.filter((name) => name.startsWith(stagingPrefix)); + if (leftovers.length > 0) { + writeStderr( + `relayout: warning: staging dirs from an earlier run sit beside ${target}; ` + + `delete them by hand when no run is using them: ${leftovers + .map((name) => path.join(path.dirname(target), name)) + .join(", ")}`, + ); } let state; @@ -1155,43 +1072,12 @@ export async function relayoutSlice({ // Build the whole layout in a sibling staging directory and move it into // place only after the link check passes, so a failure never leaves a // partial target. - // Without --replace on POSIX, claim the target before staging: an exclusive - // mkdir (EEXIST when anything is there, an empty directory included), then a - // marker naming this run, so an interrupted run's claim is recognizable and - // recoverable at the next start. POSIX rename replaces an empty directory, - // so the install removes the marker and renames onto the claim; anything - // someone else writes into the claim makes that rename fail. Windows rename - // never replaces an existing directory, so there the rename is the claim. - const claim = !replace && process.platform !== "win32"; - let claimed = false; - const releaseClaim = () => { - if (!claimed) return; - fs.rmSync(path.join(target, CLAIM_MARKER), { force: true }); - try { - fs.rmdirSync(target); - } catch { - // Not empty: someone else's files now sit in the claim; leave them. - } - }; let staging; try { fs.mkdirSync(path.dirname(target), { recursive: true }); - if (claim) { - fs.mkdirSync(target); - claimed = true; - fs.writeFileSync( - path.join(target, CLAIM_MARKER), - `${JSON.stringify({ pid: process.pid, claimedAt: new Date().toISOString() })}\n`, - ); - } staging = fs.mkdtempSync(path.join(path.dirname(target), stagingPrefix)); } catch (error) { - const code = /** @type {NodeJS.ErrnoException} */ (error).code; - if (claim && !claimed && code === "EEXIST") { - return refuse(`${target} appeared during the run; pass --replace to replace it`); - } - releaseClaim(); - return refuse(`cannot stage beside ${target}: ${/** @type {Error} */ (error).message}`); + return refuse(`cannot stage beside ${target}: ${errorMessage(error)}`); } const discard = (/** @type {string} */ dir) => fs.rmSync(dir, { recursive: true, force: true }); let rewrites = 0; @@ -1252,38 +1138,30 @@ export async function relayoutSlice({ unresolved = checkLayoutLinks(staging, { sliceDirs: new Set([...sliceEntries].filter((entry) => !sliceFiles.includes(entry))), }); - // Marks the staged layout as ready to install, so a run killed during the - // install can be finished by the next one. - if (unresolved.length === 0) { - fs.writeFileSync( - path.join(staging, COMPLETE_MARKER), - `${JSON.stringify({ pid: process.pid, completedAt: new Date().toISOString() })}\n`, - ); - } } catch (error) { discard(staging); - releaseClaim(); return refuse(`copy failed, target left untouched: ${/** @type {Error} */ (error).message}`); } if (unresolved.length > 0) { discard(staging); - releaseClaim(); return { exitCode: 1, copied: 0, rewrites, unresolved }; } - // The backup keeps a fixed name so a run killed between the two renames is - // recovered at the next start (see the backup check above). + // With --replace the old target moves to a fixed-name backup, so a run + // killed between the two renames is recovered at the next start (see the + // backup check above). Without --replace the install is one rename, which + // fails (ENOTEMPTY or EEXIST on POSIX, EPERM on Windows) when a target with + // content appeared during the run. POSIX rename replaces an empty directory + // created meanwhile; that is accepted, since an empty directory holds no data. const hadTarget = replace && fs.existsSync(target); try { if (hadTarget) fs.renameSync(target, backup); - if (claimed) fs.rmSync(path.join(target, CLAIM_MARKER)); fs.renameSync(staging, target); } catch (error) { try { if (hadTarget && !fs.existsSync(target) && fs.existsSync(backup)) { fs.renameSync(backup, target); } - releaseClaim(); discard(staging); } catch (cleanupError) { return refuse( @@ -1297,7 +1175,6 @@ export async function relayoutSlice({ return refuse(`could not move the layout into place: ${errorMessage(error)}`); } try { - fs.rmSync(path.join(target, COMPLETE_MARKER), { force: true }); if (hadTarget) discard(backup); } catch (error) { writeStderr(`relayout: installed ${target}, but cleanup failed: ${errorMessage(error)}`); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 42469b83cf4..0a55cb7d9eb 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -4,13 +4,7 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { - CLAIM_MARKER, - COMPLETE_MARKER, - checkLayoutLinks, - inlineLinks, - relayoutSlice, -} from "./relayout-slice.js"; +import { checkLayoutLinks, inlineLinks, relayoutSlice } from "./relayout-slice.js"; /** @type {string[]} */ const scratch = []; @@ -159,8 +153,7 @@ describe("relayoutSlice", () => { } expect(fs.existsSync(path.join(targetDir, "key-frames"))).toBe(false); expect(fs.existsSync(path.join(targetDir, "run-state"))).toBe(false); - expect(fs.existsSync(path.join(targetDir, COMPLETE_MARKER))).toBe(false); - expect(fs.existsSync(path.join(targetDir, CLAIM_MARKER))).toBe(false); + expect(fs.readdirSync(targetDir).filter((name) => name.startsWith("."))).toEqual([]); }); it("trims info.json to the corpus copy", async () => { @@ -319,44 +312,10 @@ describe("relayoutSlice", () => { expect(result.unresolved).toEqual([{ file: "frames/visual-frames.md", ref: "scene_0099.png" }]); }); - it.skipIf(process.platform === "win32")( - "holds the target with a claim while staging, so a late mkdir of it fails, and installs without the marker", - async () => { - const { sliceDir, targetDir } = makeFixture(); - /** @type {string[]} */ - let seenDuringCopy = []; - let lateMkdir = ""; - - const result = await relayoutSlice({ - sliceDir, - targetDir, - copyFile: (from, to) => { - if (seenDuringCopy.length === 0) { - seenDuringCopy = fs.readdirSync(targetDir); - try { - fs.mkdirSync(targetDir); - } catch (error) { - lateMkdir = /** @type {NodeJS.ErrnoException} */ (error).code ?? ""; - } - } - fs.copyFileSync(from, to); - }, - }); - - expect(seenDuringCopy).toEqual([CLAIM_MARKER]); - expect(lateMkdir).toBe("EEXIST"); - expect(result.exitCode).toBe(0); - expect(fs.existsSync(path.join(targetDir, CLAIM_MARKER))).toBe(false); - expect(fs.existsSync(path.join(targetDir, "README.md"))).toBe(true); - }, - ); - - it("removes a stale claim an interrupted run left, says so, and proceeds", async () => { + it("warns about leftover staging dirs and leaves them alone", async () => { const { sliceDir, targetDir } = makeFixture(); - // A pid far above any live one on a test host: the claiming run is gone. - writeTree(targetDir, { - [CLAIM_MARKER]: JSON.stringify({ pid: 2 ** 22 + 12345, claimedAt: "2026-10-10T00:00:00Z" }), - }); + const leftover = path.join(path.dirname(targetDir), ".out.relayout-abc123"); + writeTree(leftover, { "README.md": "# From a killed run\n" }); const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); const result = await relayoutSlice({ sliceDir, targetDir }); @@ -364,109 +323,46 @@ describe("relayoutSlice", () => { stderr.mockRestore(); expect(result.exitCode).toBe(0); - expect(message).toContain("stale claim"); - expect(fs.existsSync(path.join(targetDir, CLAIM_MARKER))).toBe(false); - }); - - it("refuses a claim held by a process that is still running", async () => { - const { sliceDir, targetDir } = makeFixture(); - writeTree(targetDir, { [CLAIM_MARKER]: JSON.stringify({ pid: process.ppid }) }); - - const result = await relayoutSlice({ sliceDir, targetDir }); - - expect(result.exitCode).toBe(1); - expect(fs.readdirSync(targetDir)).toEqual([CLAIM_MARKER]); - }); - - it("refuses a target holding the claim marker beside other files", async () => { - const { sliceDir, targetDir } = makeFixture(); - writeTree(targetDir, { [CLAIM_MARKER]: "{}", "theirs.md": "theirs\n" }); - - const result = await relayoutSlice({ sliceDir, targetDir }); - - expect(result.exitCode).toBe(1); - expect(fs.readdirSync(targetDir).sort()).toEqual([CLAIM_MARKER, "theirs.md"]); + expect(message).toContain(leftover); + expect(read(leftover, "README.md")).toBe("# From a killed run\n"); + expect(read(targetDir, "README.md")).not.toBe("# From a killed run\n"); }); - /** - * The state a run killed during its install leaves: a staging dir beside - * the target, marked complete or not, and an empty target (or one holding - * only a dead run's claim marker). - * - * @param {string} targetDir - * @param {{ complete: boolean, claim?: boolean, name?: string }} options - */ - function interruptedInstall(targetDir, { complete, claim = false, name = "abc123" }) { - const staging = path.join(path.dirname(targetDir), `.out.relayout-${name}`); - writeTree(staging, { - "README.md": "# Staged\n", - ...(complete ? { [COMPLETE_MARKER]: "{}" } : {}), - }); - fs.mkdirSync(targetDir, { recursive: true }); - if (claim) { - fs.writeFileSync( - path.join(targetDir, CLAIM_MARKER), - JSON.stringify({ pid: 2 ** 22 + 12345 }), - ); - } - return staging; - } - - it.each([ - ["an empty target", false], - ["a target holding only a dead run's claim marker", true], - ])( - "finishes an install interrupted with %s beside a completed staging dir", - async (_label, claim) => { + it.skipIf(process.platform === "win32")( + "installs over an empty target directory created during the run (it holds no data)", + async () => { const { sliceDir, targetDir } = makeFixture(); - const staging = interruptedInstall(targetDir, { complete: true, claim }); - const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + let calls = 0; - const result = await relayoutSlice({ sliceDir, targetDir }); - const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); - stderr.mockRestore(); + const result = await relayoutSlice({ + sliceDir, + targetDir, + copyFile: (from, to) => { + calls += 1; + if (calls === 1) fs.mkdirSync(targetDir); + fs.copyFileSync(from, to); + }, + }); expect(result.exitCode).toBe(0); - expect(message).toContain("finished the install an interrupted run left"); - expect(fs.readdirSync(targetDir)).toEqual(["README.md"]); - expect(read(targetDir, "README.md")).toBe("# Staged\n"); - expect(fs.existsSync(staging)).toBe(false); + expect(fs.existsSync(path.join(targetDir, "analysis/RESEARCH.md"))).toBe(true); }, ); - it.each([ - ["an incomplete staging dir", [{ complete: false }]], - [ - "two staging dirs", - [ - { complete: true, name: "aaa111" }, - { complete: false, name: "bbb222" }, - ], - ], - ])("refuses an empty target beside %s and changes nothing", async (_label, stagings) => { - const { sliceDir, targetDir } = makeFixture(); - const dirs = stagings.map((options) => interruptedInstall(targetDir, options)); - - const result = await relayoutSlice({ sliceDir, targetDir }); - - expect(result.exitCode).toBe(1); - expect(fs.readdirSync(targetDir)).toEqual([]); - for (const dir of dirs) expect(fs.existsSync(dir)).toBe(true); - }); - it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( - "refuses, without throwing, when a stale claim cannot be removed", + "refuses, without throwing, when the backup restore fails", async () => { const { sliceDir, targetDir } = makeFixture(); - writeTree(targetDir, { [CLAIM_MARKER]: JSON.stringify({ pid: 2 ** 22 + 12345 }) }); - fs.chmodSync(targetDir, 0o555); + writeTree(`${targetDir}.relayout-backup`, { "README.md": "# Curated\n" }); + const parent = path.dirname(targetDir); + fs.chmodSync(parent, 0o555); try { - const result = await relayoutSlice({ sliceDir, targetDir }); + const result = await relayoutSlice({ sliceDir, targetDir, replace: true }); expect(result.exitCode).toBe(1); - expect(fs.readdirSync(targetDir)).toEqual([CLAIM_MARKER]); + expect(read(`${targetDir}.relayout-backup`, "README.md")).toBe("# Curated\n"); } finally { - fs.chmodSync(targetDir, 0o755); + fs.chmodSync(parent, 0o755); } }, ); @@ -986,6 +882,25 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it("reads a link with a parenthesized title and rewrites its destination", async () => { + expect(inlineLinks("[x](a.md (Title)) and [y](b.md)").map(({ dest }) => dest)).toEqual([ + "a.md", + "b.md", + ]); + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "key-frames/visual-frames.md": "[t](../source/transcript.txt (The transcript))\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "[t](../transcript/transcript.txt (The transcript))\n", + ); + }); + it("reads an image nested in a link as two links with the right labels", () => { expect( inlineLinks("[![alt](a.png)](b.md)").map(({ dest, label }) => ({ dest, label })), From fc223a92f1ad8304629691c3006ce27c94cd14a2 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:24:27 -0400 Subject: [PATCH 20/73] fix(knowledge): match apostrophe variants in caption dedupe; skip escapes in link titles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The boundary-word comparison drops every apostrophe form (straight, curly, modifier letter), so don't, don’t and dont compare equal. - The link scanner skips backslash-escaped delimiters inside quoted and parenthesized titles. - output-contract.md: "fix what the check lists and re-run" moves to the link-check sentence; the leftover-staging warning says it does not stop the run. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 9 +++++---- .../extraction/transcript/write-transcript.js | 3 ++- .../transcript/write-transcript.test.js | 20 +++++++++++++++++++ .../extraction/watch/relayout-slice.js | 14 ++++++++++++- .../extraction/watch/relayout-slice.test.js | 8 ++++++++ 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index b00daeab0f1..c97ca1723ba 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -114,15 +114,16 @@ name and a path under an original lane whose parent directory the slice had (`so Not checked: paths under `plugins/`, `docs/`, `templates/` and `.claude/`, `AGENTS.md`, paths that leave the target, and backticked paths that are neither under such a lane directory nor under a top-level directory of the target or an entry beside the naming file (paths into another -repository, such as `research/deepening/...` from an upstream repo). Only a passing check moves -the staging directory into place; on any failure the staging directory is removed and the target -is left as it was. Without `--replace` the install is a single rename of the staging dir onto the +repository, such as `research/deepening/...` from an upstream repo). Fix what the check lists in +the slice and re-run. Only a passing check moves the staging directory into place; on any failure +the staging directory is removed and the target is left as it was. Without `--replace` the install is a single rename of the staging dir onto the target path, which fails, and the run refuses, when a target with content appeared during the run. On POSIX that rename replaces an empty directory someone created meanwhile; that is accepted, since an empty directory holds no data. A run killed mid-copy or mid-install leaves its `..relayout-*` staging dir behind; the next run names any such dir in a warning for deletion by hand and never installs or deletes one, since it may belong to another slice's run -or to a run still going. Fix what it lists in the slice and re-run. +or to a run still going. The warning does not stop the run; delete a named dir once no run is +using it. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index 100e785e4b2..aeb30b188ad 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -46,7 +46,8 @@ function tokens(text) { /** @param {string} word */ function comparableWord(word) { - return word.toLowerCase().replace(/[^\p{L}\p{N}']/gu, ""); + // Apostrophes in any form (', ’, ʼ) drop out, so don't, don’t and dont match. + return word.toLowerCase().replace(/\u02BC|[^\p{L}\p{N}]/gu, ""); } /** diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index 24fb8d21a41..4cbb068e6c3 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -118,6 +118,26 @@ Very good indeed, thanks. expect(result.transcript.split("\n\n")[1]).toMatch(/^\[0:05\] Very good question/); }); + it.each([ + ["a curly apostrophe", "don’t"], + ["a modifier-letter apostrophe", "donʼt"], + ["no apostrophe", "dont"], + ])("matches don't against %s when dropping a 3-word repeat", (_label, variant) => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +so we + +00:00:03.000 --> 00:00:05.000 +really don't care. + +00:00:05.000 --> 00:00:07.000 +Really ${variant} care about it. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe("[0:01] so we really don't care.\n\n[0:05] about it."); + }); + it("does not count a punctuation-only token like >> as a repeated word", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index bec25b23c9e..52abcf671b3 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -390,7 +390,19 @@ export function inlineLinks(text) { // A title is "...", '...' or (...), set off from the destination by space. const quote = text[at]; if (at > destEnd && (quote === '"' || quote === "'" || quote === "(")) { - const closeQuote = text.indexOf(quote === "(" ? ")" : quote, at + 1); + // The closing delimiter, skipping backslash-escaped characters. + const closer = quote === "(" ? ")" : quote; + let closeQuote = -1; + for (let scan = at + 1; scan < text.length; scan += 1) { + if (text[scan] === "\\") { + scan += 1; + continue; + } + if (text[scan] === closer) { + closeQuote = scan; + break; + } + } if (closeQuote === -1) continue; at = closeQuote + 1; while (text[at] === " " || text[at] === "\t") at += 1; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 0a55cb7d9eb..ab79dd5b317 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -882,6 +882,14 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it("skips backslash-escaped delimiters inside quoted and parenthesized titles", () => { + expect( + inlineLinks( + '[a](a.md "say \\"hi\\"") [b](b.md \'it\\\'s\') [c](c.md (one \\) two)) [d](d.md)', + ).map(({ dest }) => dest), + ).toEqual(["a.md", "b.md", "c.md", "d.md"]); + }); + it("reads a link with a parenthesized title and rewrites its destination", async () => { expect(inlineLinks("[x](a.md (Title)) and [y](b.md)").map(({ dest }) => dest)).toEqual([ "a.md", From 4daa3b6e923aaedd96629e9f63308d31ce185d06 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:32:10 -0400 Subject: [PATCH 21/73] fix(knowledge): read titles after a line ending; keep nested rewrites in retired links No CommonMark parser is in the extraction's dependency tree, so the scanner is extended instead of adding one: - Spaces with at most one line ending may separate a link's parts, so a title on the next line after the destination is read. - A link naming a dropped file keeps its label in place and edits only the syntax around it (the opening bracket and `](dest)`), so an image nested in the label keeps its own rewrite. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 41 +++++++++++++++---- .../extraction/watch/relayout-slice.test.js | 31 ++++++++++++++ 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 52abcf671b3..abe9b481380 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -333,10 +333,27 @@ const NOT_RETAINED = "not retained in this copy"; * @property {boolean} escaped - held backslash-escaped parentheses * @property {string} label - the link text, `!` and brackets removed * @property {number} labelStart - offset of the link's `[` (or its `!`), -1 when unmatched + * @property {number} labelEnd - offset of the `]` that closes the link text * @property {number} linkStart - offset where the whole link starts * @property {number} linkEnd - offset just past the link's closing `)` */ +/** + * The offset past spaces and tabs at `at`, including at most one line ending + * (CommonMark allows one between a link's destination and its title). + * + * @param {string} text + * @param {number} at + */ +function skipLinkSpace(text, at) { + let next = at; + while (text[next] === " " || text[next] === "\t") next += 1; + if (text[next] === "\r" && text[next + 1] === "\n") next += 2; + else if (text[next] === "\n" || text[next] === "\r") next += 1; + while (text[next] === " " || text[next] === "\t") next += 1; + return next; +} + /** * Every inline link in `text`, scanned the way CommonMark reads a * destination: `<...>`, or a run with balanced or backslash-escaped @@ -355,7 +372,7 @@ export function inlineLinks(text) { if (open === -1) return links; from = open + 2; let at = open + 2; - while (text[at] === " " || text[at] === "\t") at += 1; + at = skipLinkSpace(text, at); const destStart = at; let raw; let angle = false; @@ -386,7 +403,8 @@ export function inlineLinks(text) { } const destEnd = at; // An optional title, then the closing `)`. - while (text[at] === " " || text[at] === "\t") at += 1; + // Spaces with at most one line ending may separate the parts (CommonMark). + at = skipLinkSpace(text, at); // A title is "...", '...' or (...), set off from the destination by space. const quote = text[at]; if (at > destEnd && (quote === '"' || quote === "'" || quote === "(")) { @@ -405,7 +423,7 @@ export function inlineLinks(text) { } if (closeQuote === -1) continue; at = closeQuote + 1; - while (text[at] === " " || text[at] === "\t") at += 1; + at = skipLinkSpace(text, at); } if (text[at] !== ")") continue; const linkEnd = at + 1; @@ -433,6 +451,7 @@ export function inlineLinks(text) { escaped: /\\[()]/.test(raw), label, labelStart: linkStart, + labelEnd: open, linkStart, linkEnd, }); @@ -475,19 +494,25 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { return next; }; // Inline links: a link naming a file the layout drops becomes its label plus - // not-retained text; any other destination takes the path rewrite. Edits - // apply back to front; one overlapping an earlier-starting edit (an image - // inside a link that is itself replaced) is dropped. + // not-retained text; any other destination takes the path rewrite. Every + // edit touches only link syntax or a destination, never a label, so edits + // for nested links do not overlap; they apply back to front, and any + // overlap that still occurred would drop the later-starting edit. /** @type {{ from: number, to: number, text: string }[]} */ const edits = []; for (const link of inlineLinks(text)) { const found = lookUpReference(splitReference(link.dest).refPath, sourceFile, map); if (found && "retired" in found && link.labelStart !== -1) { + // Keep the label text in place and edit only the link syntax around it: + // drop the opening `[` (or `![`), and replace `](dest)` with the note. An + // image or link nested in the label keeps its own rewrite. rewrites += 1; + const bracket = text[link.linkStart] === "!" ? link.linkStart + 2 : link.linkStart + 1; + edits.push({ from: link.linkStart, to: bracket, text: "" }); edits.push({ - from: link.linkStart, + from: link.labelEnd, to: link.linkEnd, - text: `${link.label} (${found.retired}, ${found.note})`, + text: ` (${found.retired}, ${found.note})`, }); continue; } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index ab79dd5b317..c9f8cf9a161 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -909,6 +909,37 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ); }); + it("reads a title on the next line after the destination", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "key-frames/visual-frames.md": '[t](../source/transcript.txt\n "The transcript")\n', + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + '[t](../transcript/transcript.txt\n "The transcript")\n', + ); + }); + + it("keeps a nested image's rewrite when the outer link names a dropped file", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "key-frames/visual-frames.md": + "[![slide](frames/title-slide.png)](../run-state/watch.json)\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "![slide](key/title-slide.png) (run-state/watch.json, not retained in this copy)\n", + ); + }); + it("reads an image nested in a link as two links with the right labels", () => { expect( inlineLinks("[![alt](a.png)](b.md)").map(({ dest, label }) => ({ dest, label })), From 36d465375acc7961f35346c35d9ceaa9920f944c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:41:48 -0400 Subject: [PATCH 22/73] fix(knowledge): detect slice-root references by location, not a matching segment A path segment equal to the slice's name no longer marks a reference as slice-internal. A relative path that leaves the naming file's directory is resolved from the file's real location and counts as the slice's only when it lands inside the slice dir; otherwise it is rebased as an outside link. The .work///... form is recognized only with the slice's own work-root path. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 42 +++++++++++---- .../extraction/watch/relayout-slice.test.js | 52 +++++++++++++++++++ 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index abe9b481380..d06b3b06efe 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -192,7 +192,7 @@ function deriveDirectoryMap(fileMap) { * @property {Map} dirs - slice-relative directory → target-relative directory * @property {Map} tempFrames - temp frame file name → target-relative file * @property {boolean} framesDropped - the layout keeps no extracted frames (`--no-media`) - * @property {string} sliceName - the slice directory's own name + * @property {string|null} sliceWorkPath - the slice's `.work//` path under its work root, or null when it sits under none * @property {Set} sliceEntries - every slice-relative file and directory * @property {string} sliceAbs - absolute slice directory * @property {string} targetAbs - absolute final target directory @@ -265,20 +265,29 @@ function lookUpReference(refPath, sourceFile, map) { }; /** @type {(string|null)[]} */ let keys; - // `SLICE/` and `.work///` name the slice root explicitly. - const marker = `/${map.sliceName}/`; + // `SLICE/` names the slice root explicitly, and so does the work-root + // form `.work///`, but only with this slice's own prefix: + // a path segment that merely equals the slice's name says nothing. if (bare.startsWith("SLICE/")) { keys = [path.posix.normalize(bare.slice("SLICE/".length))]; - } else if (bare.includes(marker)) { - keys = [path.posix.normalize(bare.slice(bare.indexOf(marker) + marker.length))]; + } else if (map.sliceWorkPath && bare.startsWith(`${map.sliceWorkPath}/`)) { + keys = [path.posix.normalize(bare.slice(map.sliceWorkPath.length + 1))]; } else { const fromFile = path.posix.normalize(path.posix.join(path.posix.dirname(sourceFile), bare)); - // A relative path that leaves the slice (a sibling slice, the repo) keeps - // pointing at the same place from the file's new, deeper location. if (fromFile === ".." || fromFile.startsWith("../")) { - return { outside: path.resolve(map.sliceAbs, ...fromFile.split("/")) }; + // Resolved from the naming file's real location: a path that comes back + // into the slice (`..//x` from the root) is the slice's; any other + // (a sibling slice, the repo) keeps pointing at the same place from the + // file's new, deeper location. + const absolute = path.resolve(map.sliceAbs, ...fromFile.split("/")); + const inside = path.relative(map.sliceAbs, absolute); + if (inside === "" || inside.startsWith("..") || path.isAbsolute(inside)) { + return { outside: absolute }; + } + keys = [inside.split(path.sep).join("/")]; + } else { + keys = [fromFile, bare.startsWith(".") ? null : path.posix.normalize(bare)]; } - keys = [fromFile, bare.startsWith(".") ? null : path.posix.normalize(bare)]; } for (const key of keys) { const found = lookup(key); @@ -704,6 +713,19 @@ function readJsonObject(file) { : null; } +/** + * The slice's path under its work root, from the last `.work` segment of its + * absolute path (`.work//`), or null when no `.work` segment holds it. + * + * @param {string} slice - absolute slice directory + * @returns {string|null} + */ +function workRootRelative(slice) { + const segments = slice.split(path.sep); + const at = segments.lastIndexOf(".work"); + return at === -1 || at === segments.length - 1 ? null : segments.slice(at).join("/"); +} + /** @param {unknown} error */ function errorMessage(error) { return error instanceof Error ? error.message : String(error); @@ -1100,7 +1122,7 @@ export async function relayoutSlice({ dirs: deriveDirectoryMap(files), tempFrames, framesDropped: noMedia, - sliceName: path.basename(slice), + sliceWorkPath: workRootRelative(slice), sliceEntries, sliceAbs: slice, targetAbs: target, diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index c9f8cf9a161..712447eb564 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1035,6 +1035,58 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it("treats an outside path whose directory shares the slice's name as outside", async () => { + const { sliceDir, targetDir } = makeFixture(); + const slug = path.basename(sliceDir); + const outsideRef = `../../repos/${slug}/source/transcript.txt`; + fs.writeFileSync(path.join(sliceDir, "research", "claim-inventory.md"), `[x](${outsideRef})\n`); + const pointedAt = path.resolve(sliceDir, "research", outsideRef); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + const link = /\]\(([^)]+)\)/.exec(read(targetDir, "analysis/research/claim-inventory.md"))?.[1]; + expect(path.resolve(targetDir, "analysis/research", String(link))).toBe(pointedAt); + }); + + it("reads a path that leaves and re-enters the slice as the slice's", async () => { + const { sliceDir, targetDir } = makeFixture(); + const slug = path.basename(sliceDir); + fs.writeFileSync(path.join(sliceDir, "RESEARCH.md"), `[t](../${slug}/source/transcript.txt)\n`); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "analysis/RESEARCH.md")).toBe("[t](../transcript/transcript.txt)\n"); + }); + + it("reads the .work// form only with the slice's own work path", async () => { + const { sliceDir: original, targetDir } = makeFixture(); + const slug = path.basename(original); + const sliceDir = path.join(makeDir("relayout-root-"), ".work", "youtube-watch", slug); + fs.mkdirSync(path.dirname(sliceDir), { recursive: true }); + fs.renameSync(original, sliceDir); + fs.writeFileSync( + path.join(sliceDir, "recommendations", "questions.md"), + [ + `Own: \`.work/youtube-watch/${slug}/source/transcript.txt\`.`, + `Other epic: \`.work/other-epic/${slug}/source/transcript.txt\`.`, + "", + ].join("\n"), + ); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "analysis/recommendations/questions.md")).toBe( + [ + "Own: `../../transcript/transcript.txt`.", + `Other epic: \`.work/other-epic/${slug}/source/transcript.txt\`.`, + "", + ].join("\n"), + ); + }); + it("keeps a link that leaves the slice pointing at the same file from its deeper location", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "research/claim-inventory.md": "[other](../../other-slice/RESEARCH.md)\n" }, From 8afdedd3b47734d49c9a49c07453239fb32e7762 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:50:20 -0400 Subject: [PATCH 23/73] fix(knowledge): rewrite references to the slice dir itself; validate frame-times records - A reference resolving to the slice directory itself (..//, or ../ from a lane file) becomes plain text naming where the slice's kept files went, like any split directory. - Each frame-times.json record must be an object whose timestampSec is a finite number or null, matching scene detection's writeFrameTimes; malformed records are refused and named. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 3 +- .../extraction/watch/relayout-slice.js | 36 +++++++++++- .../extraction/watch/relayout-slice.test.js | 57 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index c97ca1723ba..2a101121b38 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -77,7 +77,8 @@ It refuses a `watch.json` that is not an object, whose `videoId` is not a plain `media/.`), that lacks a `title` or `sourceUrl`, or whose temp session dirs are not strings. Without `--no-media` it also refuses a partly removed temp session, naming the missing piece: the video, an empty video, caption track or frame image, the info JSON or -`frame-times.json` (or either one not holding a JSON object), any frame image, a frame +`frame-times.json` (or either one not holding a JSON object, or a `frame-times.json` record +without a numeric or null `timestampSec`), any frame image, a frame `frame-times.json` lists, a `frame-times.json` entry for a scene or interval frame on disk (anchor frames carry their time in their name), or the caption tracks. It refuses a temp session holding several videos (an X post with more than one), naming them, since the watch does not diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index d06b3b06efe..09723aa67c1 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -240,6 +240,23 @@ function splitReference(ref) { return { refPath, suffix, encoded }; } +/** + * A reference to the slice directory itself: the slice has no single new + * location, so name where its kept files went, as for any split directory. + * + * @param {PathMap} map + * @returns {Lookup} + */ +function sliceRootLookup(map) { + const splitInto = [ + ...new Set([...map.files.values()].map((to) => `${to.split("/")[0]}/`)), + ].sort(); + return { + retired: path.basename(map.sliceAbs), + note: `split across ${splitInto.join(", ")} in this copy`, + }; +} + /** * What a reference names: a copied path (its target), a slice file or * directory the layout does not keep (`retired`), or null when it names @@ -281,7 +298,8 @@ function lookUpReference(refPath, sourceFile, map) { // file's new, deeper location. const absolute = path.resolve(map.sliceAbs, ...fromFile.split("/")); const inside = path.relative(map.sliceAbs, absolute); - if (inside === "" || inside.startsWith("..") || path.isAbsolute(inside)) { + if (inside === "") return sliceRootLookup(map); + if (inside.startsWith("..") || path.isAbsolute(inside)) { return { outside: absolute }; } keys = [inside.split(path.sep).join("/")]; @@ -289,6 +307,7 @@ function lookUpReference(refPath, sourceFile, map) { keys = [fromFile, bare.startsWith(".") ? null : path.posix.normalize(bare)]; } } + if (keys[0] === ".") return sliceRootLookup(map); for (const key of keys) { const found = lookup(key); if (found) return found; @@ -805,6 +824,21 @@ function missingFrames(framesDir) { if (isTimesObject) { const listing = /** @type {Record} */ (times); const listed = Object.keys(listing); + // Scene detection's writeFrameTimes writes each record as + // { timestampSec, timestampSource, timestampMethod, timestampErrorSec }, + // with timestampSec a number, or null for an untimed frame. + const malformed = listed.filter((name) => { + const record = listing[name]; + if (!record || typeof record !== "object" || Array.isArray(record)) return true; + const seconds = /** @type {Record} */ (record).timestampSec; + return !(seconds === null || (typeof seconds === "number" && Number.isFinite(seconds))); + }); + if (malformed.length > 0) { + missing.push( + `frame-times.json records with an object and a finite or null timestampSec ` + + `(malformed: ${malformed.sort().join(", ")})`, + ); + } const gone = listed.filter((name) => !names.has(name)); if (gone.length > 0) missing.push(`the frames frame-times.json lists: ${gone.join(", ")}`); // Scene detection records every scene and interval frame it writes; an diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 712447eb564..5e4f278d202 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -625,6 +625,27 @@ describe("relayoutSlice", () => { (/** @type {string} */ dir) => fs.writeFileSync(path.join(dir, "frame-times.json"), '["scene_0001.png"]'), ], + [ + "a frame-times.json record is not an object", + (/** @type {string} */ dir) => + fs.writeFileSync(path.join(dir, "frame-times.json"), '{"scene_0001.png": 1}'), + ], + [ + "a frame-times.json record has no timestampSec", + (/** @type {string} */ dir) => + fs.writeFileSync( + path.join(dir, "frame-times.json"), + '{"scene_0001.png": {"timestampSource": "scene-detection"}}', + ), + ], + [ + "a frame-times.json record has a non-numeric timestampSec", + (/** @type {string} */ dir) => + fs.writeFileSync( + path.join(dir, "frame-times.json"), + '{"scene_0001.png": {"timestampSec": "1.5"}}', + ), + ], [ "only the sidecar is left", (/** @type {string} */ dir) => { @@ -1049,6 +1070,42 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(path.resolve(targetDir, "analysis/research", String(link))).toBe(pointedAt); }); + it("accepts an untimed frame-times.json record (timestampSec null, as the writer emits)", async () => { + const { sliceDir, targetDir, framesDir } = makeFixture(); + fs.writeFileSync( + path.join(framesDir, "frame-times.json"), + JSON.stringify({ "scene_0001.png": { timestampSec: null, timestampSource: null } }), + ); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + }); + + it.each([ + [ + "..// from the slice root", + "RESEARCH.md", + (/** @type {string} */ slug) => `../${slug}/`, + ], + ["../ from a lane file", "research/claim-inventory.md", () => "../"], + ])( + "rewrites a reference to the slice dir itself (%s) as split text", + async (_label, file, ref) => { + const { sliceDir, targetDir } = makeFixture(); + const slug = path.basename(sliceDir); + fs.writeFileSync(path.join(sliceDir, file), `Whole slice: [slice](${ref(slug)}).\n`); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + const target = file === "RESEARCH.md" ? "analysis/RESEARCH.md" : `analysis/${file}`; + expect(read(targetDir, target)).toBe( + `Whole slice: slice (${slug}, split across analysis/, frames/, metadata/, transcript/ in this copy).\n`, + ); + }, + ); + it("reads a path that leaves and re-enters the slice as the slice's", async () => { const { sliceDir, targetDir } = makeFixture(); const slug = path.basename(sliceDir); From 1396517632010327ba665e53ee4fe08701548960 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:02:18 -0400 Subject: [PATCH 24/73] fix(knowledge): case-insensitive Markdown files; absent-folder prefixes; root fallback - Markdown files are selected by .md or .markdown in any case, in both the rewrite and the link check. - A path below a known lane folder the slice never produced (source/decks/s1/slides.pdf) also becomes "(none in this slice)" text. - sliceRootLookup falls back to the not-retained text when nothing was kept. - output-contract.md: "finite number or null timestampSec". Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 2 +- .../extraction/watch/relayout-slice.js | 17 ++++++--- .../extraction/watch/relayout-slice.test.js | 35 +++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 2a101121b38..0d9088f3453 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -78,7 +78,7 @@ It refuses a `watch.json` that is not an object, whose `videoId` is not a plain strings. Without `--no-media` it also refuses a partly removed temp session, naming the missing piece: the video, an empty video, caption track or frame image, the info JSON or `frame-times.json` (or either one not holding a JSON object, or a `frame-times.json` record -without a numeric or null `timestampSec`), any frame image, a frame +without a finite number or null `timestampSec`), any frame image, a frame `frame-times.json` lists, a `frame-times.json` entry for a scene or interval frame on disk (anchor frames carry their time in their name), or the caption tracks. It refuses a temp session holding several videos (an X post with more than one), naming them, since the watch does not diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 09723aa67c1..7ebaa937ae3 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -104,6 +104,8 @@ const SOURCE_METADATA = new Set([ ]); const TRANSCRIPT_TEXT = /^transcript(?:-\d+)?\.txt$/; +/** Markdown files, by extension, in any case. */ +const MARKDOWN_FILE = /\.(?:md|markdown)$/i; const VIDEO_FILE = /\.(?:mp4|mkv|webm)$/; const FRAME_IMAGE = /\.(?:png|jpe?g)$/i; /** The slice lane directories, plus the `SLICE/` root alias. */ @@ -253,7 +255,7 @@ function sliceRootLookup(map) { ].sort(); return { retired: path.basename(map.sliceAbs), - note: `split across ${splitInto.join(", ")} in this copy`, + note: splitInto.length > 0 ? `split across ${splitInto.join(", ")} in this copy` : NOT_RETAINED, }; } @@ -314,8 +316,14 @@ function lookUpReference(refPath, sourceFile, map) { } // A lane folder the layout knows but this slice never produced (prose such // as "no deck in `source/decks/`") names nothing to point at. + // A path below such a folder (`source/decks/slides.pdf`) names nothing either. const absentFolder = keys.find( - (key) => key !== null && KNOWN_LANE_FOLDERS.has(key) && !map.sliceEntries.has(key), + (key) => + key !== null && + [...KNOWN_LANE_FOLDERS].some( + (folder) => + (key === folder || key.startsWith(`${folder}/`)) && !map.sliceEntries.has(folder), + ), ); if (absentFolder) return { retired: absentFolder, note: "none in this slice" }; const retired = keys.find((key) => key !== null && map.sliceEntries.has(key)); @@ -634,7 +642,8 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { /** @type {{ file: string, ref: string }[]} */ const unresolved = []; const markdown = listFiles(root).filter( - (file) => file.endsWith(".md") && !file.startsWith("frames/all/") && !file.startsWith("media/"), + (file) => + MARKDOWN_FILE.test(file) && !file.startsWith("frames/all/") && !file.startsWith("media/"), ); for (const file of markdown) { const fileDir = path.dirname(path.join(root, file)); @@ -1185,7 +1194,7 @@ export async function relayoutSlice({ copied += 1; }; for (const entry of plan) { - if (entry.sourceFile?.endsWith(".md")) { + if (entry.sourceFile && MARKDOWN_FILE.test(entry.sourceFile)) { const result = rewriteSliceReferences( fs.readFileSync(entry.from, "utf8"), entry.sourceFile, diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 5e4f278d202..9d79ebac322 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1227,6 +1227,41 @@ describe("checkLayoutLinks", () => { ]); }); + it("says so for a path below a lane folder this slice never produced", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/vision-plan.md": "Slides: `source/decks/s1/slides.pdf`.\n" }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/vision-plan.md")).toBe( + "Slides: source/decks/s1/slides.pdf (none in this slice).\n", + ); + }); + + it("rewrites and checks Markdown files whatever the extension's case", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/notes.MD": "See `source/transcript.txt`.\n", + "research/more.markdown": "See [t](../source/transcript.txt).\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "analysis/research/notes.MD")).toBe( + "See `../../transcript/transcript.txt`.\n", + ); + expect(read(targetDir, "analysis/research/more.markdown")).toBe( + "See [t](../../transcript/transcript.txt).\n", + ); + const root = makeDir("relayout-check-"); + writeTree(root, { "frames/log.MARKDOWN": "[gone](key/gone.png)\n" }); + expect(checkLayoutLinks(root)).toEqual([{ file: "frames/log.MARKDOWN", ref: "key/gone.png" }]); + }); + it("says so when prose names a lane folder this slice never produced", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "key-frames/vision-plan.md": "No deck in the `source/decks/` lane.\n" }, From 06a21e598a6739fae4ac155c34f65ca7ed85b1bf Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:17:47 -0400 Subject: [PATCH 25/73] fix(knowledge): recognize root aliases and every backticked path form in relayout - The exact slice-root aliases (SLICE, SLICE/, .work//, with or without a trailing slash) become the slice's split text. - Code spans are matched by CommonMark backtick runs (double-backtick spans included) with the one-space padding kept, quoted paths are unwrapped and re-wrapped, spaces inside a path segment are allowed (My Slides.pdf) while prose spans with spaces around separators are left alone, backslash paths read as paths, and :line:column suffixes carry over. - A reference definition whose destination is not path-like (a line such as "[0:33]: implementing ...") is no longer link-checked. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 52 +++++++++++--- .../extraction/watch/relayout-slice.test.js | 71 +++++++++++++++++++ 2 files changed, 113 insertions(+), 10 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 7ebaa937ae3..aa2b045b340 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -112,14 +112,37 @@ const FRAME_IMAGE = /\.(?:png|jpe?g)$/i; const SLICE_LANE_NAMES = new Set([...Object.values(LANES), "SLICE"]); const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$]/; -const LINE_SUFFIX = /:\d+(?:[-,]\d+)*$/; -const BACKTICK_SPAN = /`([^`\s]+)`/g; +/** `:12`, `:12-20`, `:12,15` and `:12:5` (line and column) suffixes. */ +const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; +/** A code span: a run of backticks, its content, and the same run closing it. */ +const BACKTICK_SPAN = /(?\n]+>|\S+)/gm; /** `src` and `href` in inline HTML, double-, single- or unquoted. */ const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/gi; /** A backticked span names a path when it holds a separator or ends in an extension. */ -const PATH_LIKE = /\/|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)?(?:#.*)?$/; +const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:#.*)?$/; + +/** + * A code span's content split for path handling: CommonMark strips one space + * on each side when both are there (`` ` x ` ``), and a path may sit in + * quotes. `core` is null when the content does not read as a single path: + * spaces are allowed inside a segment (`My Slides.pdf`) but not at either + * end or next to a separator, which is prose or a command. + * + * @param {string} content + * @returns {{ pad: string, quote: string, core: string|null }} + */ +function spanPath(content) { + const pad = /^ .*[^ ].* $/.test(content) ? " " : ""; + let core = pad ? content.slice(1, -1) : content; + const quoted = /^(["'])(.+)\1$/.exec(core); + const quote = quoted ? quoted[1] : ""; + if (quoted) core = quoted[2]; + const readsAsPath = + core === "SLICE" || (PATH_LIKE.test(core) && !/^\s|\s$|\s[/\\]|[/\\]\s|\s{2}/.test(core)); + return { pad, quote, core: readsAsPath ? core : null }; +} /** * Corpus-layout path for a slice-relative file, or null when the corpus keeps @@ -286,7 +309,11 @@ function lookUpReference(refPath, sourceFile, map) { let keys; // `SLICE/` names the slice root explicitly, and so does the work-root // form `.work///`, but only with this slice's own prefix: - // a path segment that merely equals the slice's name says nothing. + // a path segment that merely equals the slice's name says nothing. The bare + // aliases (`SLICE`, `SLICE/`, `.work///`) name the root itself. + if (bare === "SLICE" || (map.sliceWorkPath !== null && bare === map.sliceWorkPath)) { + return sliceRootLookup(map); + } if (bare.startsWith("SLICE/")) { keys = [path.posix.normalize(bare.slice("SLICE/".length))]; } else if (map.sliceWorkPath && bare.startsWith(`${map.sliceWorkPath}/`)) { @@ -582,14 +609,15 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const quote = doubled !== undefined ? '"' : single !== undefined ? "'" : ""; return `${open}${quote}${rewrite(doubled ?? single ?? bare)}${quote}`; }) - .replace(BACKTICK_SPAN, (match, span) => { - if (!PATH_LIKE.test(span)) return match; - const found = lookUpReference(splitReference(span).refPath, sourceFile, map); + .replace(BACKTICK_SPAN, (match, ticks, content) => { + const { pad, quote, core } = spanPath(content); + if (core === null) return match; + const found = lookUpReference(splitReference(core).refPath, sourceFile, map); if (found && "retired" in found) { rewrites += 1; return `${found.retired} (${found.note})`; } - return `\`${rewrite(span)}\``; + return `${ticks}${pad}${quote}${rewrite(core)}${quote}${pad}${ticks}`; }); return { text: rewritten, rewrites }; } @@ -661,13 +689,17 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { }; for (const link of inlineLinks(text)) check(link.dest, true); for (const match of text.matchAll(REFERENCE_DEFINITION)) { - check(match[2].replace(/^<|>$/g, ""), true); + // A line such as `[0:33]: implementing ...` parses as a definition but is + // prose; only a destination that reads as a path is checked. + const destination = match[2].replace(/^<|>$/g, ""); + if (PATH_LIKE.test(destination)) check(destination, true); } for (const match of text.matchAll(HTML_PATH_ATTRIBUTE)) { check(match[2] ?? match[3] ?? match[4], true); } for (const match of text.matchAll(BACKTICK_SPAN)) { - check(match[1], false); + const { core } = spanPath(match[2]); + if (core !== null) check(core, false); } } return unresolved; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 9d79ebac322..f477b68bdf4 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1227,6 +1227,77 @@ describe("checkLayoutLinks", () => { ]); }); + it("rewrites every backticked path form and leaves prose spans alone", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/s1/My Slides.pdf": "pdf", + "recommendations/questions.md": [ + "Spaced: `../source/decks/s1/My Slides.pdf`.", + "Double: `` source/transcript.txt `` and ``a `tick` in source/transcript.txt``.", + 'Quoted: `"source/transcript.txt"`.', + "Line and column: `source/transcript.txt:3:5`.", + "Backslash: `source\\transcript.txt`.", + "Trailing slash: `research/findings/`.", + "Dot prefix: `./menu.md`.", + "Lane only: `research`.", + "Prose: `run the source / research step`.", + "", + ].join("\n"), + "recommendations/menu.md": "# Menu\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "analysis/recommendations/questions.md")).toBe( + [ + "Spaced: `../../metadata/decks/s1/My Slides.pdf`.", + "Double: `` ../../transcript/transcript.txt `` and ``a `tick` in source/transcript.txt``.", + 'Quoted: `"../../transcript/transcript.txt"`.', + "Line and column: `../../transcript/transcript.txt:3:5`.", + "Backslash: `../../transcript/transcript.txt`.", + "Trailing slash: `../research/findings/`.", + "Dot prefix: `menu.md`.", + "Lane only: `research`.", + "Prose: `run the source / research step`.", + "", + ].join("\n"), + ); + }); + + it("rewrites the exact slice-root aliases as split text", async () => { + const { sliceDir: original, targetDir } = makeFixture(); + const slug = path.basename(original); + const sliceDir = path.join(makeDir("relayout-root-"), ".work", "youtube-watch", slug); + fs.mkdirSync(path.dirname(sliceDir), { recursive: true }); + fs.renameSync(original, sliceDir); + fs.writeFileSync( + path.join(sliceDir, "recommendations", "questions.md"), + `\`SLICE\` \`SLICE/\` \`.work/youtube-watch/${slug}\` \`.work/youtube-watch/${slug}/\`\n`, + ); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + const split = `${slug} (split across analysis/, frames/, metadata/, transcript/ in this copy)`; + expect(read(targetDir, "analysis/recommendations/questions.md")).toBe( + `${split} ${split} ${split} ${split}\n`, + ); + }); + + it("does not check a reference definition whose destination is prose", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/claim-inventory.md": "[0:33]: implementing a whole spec in one session\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + }); + it("says so for a path below a lane folder this slice never produced", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "key-frames/vision-plan.md": "Slides: `source/decks/s1/slides.pdf`.\n" }, From fd679997898ed429322b6f8b45ac2929f141754e Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:27:35 -0400 Subject: [PATCH 26/73] fix(knowledge): check the temp video's id; skip code blocks; validate contact-sheets dir - In media mode the one video must be this slice's: its yt-dlp id is watch.json's videoId, or, for an X post, its info JSON's display_id is the recorded post id. A mismatch is refused, naming both ids. - Link scanning and rewriting skip fenced code blocks (``` and ~~~) and read no link syntax inside inline code spans; every pass now records edits by offset against masked copies of the text. Backticked paths in inline code are still rewritten. - The recorded contact-sheets dir gets the same OS-temp-dir and slice overlap checks as the work and frames dirs. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 11 +- .../extraction/watch/relayout-slice.js | 152 +++++++++++++----- .../extraction/watch/relayout-slice.test.js | 96 +++++++++++ 3 files changed, 216 insertions(+), 43 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 0d9088f3453..487aec685d4 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -82,8 +82,9 @@ without a finite number or null `timestampSec`), any frame image, a frame `frame-times.json` lists, a `frame-times.json` entry for a scene or interval frame on disk (anchor frames carry their time in their name), or the caption tracks. It refuses a temp session holding several videos (an X post with more than one), naming them, since the watch does not -record which one is primary; a recorded temp dir outside the OS temp dir or overlapping the -slice; and a symlink in the slice or the temp session, since a copy would follow it. It refuses +record which one is primary; a video whose id is not this slice's (for an X post, whose +info JSON does not name this post as its `display_id`); a recorded work, frames or +contact-sheets dir outside the OS temp dir or overlapping the slice; and a symlink in the slice or the temp session, since a copy would follow it. It refuses when the slice is, holds, or sits inside a path the run writes: the target, `.relayout-backup`, or a `..relayout-*` staging dir beside the target. In every mode, `--no-media` included, it refuses when one of those paths is, holds, or sits inside a @@ -104,7 +105,8 @@ the copied markdown names is rewritten through a path map relative to the file's inline links (balanced or escaped parentheses in the destination, quoted or parenthesized titles), reference-style links, HTML `src`/`href` in any quoting or case, and backticked paths, with each segment percent-decoded (and re-encoded on rewrite) and `\` separators read as `/`. -A relative link that leaves the slice keeps pointing at the same file. +A relative link that leaves the slice keeps pointing at the same file. Fenced code blocks are +left untouched, and link syntax inside an inline code span is not read as a link. One naming a pipeline file the layout does not keep (such as `run-state/watch.json`, the slice's own journey `README.md`, or an extracted frame's bare name under `--no-media`) becomes plain text saying it is not retained; one naming a directory whose files went to several places (`source/`) @@ -117,7 +119,8 @@ that leave the target, and backticked paths that are neither under such a lane d under a top-level directory of the target or an entry beside the naming file (paths into another repository, such as `research/deepening/...` from an upstream repo). Fix what the check lists in the slice and re-run. Only a passing check moves the staging directory into place; on any failure -the staging directory is removed and the target is left as it was. Without `--replace` the install is a single rename of the staging dir onto the +the staging directory is removed and the target is left as it was. Without `--replace` the +install is a single rename of the staging dir onto the target path, which fails, and the run refuses, when a target with content appeared during the run. On POSIX that rename replaces an empty directory someone created meanwhile; that is accepted, since an empty directory holds no data. A run killed mid-copy or mid-install leaves its diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index aa2b045b340..a0a8605576d 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -117,9 +117,9 @@ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; /** A code span: a run of backticks, its content, and the same run closing it. */ const BACKTICK_SPAN = /(?\n]+>|\S+)/gm; +const REFERENCE_DEFINITION = /^( {0,3}\[[^\]\n]+\]:[ \t]*)(<[^>\n]+>|\S+)/dgm; /** `src` and `href` in inline HTML, double-, single- or unquoted. */ -const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/gi; +const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/dgi; /** A backticked span names a path when it holds a separator or ends in an extension. */ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:#.*)?$/; @@ -556,19 +556,20 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { if (next !== ref) rewrites += 1; return next; }; - // Inline links: a link naming a file the layout drops becomes its label plus - // not-retained text; any other destination takes the path rewrite. Every - // edit touches only link syntax or a destination, never a label, so edits - // for nested links do not overlap; they apply back to front, and any - // overlap that still occurred would drop the later-starting edit. + const { fenceMasked, linkMasked } = maskCode(text); + // Every pass records edits by offset into the original text; the scans run + // over masked copies, so nothing inside a fenced code block is touched and + // no link syntax inside an inline code span is read as a link. Inline code + // spans themselves are read for paths (the backticked-path rewrite). /** @type {{ from: number, to: number, text: string }[]} */ const edits = []; - for (const link of inlineLinks(text)) { + // Inline links: a link naming a file the layout drops keeps its label in + // place and loses only its syntax: the opening `[` (or `![`) goes and + // `](dest)` becomes the note, so an image nested in the label keeps its + // own rewrite. Any other destination takes the path rewrite. + for (const link of inlineLinks(linkMasked)) { const found = lookUpReference(splitReference(link.dest).refPath, sourceFile, map); if (found && "retired" in found && link.labelStart !== -1) { - // Keep the label text in place and edit only the link syntax around it: - // drop the opening `[` (or `![`), and replace `](dest)` with the note. An - // image or link nested in the label keeps its own rewrite. rewrites += 1; const bracket = text[link.linkStart] === "!" ? link.linkStart + 2 : link.linkStart + 1; edits.push({ from: link.linkStart, to: bracket, text: "" }); @@ -588,40 +589,99 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { text: link.angle ? `<${written}>` : written, }); } + // Reference-style definitions and HTML src/href values take the same path + // rewrite. One naming a file the layout drops is left for the link check to + // flag: a definition or attribute has no prose to rewrite into. + for (const match of linkMasked.matchAll(REFERENCE_DEFINITION)) { + const [from, to] = /** @type {[number, number]} */ (match.indices?.[2]); + const target = text.slice(from, to); + const next = target.startsWith("<") ? `<${rewrite(unwrap(target))}>` : rewrite(target); + if (next !== target) edits.push({ from, to, text: next }); + } + for (const match of linkMasked.matchAll(HTML_PATH_ATTRIBUTE)) { + const group = [2, 3, 4].find((at) => match[at] !== undefined) ?? 2; + const [from, to] = /** @type {[number, number]} */ (match.indices?.[group]); + const value = text.slice(from, to); + const next = rewrite(value); + if (next !== value) edits.push({ from, to, text: next }); + } + for (const match of fenceMasked.matchAll(BACKTICK_SPAN)) { + const [whole, ticks, content] = match; + const from = /** @type {number} */ (match.index); + const { pad, quote, core } = spanPath(content); + if (core === null) continue; + const found = lookUpReference(splitReference(core).refPath, sourceFile, map); + let next; + if (found && "retired" in found) { + rewrites += 1; + next = `${found.retired} (${found.note})`; + } else { + next = `${ticks}${pad}${quote}${rewrite(core)}${quote}${pad}${ticks}`; + } + if (next !== whole) edits.push({ from, to: from + whole.length, text: next }); + } edits.sort((a, b) => a.from - b.from); /** @type {typeof edits} */ const kept = []; for (const edit of edits) { if (kept.length === 0 || edit.from >= kept[kept.length - 1].to) kept.push(edit); } - let linked = text; + let rewritten = text; for (const edit of kept.reverse()) { - linked = linked.slice(0, edit.from) + edit.text + linked.slice(edit.to); + rewritten = rewritten.slice(0, edit.from) + edit.text + rewritten.slice(edit.to); } - const rewritten = linked - // Reference-style definitions and HTML src/href attributes take the same - // path rewrite. One naming a file the layout drops is left for the link - // check to flag: a definition or attribute has no prose to rewrite into. - .replace(REFERENCE_DEFINITION, (_match, open, target) => - target.startsWith("<") ? `${open}<${rewrite(unwrap(target))}>` : `${open}${rewrite(target)}`, - ) - .replace(HTML_PATH_ATTRIBUTE, (_match, open, doubled, single, bare) => { - const quote = doubled !== undefined ? '"' : single !== undefined ? "'" : ""; - return `${open}${quote}${rewrite(doubled ?? single ?? bare)}${quote}`; - }) - .replace(BACKTICK_SPAN, (match, ticks, content) => { - const { pad, quote, core } = spanPath(content); - if (core === null) return match; - const found = lookUpReference(splitReference(core).refPath, sourceFile, map); - if (found && "retired" in found) { - rewrites += 1; - return `${found.retired} (${found.note})`; - } - return `${ticks}${pad}${quote}${rewrite(core)}${quote}${pad}${ticks}`; - }); return { text: rewritten, rewrites }; } +/** + * Copies of `text` with code masked by spaces (line endings kept, so every + * offset still matches the original): `fenceMasked` blanks fenced code blocks + * (``` or ~~~, per CommonMark; an unclosed fence runs to the end), and + * `linkMasked` also blanks inline code spans. Indented code blocks are not + * masked: telling them from indented list content needs a full parser. + * + * @param {string} text + * @returns {{ fenceMasked: string, linkMasked: string }} + */ +export function maskCode(text) { + const blank = (/** @type {string} */ source, /** @type {[number, number][]} */ ranges) => { + let out = source; + for (const [from, to] of ranges) { + out = out.slice(0, from) + out.slice(from, to).replace(/[^\n]/g, " ") + out.slice(to); + } + return out; + }; + /** @type {[number, number][]} */ + const fences = []; + let offset = 0; + /** @type {{ char: string, length: number, start: number }|null} */ + let open = null; + for (const line of text.split(/(?<=\n)/)) { + const body = line.replace(/\r?\n$/, ""); + if (open === null) { + const opening = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(body); + if (opening && !(opening[1][0] === "`" && opening[2].includes("`"))) { + open = { char: opening[1][0], length: opening[1].length, start: offset }; + } + } else { + const closing = /^ {0,3}(`{3,}|~{3,})[ \t]*$/.exec(body); + if (closing && closing[1][0] === open.char && closing[1].length >= open.length) { + fences.push([open.start, offset + line.length]); + open = null; + } + } + offset += line.length; + } + if (open !== null) fences.push([open.start, text.length]); + const fenceMasked = blank(text, fences); + /** @type {[number, number][]} */ + const spans = [...fenceMasked.matchAll(BACKTICK_SPAN)].map((match) => [ + /** @type {number} */ (match.index), + /** @type {number} */ (match.index) + match[0].length, + ]); + return { fenceMasked, linkMasked: blank(fenceMasked, spans) }; +} + /** * True when a backticked span reads as a path into the slice: `SLICE/`, an * explicit `./` or `../` path, or one whose first segment is a top-level @@ -687,17 +747,18 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { if (rel.startsWith("..") || path.isAbsolute(rel)) return; if (!fs.existsSync(resolved)) unresolved.push({ file, ref }); }; - for (const link of inlineLinks(text)) check(link.dest, true); - for (const match of text.matchAll(REFERENCE_DEFINITION)) { + const { fenceMasked, linkMasked } = maskCode(text); + for (const link of inlineLinks(linkMasked)) check(link.dest, true); + for (const match of linkMasked.matchAll(REFERENCE_DEFINITION)) { // A line such as `[0:33]: implementing ...` parses as a definition but is // prose; only a destination that reads as a path is checked. const destination = match[2].replace(/^<|>$/g, ""); if (PATH_LIKE.test(destination)) check(destination, true); } - for (const match of text.matchAll(HTML_PATH_ATTRIBUTE)) { + for (const match of linkMasked.matchAll(HTML_PATH_ATTRIBUTE)) { check(match[2] ?? match[3] ?? match[4], true); } - for (const match of text.matchAll(BACKTICK_SPAN)) { + for (const match of fenceMasked.matchAll(BACKTICK_SPAN)) { const { core } = spanPath(match[2]); if (core !== null) check(core, false); } @@ -1085,7 +1146,7 @@ export async function relayoutSlice({ // OS temp dir; anything else (a hand-edited path, the slice, the target) is // not a temp session to read from. const realTmp = fs.realpathSync.native(os.tmpdir()); - for (const dir of [temp.workDir, temp.framesDir]) { + for (const dir of [temp.workDir, temp.framesDir, temp.contactSheetsDir]) { if (!dir) continue; if (!isSameOrInside(realPathOf(dir), realTmp) || realPathOf(dir) === realTmp) { return refuse(`temp session dir ${dir} is not inside the OS temp dir`); @@ -1114,6 +1175,19 @@ export async function relayoutSlice({ captionPaths: workFiles.filter((file) => ofMedia(file) && file.endsWith(".vtt")), metadataPath: workFiles.find((file) => ofMedia(file) && file.endsWith(".info.json")) ?? "", }; + // The one video must be this slice's: its yt-dlp id is watch.json's videoId + // (YouTube), or, for an X post, whose media id differs from the post id, its + // info JSON's display_id is the post id the watch recorded. + if (!noMedia && mediaId !== null && mediaId !== state.videoId) { + const displayId = media.metadataPath ? readJsonObject(media.metadataPath)?.display_id : null; + const postIds = [state.videoId, state.sourceMetadata?.["source:displayId"]].filter(Boolean); + if (!(typeof displayId === "string" && postIds.includes(displayId))) { + return refuse( + `the temp session's video is ${mediaId}, not this slice's ${state.videoId}` + + (typeof displayId === "string" ? ` (its info JSON names post ${displayId})` : ""), + ); + } + } if (!noMedia) { // A partly removed temp session would yield a layout silently missing a part. // Captions are required unless the transcript phase records that no diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index f477b68bdf4..b4688287a36 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -722,6 +722,102 @@ describe("relayoutSlice", () => { expect(fs.existsSync(contactSheetsDir)).toBe(false); }); + /** + * Rename the temp session's video, info JSON and caption tracks to another + * yt-dlp id, optionally writing an info JSON with `display_id`. + * + * @param {string} workDir + * @param {string} id + * @param {Record} [info] + */ + function renameMedia(workDir, id, info) { + for (const name of fs.readdirSync(workDir)) { + fs.renameSync(path.join(workDir, name), path.join(workDir, name.replace(VIDEO_ID, id))); + } + if (info) fs.writeFileSync(path.join(workDir, `${id}.info.json`), JSON.stringify(info)); + } + + it("refuses a temp session whose video is another id, naming both", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + renameMedia(workDir, "otherVideo1", { id: "otherVideo1", display_id: "otherVideo1" }); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + const result = await relayoutSlice({ sliceDir, targetDir }); + const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); + stderr.mockRestore(); + + expect(result.exitCode).toBe(1); + expect(message).toContain("otherVideo1"); + expect(message).toContain(VIDEO_ID); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("accepts an X media id whose info JSON names this slice's post as display_id", async () => { + const { sliceDir, targetDir, workDir } = makeFixture(); + renameMedia(workDir, "1800000000000000001", { + id: "1800000000000000001", + display_id: VIDEO_ID, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(fs.existsSync(path.join(targetDir, `media/${VIDEO_ID}.mp4`))).toBe(true); + }); + + it("refuses a recorded contact-sheets dir outside the OS temp dir", async () => { + const { sliceDir, targetDir } = makeFixture(); + const statePath = path.join(sliceDir, "run-state", "watch.json"); + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + state.tempSession.contactSheetsDir = path.join(path.parse(os.tmpdir()).root, "relayout-sheets"); + fs.writeFileSync(statePath, JSON.stringify(state)); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("refuses a recorded contact-sheets dir that overlaps the slice", async () => { + const { sliceDir, targetDir } = makeFixture(); + const statePath = path.join(sliceDir, "run-state", "watch.json"); + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + state.tempSession.contactSheetsDir = path.join(sliceDir, "key-frames"); + fs.writeFileSync(statePath, JSON.stringify(state)); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + it("leaves fenced code blocks alone and reads no link inside inline code", async () => { + const body = [ + "Outside: [t](../source/transcript.txt).", + "```", + "[t](../source/transcript.txt) `source/transcript.txt` [gone](../research/gone.md)", + "```", + "~~~~ text", + "[t](../source/transcript.txt)", + "~~~~", + "Inline: `[gone](../research/gone.md)`.", + "", + ].join("\n"); + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": body }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + body.replace( + "Outside: [t](../source/transcript.txt)", + "Outside: [t](../transcript/transcript.txt)", + ), + ); + }); + it("refuses an empty video file", async () => { const { sliceDir, targetDir, workDir } = makeFixture(); fs.writeFileSync(path.join(workDir, `${VIDEO_ID}.mp4`), ""); From 3a788a6b8b5babe0a4e1f15b7faefeb5e25d8859 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:28:37 -0400 Subject: [PATCH 27/73] docs(knowledge): rewrap the relayout refusal list in output-contract.md Co-Authored-By: Claude Opus 5.5 --- .../knowledge/skills/video-digest/context/output-contract.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 487aec685d4..20d5d1377d8 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -84,8 +84,8 @@ without a finite number or null `timestampSec`), any frame image, a frame holding several videos (an X post with more than one), naming them, since the watch does not record which one is primary; a video whose id is not this slice's (for an X post, whose info JSON does not name this post as its `display_id`); a recorded work, frames or -contact-sheets dir outside the OS temp dir or overlapping the slice; and a symlink in the slice or the temp session, since a copy would follow it. It refuses -when the slice is, holds, or sits inside a path the run writes: the target, +contact-sheets dir outside the OS temp dir or overlapping the slice; and a symlink in the slice +or the temp session, since a copy would follow it. It refuses when the slice is, holds, or sits inside a path the run writes: the target, `.relayout-backup`, or a `..relayout-*` staging dir beside the target. In every mode, `--no-media` included, it refuses when one of those paths is, holds, or sits inside a temp dir `watch.json` records, existing or not, since `close` deletes those. It refuses an From b2adecd35004f1f0bd80f5c22b53ad5300078487 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:28:47 -0400 Subject: [PATCH 28/73] docs(knowledge): finish rewrapping the relayout refusal list Co-Authored-By: Claude Opus 5.5 --- .../knowledge/skills/video-digest/context/output-contract.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 20d5d1377d8..ab11473295a 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -85,8 +85,8 @@ holding several videos (an X post with more than one), naming them, since the wa record which one is primary; a video whose id is not this slice's (for an X post, whose info JSON does not name this post as its `display_id`); a recorded work, frames or contact-sheets dir outside the OS temp dir or overlapping the slice; and a symlink in the slice -or the temp session, since a copy would follow it. It refuses when the slice is, holds, or sits inside a path the run writes: the target, -`.relayout-backup`, or a `..relayout-*` staging dir beside the target. In +or the temp session, since a copy would follow it. It refuses when the slice is, holds, or sits +inside a path the run writes: the target, `.relayout-backup`, or a `..relayout-*` staging dir beside the target. In every mode, `--no-media` included, it refuses when one of those paths is, holds, or sits inside a temp dir `watch.json` records, existing or not, since `close` deletes those. It refuses an existing target unless `--replace` is passed, which replaces the target and keeps its From 2e12428804aed1956e3ad2f364a46928ffb24c1d Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:29:06 -0400 Subject: [PATCH 29/73] docs(knowledge): wrap the last long line in the relayout refusal list Co-Authored-By: Claude Opus 5.5 --- .../knowledge/skills/video-digest/context/output-contract.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index ab11473295a..05d872df343 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -86,7 +86,8 @@ record which one is primary; a video whose id is not this slice's (for an X post info JSON does not name this post as its `display_id`); a recorded work, frames or contact-sheets dir outside the OS temp dir or overlapping the slice; and a symlink in the slice or the temp session, since a copy would follow it. It refuses when the slice is, holds, or sits -inside a path the run writes: the target, `.relayout-backup`, or a `..relayout-*` staging dir beside the target. In +inside a path the run writes: the target, `.relayout-backup`, or a +`..relayout-*` staging dir beside the target. In every mode, `--no-media` included, it refuses when one of those paths is, holds, or sits inside a temp dir `watch.json` records, existing or not, since `close` deletes those. It refuses an existing target unless `--replace` is passed, which replaces the target and keeps its From f77c63320515eda28e11e67010161edecb9f7abd Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:38:04 -0400 Subject: [PATCH 30/73] fix(knowledge): read multi-line code spans, escaped labels and command spans correctly - Code spans are found by a CommonMark-style scanner: an unescaped backtick run closed by the next run of the same length, across lines within a paragraph. Masking, the backticked-path rewrite and the link check all use it, so a lone backtick no longer mispairs later spans. - A `](...)` with an escaped `]`, or whose opening `[` is escaped or missing (`\[watch](...)`), is not read as a link. - A code span with spaces counts as a path only when the whole span resolves through the path map, and is never link-checked, so commands such as `source/transcript.txt --verbose` are left alone. - output-contract.md gains a short "Known limits of the re-layout" subsection: constructs the scanner does not model, with the link check as the backstop. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 7 + .../extraction/watch/relayout-slice.js | 128 +++++++++++++++--- .../extraction/watch/relayout-slice.test.js | 46 +++++++ 3 files changed, 159 insertions(+), 22 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 05d872df343..e134e3b8b67 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -130,6 +130,13 @@ deletion by hand and never installs or deletes one, since it may belong to anoth or to a run still going. The warning does not stop the run; delete a named dir once no run is using it. +### Known limits of the re-layout + +The reference scanner is not a full CommonMark parser. It does not model indented code blocks, +HTML blocks, link labels nested more than one level, or link reference definitions inside +containers (lists, block quotes). References in those constructs are left unrewritten, and the +link check is the backstop: it refuses a dangling slice-internal path it can see. + ## Agent-written artifacts share the same root **Do not split the slice.** Every `.work//…` path in this skill and its `context/` diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index a0a8605576d..ce0da5ba728 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -114,8 +114,6 @@ const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$]/; /** `:12`, `:12-20`, `:12,15` and `:12:5` (line and column) suffixes. */ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; -/** A code span: a run of backticks, its content, and the same run closing it. */ -const BACKTICK_SPAN = /(?\n]+>|\S+)/dgm; /** `src` and `href` in inline HTML, double-, single- or unquoted. */ @@ -126,12 +124,13 @@ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:#.*)?$/; /** * A code span's content split for path handling: CommonMark strips one space * on each side when both are there (`` ` x ` ``), and a path may sit in - * quotes. `core` is null when the content does not read as a single path: - * spaces are allowed inside a segment (`My Slides.pdf`) but not at either - * end or next to a separator, which is prose or a command. + * quotes. `core` is null when the content cannot be one path. `spaced` marks + * a candidate holding spaces (`My Slides.pdf`, or a command such as + * `node run.mjs x.js --no-media`): it counts as a path only when the whole + * span resolves through the path map, and it is never link-checked. * * @param {string} content - * @returns {{ pad: string, quote: string, core: string|null }} + * @returns {{ pad: string, quote: string, core: string|null, spaced: boolean }} */ function spanPath(content) { const pad = /^ .*[^ ].* $/.test(content) ? " " : ""; @@ -140,8 +139,82 @@ function spanPath(content) { const quote = quoted ? quoted[1] : ""; if (quoted) core = quoted[2]; const readsAsPath = - core === "SLICE" || (PATH_LIKE.test(core) && !/^\s|\s$|\s[/\\]|[/\\]\s|\s{2}/.test(core)); - return { pad, quote, core: readsAsPath ? core : null }; + !core.includes("\n") && + (core === "SLICE" || (PATH_LIKE.test(core) && !/^\s|\s$|\s[/\\]|[/\\]\s|\s{2}/.test(core))); + return { pad, quote, core: readsAsPath ? core : null, spaced: /\s/.test(core) }; +} + +/** + * @typedef {Object} CodeSpan + * @property {number} start - offset of the opening backtick run + * @property {number} end - offset just past the closing run + * @property {string} ticks - the backtick run + * @property {string} content - what sits between the runs + */ + +/** + * Every inline code span in `text`, read as CommonMark does: an unescaped run + * of backticks closed by the next run of exactly the same length, possibly on + * a later line of the same paragraph (no blank line between). A run with no + * such closer is literal backticks. + * + * @param {string} text + * @returns {CodeSpan[]} + */ +export function codeSpans(text) { + /** @type {CodeSpan[]} */ + const spans = []; + let at = 0; + while (at < text.length) { + const open = text.indexOf("`", at); + if (open === -1) break; + let runEnd = open; + while (text[runEnd] === "`") runEnd += 1; + const ticks = text.slice(open, runEnd); + if (isEscaped(text, open)) { + at = open + 1; + continue; + } + let close = -1; + let scan = runEnd; + while (scan < text.length) { + const next = text.indexOf("`", scan); + if (next === -1) break; + if (/\n[ \t]*\n/.test(text.slice(runEnd, next))) break; + let nextEnd = next; + while (text[nextEnd] === "`") nextEnd += 1; + if (nextEnd - next === ticks.length) { + close = next; + break; + } + scan = nextEnd; + } + if (close === -1) { + at = runEnd; + continue; + } + spans.push({ + start: open, + end: close + ticks.length, + ticks, + content: text.slice(runEnd, close), + }); + at = close + ticks.length; + } + return spans; +} + +/** + * True when the character at `at` is backslash-escaped: an odd number of + * backslashes sits right before it. + * + * @param {string} text + * @param {number} at + */ +function isEscaped(text, at) { + let count = 0; + for (let back = at - 1; back >= 0 && text[back] === "\\"; back -= 1) count += 1; + return count % 2 === 1; } /** @@ -490,11 +563,18 @@ export function inlineLinks(text) { } if (text[at] !== ")") continue; const linkEnd = at + 1; - // The `[` that opens this link's text, matching nested brackets. + // The `[` that opens this link's text, matching nested brackets. An + // escaped `]`, or an opening `[` that is escaped (`\[watch](...)`) or + // missing, means this is not a link. + if (isEscaped(text, open)) continue; let labelStart = -1; let depth = 0; for (let back = open - 1; back >= 0; back -= 1) { - if (text[back - 1] === "\\") continue; + if (text[back] === "\n" && text[back - 1] === "\n") break; + if ((text[back] === "[" || text[back] === "]") && isEscaped(text, back)) { + if (text[back] === "[" && depth === 0) break; + continue; + } if (text[back] === "]") depth += 1; else if (text[back] === "[") { if (depth === 0) { @@ -502,8 +582,9 @@ export function inlineLinks(text) { break; } depth -= 1; - } else if (text[back] === "\n" && text[back - 1] === "\n") break; + } } + if (labelStart === -1) continue; const label = labelStart === -1 ? "" : text.slice(labelStart + 1, open); const linkStart = labelStart > 0 && text[labelStart - 1] === "!" ? labelStart - 1 : labelStart; links.push({ @@ -605,12 +686,15 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const next = rewrite(value); if (next !== value) edits.push({ from, to, text: next }); } - for (const match of fenceMasked.matchAll(BACKTICK_SPAN)) { - const [whole, ticks, content] = match; - const from = /** @type {number} */ (match.index); - const { pad, quote, core } = spanPath(content); + for (const span of codeSpans(fenceMasked)) { + const from = span.start; + const whole = text.slice(span.start, span.end); + const { ticks, content } = span; + const { pad, quote, core, spaced } = spanPath(content); if (core === null) continue; const found = lookUpReference(splitReference(core).refPath, sourceFile, map); + // A span with spaces is a path only when it resolves as one whole path. + if (spaced && !found) continue; let next; if (found && "retired" in found) { rewrites += 1; @@ -675,10 +759,8 @@ export function maskCode(text) { if (open !== null) fences.push([open.start, text.length]); const fenceMasked = blank(text, fences); /** @type {[number, number][]} */ - const spans = [...fenceMasked.matchAll(BACKTICK_SPAN)].map((match) => [ - /** @type {number} */ (match.index), - /** @type {number} */ (match.index) + match[0].length, - ]); + /** @type {[number, number][]} */ + const spans = codeSpans(fenceMasked).map((span) => [span.start, span.end]); return { fenceMasked, linkMasked: blank(fenceMasked, spans) }; } @@ -758,9 +840,11 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { for (const match of linkMasked.matchAll(HTML_PATH_ATTRIBUTE)) { check(match[2] ?? match[3] ?? match[4], true); } - for (const match of fenceMasked.matchAll(BACKTICK_SPAN)) { - const { core } = spanPath(match[2]); - if (core !== null) check(core, false); + for (const span of codeSpans(fenceMasked)) { + // A span with spaces may be a command (`node run.mjs x.js --flag`): only + // single-token spans are link-checked. + const { core, spaced } = spanPath(span.content); + if (core !== null && !spaced) check(core, false); } } return unresolved; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index b4688287a36..3ee9fdb9f0f 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -791,6 +791,52 @@ describe("relayoutSlice", () => { expect(fs.existsSync(targetDir)).toBe(false); }); + it("reads no link inside a code span that spans lines, and treats a lone backtick as text", async () => { + const body = [ + "Code: `see [gone](../research/gone.md)", + "and more` then [t](../source/transcript.txt).", + "A lone ` backtick and [u](../source/transcript.txt).", + "", + ].join("\n"); + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": body }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + body.replaceAll("](../source/transcript.txt)", "](../transcript/transcript.txt)"), + ); + }); + + it("does not read an escaped \\[label](...) as a link", async () => { + const body = "Literal: \\[watch](../run-state/watch.json) and \\[x\\](../research/gone.md).\n"; + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": body }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe(body); + }); + + it("leaves a command span alone and does not link-check it", async () => { + const body = [ + "Run `node run.mjs source/transcript.txt --no-media` or `source/transcript.txt --verbose`.", + "", + ].join("\n"); + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "recommendations/questions.md": body }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "analysis/recommendations/questions.md")).toBe(body); + }); + it("leaves fenced code blocks alone and reads no link inside inline code", async () => { const body = [ "Outside: [t](../source/transcript.txt).", From c27312255b121c676e0db9ce609f13cf8b2d6671 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:45:31 -0400 Subject: [PATCH 31/73] fix(knowledge): honor escaped image markers; leave container fences visible to the link check - A `!` preceded by an odd number of backslashes is not an image marker: `\![x](dest)` is a plain link, and retiring it keeps the `\!`. - Only fences opening at column 0 are masked. A fence inside a block quote or list item (or indented) is a documented known limit: its backtick line is no longer mistaken for a multi-line code span, so the links it holds stay visible and the link check refuses a dangling one. The Known limits subsection now names these fences. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 5 ++- .../extraction/watch/relayout-slice.js | 26 ++++++++--- .../extraction/watch/relayout-slice.test.js | 44 +++++++++++++++++++ 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index e134e3b8b67..0cce0e96f80 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -133,8 +133,9 @@ using it. ### Known limits of the re-layout The reference scanner is not a full CommonMark parser. It does not model indented code blocks, -HTML blocks, link labels nested more than one level, or link reference definitions inside -containers (lists, block quotes). References in those constructs are left unrewritten, and the +fenced code blocks that are indented or inside block quotes or list items, HTML blocks, link +labels nested more than one level, or link reference definitions inside containers (lists, block +quotes). References in those constructs may be left unrewritten or read as ordinary text, and the link check is the backstop: it refuses a dangling slice-internal path it can see. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index ce0da5ba728..c7541e0844b 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -175,6 +175,15 @@ export function codeSpans(text) { at = open + 1; continue; } + // Three or more backticks opening a line (after any block-quote or list + // markers) are a fence, not a code span. Fences inside containers are not + // masked (a known limit), so what they hold stays visible to the check. + const lineStart = text.lastIndexOf("\n", open - 1) + 1; + const lead = text.slice(lineStart, open); + if (ticks.length >= 3 && /^[ \t]*(?:>[ \t]*)*(?:(?:[-*+]|\d+[.)])[ \t]+)?[ \t]*$/.test(lead)) { + at = runEnd; + continue; + } let close = -1; let scan = runEnd; while (scan < text.length) { @@ -586,7 +595,10 @@ export function inlineLinks(text) { } if (labelStart === -1) continue; const label = labelStart === -1 ? "" : text.slice(labelStart + 1, open); - const linkStart = labelStart > 0 && text[labelStart - 1] === "!" ? labelStart - 1 : labelStart; + // A `!` right before the label marks an image, unless it is escaped (`\![x](dest)`). + const image = + labelStart > 0 && text[labelStart - 1] === "!" && !isEscaped(text, labelStart - 1); + const linkStart = image ? labelStart - 1 : labelStart; links.push({ dest: raw.replace(/\\([()])/g, "$1"), destStart, @@ -719,10 +731,12 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { /** * Copies of `text` with code masked by spaces (line endings kept, so every - * offset still matches the original): `fenceMasked` blanks fenced code blocks - * (``` or ~~~, per CommonMark; an unclosed fence runs to the end), and - * `linkMasked` also blanks inline code spans. Indented code blocks are not - * masked: telling them from indented list content needs a full parser. + * offset still matches the original): `fenceMasked` blanks top-level fenced + * code blocks (``` or ~~~ opening at column 0; an unclosed fence runs to the + * end), and `linkMasked` also blanks inline code spans. Fences that are + * indented or inside a block quote or list item, and indented code blocks, + * are not masked: telling them apart needs container parsing. What they hold + * is read as ordinary text, so the link check still sees a dangling path. * * @param {string} text * @returns {{ fenceMasked: string, linkMasked: string }} @@ -743,7 +757,7 @@ export function maskCode(text) { for (const line of text.split(/(?<=\n)/)) { const body = line.replace(/\r?\n$/, ""); if (open === null) { - const opening = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(body); + const opening = /^(`{3,}|~{3,})(.*)$/.exec(body); if (opening && !(opening[1][0] === "`" && opening[2].includes("`"))) { open = { char: opening[1][0], length: opening[1].length, start: offset }; } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 3ee9fdb9f0f..c6674631676 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -810,6 +810,50 @@ describe("relayoutSlice", () => { ); }); + it("reads \\![x](dest) as a plain link and keeps the \\! when retiring it", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "key-frames/visual-frames.md": + "Bang: \\![state](../run-state/watch.json) and \\![t](../source/transcript.txt).\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "Bang: \\!state (run-state/watch.json, not retained in this copy) and \\![t](../transcript/transcript.txt).\n", + ); + }); + + it("catches, through the link check, a dangling path in a fence inside a block quote or list", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "recommendations/questions.md": [ + "> ```", + "> [gone](../research/gone.md)", + "> ```", + "", + "- item", + "", + " ```", + " [also gone](../research/also-gone.md)", + " ```", + "", + ].join("\n"), + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/recommendations/questions.md", ref: "../research/gone.md" }, + { file: "analysis/recommendations/questions.md", ref: "../research/also-gone.md" }, + ]); + expect(fs.existsSync(targetDir)).toBe(false); + }); + it("does not read an escaped \\[label](...) as a link", async () => { const body = "Literal: \\[watch](../run-state/watch.json) and \\[x\\](../research/gone.md).\n"; const { sliceDir, targetDir } = makeFixture({ From cc92c63c903b5a14e2cf6fab4b969e05b66647d3 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:54:51 -0400 Subject: [PATCH 32/73] fix(knowledge): guard temp-dir reads; match raw #/? names; cite relayout evidence - A recorded work or frames dir that is not a readable directory is refused, naming it, instead of throwing in readdirSync. Symlink scans tolerate a missing dir, and file-size reads on the temp video, caption tracks and frames treat an unreadable file as empty. - A backticked path is first matched whole, so a file named notes#1.md resolves; only when the whole span names nothing is a #/? suffix read as a fragment or query, in both the rewrite and the link check. - output-contract.md names the observed stumbles behind the relayout step: knowledge-corpus#42, knowledge-corpus#46 (after #37), #6902. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 4 + .../extraction/watch/relayout-slice.js | 88 +++++++++++++++---- .../extraction/watch/relayout-slice.test.js | 39 ++++++++ 3 files changed, 116 insertions(+), 15 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 0cce0e96f80..b866e3961a6 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -62,6 +62,10 @@ material at a differently-shaped path (for example `sources///`) lan node "/extraction/run.mjs" --data-dir "" watch/relayout-slice.js "" "" [--no-media] [--replace] ``` +Evidence for this step (#6902): re-laying out by hand broke relative paths twice. +knowledge-corpus#42 took four review rounds and six threads to clear them, and knowledge-corpus#46 +fixed 41 broken paths in the slice #37 had landed. + The media, every extracted frame, the served caption tracks and the info JSON come from the temp session `watch.json` records, which `close` removes. To keep them, run in this order: diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index c7541e0844b..f0042ed2d33 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -309,13 +309,15 @@ function deriveDirectoryMap(fileMap) { /** * Split a reference into its path and the suffix to carry over unchanged - * (`#anchor`, `?query`, `:line`). + * (`#anchor`, `?query`, `:line`). With `fragments: false`, a raw `#` or `?` + * stays part of the path (a file named `notes#1.md`). * * @param {string} ref + * @param {{ fragments?: boolean }} [options] * @returns {{ refPath: string, suffix: string, encoded: boolean }} */ -function splitReference(ref) { - const hash = ref.search(/[#?]/); +function splitReference(ref, { fragments = true } = {}) { + const hash = fragments ? ref.search(/[#?]/) : -1; let refPath = hash === -1 ? ref : ref.slice(0, hash); let suffix = hash === -1 ? "" : ref.slice(hash); const line = LINE_SUFFIX.exec(refPath); @@ -630,8 +632,8 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { let rewrites = 0; const unwrap = (/** @type {string} */ target) => target.startsWith("<") ? target.slice(1, -1) : target; - const rewrite = (/** @type {string} */ ref) => { - const { refPath, suffix, encoded } = splitReference(ref); + const rewrite = (/** @type {string} */ ref, fragments = true) => { + const { refPath, suffix, encoded } = splitReference(ref, { fragments }); const found = lookUpReference(refPath, sourceFile, map); if (!found || "retired" in found) return ref; let relative; @@ -704,7 +706,15 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const { ticks, content } = span; const { pad, quote, core, spaced } = spanPath(content); if (core === null) continue; - const found = lookUpReference(splitReference(core).refPath, sourceFile, map); + // The whole span, a raw `#` or `?` included (`notes#1.md`), is tried + // first; only when it names nothing is the suffix read as a fragment. + const wholeFound = lookUpReference( + splitReference(core, { fragments: false }).refPath, + sourceFile, + map, + ); + const fragments = wholeFound === null; + const found = wholeFound ?? lookUpReference(splitReference(core).refPath, sourceFile, map); // A span with spaces is a path only when it resolves as one whole path. if (spaced && !found) continue; let next; @@ -712,7 +722,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { rewrites += 1; next = `${found.retired} (${found.note})`; } else { - next = `${ticks}${pad}${quote}${rewrite(core)}${quote}${pad}${ticks}`; + next = `${ticks}${pad}${quote}${rewrite(core, fragments)}${quote}${pad}${ticks}`; } if (next !== whole) edits.push({ from, to: from + whole.length, text: next }); } @@ -833,6 +843,14 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { const fileDir = path.dirname(path.join(root, file)); const text = fs.readFileSync(path.join(root, file), "utf8"); const check = (/** @type {string} */ ref, /** @type {boolean} */ isLink) => { + // A backticked path may hold a raw `#` or `?` in its name: if the whole + // span resolves, it is fine; otherwise the suffix is a fragment. + if (!isLink) { + const whole = splitReference(ref, { fragments: false }).refPath; + if (whole && !PLACEHOLDER.test(whole) && fs.existsSync(path.resolve(fileDir, whole))) { + return; + } + } const { refPath } = splitReference(ref); if (!refPath || URL_SCHEME.test(refPath) || PLACEHOLDER.test(refPath)) return; if (refPath.startsWith("/") || refPath.startsWith("~")) return; @@ -993,8 +1011,9 @@ function watchStateShapeProblem(state) { * @returns {string[]} */ function missingFrames(framesDir) { - if (!framesDir || !fs.existsSync(framesDir)) { - return [`the frames (${framesDir ?? "no tempSession.framesDir"})`]; + if (!framesDir) return ["the frames (no tempSession.framesDir)"]; + if (!readableDirectory(framesDir)) { + return [`the frames (${framesDir} is not a readable directory)`]; } const names = new Set(fs.readdirSync(framesDir)); const missing = []; @@ -1014,7 +1033,7 @@ function missingFrames(framesDir) { missing.push(`a frame-times.json holding a JSON object (${framesDir})`); } const emptyImages = [...names].filter( - (name) => FRAME_IMAGE.test(name) && fs.statSync(path.join(framesDir, name)).size === 0, + (name) => FRAME_IMAGE.test(name) && sizeOf(path.join(framesDir, name)) === 0, ); if (emptyImages.length > 0) missing.push(`non-empty frames (empty: ${emptyImages.sort().join(", ")})`); @@ -1053,6 +1072,35 @@ function missingFrames(framesDir) { return missing; } +/** + * True when `dir` is a directory this run can list. + * + * @param {string} dir + */ +function readableDirectory(dir) { + try { + if (!fs.statSync(dir).isDirectory()) return false; + fs.readdirSync(dir); + return true; + } catch { + return false; + } +} + +/** + * Size of `file` in bytes; 0 when it cannot be read, so an unreadable + * artifact is refused like an empty one. + * + * @param {string} file + */ +function sizeOf(file) { + try { + return fs.statSync(file).size; + } catch { + return 0; + } +} + /** * posix paths, relative to `dir`, of every symlink under it. * @@ -1061,8 +1109,14 @@ function missingFrames(framesDir) { * @returns {string[]} */ function findSymlinks(dir, prefix = "") { - if (!fs.existsSync(dir)) return []; - return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + // Missing or unreadable: the readability checks name it. + return []; + } + return entries.flatMap((entry) => { const rel = prefix ? `${prefix}/${entry.name}` : entry.name; if (entry.isSymbolicLink()) return [rel]; return entry.isDirectory() ? findSymlinks(path.join(dir, entry.name), rel) : []; @@ -1299,8 +1353,12 @@ export async function relayoutSlice({ transcriptMetrics.transcriptStrategy === "asr" || transcriptMetrics.skipped === true ); const missing = [ - ...(media.videoPath ? [] : [`the video (${temp.workDir ?? "no tempSession.workDir"})`]), - ...(media.videoPath && fs.statSync(media.videoPath).size === 0 + ...(temp.workDir && !readableDirectory(temp.workDir) + ? [`the work dir (${temp.workDir} is not a readable directory)`] + : media.videoPath + ? [] + : [`the video (${temp.workDir ?? "no tempSession.workDir"})`]), + ...(media.videoPath && sizeOf(media.videoPath) === 0 ? [`a non-empty video (${path.basename(media.videoPath)} is empty)`] : []), ...(media.videoPath && !media.metadataPath ? [`the info JSON (${mediaId}.info.json)`] : []), @@ -1313,7 +1371,7 @@ export async function relayoutSlice({ .map((link) => `regular files in place of symlinks (${link})`), ...(captioned && media.captionPaths.length === 0 ? ["the caption tracks"] : []), ...media.captionPaths - .filter((caption) => fs.statSync(caption).size === 0) + .filter((caption) => sizeOf(caption) === 0) .map((caption) => `a non-empty caption track (${path.basename(caption)} is empty)`), ]; if (missing.length > 0) { diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index c6674631676..18972eca8c6 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -908,6 +908,45 @@ describe("relayoutSlice", () => { ); }); + it.each([ + ["framesDir", "framesDir"], + ["workDir", "workDir"], + ])("refuses, without throwing, a recorded %s that is a file", async (_label, field) => { + const fixture = makeFixture(); + const dir = field === "framesDir" ? fixture.framesDir : fixture.workDir; + fs.rmSync(dir, { recursive: true, force: true }); + fs.writeFileSync(dir, "not a directory"); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + const result = await relayoutSlice({ + sliceDir: fixture.sliceDir, + targetDir: fixture.targetDir, + }); + const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); + stderr.mockRestore(); + + expect(result.exitCode).toBe(1); + expect(message).toContain(`${dir} is not a readable directory`); + expect(fs.existsSync(fixture.targetDir)).toBe(false); + }); + + it("matches a backticked path whose name holds # or ? before reading a fragment", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/findings/notes#1.md": "n\n", + "recommendations/questions.md": + "Raw: `research/findings/notes#1.md`. Fragment: `research/findings/topic.md#intro`.\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "analysis/recommendations/questions.md")).toBe( + "Raw: `../research/findings/notes#1.md`. Fragment: `../research/findings/topic.md#intro`.\n", + ); + }); + it("refuses an empty video file", async () => { const { sliceDir, targetDir, workDir } = makeFixture(); fs.writeFileSync(path.join(workDir, `${VIDEO_ID}.mp4`), ""); From b4538556e2e3ed4f3e5bcd2921e5552881a7c024 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:02:49 -0400 Subject: [PATCH 33/73] fix(knowledge): keep relayout evidence out of the skill; read multi-line reference definitions - output-contract.md states the relayout step's reason in one sentence with no issue numbers, per skill-criteria's history and provenance rule; the concrete evidence (knowledge-corpus#42, #46 after #37, #6902) moves to the changelog fragment and the PR body. - A reference definition's destination may follow one line ending (`[deck]:` then the path on the next line), in both the rewrite and the link check. Co-Authored-By: Claude Opus 5.5 --- ...st-relayout-and-caption-dedupe-59495b96.md | 2 +- .../video-digest/context/output-contract.md | 5 ++-- .../extraction/watch/relayout-slice.js | 4 ++-- .../extraction/watch/relayout-slice.test.js | 23 +++++++++++++++++++ 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md index 7da2a67ed26..32ec878de2e 100644 --- a/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md +++ b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md @@ -4,7 +4,7 @@ bump: minor ### Added -- video-digest: `watch/relayout-slice.js` copies a slice into the knowledge-corpus layout (`transcript/`, `metadata/`, `frames/all/`, `frames/key/`, `media/`, `analysis/`, provenance `README.md`), rewrites the slice paths its markdown names through a path map (a pipeline file the layout does not keep becomes plain text saying so), and exits 1 listing every slice-internal path that still does not resolve. It runs on a closed slice, or before `close` on one whose outcome checks pass so the temp media is still there, and refuses a slice whose temp media is gone unless `--no-media`, which takes nothing from the temp session. It builds the layout in a staging directory and moves it into place only when the link check passes; an existing target needs `--replace` ([#6902](https://github.com/melodic-software/claude-code-plugins/issues/6902)). +- video-digest: `watch/relayout-slice.js` copies a slice into the knowledge-corpus layout (`transcript/`, `metadata/`, `frames/all/`, `frames/key/`, `media/`, `analysis/`, provenance `README.md`), rewrites the slice paths its markdown names through a path map (a pipeline file the layout does not keep becomes plain text saying so), and exits 1 listing every slice-internal path that still does not resolve. It runs on a closed slice, or before `close` on one whose outcome checks pass so the temp media is still there, and refuses a slice whose temp media is gone unless `--no-media`, which takes nothing from the temp session. It builds the layout in a staging directory and moves it into place only when the link check passes; an existing target needs `--replace` ([#6902](https://github.com/melodic-software/claude-code-plugins/issues/6902)). Hand re-layouts had broken relative paths twice: melodic-software/knowledge-corpus#42 needed four review rounds and six threads to clear them, and knowledge-corpus#46 fixed 41 broken paths in the slice knowledge-corpus#37 had landed. ### Fixed diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index b866e3961a6..2a511a9bbb4 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -62,9 +62,8 @@ material at a differently-shaped path (for example `sources///`) lan node "/extraction/run.mjs" --data-dir "" watch/relayout-slice.js "" "" [--no-media] [--replace] ``` -Evidence for this step (#6902): re-laying out by hand broke relative paths twice. -knowledge-corpus#42 took four review rounds and six threads to clear them, and knowledge-corpus#46 -fixed 41 broken paths in the slice #37 had landed. +Re-laying out a slice by hand has repeatedly left broken relative paths in the corpus; the +script's rewrite and link check are what catch them. The media, every extracted frame, the served caption tracks and the info JSON come from the temp session `watch.json` records, which `close` removes. To keep them, run in this order: diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index f0042ed2d33..3f64486a4fb 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -114,8 +114,8 @@ const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$]/; /** `:12`, `:12-20`, `:12,15` and `:12:5` (line and column) suffixes. */ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; -/** `[label]: path` reference-style link definitions. */ -const REFERENCE_DEFINITION = /^( {0,3}\[[^\]\n]+\]:[ \t]*)(<[^>\n]+>|\S+)/dgm; +/** `[label]: path` reference-style link definitions; the path may follow one line ending. */ +const REFERENCE_DEFINITION = /^( {0,3}\[[^\]\n]+\]:[ \t]*(?:\r?\n[ \t]*)?)(<[^>\n]+>|\S+)/dgm; /** `src` and `href` in inline HTML, double-, single- or unquoted. */ const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/dgi; /** A backticked span names a path when it holds a separator or ends in an extension. */ diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 18972eca8c6..4f3a0e1c8de 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1268,6 +1268,29 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(fs.readdirSync(path.dirname(targetDir))).toEqual([]); }); + it("reads a reference definition whose destination follows a line ending", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/slides.pdf": "pdf", + "key-frames/vision-plan.md": "[deck][d]\n\n[d]:\n ../source/decks/slides.pdf\n", + "recommendations/questions.md": "[g][g]\n\n[g]:\n ../research/gone.md\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/recommendations/questions.md", ref: "../research/gone.md" }, + ]); + fs.rmSync(path.join(sliceDir, "recommendations", "questions.md")); + const fixed = await relayoutSlice({ sliceDir, targetDir }); + expect(fixed.exitCode).toBe(0); + expect(read(targetDir, "frames/vision-plan.md")).toBe( + "[deck][d]\n\n[d]:\n ../metadata/decks/slides.pdf\n", + ); + }); + it("flags a reference-style definition to a path that does not exist", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "research/claim-inventory.md": "[gone][g]\n\n[g]: findings/gone.md\n" }, From 11e857148ca6967559bfbda5b60df58ab0bd0589 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:10:35 -0400 Subject: [PATCH 34/73] fix(knowledge): decode HTML references in src/href; require regular temp files - HTML src/href values have their character references decoded (named & < > " ', decimal and hex) before path mapping and link checking; a rewritten value writes & back as & when the original did. - Every required temp artifact must be a regular file (lstat isFile): frame images, the frame-times sidecar, the video, the caption tracks and the info JSON. Anything else is refused, naming it. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 6 +- .../extraction/watch/relayout-slice.js | 61 ++++++++++++++++++- .../extraction/watch/relayout-slice.test.js | 52 ++++++++++++++++ 3 files changed, 114 insertions(+), 5 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 2a511a9bbb4..0c8cc49bac1 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -79,7 +79,8 @@ tracks or info JSON) even when it still exists; after a normal close that is the It refuses a `watch.json` that is not an object, whose `videoId` is not a plain id (it names `media/.`), that lacks a `title` or `sourceUrl`, or whose temp session dirs are not strings. Without `--no-media` it also refuses a partly removed temp session, naming the missing -piece: the video, an empty video, caption track or frame image, the info JSON or +piece: the video, an empty video, caption track or frame image, any of those (or the +sidecar) that is not a regular file, the info JSON or `frame-times.json` (or either one not holding a JSON object, or a `frame-times.json` record without a finite number or null `timestampSec`), any frame image, a frame `frame-times.json` lists, a `frame-times.json` entry for a scene or interval frame on disk @@ -107,7 +108,8 @@ the companion sources brief, and the fetched `decks/` and `attachments/`), `fram paths yt-dlp records. The script's header comment is the authoritative mapping. Each slice path the copied markdown names is rewritten through a path map relative to the file's new location: inline links (balanced or escaped parentheses in the destination, quoted or parenthesized titles), -reference-style links, HTML `src`/`href` in any quoting or case, and backticked paths, with each +reference-style links, HTML `src`/`href` in any quoting or case (character references decoded), +and backticked paths, with each segment percent-decoded (and re-encoded on rewrite) and `\` separators read as `/`. A relative link that leaves the slice keeps pointing at the same file. Fenced code blocks are left untouched, and link syntax inside an inline code span is not read as a link. diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 3f64486a4fb..d4146f3c5cc 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -121,6 +121,31 @@ const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|( /** A backticked span names a path when it holds a separator or ends in an extension. */ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:#.*)?$/; +const HTML_NAMED_REFERENCES = /** @type {Record} */ ({ + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", +}); + +/** + * An HTML attribute value with its character references decoded: the named + * ones a path can hold (`&` `<` `>` `"` `'`) and numeric + * ones (`'`, `'`). Anything else stays as written. + * + * @param {string} value + */ +function decodeHtmlReferences(value) { + return value.replace(/&(?:#(\d+)|#[xX]([0-9A-Fa-f]+)|([A-Za-z]+));/g, (match, dec, hex, name) => { + if (dec !== undefined || hex !== undefined) { + const code = dec !== undefined ? Number(dec) : Number.parseInt(hex, 16); + return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match; + } + return HTML_NAMED_REFERENCES[name] ?? match; + }); +} + /** * A code span's content split for path handling: CommonMark strips one space * on each side when both are there (`` ` x ` ``), and a path may sit in @@ -696,9 +721,13 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { for (const match of linkMasked.matchAll(HTML_PATH_ATTRIBUTE)) { const group = [2, 3, 4].find((at) => match[at] !== undefined) ?? 2; const [from, to] = /** @type {[number, number]} */ (match.indices?.[group]); + // An attribute value may hold character references (`&`): map the + // decoded path, and write `&` back as `&` when the original did. const value = text.slice(from, to); - const next = rewrite(value); - if (next !== value) edits.push({ from, to, text: next }); + const decoded = decodeHtmlReferences(value); + const next = rewrite(decoded); + if (next === decoded) continue; + edits.push({ from, to, text: value.includes("&") ? next.replaceAll("&", "&") : next }); } for (const span of codeSpans(fenceMasked)) { const from = span.start; @@ -870,7 +899,7 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { if (PATH_LIKE.test(destination)) check(destination, true); } for (const match of linkMasked.matchAll(HTML_PATH_ATTRIBUTE)) { - check(match[2] ?? match[3] ?? match[4], true); + check(decodeHtmlReferences(match[2] ?? match[3] ?? match[4]), true); } for (const span of codeSpans(fenceMasked)) { // A span with spaces may be a command (`node run.mjs x.js --flag`): only @@ -1017,6 +1046,15 @@ function missingFrames(framesDir) { } const names = new Set(fs.readdirSync(framesDir)); const missing = []; + // A frame image or the sidecar must be a regular file, not just a name. + const notFiles = [...names].filter( + (name) => + (FRAME_IMAGE.test(name) || name === "frame-times.json") && + !isRegularFile(path.join(framesDir, name)), + ); + if (notFiles.length > 0) { + missing.push(`regular files for the frames entries ${notFiles.sort().join(", ")}`); + } /** @type {unknown} */ let times = null; let readable = true; @@ -1087,6 +1125,19 @@ function readableDirectory(dir) { } } +/** + * True when `file` is a regular file (not a directory, symlink or device). + * + * @param {string} file + */ +function isRegularFile(file) { + try { + return fs.lstatSync(file).isFile(); + } catch { + return false; + } +} + /** * Size of `file` in bytes; 0 when it cannot be read, so an unreadable * artifact is refused like an empty one. @@ -1094,6 +1145,7 @@ function readableDirectory(dir) { * @param {string} file */ function sizeOf(file) { + if (!isRegularFile(file)) return 0; try { return fs.statSync(file).size; } catch { @@ -1373,6 +1425,9 @@ export async function relayoutSlice({ ...media.captionPaths .filter((caption) => sizeOf(caption) === 0) .map((caption) => `a non-empty caption track (${path.basename(caption)} is empty)`), + ...[media.videoPath, media.metadataPath, ...media.captionPaths] + .filter((file) => file && fs.existsSync(file) && !isRegularFile(file)) + .map((file) => `a regular file at ${path.basename(file)}`), ]; if (missing.length > 0) { return refuse( diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 4f3a0e1c8de..d6be50e161a 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -947,6 +947,36 @@ describe("relayoutSlice", () => { ); }); + it.each([ + [ + "a frame name that is a directory", + (/** @type {Record} */ d) => + fs.mkdirSync(path.join(d.framesDir, "anchor_00030000_0002.png")), + ], + [ + "a frame-times.json that is a directory", + (/** @type {Record} */ d) => { + fs.rmSync(path.join(d.framesDir, "frame-times.json")); + fs.mkdirSync(path.join(d.framesDir, "frame-times.json")); + }, + ], + ])("refuses %s, naming it", async (label, damage) => { + const { sliceDir, targetDir, workDir, framesDir } = makeFixture(); + damage({ workDir, framesDir }); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + const result = await relayoutSlice({ sliceDir, targetDir }); + const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); + stderr.mockRestore(); + + expect(result.exitCode).toBe(1); + expect(message).toContain("regular files for the frames entries"); + expect(message).toContain( + label.includes("frame-times") ? "frame-times.json" : "anchor_00030000_0002.png", + ); + expect(fs.existsSync(targetDir)).toBe(false); + }); + it("refuses an empty video file", async () => { const { sliceDir, targetDir, workDir } = makeFixture(); fs.writeFileSync(path.join(workDir, `${VIDEO_ID}.mp4`), ""); @@ -1229,6 +1259,28 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ); }); + it("decodes HTML character references in src and href, re-encoding & as the original did", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/s1/a&b.pdf": "pdf", + "key-frames/visual-frames.md": + 'deck \n', + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + 'deck \n', + ); + const root = makeDir("relayout-check-"); + writeTree(root, { "frames/visual-frames.md": '\n' }); + expect(checkLayoutLinks(root)).toEqual([ + { file: "frames/visual-frames.md", ref: "key/x&y.png" }, + ]); + }); + it("rewrites and checks upper-case HTML attribute names", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "key-frames/visual-frames.md": '\n' }, From ef2a06f5528b881b1f3b2137fcb2a917d08cd79c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:16:09 -0400 Subject: [PATCH 35/73] fix(knowledge): escape rewritten HTML src/href values for their quoting A rewritten attribute value always writes & as &, writes " as " inside double quotes and ' as ' inside single quotes, and an unquoted value that now holds whitespace, a quote, =, <, > or a backtick is wrapped in double quotes. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 10 ++++-- .../extraction/watch/relayout-slice.test.js | 33 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index d4146f3c5cc..1e850f32ee2 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -722,12 +722,18 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const group = [2, 3, 4].find((at) => match[at] !== undefined) ?? 2; const [from, to] = /** @type {[number, number]} */ (match.indices?.[group]); // An attribute value may hold character references (`&`): map the - // decoded path, and write `&` back as `&` when the original did. + // decoded path, then escape the new value for its quoting: `&` always, + // the enclosing quote character, and an unquoted value that now needs + // quotes gets double quotes. const value = text.slice(from, to); const decoded = decodeHtmlReferences(value); const next = rewrite(decoded); if (next === decoded) continue; - edits.push({ from, to, text: value.includes("&") ? next.replaceAll("&", "&") : next }); + let escaped = next.replaceAll("&", "&"); + if (group === 2) escaped = escaped.replaceAll('"', """); + else if (group === 3) escaped = escaped.replaceAll("'", "'"); + else if (/[\s"'=<>`]/.test(escaped)) escaped = `"${escaped.replaceAll('"', """)}"`; + edits.push({ from, to, text: escaped }); } for (const span of codeSpans(fenceMasked)) { const from = span.start; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index d6be50e161a..a5f734a8040 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1281,6 +1281,39 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it.skipIf(process.platform === "win32")( + "escapes a rewritten src/href value for its quoting", + async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + 'source/decks/s1/a"b.pdf': "pdf", + "source/decks/s1/it's.pdf": "pdf", + "source/decks/s1/a&b.pdf": "pdf", + "key-frames/visual-frames.md": [ + 'dq', + "sq", + "bare", + 'raw amp', + "", + ].join("\n"), + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + [ + 'dq', + "sq", + 'bare', + 'raw amp', + "", + ].join("\n"), + ); + }, + ); + it("rewrites and checks upper-case HTML attribute names", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "key-frames/visual-frames.md": '\n' }, From 5f5fcfa8436ff620e166731738aaec102417af9a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:24:45 -0400 Subject: [PATCH 36/73] fix(knowledge): decode backslash escapes in destinations; refuse unreadable slice dirs - Link and definition destinations have their CommonMark backslash escapes of ASCII punctuation decoded before a fragment or query is split off, so \# and \? are literal parts of the path (notes\#1.md). A rewrite re-escapes the characters the original escaped. - The slice tree is read once before staging; an unreadable nested dir is refused, naming it, instead of throwing mid-run. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 72 +++++++++++++++---- .../extraction/watch/relayout-slice.test.js | 49 +++++++++++++ 2 files changed, 107 insertions(+), 14 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 1e850f32ee2..b7e11ac4ecf 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -332,6 +332,36 @@ function deriveDirectoryMap(fileMap) { /** @typedef {{ target: string, isDir: boolean } | { retired: string, note: string } | { outside: string }} Lookup */ +/** Stand-ins for a backslash-escaped `#` or `?`, which are part of the path. */ +const LITERAL_HASH = ""; +const LITERAL_QUERY = ""; + +/** + * A link or definition destination with its CommonMark backslash escapes of + * ASCII punctuation decoded. An escaped `#` or `?` is literal, not a fragment + * or query, so it becomes a stand-in that splitReference keeps in the path. + * `escapedChars` lists the characters the original escaped, to re-escape on + * rewrite. + * + * @param {string} raw + * @returns {{ dest: string, escapedChars: string }} + */ +function unescapeDestination(raw) { + let escapedChars = ""; + const dest = raw.replace(/\\([!-/:-@[-`{-~])/g, (_match, char) => { + if (!escapedChars.includes(char)) escapedChars += char; + if (char === "#") return LITERAL_HASH; + if (char === "?") return LITERAL_QUERY; + return char; + }); + return { dest, escapedChars }; +} + +/** @param {string} value */ +function literalText(value) { + return value.replaceAll(LITERAL_HASH, "#").replaceAll(LITERAL_QUERY, "?"); +} + /** * Split a reference into its path and the suffix to carry over unchanged * (`#anchor`, `?query`, `:line`). With `fragments: false`, a raw `#` or `?` @@ -371,7 +401,7 @@ function splitReference(ref, { fragments = true } = {}) { encoded = decoded !== refPath; refPath = decoded; } - return { refPath, suffix, encoded }; + return { refPath: literalText(refPath), suffix: literalText(suffix), encoded }; } /** @@ -498,11 +528,12 @@ const NOT_RETAINED = "not retained in this copy"; /** * @typedef {Object} InlineLink - * @property {string} dest - the destination, angle brackets removed and `\(` `\)` unescaped + * @property {string} dest - the destination, angle brackets removed and backslash escapes decoded + * (an escaped `#` or `?` as a literal stand-in, see unescapeDestination) * @property {number} destStart - offset of the destination as written (its `<` when angled) * @property {number} destEnd - offset just past it * @property {boolean} angle - written as `<...>` - * @property {boolean} escaped - held backslash-escaped parentheses + * @property {string} escapedChars - the punctuation the destination backslash-escaped * @property {string} label - the link text, `!` and brackets removed * @property {number} labelStart - offset of the link's `[` (or its `!`), -1 when unmatched * @property {number} labelEnd - offset of the `]` that closes the link text @@ -627,11 +658,10 @@ export function inlineLinks(text) { labelStart > 0 && text[labelStart - 1] === "!" && !isEscaped(text, labelStart - 1); const linkStart = image ? labelStart - 1 : labelStart; links.push({ - dest: raw.replace(/\\([()])/g, "$1"), + ...unescapeDestination(raw), destStart, destEnd, angle, - escaped: /\\[()]/.test(raw), label, labelStart: linkStart, labelEnd: open, @@ -657,7 +687,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { let rewrites = 0; const unwrap = (/** @type {string} */ target) => target.startsWith("<") ? target.slice(1, -1) : target; - const rewrite = (/** @type {string} */ ref, fragments = true) => { + const rewrite = (/** @type {string} */ ref, fragments = true, escapedChars = "") => { const { refPath, suffix, encoded } = splitReference(ref, { fragments }); const found = lookUpReference(refPath, sourceFile, map); if (!found || "retired" in found) return ref; @@ -671,7 +701,9 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { relative = path.posix.relative(path.posix.dirname(targetFile), found.target) || "."; trailing = found.isDir && refPath.endsWith("/") ? "/" : ""; } - const written = encoded ? relative.split("/").map(encodeURIComponent).join("/") : relative; + let written = encoded ? relative.split("/").map(encodeURIComponent).join("/") : relative; + // Re-escape what the original destination backslash-escaped. + for (const char of escapedChars) written = written.replaceAll(char, `\\${char}`); const next = `${written}${trailing}${suffix}`; if (next !== ref) rewrites += 1; return next; @@ -700,9 +732,9 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { }); continue; } - const next = rewrite(link.dest); + const next = rewrite(link.dest, true, link.escapedChars); if (next === link.dest) continue; - const written = link.escaped ? next.replace(/[()]/g, "\\$&") : next; + const written = next; edits.push({ from: link.destStart, to: link.destEnd, @@ -715,8 +747,10 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { for (const match of linkMasked.matchAll(REFERENCE_DEFINITION)) { const [from, to] = /** @type {[number, number]} */ (match.indices?.[2]); const target = text.slice(from, to); - const next = target.startsWith("<") ? `<${rewrite(unwrap(target))}>` : rewrite(target); - if (next !== target) edits.push({ from, to, text: next }); + const { dest, escapedChars } = unescapeDestination(unwrap(target)); + const rewritten = rewrite(dest, true, escapedChars); + if (rewritten === dest) continue; + edits.push({ from, to, text: target.startsWith("<") ? `<${rewritten}>` : rewritten }); } for (const match of linkMasked.matchAll(HTML_PATH_ATTRIBUTE)) { const group = [2, 3, 4].find((at) => match[at] !== undefined) ?? 2; @@ -894,14 +928,14 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { const resolved = path.resolve(fileDir, refPath); const rel = path.relative(root, resolved); if (rel.startsWith("..") || path.isAbsolute(rel)) return; - if (!fs.existsSync(resolved)) unresolved.push({ file, ref }); + if (!fs.existsSync(resolved)) unresolved.push({ file, ref: literalText(ref) }); }; const { fenceMasked, linkMasked } = maskCode(text); for (const link of inlineLinks(linkMasked)) check(link.dest, true); for (const match of linkMasked.matchAll(REFERENCE_DEFINITION)) { // A line such as `[0:33]: implementing ...` parses as a definition but is // prose; only a destination that reads as a path is checked. - const destination = match[2].replace(/^<|>$/g, ""); + const destination = unescapeDestination(match[2].replace(/^<|>$/g, "")).dest; if (PATH_LIKE.test(destination)) check(destination, true); } for (const match of linkMasked.matchAll(HTML_PATH_ATTRIBUTE)) { @@ -1266,6 +1300,16 @@ export async function relayoutSlice({ if (sliceInStaging) { return refuse(`the slice ${slice} sits in a relayout staging dir of ${target}`); } + // Read the whole slice tree once, up front: an unreadable directory in it + // is refused here, naming it, rather than failing mid-copy. + /** @type {string[]} */ + let sliceFiles; + try { + sliceFiles = listFiles(slice); + } catch (error) { + const where = /** @type {NodeJS.ErrnoException} */ (error).path ?? slice; + return refuse(`cannot read ${where} in the slice: ${errorMessage(error)}`); + } const symlinks = findSymlinks(slice); if (symlinks.length > 0) { return refuse(`the slice holds symlinks, which a copy would follow: ${symlinks.join(", ")}`); @@ -1445,7 +1489,7 @@ export async function relayoutSlice({ /** @type {{ from: string, to: string, sourceFile?: string }[]} */ const plan = []; - const sliceFiles = listFiles(slice); + const sliceEntries = new Set( sliceFiles.flatMap((file) => file.split("/").map((_part, index, parts) => parts.slice(0, index + 1).join("/")), diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index a5f734a8040..5863af0d5c4 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -930,6 +930,55 @@ describe("relayoutSlice", () => { expect(fs.existsSync(fixture.targetDir)).toBe(false); }); + it("decodes backslash escapes in link and definition destinations, keeping \\# in the path", async () => { + const body = [ + "[n](../research/findings/notes\\#1.md#intro) [q](../research/findings/what\\?.md)", + "", + "[def]: ../research/findings/notes\\#1.md", + "", + ].join("\n"); + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/findings/notes#1.md": "n\n", + "research/findings/what?.md": "w\n", + "key-frames/visual-frames.md": body, + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + [ + "[n](../analysis/research/findings/notes\\#1.md#intro) [q](../analysis/research/findings/what\\?.md)", + "", + "[def]: ../analysis/research/findings/notes\\#1.md", + "", + ].join("\n"), + ); + }); + + it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "refuses, naming it, a slice holding an unreadable nested dir", + async () => { + const { sliceDir, targetDir } = makeFixture(); + const locked = path.join(sliceDir, "research", "findings"); + fs.chmodSync(locked, 0o000); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + try { + const result = await relayoutSlice({ sliceDir, targetDir }); + const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); + + expect(result.exitCode).toBe(1); + expect(message).toContain(locked); + expect(fs.readdirSync(path.dirname(targetDir))).toEqual([]); + } finally { + stderr.mockRestore(); + fs.chmodSync(locked, 0o755); + } + }, + ); + it("matches a backticked path whose name holds # or ? before reading a fragment", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From 4f10998db250e46b112c6603da78f23a8cfc67ef Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:31:27 -0400 Subject: [PATCH 37/73] fix(knowledge): allow escaped brackets in reference-definition labels A definition label may hold backslash-escaped brackets ([deck\]]: ...), matched with (?:\\.|[^\]\\])+. Inline link labels already skip escaped brackets when scanning back for the opening bracket; a test now covers both. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 5 +-- .../extraction/watch/relayout-slice.test.js | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index b7e11ac4ecf..a2a164224a2 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -114,8 +114,9 @@ const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$]/; /** `:12`, `:12-20`, `:12,15` and `:12:5` (line and column) suffixes. */ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; -/** `[label]: path` reference-style link definitions; the path may follow one line ending. */ -const REFERENCE_DEFINITION = /^( {0,3}\[[^\]\n]+\]:[ \t]*(?:\r?\n[ \t]*)?)(<[^>\n]+>|\S+)/dgm; +/** `[label]: path` reference-style link definitions (a label may hold escaped brackets, `[a\]b]`); the path may follow one line ending. */ +const REFERENCE_DEFINITION = + /^( {0,3}\[(?:\\[^\n]|[^\]\\\n])+\]:[ \t]*(?:\r?\n[ \t]*)?)(<[^>\n]+>|\S+)/dgm; /** `src` and `href` in inline HTML, double-, single- or unquoted. */ const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/dgi; /** A backticked span names a path when it holds a separator or ends in an extension. */ diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 5863af0d5c4..5e61e67fedd 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1402,6 +1402,40 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(fs.readdirSync(path.dirname(targetDir))).toEqual([]); }); + it("reads labels holding escaped brackets in definitions and inline links", async () => { + const body = [ + "[deck\\]][d] and [a\\]b](../source/transcript.txt)", + "", + "[deck\\]]: ../source/transcript.txt", + "[x\\[y]: ../research/gone.md", + "", + ].join("\n"); + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": body }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "frames/visual-frames.md", ref: "../analysis/research/gone.md" }, + ]); + fs.writeFileSync( + path.join(sliceDir, "key-frames", "visual-frames.md"), + body.replace("[x\\[y]: ../research/gone.md\n", ""), + ); + const fixed = await relayoutSlice({ sliceDir, targetDir }); + expect(fixed.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + [ + "[deck\\]][d] and [a\\]b](../transcript/transcript.txt)", + "", + "[deck\\]]: ../transcript/transcript.txt", + "", + ].join("\n"), + ); + }); + it("reads a reference definition whose destination follows a line ending", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From 11fce25d78d8af0bfba6063269e270b436fbbb87 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:39:01 -0400 Subject: [PATCH 38/73] fix(knowledge): follow CommonMark link-label rules in relayout - Reference-definition labels may span line endings but not a blank line, are capped at 999 characters, may not hold an unescaped bracket, and must hold a non-space character. - Inline link text no longer crosses a blank line that holds only spaces or tabs. - References are not matched to definitions by label, so case-folded label matching does not apply. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 31 ++++++++++++++++-- .../extraction/watch/relayout-slice.test.js | 32 +++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index a2a164224a2..90afbb9704c 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -114,9 +114,26 @@ const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /[<>{}*$]/; /** `:12`, `:12-20`, `:12,15` and `:12:5` (line and column) suffixes. */ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; -/** `[label]: path` reference-style link definitions (a label may hold escaped brackets, `[a\]b]`); the path may follow one line ending. */ +/** + * `[label]: path` reference-style link definitions, with CommonMark's label + * rules: up to 999 characters, backslash escapes allowed (`[a\]b]`), no + * unescaped bracket, line endings allowed but not a blank line. The path may + * follow one line ending. A label must also hold a non-space character, + * checked by definitionLabelOk. + */ const REFERENCE_DEFINITION = - /^( {0,3}\[(?:\\[^\n]|[^\]\\\n])+\]:[ \t]*(?:\r?\n[ \t]*)?)(<[^>\n]+>|\S+)/dgm; + /^( {0,3}\[(?:\\[^\n]|[^[\]\\\n]|\r?\n(?![ \t]*\r?\n)){1,999}\]:[ \t]*(?:\r?\n[ \t]*)?)(<[^>\n]+>|\S+)/dgm; + +/** + * True when a REFERENCE_DEFINITION match has a label holding a non-space + * character (CommonMark rejects an all-whitespace label). + * + * @param {RegExpMatchArray} match + */ +function definitionLabelOk(match) { + const prefix = match[1]; + return /\S/.test(prefix.slice(prefix.indexOf("[") + 1, prefix.lastIndexOf("]:"))); +} /** `src` and `href` in inline HTML, double-, single- or unquoted. */ const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/dgi; /** A backticked span names a path when it holds a separator or ends in an extension. */ @@ -638,7 +655,13 @@ export function inlineLinks(text) { let labelStart = -1; let depth = 0; for (let back = open - 1; back >= 0; back -= 1) { - if (text[back] === "\n" && text[back - 1] === "\n") break; + // Link text never crosses a blank line (one holding only spaces counts). + if ( + text[back] === "\n" && + /^[ \t\r]*$/.test(text.slice(text.lastIndexOf("\n", back - 1) + 1, back)) + ) { + break; + } if ((text[back] === "[" || text[back] === "]") && isEscaped(text, back)) { if (text[back] === "[" && depth === 0) break; continue; @@ -746,6 +769,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { // rewrite. One naming a file the layout drops is left for the link check to // flag: a definition or attribute has no prose to rewrite into. for (const match of linkMasked.matchAll(REFERENCE_DEFINITION)) { + if (!definitionLabelOk(match)) continue; const [from, to] = /** @type {[number, number]} */ (match.indices?.[2]); const target = text.slice(from, to); const { dest, escapedChars } = unescapeDestination(unwrap(target)); @@ -934,6 +958,7 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { const { fenceMasked, linkMasked } = maskCode(text); for (const link of inlineLinks(linkMasked)) check(link.dest, true); for (const match of linkMasked.matchAll(REFERENCE_DEFINITION)) { + if (!definitionLabelOk(match)) continue; // A line such as `[0:33]: implementing ...` parses as a definition but is // prose; only a destination that reads as a path is checked. const destination = unescapeDestination(match[2].replace(/^<|>$/g, "")).dest; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 5e61e67fedd..24c833f57ae 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1402,6 +1402,38 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(fs.readdirSync(path.dirname(targetDir))).toEqual([]); }); + it("applies CommonMark's label rules: line endings yes, blank lines, brackets, blanks and 1000 characters no", async () => { + const body = [ + "[deck", + "slides]: ../source/transcript.txt", + "", + "[long]: ../source/transcript.txt", + `[${"x".repeat(999)}]: ../source/transcript.txt`, + `[${"x".repeat(1000)}]: ../research/gone.md`, + "[ ]: ../research/gone.md", + "[a[b]: ../research/gone.md", + "", + "[split", + "", + "label]: ../research/gone.md", + "", + "[text", + " ", + "more](../research/gone.md)", + "", + ].join("\n"); + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": body }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + body.replaceAll("]: ../source/transcript.txt", "]: ../transcript/transcript.txt"), + ); + }); + it("reads labels holding escaped brackets in definitions and inline links", async () => { const body = [ "[deck\\]][d] and [a\\]b](../source/transcript.txt)", From 06984eef648047ebef38a0017ceec3d405b86080 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:49:26 -0400 Subject: [PATCH 39/73] fix(knowledge): require carry-over evidence for every caption repeat; tighten relayout scanning - dropParagraphBoundaryRepeats drops a boundary repeat of any length only with cue-level carry-over evidence from the single opener cue (time overlap, or a verbatim continuation of the previous cue); a repeated phrase such as "I don't know." / "I don't know why." stays. - Placeholders are recognized as complete syntax (, ${NAME}, {name}, a glob segment with *), so a real budget$2026.pdf is mapped and checked. - A spaced code span rewritten through the unknown-child fallback is handed to the link check, which otherwise skips spaced spans. - src/href are read only inside HTML start tags, never in prose. Co-Authored-By: Claude Opus 5.5 --- ...st-relayout-and-caption-dedupe-59495b96.md | 2 +- .../extraction/transcript/write-transcript.js | 24 +++---- .../transcript/write-transcript.test.js | 24 +++++-- .../extraction/watch/relayout-slice.js | 65 +++++++++++++---- .../extraction/watch/relayout-slice.test.js | 69 +++++++++++++++++++ 5 files changed, 154 insertions(+), 30 deletions(-) diff --git a/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md index 32ec878de2e..e71a572a330 100644 --- a/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md +++ b/.changes/knowledge/fix-video-digest-relayout-and-caption-dedupe-59495b96.md @@ -8,4 +8,4 @@ bump: minor ### Fixed -- video-digest: an auto-caption transcript paragraph no longer repeats the previous paragraph's closing words that a rolling caption carried over: 3 to 12 words compared case- and punctuation-insensitively, or 2 when the opening cue carries them from the cue before it, so a phrase the speaker genuinely repeats stays ([#6992](https://github.com/melodic-software/claude-code-plugins/issues/6992)). +- video-digest: an auto-caption transcript paragraph no longer repeats the previous paragraph's closing words that a rolling caption carried over: 2 to 12 words compared case- and punctuation-insensitively, dropped only when the opening cue carries them from the cue before it (the cues overlap in time, or the cue starts with the previous cue's words verbatim), so a phrase the speaker genuinely repeats stays ([#6992](https://github.com/melodic-software/claude-code-plugins/issues/6992)). diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index aeb30b188ad..6b07e548542 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -34,8 +34,7 @@ function countParagraphs(transcript) { return transcript ? transcript.split("\n\n").length : 0; } -const BOUNDARY_REPEAT_MIN_WORDS = 3; -const BOUNDARY_CARRY_OVER_WORDS = 2; +const BOUNDARY_REPEAT_MIN_WORDS = 2; const BOUNDARY_REPEAT_MAX_WORDS = 12; const PARAGRAPH_STAMP = /^(\[[^\]]*\]\s*)([\s\S]*)$/; @@ -94,13 +93,14 @@ function carriesOverFromPreviousCue(stamp, run, cues) { /** * Drop the words a paragraph opens with when they repeat the previous - * paragraph's closing words, the longest run first: 3 to 12 words compared - * case- and punctuation-insensitively, or 2 words when the opening cue carries + * paragraph's closing words, the longest run first: 2 to 12 words compared + * case- and punctuation-insensitively, and only when the opening cue carries * them over from the cue before it ({@link carriesOverFromPreviousCue}). A * rolling auto caption carries its tail into the next cue, and the vendor * formatter only drops overlaps of 3 or more words it matches ignoring case - * alone. A two-word repeat the speaker actually said ("very good." / "Very - * good question") stays. A paragraph always keeps at least one word. + * alone. A repeat the speaker actually said ("very good." / "Very good + * question", "I don't know." / "I don't know why.") stays. A paragraph always + * keeps at least one word. * * @param {string} transcript - paragraphs joined by a blank line, each `[M:SS] text` * @param {readonly { startSec: number, endSec: number, text: string }[]} cues - the @@ -122,15 +122,13 @@ export function dropParagraphBoundaryRepeats(transcript, cues) { previousWords.length, currentWords.length - 1, ); - for (let length = longest; length >= BOUNDARY_CARRY_OVER_WORDS; length--) { + for (let length = longest; length >= BOUNDARY_REPEAT_MIN_WORDS; length--) { const run = currentWords.slice(0, length); if (comparableRun(previousWords.slice(-length)) !== comparableRun(run)) continue; - if ( - length < BOUNDARY_REPEAT_MIN_WORDS && - !carriesOverFromPreviousCue(current[1].trim(), run, cues) - ) { - continue; - } + // Every match, whatever its length, needs cue-level carry-over evidence: + // a speaker may genuinely repeat several words ("I don't know." / "I + // don't know why."). + if (!carriesOverFromPreviousCue(current[1].trim(), run, cues)) continue; // Keep punctuation-only tokens before the dropped words (a `>>` speaker // marker); drop any between them. const kept = [ diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index 4cbb068e6c3..84a7530afac 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -79,13 +79,13 @@ Benchmarks too and then we moved on. ); }); - it("drops a repeat of three or more words compared case- and punctuation-insensitively", () => { + it("drops a repeat of three or more words compared case- and punctuation-insensitively when the cues overlap", () => { const autoVtt = `WEBVTT 00:00:01.000 --> 00:00:03.000 we ran the -00:00:03.000 --> 00:00:05.000 +00:00:03.000 --> 00:00:05.500 numbers on benchmarks, too. 00:00:05.000 --> 00:00:07.000 @@ -122,13 +122,13 @@ Very good indeed, thanks. ["a curly apostrophe", "don’t"], ["a modifier-letter apostrophe", "donʼt"], ["no apostrophe", "dont"], - ])("matches don't against %s when dropping a 3-word repeat", (_label, variant) => { + ])("matches don't against %s when dropping a 3-word repeat across overlapping cues", (_label, variant) => { const autoVtt = `WEBVTT 00:00:01.000 --> 00:00:03.000 so we -00:00:03.000 --> 00:00:05.000 +00:00:03.000 --> 00:00:05.500 really don't care. 00:00:05.000 --> 00:00:07.000 @@ -138,6 +138,22 @@ Really ${variant} care about it. expect(result.transcript).toBe("[0:01] so we really don't care.\n\n[0:05] about it."); }); + it("keeps a three-word repeat the speaker said when no cue carries it over", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +honestly + +00:00:03.000 --> 00:00:05.000 +I don't know. + +00:00:05.000 --> 00:00:07.000 +I don't know why. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe("[0:01] honestly I don't know.\n\n[0:05] I don't know why."); + }); + it("does not count a punctuation-only token like >> as a repeated word", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 90afbb9704c..ccb1eb5bcf7 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -111,7 +111,8 @@ const FRAME_IMAGE = /\.(?:png|jpe?g)$/i; /** The slice lane directories, plus the `SLICE/` root alias. */ const SLICE_LANE_NAMES = new Set([...Object.values(LANES), "SLICE"]); const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; -const PLACEHOLDER = /[<>{}*$]/; +/** Placeholder syntax, not a real path: ``, `${NAME}`, `{name}`, or a glob segment with `*`. */ +const PLACEHOLDER = /<[^<>]*>|\$\{[^}]*\}|\{[^{}]*\}|(?:^|\/)[^/]*\*/; /** `:12`, `:12-20`, `:12,15` and `:12:5` (line and column) suffixes. */ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; /** @@ -139,6 +140,31 @@ const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|( /** A backticked span names a path when it holds a separator or ends in an extension. */ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:#.*)?$/; +/** An HTML start tag: ``. */ +const HTML_START_TAG = /<[A-Za-z][A-Za-z0-9-]*(?:\s(?:"[^"]*"|'[^']*'|[^<>"'])*)?\/?>/g; + +/** + * Offsets (into `text`) of every `src`/`href` value inside an HTML start tag, + * with the regex group that matched (2 double-quoted, 3 single-quoted, + * 4 unquoted). Prose such as `set href=../x` outside a tag is not read. + * + * @param {string} text + * @returns {{ from: number, to: number, group: number }[]} + */ +function htmlAttributeValues(text) { + /** @type {{ from: number, to: number, group: number }[]} */ + const values = []; + for (const tag of text.matchAll(HTML_START_TAG)) { + const base = /** @type {number} */ (tag.index); + for (const match of tag[0].matchAll(HTML_PATH_ATTRIBUTE)) { + const group = [2, 3, 4].find((at) => match[at] !== undefined) ?? 2; + const [from, to] = /** @type {[number, number]} */ (match.indices?.[group]); + values.push({ from: base + from, to: base + to, group }); + } + } + return values; +} + const HTML_NAMED_REFERENCES = /** @type {Record} */ ({ amp: "&", lt: "<", @@ -705,10 +731,13 @@ export function inlineLinks(text) { * @param {string} sourceFile - slice-relative path of the file * @param {string} targetFile - target-relative path the file is copied to * @param {PathMap} map - * @returns {{ text: string, rewrites: number }} + * @returns {{ text: string, rewrites: number, spacedRefs: string[] }} - `spacedRefs`, the + * rewritten code spans holding spaces, for the link check to verify */ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { let rewrites = 0; + /** @type {string[]} */ + const spacedRefs = []; const unwrap = (/** @type {string} */ target) => target.startsWith("<") ? target.slice(1, -1) : target; const rewrite = (/** @type {string} */ ref, fragments = true, escapedChars = "") => { @@ -777,9 +806,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { if (rewritten === dest) continue; edits.push({ from, to, text: target.startsWith("<") ? `<${rewritten}>` : rewritten }); } - for (const match of linkMasked.matchAll(HTML_PATH_ATTRIBUTE)) { - const group = [2, 3, 4].find((at) => match[at] !== undefined) ?? 2; - const [from, to] = /** @type {[number, number]} */ (match.indices?.[group]); + for (const { from, to, group } of htmlAttributeValues(linkMasked)) { // An attribute value may hold character references (`&`): map the // decoded path, then escape the new value for its quoting: `&` always, // the enclosing quote character, and an unquoted value that now needs @@ -816,7 +843,11 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { rewrites += 1; next = `${found.retired} (${found.note})`; } else { - next = `${ticks}${pad}${quote}${rewrite(core, fragments)}${quote}${pad}${ticks}`; + const rewritten = rewrite(core, fragments); + next = `${ticks}${pad}${quote}${rewritten}${quote}${pad}${ticks}`; + // The link check skips spans with spaces (they may be commands), so a + // spaced path this rewrite placed is handed to it explicitly. + if (spaced) spacedRefs.push(rewritten); } if (next !== whole) edits.push({ from, to: from + whole.length, text: next }); } @@ -830,7 +861,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { for (const edit of kept.reverse()) { rewritten = rewritten.slice(0, edit.from) + edit.text + rewritten.slice(edit.to); } - return { text: rewritten, rewrites }; + return { text: rewritten, rewrites, spacedRefs }; } /** @@ -920,11 +951,14 @@ function looksSliceInternal(refPath, fileDir, topLevel, sliceDirs) { * slice (another repository) and the allowed-miss prefixes are skipped. * * @param {string} rootDir - * @param {{ sliceDirs?: Set }} [options] - `sliceDirs`, the source - * slice's directories, narrows which lane-named paths count as the slice's + * @param {{ sliceDirs?: Set, spacedRefs?: Map }} [options] + * `sliceDirs`, the source slice's directories, narrows which lane-named paths + * count as the slice's; + * `spacedRefs` maps a target-relative file to the spaced code-span paths a + * rewrite placed in it, which are checked like links * @returns {{ file: string, ref: string }[]} unresolved references */ -export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { +export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } = {}) { const root = path.resolve(rootDir); const topLevel = new Set(fs.existsSync(root) ? fs.readdirSync(root) : []); /** @type {{ file: string, ref: string }[]} */ @@ -964,8 +998,8 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { const destination = unescapeDestination(match[2].replace(/^<|>$/g, "")).dest; if (PATH_LIKE.test(destination)) check(destination, true); } - for (const match of linkMasked.matchAll(HTML_PATH_ATTRIBUTE)) { - check(decodeHtmlReferences(match[2] ?? match[3] ?? match[4]), true); + for (const { from, to } of htmlAttributeValues(linkMasked)) { + check(decodeHtmlReferences(text.slice(from, to)), true); } for (const span of codeSpans(fenceMasked)) { // A span with spaces may be a command (`node run.mjs x.js --flag`): only @@ -973,6 +1007,9 @@ export function checkLayoutLinks(rootDir, { sliceDirs } = {}) { const { core, spaced } = spanPath(span.content); if (core !== null && !spaced) check(core, false); } + // Spaced spans a rewrite placed (see rewriteSliceReferences) are checked + // like links: they are known to be paths. + for (const ref of spacedRefs.get(file) ?? []) check(ref, true); } return unresolved; } @@ -1576,6 +1613,8 @@ export async function relayoutSlice({ } const discard = (/** @type {string} */ dir) => fs.rmSync(dir, { recursive: true, force: true }); let rewrites = 0; + /** @type {Map} */ + const spacedRefs = new Map(); let copied = 0; /** @type {{ file: string, ref: string }[]} */ let unresolved; @@ -1595,6 +1634,7 @@ export async function relayoutSlice({ map, ); rewrites += result.rewrites; + if (result.spacedRefs.length > 0) spacedRefs.set(entry.to, result.spacedRefs); write(entry.to, result.text); } else { fs.mkdirSync(path.dirname(path.join(staging, entry.to)), { recursive: true }); @@ -1632,6 +1672,7 @@ export async function relayoutSlice({ unresolved = checkLayoutLinks(staging, { sliceDirs: new Set([...sliceEntries].filter((entry) => !sliceFiles.includes(entry))), + spacedRefs, }); } catch (error) { discard(staging); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 24c833f57ae..2464babd0eb 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1363,6 +1363,75 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( }, ); + it("maps a real path holding $ and skips only complete placeholder syntax", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/budget$2026.pdf": "pdf", + "key-frames/visual-frames.md": "[b](../source/decks/budget$2026.pdf)\n", + "recommendations/questions.md": [ + "Missing: `research/budget$2025.md`.", + "Placeholders: `/research/x.md` `${ROOT}/research/x.md` `{tmp}/x.png` `research/*.md`.", + "", + ].join("\n"), + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/recommendations/questions.md", ref: "../research/budget$2025.md" }, + ]); + fs.writeFileSync( + path.join(sliceDir, "recommendations", "questions.md"), + "Placeholders: `/research/x.md` `${ROOT}/research/x.md` `research/*.md`.\n", + ); + const fixed = await relayoutSlice({ sliceDir, targetDir }); + expect(fixed.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "[b](../metadata/decks/budget$2026.pdf)\n", + ); + }); + + it("link-checks a spaced code span its rewrite placed through the unknown-child fallback", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "recommendations/questions.md": "Notes: `research/findings/My Missing Notes.md`.\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { + file: "analysis/recommendations/questions.md", + ref: "../research/findings/My Missing Notes.md", + }, + ]); + }); + + it("reads src and href only inside HTML tags, never in prose", async () => { + const body = [ + "Set href=../research/gone.md or src='../source/transcript.txt' in the config.", + 't', + "", + ].join("\n"); + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": body }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + body.replace( + '', + '', + ), + ); + }); + it("rewrites and checks upper-case HTML attribute names", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "key-frames/visual-frames.md": '\n' }, From 3f95aef8dee0a353e064d1c7518aad356315ef71 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 05:58:12 -0400 Subject: [PATCH 40/73] fix(knowledge): resolve before treating a path as a placeholder; exact allowed misses; escaped angle ends - The path map (in the rewrite) and the filesystem (in the link check) are consulted first; a reference counts as a placeholder only when it names no real file, so report-{draft}.pdf and report-.html are mapped. - The AGENTS.md allowed miss matches that path exactly (with an optional ./ and fragment); the plugins/, docs/, templates/ and .claude/ misses match whole leading segments, so AGENTS.md.old and docs-old/ are checked. - An angle-bracket destination ends at the first unescaped >; escaped < and > are skipped, and an unescaped < or line ending ends the attempt. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 56 ++++++++++++++---- .../extraction/watch/relayout-slice.test.js | 57 +++++++++++++++++++ 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index ccb1eb5bcf7..7fde109d469 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -63,7 +63,10 @@ export const INFO_JSON_DROPPED_KEYS = Object.freeze([ export const INFO_JSON_COMMENT_CAP = 20; -/** Path prefixes that name files in another repository, never in the slice. */ +/** + * Paths that name files in another repository, never in the slice: a + * directory (trailing `/`) matches any path under it, a file only exactly. + */ export const ALLOWED_MISS_PREFIXES = Object.freeze([ "plugins/", "docs/", @@ -72,6 +75,20 @@ export const ALLOWED_MISS_PREFIXES = Object.freeze([ "AGENTS.md", ]); +/** + * True when `refPath` (fragment already split off) is an allowed miss: + * under one of the directories, or exactly one of the files, with any + * leading `./` ignored. `AGENTS.md.old` or `docs-old/x` does not match. + * + * @param {string} refPath + */ +function isAllowedMiss(refPath) { + const plain = refPath.replace(/^(?:\.\/)+/, ""); + return ALLOWED_MISS_PREFIXES.some((entry) => + entry.endsWith("/") ? plain.startsWith(entry) : plain === entry, + ); +} + /** Source-lane folders the layout keeps, and where each one goes. */ const SOURCE_FOLDERS = new Map([ ["companion-digest", "analysis/companion-digest"], @@ -477,7 +494,7 @@ function sliceRootLookup(map) { */ function lookUpReference(refPath, sourceFile, map) { const bare = refPath.replace(/\/+$/, ""); - if (!bare || URL_SCHEME.test(bare) || PLACEHOLDER.test(bare) || bare.startsWith("/")) { + if (!bare || URL_SCHEME.test(bare) || bare.startsWith("/")) { return null; } const lookup = (/** @type {string|null} */ key) => { @@ -524,6 +541,10 @@ function lookUpReference(refPath, sourceFile, map) { const found = lookup(key); if (found) return found; } + // A placeholder (`/x`, `${ROOT}/x`, `{tmp}/x`, `research/*.md`) + // names no real file. It counts as one only now, after the path map found + // nothing, so a real `report-{draft}.pdf` is still mapped. + if (PLACEHOLDER.test(bare)) return null; // A lane folder the layout knows but this slice never produced (prose such // as "no deck in `source/decks/`") names nothing to point at. // A path below such a folder (`source/decks/slides.pdf`) names nothing either. @@ -624,8 +645,22 @@ export function inlineLinks(text) { let raw; let angle = false; if (text[at] === "<") { - const close = text.indexOf(">", at); - if (close === -1 || text.slice(at, close).includes("\n")) continue; + // The closing `>`, skipping backslash-escaped `<` and `>`; an unescaped + // `<` or a line ending inside means this is not an angle destination. + let close = -1; + for (let scan = at + 1; scan < text.length; scan += 1) { + const char = text[scan]; + if (char === "\\") { + scan += 1; + continue; + } + if (char === "\n" || char === "<") break; + if (char === ">") { + close = scan; + break; + } + } + if (close === -1) continue; angle = true; raw = text.slice(at + 1, close); at = close + 1; @@ -975,19 +1010,20 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } // span resolves, it is fine; otherwise the suffix is a fragment. if (!isLink) { const whole = splitReference(ref, { fragments: false }).refPath; - if (whole && !PLACEHOLDER.test(whole) && fs.existsSync(path.resolve(fileDir, whole))) { - return; - } + if (whole && fs.existsSync(path.resolve(fileDir, whole))) return; } const { refPath } = splitReference(ref); - if (!refPath || URL_SCHEME.test(refPath) || PLACEHOLDER.test(refPath)) return; + if (!refPath || URL_SCHEME.test(refPath)) return; if (refPath.startsWith("/") || refPath.startsWith("~")) return; - if (ALLOWED_MISS_PREFIXES.some((prefix) => refPath.startsWith(prefix))) return; + if (isAllowedMiss(refPath)) return; if (!isLink && !looksSliceInternal(refPath, fileDir, topLevel, sliceDirs)) return; const resolved = path.resolve(fileDir, refPath); const rel = path.relative(root, resolved); if (rel.startsWith("..") || path.isAbsolute(rel)) return; - if (!fs.existsSync(resolved)) unresolved.push({ file, ref: literalText(ref) }); + if (fs.existsSync(resolved)) return; + // A placeholder that names no real file is not a dangling path. + if (PLACEHOLDER.test(refPath)) return; + unresolved.push({ file, ref: literalText(ref) }); }; const { fenceMasked, linkMasked } = maskCode(text); for (const link of inlineLinks(linkMasked)) check(link.dest, true); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 2464babd0eb..5b8cbffc4bd 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1363,6 +1363,63 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( }, ); + it.skipIf(process.platform === "win32")( + "maps a real file whose name looks like a placeholder before treating it as one", + async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/findings/report-{draft}.pdf": "pdf", + "research/findings/report-.html": "html", + "recommendations/questions.md": + "Draft `research/findings/report-{draft}.pdf`, final `research/findings/report-.html`, template `research/findings/report-{name}.pdf`.\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "analysis/recommendations/questions.md")).toBe( + "Draft `../research/findings/report-{draft}.pdf`, final `../research/findings/report-.html`, template `research/findings/report-{name}.pdf`.\n", + ); + }, + ); + + it("matches the AGENTS.md allowed miss exactly and the directories by whole segment", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/claim-inventory.md": + "[a](./AGENTS.md#rules) [b](AGENTS.md) [c](AGENTS.md.old) [d](docs-old/x.md) [e](docs/x.md)\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/research/claim-inventory.md", ref: "AGENTS.md.old" }, + { file: "analysis/research/claim-inventory.md", ref: "docs-old/x.md" }, + ]); + }); + + it.skipIf(process.platform === "win32")( + "skips backslash-escaped < and > when finding the end of an angle destination", + async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/findings/a>b.md": "a\n", + "key-frames/visual-frames.md": "[x](<../research/findings/a\\>b.md>)\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "[x](<../analysis/research/findings/a\\>b.md>)\n", + ); + }, + ); + it("maps a real path holding $ and skips only complete placeholder syntax", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From bf3a72e3fd6bda6ee7c044fafb520ae6160bac5a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:08:09 -0400 Subject: [PATCH 41/73] fix(knowledge): share one destination parser and check spaced code paths as paths Reference definitions now read their destination with parseDestination, the same parser inline links use, so an escaped `>` inside `<...>` no longer ends the destination. Spaced code-span paths a rewrite placed are link-checked by the backticked-path rules: whole span first, a raw `#` or `?` kept in the name. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 156 ++++++++++-------- .../extraction/watch/relayout-slice.test.js | 36 ++++ 2 files changed, 122 insertions(+), 70 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 7fde109d469..291984a51d6 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -133,24 +133,33 @@ const PLACEHOLDER = /<[^<>]*>|\$\{[^}]*\}|\{[^{}]*\}|(?:^|\/)[^/]*\*/; /** `:12`, `:12-20`, `:12,15` and `:12:5` (line and column) suffixes. */ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; /** - * `[label]: path` reference-style link definitions, with CommonMark's label - * rules: up to 999 characters, backslash escapes allowed (`[a\]b]`), no - * unescaped bracket, line endings allowed but not a blank line. The path may - * follow one line ending. A label must also hold a non-space character, - * checked by definitionLabelOk. + * The `[label]:` opening of a reference-style link definition, with + * CommonMark's label rules: up to 999 characters, backslash escapes allowed + * (`[a\]b]`), no unescaped bracket, line endings allowed but not a blank line. + * The destination may follow one line ending; parseDestination reads it. */ -const REFERENCE_DEFINITION = - /^( {0,3}\[(?:\\[^\n]|[^[\]\\\n]|\r?\n(?![ \t]*\r?\n)){1,999}\]:[ \t]*(?:\r?\n[ \t]*)?)(<[^>\n]+>|\S+)/dgm; +const REFERENCE_DEFINITION_LABEL = + /^ {0,3}\[((?:\\[^\n]|[^[\]\\\n]|\r?\n(?![ \t]*\r?\n)){1,999})\]:[ \t]*(?:\r?\n[ \t]*)?/gm; /** - * True when a REFERENCE_DEFINITION match has a label holding a non-space - * character (CommonMark rejects an all-whitespace label). + * Every reference-style link definition in `text` whose label holds a + * non-space character (CommonMark rejects an all-whitespace label), its + * destination read by the same parser as inline links and followed by + * whitespace or the end of the text. * - * @param {RegExpMatchArray} match + * @param {string} text + * @returns {{ from: number, to: number, raw: string, angle: boolean }[]} */ -function definitionLabelOk(match) { - const prefix = match[1]; - return /\S/.test(prefix.slice(prefix.indexOf("[") + 1, prefix.lastIndexOf("]:"))); +function referenceDefinitions(text) { + const found = []; + for (const match of text.matchAll(REFERENCE_DEFINITION_LABEL)) { + if (!/\S/.test(match[1])) continue; + const from = /** @type {number} */ (match.index) + match[0].length; + const parsed = parseDestination(text, from); + if (parsed === null || parsed.raw === "" || /\S/.test(text[parsed.end] ?? " ")) continue; + found.push({ from, to: parsed.end, raw: parsed.raw, angle: parsed.angle }); + } + return found; } /** `src` and `href` in inline HTML, double-, single- or unquoted. */ const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/dgi; @@ -622,6 +631,51 @@ function skipLinkSpace(text, at) { return next; } +/** + * The link destination starting at `at`, read as CommonMark does for both + * inline links and reference definitions: `<...>` up to the first unescaped + * `>` (an unescaped `<` or a line ending makes it no destination), or a bare + * run with balanced parentheses up to whitespace or an unbalanced `)`, where + * a backslash escapes any ASCII punctuation. `raw` is the destination as + * written, without the angle brackets. + * + * @param {string} text + * @param {number} at + * @returns {{ raw: string, angle: boolean, end: number }|null} + */ +export function parseDestination(text, at) { + if (text[at] === "<") { + for (let scan = at + 1; scan < text.length; scan += 1) { + const char = text[scan]; + if (char === "\\") { + scan += 1; + continue; + } + if (char === "\n" || char === "<") return null; + if (char === ">") return { raw: text.slice(at + 1, scan), angle: true, end: scan + 1 }; + } + return null; + } + let end = at; + let depth = 0; + while (end < text.length) { + const char = text[end]; + if (char === "\\" && /[!-/:-@[-`{-~]/.test(text[end + 1] ?? "")) { + end += 2; + continue; + } + if (/\s/.test(char)) break; + if (char === "(") depth += 1; + else if (char === ")") { + if (depth === 0) break; + depth -= 1; + } + end += 1; + } + if (end === at || depth !== 0) return null; + return { raw: text.slice(at, end), angle: false, end }; +} + /** * Every inline link in `text`, scanned the way CommonMark reads a * destination: `<...>`, or a run with balanced or backslash-escaped @@ -642,47 +696,10 @@ export function inlineLinks(text) { let at = open + 2; at = skipLinkSpace(text, at); const destStart = at; - let raw; - let angle = false; - if (text[at] === "<") { - // The closing `>`, skipping backslash-escaped `<` and `>`; an unescaped - // `<` or a line ending inside means this is not an angle destination. - let close = -1; - for (let scan = at + 1; scan < text.length; scan += 1) { - const char = text[scan]; - if (char === "\\") { - scan += 1; - continue; - } - if (char === "\n" || char === "<") break; - if (char === ">") { - close = scan; - break; - } - } - if (close === -1) continue; - angle = true; - raw = text.slice(at + 1, close); - at = close + 1; - } else { - let depth = 0; - while (at < text.length) { - const ch = text[at]; - if (ch === "\\" && (text[at + 1] === "(" || text[at + 1] === ")")) { - at += 2; - continue; - } - if (/\s/.test(ch)) break; - if (ch === "(") depth += 1; - else if (ch === ")") { - if (depth === 0) break; - depth -= 1; - } - at += 1; - } - if (at === destStart || depth !== 0) continue; - raw = text.slice(destStart, at); - } + const parsed = parseDestination(text, at); + if (parsed === null) continue; + const { raw, angle } = parsed; + at = parsed.end; const destEnd = at; // An optional title, then the closing `)`. // Spaces with at most one line ending may separate the parts (CommonMark). @@ -773,8 +790,6 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { let rewrites = 0; /** @type {string[]} */ const spacedRefs = []; - const unwrap = (/** @type {string} */ target) => - target.startsWith("<") ? target.slice(1, -1) : target; const rewrite = (/** @type {string} */ ref, fragments = true, escapedChars = "") => { const { refPath, suffix, encoded } = splitReference(ref, { fragments }); const found = lookUpReference(refPath, sourceFile, map); @@ -832,14 +847,12 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { // Reference-style definitions and HTML src/href values take the same path // rewrite. One naming a file the layout drops is left for the link check to // flag: a definition or attribute has no prose to rewrite into. - for (const match of linkMasked.matchAll(REFERENCE_DEFINITION)) { - if (!definitionLabelOk(match)) continue; - const [from, to] = /** @type {[number, number]} */ (match.indices?.[2]); - const target = text.slice(from, to); - const { dest, escapedChars } = unescapeDestination(unwrap(target)); + for (const { from, to, angle } of referenceDefinitions(linkMasked)) { + const raw = text.slice(angle ? from + 1 : from, angle ? to - 1 : to); + const { dest, escapedChars } = unescapeDestination(raw); const rewritten = rewrite(dest, true, escapedChars); if (rewritten === dest) continue; - edits.push({ from, to, text: target.startsWith("<") ? `<${rewritten}>` : rewritten }); + edits.push({ from, to, text: angle ? `<${rewritten}>` : rewritten }); } for (const { from, to, group } of htmlAttributeValues(linkMasked)) { // An attribute value may hold character references (`&`): map the @@ -1005,7 +1018,11 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } for (const file of markdown) { const fileDir = path.dirname(path.join(root, file)); const text = fs.readFileSync(path.join(root, file), "utf8"); - const check = (/** @type {string} */ ref, /** @type {boolean} */ isLink) => { + const check = ( + /** @type {string} */ ref, + /** @type {boolean} */ isLink, + /** @type {boolean} */ knownPath = isLink, + ) => { // A backticked path may hold a raw `#` or `?` in its name: if the whole // span resolves, it is fine; otherwise the suffix is a fragment. if (!isLink) { @@ -1016,7 +1033,7 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } if (!refPath || URL_SCHEME.test(refPath)) return; if (refPath.startsWith("/") || refPath.startsWith("~")) return; if (isAllowedMiss(refPath)) return; - if (!isLink && !looksSliceInternal(refPath, fileDir, topLevel, sliceDirs)) return; + if (!knownPath && !looksSliceInternal(refPath, fileDir, topLevel, sliceDirs)) return; const resolved = path.resolve(fileDir, refPath); const rel = path.relative(root, resolved); if (rel.startsWith("..") || path.isAbsolute(rel)) return; @@ -1027,11 +1044,10 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } }; const { fenceMasked, linkMasked } = maskCode(text); for (const link of inlineLinks(linkMasked)) check(link.dest, true); - for (const match of linkMasked.matchAll(REFERENCE_DEFINITION)) { - if (!definitionLabelOk(match)) continue; + for (const { raw } of referenceDefinitions(linkMasked)) { // A line such as `[0:33]: implementing ...` parses as a definition but is // prose; only a destination that reads as a path is checked. - const destination = unescapeDestination(match[2].replace(/^<|>$/g, "")).dest; + const destination = unescapeDestination(raw).dest; if (PATH_LIKE.test(destination)) check(destination, true); } for (const { from, to } of htmlAttributeValues(linkMasked)) { @@ -1043,9 +1059,9 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } const { core, spaced } = spanPath(span.content); if (core !== null && !spaced) check(core, false); } - // Spaced spans a rewrite placed (see rewriteSliceReferences) are checked - // like links: they are known to be paths. - for (const ref of spacedRefs.get(file) ?? []) check(ref, true); + // Spaced spans a rewrite placed (see rewriteSliceReferences) are known to + // be paths: checked by the backticked-path rules, whole span first. + for (const ref of spacedRefs.get(file) ?? []) check(ref, false, true); } return unresolved; } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 5b8cbffc4bd..5e99359f87c 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1420,6 +1420,42 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( }, ); + it.skipIf(process.platform === "win32")( + "reads a reference definition's angle destination past a backslash-escaped >", + async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/a>b.pdf": "pdf", + "key-frames/vision-plan.md": "[deck][deck]\n\n[deck]: <../source/decks/a\\>b.pdf>\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/vision-plan.md")).toBe( + "[deck][deck]\n\n[deck]: <../metadata/decks/a\\>b.pdf>\n", + ); + }, + ); + + it("link-checks a spaced code-span path with a raw # as one whole path", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/My #1.pdf": "pdf", + "key-frames/vision-plan.md": "Deck: `../source/decks/My #1.pdf`.\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.unresolved).toEqual([]); + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/vision-plan.md")).toBe( + "Deck: `../metadata/decks/My #1.pdf`.\n", + ); + }); + it("maps a real path holding $ and skips only complete placeholder syntax", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From a1455e1ae941444c71b1c96089ca407c47346116 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:08:59 -0400 Subject: [PATCH 42/73] fix(knowledge): skip an HTML attribute match with no captured span instead of throwing Co-Authored-By: Claude Opus 5.5 --- .../skills/video-digest/extraction/watch/relayout-slice.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 291984a51d6..8717eb88775 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -184,8 +184,9 @@ function htmlAttributeValues(text) { const base = /** @type {number} */ (tag.index); for (const match of tag[0].matchAll(HTML_PATH_ATTRIBUTE)) { const group = [2, 3, 4].find((at) => match[at] !== undefined) ?? 2; - const [from, to] = /** @type {[number, number]} */ (match.indices?.[group]); - values.push({ from: base + from, to: base + to, group }); + const span = match.indices?.[group]; + if (!span) continue; + values.push({ from: base + span[0], to: base + span[1], group }); } } return values; From 1649d5c556b6966a686960f6299eb3b1dc2587dc Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:19:47 -0400 Subject: [PATCH 43/73] fix(knowledge): treat HTML tag text as data and require whole-line reference definitions Mask HTML start tags and comments from the inline-link, reference-definition and backticked-path passes; only the src/href pass reads tags. Tokenize a tag's attributes left to right so src/href match only as attribute names. Read a reference definition only when its line holds nothing after the destination but an optional title. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 6 +- .../extraction/watch/relayout-slice.js | 93 +++++++++++++------ .../extraction/watch/relayout-slice.test.js | 77 +++++++++++++++ 3 files changed, 147 insertions(+), 29 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 0c8cc49bac1..3db9e041ad7 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -108,11 +108,13 @@ the companion sources brief, and the fetched `decks/` and `attachments/`), `fram paths yt-dlp records. The script's header comment is the authoritative mapping. Each slice path the copied markdown names is rewritten through a path map relative to the file's new location: inline links (balanced or escaped parentheses in the destination, quoted or parenthesized titles), -reference-style links, HTML `src`/`href` in any quoting or case (character references decoded), +reference-style definitions whose line holds nothing after the destination but an optional +title, HTML `src`/`href` attributes in any quoting or case (character references decoded), and backticked paths, with each segment percent-decoded (and re-encoded on rewrite) and `\` separators read as `/`. A relative link that leaves the slice keeps pointing at the same file. Fenced code blocks are -left untouched, and link syntax inside an inline code span is not read as a link. +left untouched, and link syntax inside an inline code span is not read as a link. Text inside an +HTML tag or comment is data: only a tag's own `src`/`href` attributes are read. One naming a pipeline file the layout does not keep (such as `run-state/watch.json`, the slice's own journey `README.md`, or an extracted frame's bare name under `--no-media`) becomes plain text saying it is not retained; one naming a directory whose files went to several places (`source/`) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 8717eb88775..c2a330b4055 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -140,12 +140,16 @@ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; */ const REFERENCE_DEFINITION_LABEL = /^ {0,3}\[((?:\\[^\n]|[^[\]\\\n]|\r?\n(?![ \t]*\r?\n)){1,999})\]:[ \t]*(?:\r?\n[ \t]*)?/gm; +/** What may follow a definition's destination: a line ending, or a title then one. */ +const DEFINITION_TAIL = + /^(?:[ \t]+(?:"(?:\\[^]|[^"\\])*"|'(?:\\[^]|[^'\\])*'|\((?:\\[^]|[^()\\])*\)))?[ \t]*(?:\r?\n|$)/; /** * Every reference-style link definition in `text` whose label holds a * non-space character (CommonMark rejects an all-whitespace label), its - * destination read by the same parser as inline links and followed by - * whitespace or the end of the text. + * destination read by the same parser as inline links and followed, on its + * line, by nothing but whitespace and an optional title. A line such as + * `[note]: source/x.txt is illustrative` is prose, not a definition. * * @param {string} text * @returns {{ from: number, to: number, raw: string, angle: boolean }[]} @@ -156,23 +160,32 @@ function referenceDefinitions(text) { if (!/\S/.test(match[1])) continue; const from = /** @type {number} */ (match.index) + match[0].length; const parsed = parseDestination(text, from); - if (parsed === null || parsed.raw === "" || /\S/.test(text[parsed.end] ?? " ")) continue; + if (parsed === null || parsed.raw === "") continue; + if (!DEFINITION_TAIL.test(text.slice(parsed.end))) continue; found.push({ from, to: parsed.end, raw: parsed.raw, angle: parsed.angle }); } return found; } -/** `src` and `href` in inline HTML, double-, single- or unquoted. */ -const HTML_PATH_ATTRIBUTE = /(\b(?:src|href)\s*=\s*)(?:"([^"\n]+)"|'([^'\n]+)'|([^\s"'=<>`]+))/dgi; +/** + * One attribute of an HTML start tag, read from where the previous one ended: + * a name, then an optional double-, single- or unquoted value. Sticky, so a + * `src=` inside another attribute's quoted value is never read as a name. + */ +const HTML_ATTRIBUTE = /[\s/]*([^\s"'>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/dy; /** A backticked span names a path when it holds a separator or ends in an extension. */ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:#.*)?$/; /** An HTML start tag: ``. */ -const HTML_START_TAG = /<[A-Za-z][A-Za-z0-9-]*(?:\s(?:"[^"]*"|'[^']*'|[^<>"'])*)?\/?>/g; +const HTML_START_TAG = /<([A-Za-z][A-Za-z0-9-]*)(?:\s(?:"[^"]*"|'[^']*'|[^<>"'])*)?\/?>/g; +/** An HTML comment. */ +const HTML_COMMENT = //g; /** * Offsets (into `text`) of every `src`/`href` value inside an HTML start tag, - * with the regex group that matched (2 double-quoted, 3 single-quoted, - * 4 unquoted). Prose such as `set href=../x` outside a tag is not read. + * with the group that matched (2 double-quoted, 3 single-quoted, + * 4 unquoted). The tag's attributes are read left to right, so only a real + * attribute counts: not prose such as `set href=../x` outside a tag, not + * `data-src`, and not `src=` text inside another attribute's value. * * @param {string} text * @returns {{ from: number, to: number, group: number }[]} @@ -182,16 +195,33 @@ function htmlAttributeValues(text) { const values = []; for (const tag of text.matchAll(HTML_START_TAG)) { const base = /** @type {number} */ (tag.index); - for (const match of tag[0].matchAll(HTML_PATH_ATTRIBUTE)) { - const group = [2, 3, 4].find((at) => match[at] !== undefined) ?? 2; + HTML_ATTRIBUTE.lastIndex = 1 + tag[1].length; + for (let match; (match = HTML_ATTRIBUTE.exec(tag[0])) !== null; ) { + if (!/^(?:src|href)$/i.test(match[1])) continue; + const group = [2, 3, 4].find((at) => match?.[at] !== undefined) ?? 2; const span = match.indices?.[group]; - if (!span) continue; + if (!span || span[0] === span[1] || tag[0].slice(span[0], span[1]).includes("\n")) continue; values.push({ from: base + span[0], to: base + span[1], group }); } } return values; } +/** + * Ranges of `text` holding HTML start tags and comments: attribute and + * comment text is data, so the link, definition and backticked-path passes + * skip it; only htmlAttributeValues reads a tag. + * + * @param {string} text + * @returns {[number, number][]} + */ +function htmlRanges(text) { + return [...text.matchAll(HTML_COMMENT), ...text.matchAll(HTML_START_TAG)].map((match) => { + const from = /** @type {number} */ (match.index); + return /** @type {[number, number]} */ ([from, from + match[0].length]); + }); +} + const HTML_NAMED_REFERENCES = /** @type {Record} */ ({ amp: "&", lt: "<", @@ -812,11 +842,12 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { if (next !== ref) rewrites += 1; return next; }; - const { fenceMasked, linkMasked } = maskCode(text); + const { fenceMasked, linkMasked, tagText } = maskCode(text); // Every pass records edits by offset into the original text; the scans run - // over masked copies, so nothing inside a fenced code block is touched and - // no link syntax inside an inline code span is read as a link. Inline code - // spans themselves are read for paths (the backticked-path rewrite). + // over masked copies, so nothing inside a fenced code block, an HTML tag or + // an HTML comment is touched and no link syntax inside an inline code span + // is read as a link. Inline code spans themselves are read for paths (the + // backticked-path rewrite); tags only for their src/href values. /** @type {{ from: number, to: number, text: string }[]} */ const edits = []; // Inline links: a link naming a file the layout drops keeps its label in @@ -855,7 +886,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { if (rewritten === dest) continue; edits.push({ from, to, text: angle ? `<${rewritten}>` : rewritten }); } - for (const { from, to, group } of htmlAttributeValues(linkMasked)) { + for (const { from, to, group } of htmlAttributeValues(tagText)) { // An attribute value may hold character references (`&`): map the // decoded path, then escape the new value for its quoting: `&` always, // the enclosing quote character, and an unquoted value that now needs @@ -914,16 +945,19 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { } /** - * Copies of `text` with code masked by spaces (line endings kept, so every - * offset still matches the original): `fenceMasked` blanks top-level fenced + * Copies of `text` with code and HTML masked by spaces (line endings kept, so + * every offset still matches the original). All three blank top-level fenced * code blocks (``` or ~~~ opening at column 0; an unclosed fence runs to the - * end), and `linkMasked` also blanks inline code spans. Fences that are - * indented or inside a block quote or list item, and indented code blocks, - * are not masked: telling them apart needs container parsing. What they hold - * is read as ordinary text, so the link check still sees a dangling path. + * end). `fenceMasked` also blanks HTML start tags and comments, whose text is + * data; `linkMasked` blanks those and inline code spans; `tagText` blanks + * inline code spans and HTML comments, keeping start tags for the src/href + * pass. Fences that are indented or inside a block quote or list item, and + * indented code blocks, are not masked: telling them apart needs container + * parsing. What they hold is read as ordinary text, so the link check still + * sees a dangling path. * * @param {string} text - * @returns {{ fenceMasked: string, linkMasked: string }} + * @returns {{ fenceMasked: string, linkMasked: string, tagText: string }} */ export function maskCode(text) { const blank = (/** @type {string} */ source, /** @type {[number, number][]} */ ranges) => { @@ -957,9 +991,14 @@ export function maskCode(text) { if (open !== null) fences.push([open.start, text.length]); const fenceMasked = blank(text, fences); /** @type {[number, number][]} */ - /** @type {[number, number][]} */ const spans = codeSpans(fenceMasked).map((span) => [span.start, span.end]); - return { fenceMasked, linkMasked: blank(fenceMasked, spans) }; + const codeMasked = blank(fenceMasked, spans); + const html = htmlRanges(codeMasked); + return { + fenceMasked: blank(fenceMasked, html), + linkMasked: blank(codeMasked, html), + tagText: codeMasked.replace(HTML_COMMENT, (comment) => comment.replace(/[^\n]/g, " ")), + }; } /** @@ -1043,7 +1082,7 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } if (PLACEHOLDER.test(refPath)) return; unresolved.push({ file, ref: literalText(ref) }); }; - const { fenceMasked, linkMasked } = maskCode(text); + const { fenceMasked, linkMasked, tagText } = maskCode(text); for (const link of inlineLinks(linkMasked)) check(link.dest, true); for (const { raw } of referenceDefinitions(linkMasked)) { // A line such as `[0:33]: implementing ...` parses as a definition but is @@ -1051,7 +1090,7 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } const destination = unescapeDestination(raw).dest; if (PATH_LIKE.test(destination)) check(destination, true); } - for (const { from, to } of htmlAttributeValues(linkMasked)) { + for (const { from, to } of htmlAttributeValues(tagText)) { check(decodeHtmlReferences(text.slice(from, to)), true); } for (const span of codeSpans(fenceMasked)) { diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 5e99359f87c..b4b59b45527 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1550,6 +1550,83 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it("reads no link, definition or backticked path inside an HTML tag or comment", async () => { + const body = [ + 'x', + "t", + "", + "", + "", + ].join("\n"); + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": body }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe(body); + const root = makeDir("relayout-check-"); + writeTree(root, { + "frames/visual-frames.md": [ + 'x', + "t", + "", + "", + ].join("\n"), + }); + expect(checkLayoutLinks(root)).toEqual([]); + }); + + it("reads src and href only as attribute names, never inside another attribute's value", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "key-frames/visual-frames.md": + 't\n', + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + 't\n', + ); + const root = makeDir("relayout-check-"); + writeTree(root, { + "frames/visual-frames.md": + "\"href='key/gone.png'\"\n", + }); + expect(checkLayoutLinks(root)).toEqual([ + { file: "frames/visual-frames.md", ref: "key/missing.png" }, + ]); + }); + + it("reads a reference definition only when the rest of its line is blank or a title", async () => { + const body = "[note]: ../source/transcript.txt is illustrative\n"; + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": body }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe(body); + const root = makeDir("relayout-check-"); + writeTree(root, { + "frames/visual-frames.md": [ + "[note]: source/missing.txt is illustrative", + '[t]: key/gone.png "A title" ', + "", + ].join("\n"), + }); + expect(checkLayoutLinks(root)).toEqual([ + { file: "frames/visual-frames.md", ref: "key/gone.png" }, + ]); + }); + it.each([ ["null", "null"], ["an array", "[]"], From 777297c3d0e9e1a0c829d8bb3b368ee76b2881ed Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:27:11 -0400 Subject: [PATCH 44/73] fix(knowledge): check recorded temp dirs in no-media mode and accept query suffixes on bare paths Run the OS-temp-dir provenance and slice-overlap checks on every temp dir watch.json records, under --no-media too, since close deletes them. Let PATH_LIKE take an optional ?query before an optional #fragment so a bare file destination such as missing.md?download=1 is link-checked. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 7 +- .../extraction/watch/relayout-slice.js | 16 +++-- .../extraction/watch/relayout-slice.test.js | 66 +++++++++++++------ 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 3db9e041ad7..7a49ec333a6 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -87,9 +87,10 @@ without a finite number or null `timestampSec`), any frame image, a frame (anchor frames carry their time in their name), or the caption tracks. It refuses a temp session holding several videos (an X post with more than one), naming them, since the watch does not record which one is primary; a video whose id is not this slice's (for an X post, whose -info JSON does not name this post as its `display_id`); a recorded work, frames or -contact-sheets dir outside the OS temp dir or overlapping the slice; and a symlink in the slice -or the temp session, since a copy would follow it. It refuses when the slice is, holds, or sits +info JSON does not name this post as its `display_id`); and a symlink in the slice or the temp +session, since a copy would follow it. In every mode, `--no-media` included, it refuses a recorded +work, frames or contact-sheets dir outside the OS temp dir or overlapping the slice, existing or +not, since `close` deletes those. It refuses when the slice is, holds, or sits inside a path the run writes: the target, `.relayout-backup`, or a `..relayout-*` staging dir beside the target. In every mode, `--no-media` included, it refuses when one of those paths is, holds, or sits inside a diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index c2a330b4055..97b3fcd162c 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -172,8 +172,11 @@ function referenceDefinitions(text) { * `src=` inside another attribute's quoted value is never read as a name. */ const HTML_ATTRIBUTE = /[\s/]*([^\s"'>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/dy; -/** A backticked span names a path when it holds a separator or ends in an extension. */ -const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:#.*)?$/; +/** + * A backticked span names a path when it holds a separator or ends in an + * extension, optionally followed by a `?query`, a `#fragment`, or both. + */ +const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:\?[^#]*)?(?:#.*)?$/; /** An HTML start tag: ``. */ const HTML_START_TAG = /<([A-Za-z][A-Za-z0-9-]*)(?:\s(?:"[^"]*"|'[^']*'|[^<>"'])*)?\/?>/g; @@ -1549,13 +1552,12 @@ export async function relayoutSlice({ } } - // `--no-media` reads nothing from the temp session. - const temp = noMedia ? {} : recorded; // watch.json names the temp dirs, and the watch only ever makes them in the // OS temp dir; anything else (a hand-edited path, the slice, the target) is - // not a temp session to read from. + // not a temp session, and `close` would delete it. Checked in every mode, + // whether or not the dir still exists. const realTmp = fs.realpathSync.native(os.tmpdir()); - for (const dir of [temp.workDir, temp.framesDir, temp.contactSheetsDir]) { + for (const dir of [recorded.workDir, recorded.framesDir, recorded.contactSheetsDir]) { if (!dir) continue; if (!isSameOrInside(realPathOf(dir), realTmp) || realPathOf(dir) === realTmp) { return refuse(`temp session dir ${dir} is not inside the OS temp dir`); @@ -1564,6 +1566,8 @@ export async function relayoutSlice({ return refuse(`temp session dir ${dir} overlaps the slice`); } } + // `--no-media` reads nothing from the temp session. + const temp = noMedia ? {} : recorded; const workFiles = temp.workDir ? await listWorkDirFiles(temp.workDir) : []; // The watch does not record which work file is the primary entry's, so only // a work dir holding exactly one video is unambiguous (an X post with diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index b4b59b45527..26e46913ad1 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -765,31 +765,40 @@ describe("relayoutSlice", () => { expect(fs.existsSync(path.join(targetDir, `media/${VIDEO_ID}.mp4`))).toBe(true); }); - it("refuses a recorded contact-sheets dir outside the OS temp dir", async () => { - const { sliceDir, targetDir } = makeFixture(); - const statePath = path.join(sliceDir, "run-state", "watch.json"); - const state = JSON.parse(fs.readFileSync(statePath, "utf8")); - state.tempSession.contactSheetsDir = path.join(path.parse(os.tmpdir()).root, "relayout-sheets"); - fs.writeFileSync(statePath, JSON.stringify(state)); + it.each([false, true])( + "refuses a recorded contact-sheets dir outside the OS temp dir (noMedia: %s)", + async (noMedia) => { + const { sliceDir, targetDir } = makeFixture(); + const statePath = path.join(sliceDir, "run-state", "watch.json"); + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + state.tempSession.contactSheetsDir = path.join( + path.parse(os.tmpdir()).root, + "relayout-sheets", + ); + fs.writeFileSync(statePath, JSON.stringify(state)); - const result = await relayoutSlice({ sliceDir, targetDir }); + const result = await relayoutSlice({ sliceDir, targetDir, noMedia }); - expect(result.exitCode).toBe(1); - expect(fs.existsSync(targetDir)).toBe(false); - }); + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }, + ); - it("refuses a recorded contact-sheets dir that overlaps the slice", async () => { - const { sliceDir, targetDir } = makeFixture(); - const statePath = path.join(sliceDir, "run-state", "watch.json"); - const state = JSON.parse(fs.readFileSync(statePath, "utf8")); - state.tempSession.contactSheetsDir = path.join(sliceDir, "key-frames"); - fs.writeFileSync(statePath, JSON.stringify(state)); + it.each([false, true])( + "refuses a recorded contact-sheets dir that overlaps the slice (noMedia: %s)", + async (noMedia) => { + const { sliceDir, targetDir } = makeFixture(); + const statePath = path.join(sliceDir, "run-state", "watch.json"); + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + state.tempSession.contactSheetsDir = path.join(sliceDir, "key-frames"); + fs.writeFileSync(statePath, JSON.stringify(state)); - const result = await relayoutSlice({ sliceDir, targetDir }); + const result = await relayoutSlice({ sliceDir, targetDir, noMedia }); - expect(result.exitCode).toBe(1); - expect(fs.existsSync(targetDir)).toBe(false); - }); + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }, + ); it("reads no link inside a code span that spans lines, and treats a lone backtick as text", async () => { const body = [ @@ -1730,6 +1739,23 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ); }); + it("checks a bare-file definition destination carrying a query, a fragment or both", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/claim-inventory.md": + "[notes]: missing.md?download=1\n[both]: gone.md?download=1#top\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved.map((entry) => entry.ref)).toEqual([ + "missing.md?download=1", + "gone.md?download=1#top", + ]); + }); + it("flags a reference-style definition to a path that does not exist", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "research/claim-inventory.md": "[gone][g]\n\n[g]: findings/gone.md\n" }, From 381441d92d7a4608320591ff07a5039826a9afd9 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:33:04 -0400 Subject: [PATCH 45/73] fix(knowledge): write the upstream title as plain Markdown text in the relayout README Co-Authored-By: Claude Opus 5.5 --- .../video-digest/extraction/watch/relayout-slice.js | 13 +++++++++++-- .../extraction/watch/relayout-slice.test.js | 13 +++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 97b3fcd162c..9aac910daec 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -1132,6 +1132,15 @@ const CONTENTS_LINES = /** @type {const} */ ([ ["analysis/", "the digest's research and recommendations"], ]); +/** + * Plain text for a Markdown line: one line, with every Markdown-significant character + * backslash-escaped, so an upstream title never renders (or is link-checked) as syntax. + * @param {string} text + */ +export function markdownText(text) { + return text.replace(/\s+/g, " ").trim().replace(/[\\`*_{}[\]()<>#+\-.!|~&]/g, "\\$&"); +} + /** * @param {{ title: string, sourceUrl: string, videoId: string, acquiredAt?: string }} meta * @param {Set} presentTopDirs - target directories that received files @@ -1142,11 +1151,11 @@ function provenanceReadme({ title, sourceUrl, videoId, acquiredAt }, presentTopD ([dir, label]) => `- \`${dir}\`: ${label}`, ); return [ - `# ${title}`, + `# ${markdownText(title)}`, "", "## Provenance", "", - `- Canonical origin: <${sourceUrl}>`, + `- Canonical origin: <${sourceUrl.replace(/[\s<>]/g, encodeURIComponent)}>`, `- Video id: \`${videoId}\``, `- Fetched${fetched} through \`/knowledge:video-digest\` and re-laid out from its finished slice.`, "", diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 26e46913ad1..a8062acc789 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -119,6 +119,19 @@ afterEach(() => { }); describe("relayoutSlice", () => { + it("writes an upstream title with Markdown syntax as plain text in the README", async () => { + const { sliceDir, targetDir } = makeFixture(); + const watchPath = path.join(sliceDir, "run-state/watch.json"); + const watch = JSON.parse(fs.readFileSync(watchPath, "utf8")); + fs.writeFileSync(watchPath, JSON.stringify({ ...watch, title: "[Episode](trailer.md)\n#1" })); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + const readme = fs.readFileSync(path.join(targetDir, "README.md"), "utf8"); + expect(readme.split("\n")[0]).toBe("# \\[Episode\\]\\(trailer\\.md\\) \\#1"); + }); + it("copies a closed slice into the corpus layout", async () => { const { sliceDir, targetDir } = makeFixture(); From 61a65c8267bbf0cd693ae6d5e21a0fdc916fd925 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:44:05 -0400 Subject: [PATCH 46/73] fix(knowledge): keep the skill contract additive and decode references in link destinations Restore the output contract's root-relocation guidance and the SKILL.md routing row verbatim, and name relayout-slice.js as an additional option. relayout-slice.js now skips escaped brackets when finding a link's opening `[`, and decodes character references in Markdown inline and definition destinations before mapping and checking them, writing `&` back as `&` when the original used a reference. Co-Authored-By: Claude Opus 5.5 --- .../knowledge/skills/video-digest/SKILL.md | 2 +- .../video-digest/context/output-contract.md | 10 ++- .../extraction/watch/relayout-slice.js | 63 ++++++++++++------- .../extraction/watch/relayout-slice.test.js | 57 +++++++++++++++++ 4 files changed, 107 insertions(+), 25 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/SKILL.md b/plugins/knowledge/skills/video-digest/SKILL.md index 48fde980873..6e7071cc64f 100644 --- a/plugins/knowledge/skills/video-digest/SKILL.md +++ b/plugins/knowledge/skills/video-digest/SKILL.md @@ -43,7 +43,7 @@ Read a spoke **only when its condition holds**. These are mutually exclusive by | `context/watch-pipeline.md` | when running the **watch** action (or `resume` into it), the full per-phase procedure | | `context/workflow.md` | when you want the watch phase-flow diagram + phase summary table | | `context/watch-queue.md` | when running any **queue** action, or dequeuing via `watch` / `watch `. Claim protocol and preflight decision table | -| `context/output-contract.md` | when about to write or stage slice artifacts, when a non-default `library_dir` work root is configured, or when re-laying out a closed slice into a corpus layout | +| `context/output-contract.md` | when about to write or stage slice artifacts, or when a non-default `library_dir` work root is configured | | `context/quality-gates.md` | when grading a phase or a finished slice against binary criteria | | `context/synthesis-contract.md` | when harvesting decks, or when applying the frame promotion bar | | `templates/vision-json-shapes.md` | when briefing a vision subagent to write a sheet triage, promotion-decisions or quality-audit JSON file. Each file's keys and the rules its validators enforce | diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 7a49ec333a6..9ed19620d64 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -55,8 +55,14 @@ once per process, and the new name wins when both are set. **Scope of the setting.** `library_dir` relocates the work *root*; it does not reshape the `//` sub-path itself. A consumer whose own convention lands source -material at a differently-shaped path (for example `sources///`) lands under -`library_dir` as written, then re-lays out the slice with `relayout-slice.js`: +material at a differently-shaped path (for example `sources///`) does not get that +shape from this skill: land under `library_dir` as-written and re-lay-out by hand, or fork +the sub-path in your own automation. This skill's contract is root relocation only. +For the knowledge-corpus layout there is also `relayout-slice.js`, described next. + +### Re-laying out a slice with `relayout-slice.js` + +Run it on a slice landed under `library_dir`: ```bash node "/extraction/run.mjs" --data-dir "" watch/relayout-slice.js "" "" [--no-media] [--replace] diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 9aac910daec..86e0a8e8922 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -440,25 +440,36 @@ function deriveDirectoryMap(fileMap) { const LITERAL_HASH = ""; const LITERAL_QUERY = ""; +/** Stand-in for a backslash-escaped `&`, which starts no character reference. */ +const LITERAL_AMP = ""; + /** - * A link or definition destination with its CommonMark backslash escapes of - * ASCII punctuation decoded. An escaped `#` or `?` is literal, not a fragment - * or query, so it becomes a stand-in that splitReference keeps in the path. - * `escapedChars` lists the characters the original escaped, to re-escape on - * rewrite. + * A link or definition destination decoded as CommonMark does: backslash + * escapes of ASCII punctuation, then character references (`&`, + * `&`). An escaped `#` or `?` is literal, not a fragment or query, so it + * becomes a stand-in that splitReference keeps in the path. `escapedChars` + * lists the characters the original escaped, to re-escape on rewrite; + * `references` marks a destination that held a character reference, so a + * rewrite writes `&` as `&`. * * @param {string} raw - * @returns {{ dest: string, escapedChars: string }} + * @returns {{ dest: string, escapedChars: string, references: boolean }} */ function unescapeDestination(raw) { let escapedChars = ""; - const dest = raw.replace(/\\([!-/:-@[-`{-~])/g, (_match, char) => { + const unescaped = raw.replace(/\\([!-/:-@[-`{-~])/g, (_match, char) => { if (!escapedChars.includes(char)) escapedChars += char; if (char === "#") return LITERAL_HASH; if (char === "?") return LITERAL_QUERY; + if (char === "&") return LITERAL_AMP; return char; }); - return { dest, escapedChars }; + const decoded = decodeHtmlReferences(unescaped); + return { + dest: decoded.replaceAll(LITERAL_AMP, "&"), + escapedChars, + references: decoded !== unescaped, + }; } /** @param {string} value */ @@ -642,6 +653,7 @@ const NOT_RETAINED = "not retained in this copy"; * @property {number} destEnd - offset just past it * @property {boolean} angle - written as `<...>` * @property {string} escapedChars - the punctuation the destination backslash-escaped + * @property {boolean} references - the destination held a character reference (`&`) * @property {string} label - the link text, `!` and brackets removed * @property {number} labelStart - offset of the link's `[` (or its `!`), -1 when unmatched * @property {number} labelEnd - offset of the `]` that closes the link text @@ -761,8 +773,8 @@ export function inlineLinks(text) { if (text[at] !== ")") continue; const linkEnd = at + 1; // The `[` that opens this link's text, matching nested brackets. An - // escaped `]`, or an opening `[` that is escaped (`\[watch](...)`) or - // missing, means this is not a link. + // escaped `]`, or no unescaped opening `[` (`\[watch](...)`), means this + // is not a link. if (isEscaped(text, open)) continue; let labelStart = -1; let depth = 0; @@ -774,10 +786,8 @@ export function inlineLinks(text) { ) { break; } - if ((text[back] === "[" || text[back] === "]") && isEscaped(text, back)) { - if (text[back] === "[" && depth === 0) break; - continue; - } + // An escaped bracket is literal text and takes no part in nesting. + if ((text[back] === "[" || text[back] === "]") && isEscaped(text, back)) continue; if (text[back] === "]") depth += 1; else if (text[back] === "[") { if (depth === 0) { @@ -824,7 +834,12 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { let rewrites = 0; /** @type {string[]} */ const spacedRefs = []; - const rewrite = (/** @type {string} */ ref, fragments = true, escapedChars = "") => { + const rewrite = ( + /** @type {string} */ ref, + fragments = true, + escapedChars = "", + references = false, + ) => { const { refPath, suffix, encoded } = splitReference(ref, { fragments }); const found = lookUpReference(refPath, sourceFile, map); if (!found || "retired" in found) return ref; @@ -839,11 +854,15 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { trailing = found.isDir && refPath.endsWith("/") ? "/" : ""; } let written = encoded ? relative.split("/").map(encodeURIComponent).join("/") : relative; - // Re-escape what the original destination backslash-escaped. - for (const char of escapedChars) written = written.replaceAll(char, `\\${char}`); + // Re-escape what the original destination backslash-escaped; an `&` in a + // destination that held character references is written `&` instead. + for (const char of escapedChars) { + if (!(references && char === "&")) written = written.replaceAll(char, `\\${char}`); + } const next = `${written}${trailing}${suffix}`; - if (next !== ref) rewrites += 1; - return next; + if (next === ref) return ref; + rewrites += 1; + return references ? next.replaceAll("&", "&") : next; }; const { fenceMasked, linkMasked, tagText } = maskCode(text); // Every pass records edits by offset into the original text; the scans run @@ -870,7 +889,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { }); continue; } - const next = rewrite(link.dest, true, link.escapedChars); + const next = rewrite(link.dest, true, link.escapedChars, link.references); if (next === link.dest) continue; const written = next; edits.push({ @@ -884,8 +903,8 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { // flag: a definition or attribute has no prose to rewrite into. for (const { from, to, angle } of referenceDefinitions(linkMasked)) { const raw = text.slice(angle ? from + 1 : from, angle ? to - 1 : to); - const { dest, escapedChars } = unescapeDestination(raw); - const rewritten = rewrite(dest, true, escapedChars); + const { dest, escapedChars, references } = unescapeDestination(raw); + const rewritten = rewrite(dest, true, escapedChars, references); if (rewritten === dest) continue; edits.push({ from, to, text: angle ? `<${rewritten}>` : rewritten }); } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index a8062acc789..1fe24f19a16 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1220,6 +1220,63 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ); }); + // CommonMark: a backslash-escaped bracket is literal link text, so the link + // opens at the unescaped `[` before it. + it("reads a link whose text holds an escaped opening bracket", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": "[see \\[draft](../source/transcript.txt)\n" }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "[see \\[draft](../transcript/transcript.txt)\n", + ); + }); + + // CommonMark: entity and numeric character references are decoded in link + // destinations, so `a&b.pdf` names the file `a&b.pdf`. + it("decodes character references in link and definition destinations", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/a&b.pdf": "pdf", + "key-frames/visual-frames.md": [ + "[deck](../source/decks/a&b.pdf)", + "[hex][d]", + "", + "[d]: ../source/decks/a&b.pdf", + "", + ].join("\n"), + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + [ + "[deck](../metadata/decks/a&b.pdf)", + "[hex][d]", + "", + "[d]: ../metadata/decks/a&b.pdf", + "", + ].join("\n"), + ); + }); + + it("resolves a destination holding a character reference in the link check", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/decks/a&b.pdf": "pdf", + "metadata/deck-inventory.md": "[deck](decks/a&b.pdf) [gone](decks/c&d.pdf)\n", + }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "metadata/deck-inventory.md", ref: "decks/c&d.pdf" }, + ]); + }); + it("flags a missing destination with parentheses rather than a truncated one", () => { const root = makeDir("relayout-check-"); writeTree(root, { "metadata/deck-inventory.md": "[deck](decks/slides_(v2).pdf)\n" }); From 25274ca8d2d2e16902be2d8a863132466543ba9f Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 06:52:34 -0400 Subject: [PATCH 47/73] fix(knowledge): resolve whole names before line suffixes, refuse symlinked temp roots, document entity limit A code-span path is now tried whole, a `:12` or `:12:5` ending included, against the path map and the filesystem before a line suffix is stripped, so a real file named `slide.pdf:12` maps and passes the link check. A recorded temp dir (workDir, framesDir, contactSheetsDir) that is itself a symlink is refused via lstat before any containment or readability check follows it. Named character references beyond the five decoded ones are documented as a known limit; a test pins that such a destination fails closed. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 5 ++ .../extraction/watch/relayout-slice.js | 70 +++++++++++++------ .../extraction/watch/relayout-slice.test.js | 61 ++++++++++++++++ 3 files changed, 114 insertions(+), 22 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 9ed19620d64..1e083559642 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -152,6 +152,11 @@ labels nested more than one level, or link reference definitions inside containe quotes). References in those constructs may be left unrewritten or read as ordinary text, and the link check is the backstop: it refuses a dangling slice-internal path it can see. +It also does not decode named character references beyond `& < > " '` +(numeric references are decoded). A destination holding another one (`a@b.pdf` for the file +`a@b.pdf`) is not mapped to its file, and the link check refuses it, naming the path, so nothing +passes silently. + ## Agent-written artifacts share the same root **Do not split the slice.** Every `.work//…` path in this skill and its `context/` diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 86e0a8e8922..d2a849677aa 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -480,17 +480,18 @@ function literalText(value) { /** * Split a reference into its path and the suffix to carry over unchanged * (`#anchor`, `?query`, `:line`). With `fragments: false`, a raw `#` or `?` - * stays part of the path (a file named `notes#1.md`). + * stays part of the path (a file named `notes#1.md`); with `lines: false`, so + * does a `:12` or `:12:5` ending (a file named `slide.pdf:12`). * * @param {string} ref - * @param {{ fragments?: boolean }} [options] + * @param {{ fragments?: boolean, lines?: boolean }} [options] * @returns {{ refPath: string, suffix: string, encoded: boolean }} */ -function splitReference(ref, { fragments = true } = {}) { +function splitReference(ref, { fragments = true, lines = true } = {}) { const hash = fragments ? ref.search(/[#?]/) : -1; let refPath = hash === -1 ? ref : ref.slice(0, hash); let suffix = hash === -1 ? "" : ref.slice(hash); - const line = LINE_SUFFIX.exec(refPath); + const line = lines ? LINE_SUFFIX.exec(refPath) : null; if (line) { suffix = line[0] + suffix; refPath = refPath.slice(0, line.index); @@ -836,11 +837,11 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const spacedRefs = []; const rewrite = ( /** @type {string} */ ref, - fragments = true, + /** @type {{ fragments?: boolean, lines?: boolean }} */ split = {}, escapedChars = "", references = false, ) => { - const { refPath, suffix, encoded } = splitReference(ref, { fragments }); + const { refPath, suffix, encoded } = splitReference(ref, split); const found = lookUpReference(refPath, sourceFile, map); if (!found || "retired" in found) return ref; let relative; @@ -889,7 +890,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { }); continue; } - const next = rewrite(link.dest, true, link.escapedChars, link.references); + const next = rewrite(link.dest, {}, link.escapedChars, link.references); if (next === link.dest) continue; const written = next; edits.push({ @@ -904,7 +905,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { for (const { from, to, angle } of referenceDefinitions(linkMasked)) { const raw = text.slice(angle ? from + 1 : from, angle ? to - 1 : to); const { dest, escapedChars, references } = unescapeDestination(raw); - const rewritten = rewrite(dest, true, escapedChars, references); + const rewritten = rewrite(dest, {}, escapedChars, references); if (rewritten === dest) continue; edits.push({ from, to, text: angle ? `<${rewritten}>` : rewritten }); } @@ -929,15 +930,17 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const { ticks, content } = span; const { pad, quote, core, spaced } = spanPath(content); if (core === null) continue; - // The whole span, a raw `#` or `?` included (`notes#1.md`), is tried - // first; only when it names nothing is the suffix read as a fragment. - const wholeFound = lookUpReference( - splitReference(core, { fragments: false }).refPath, - sourceFile, - map, - ); - const fragments = wholeFound === null; - const found = wholeFound ?? lookUpReference(splitReference(core).refPath, sourceFile, map); + // The whole span, a raw `#` or `?` and a `:12` ending included + // (`notes#1.md`, `slide.pdf:12`), is tried first, then without its line + // suffix; only when neither names anything is the rest read as a fragment. + /** @type {{ fragments?: boolean, lines?: boolean }} */ + let split = {}; + let found = null; + for (const option of [{ fragments: false, lines: false }, { fragments: false }, {}]) { + split = option; + found = lookUpReference(splitReference(core, option).refPath, sourceFile, map); + if (found) break; + } // A span with spaces is a path only when it resolves as one whole path. if (spaced && !found) continue; let next; @@ -945,7 +948,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { rewrites += 1; next = `${found.retired} (${found.note})`; } else { - const rewritten = rewrite(core, fragments); + const rewritten = rewrite(core, split); next = `${ticks}${pad}${quote}${rewritten}${quote}${pad}${ticks}`; // The link check skips spans with spaces (they may be commands), so a // spaced path this rewrite placed is handed to it explicitly. @@ -1085,11 +1088,14 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } /** @type {boolean} */ isLink, /** @type {boolean} */ knownPath = isLink, ) => { - // A backticked path may hold a raw `#` or `?` in its name: if the whole - // span resolves, it is fine; otherwise the suffix is a fragment. + // A backticked path may hold a raw `#` or `?`, or a `:12` ending, in its + // name: if the whole span resolves, it is fine; then without its line + // suffix; otherwise the rest is a fragment. if (!isLink) { - const whole = splitReference(ref, { fragments: false }).refPath; - if (whole && fs.existsSync(path.resolve(fileDir, whole))) return; + for (const option of [{ fragments: false, lines: false }, { fragments: false }]) { + const whole = splitReference(ref, option).refPath; + if (whole && fs.existsSync(path.resolve(fileDir, whole))) return; + } } const { refPath } = splitReference(ref); if (!refPath || URL_SCHEME.test(refPath)) return; @@ -1364,6 +1370,19 @@ function isRegularFile(file) { } } +/** + * True when `p` itself is a symlink (not followed); false when it is absent. + * + * @param {string} p + */ +function isSymlink(p) { + try { + return fs.lstatSync(p).isSymbolicLink(); + } catch { + return false; + } +} + /** * Size of `file` in bytes; 0 when it cannot be read, so an unreadable * artifact is refused like an empty one. @@ -1567,6 +1586,13 @@ export async function relayoutSlice({ // `--no-media` included and whether or not the dir still exists, nothing // this run writes may sit in, hold, or be one of them. const recorded = resolveTempSession(state.tempSession ?? {}); + // A recorded temp dir that is itself a symlink points somewhere the watch + // never made: checked before any containment or readability check follows it. + for (const dir of [recorded.workDir, recorded.framesDir, recorded.contactSheetsDir]) { + if (dir && isSymlink(dir)) { + return refuse(`temp session dir ${dir} is a symlink`); + } + } for (const dir of [recorded.workDir, recorded.framesDir, recorded.contactSheetsDir]) { if (!dir) continue; const realDir = realPathOf(dir); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 1fe24f19a16..21301d98f57 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1277,6 +1277,67 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + // Only `& < > " '` and numeric references are decoded + // (a documented limit): any other named reference stays as written, so the + // destination names no file and the run fails closed instead of passing. + it("refuses, naming it, a destination holding a named reference it does not decode", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/a@b.pdf": "pdf", + "key-frames/visual-frames.md": "[deck](../source/decks/a@b.pdf)\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "frames/visual-frames.md", ref: "../metadata/decks/a@b.pdf" }, + ]); + expect(fs.existsSync(targetDir)).toBe(false); + }); + + // `:` is not allowed in a Windows file name. + it.skipIf(process.platform === "win32")( + "maps a code-span path whose real name ends in :12 before reading it as a line suffix", + async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/slide.pdf:12": "pdf", + "key-frames/vision-plan.md": "Deck: `source/decks/slide.pdf:12`.\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "metadata/decks/slide.pdf:12")).toBe("pdf"); + expect(read(targetDir, "frames/vision-plan.md")).toBe( + "Deck: `../metadata/decks/slide.pdf:12`.\n", + ); + }, + ); + + // Creating a symlink on Windows needs a privilege a test run may lack. + it.skipIf(process.platform === "win32")( + "refuses, naming it, a recorded temp dir that is itself a symlink", + async () => { + const { sliceDir, workDir, targetDir } = makeFixture(); + const link = path.join(makeDir("relayout-link-"), "work"); + fs.symlinkSync(workDir, link); + editState(sliceDir, (s) => ({ ...s, tempSession: { ...s.tempSession, workDir: link } })); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + const result = await relayoutSlice({ sliceDir, targetDir }); + const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); + stderr.mockRestore(); + + expect(result.exitCode).toBe(1); + expect(message).toContain(`${link} is a symlink`); + expect(fs.existsSync(targetDir)).toBe(false); + }, + ); + it("flags a missing destination with parentheses rather than a truncated one", () => { const root = makeDir("relayout-check-"); writeTree(root, { "metadata/deck-inventory.md": "[deck](decks/slides_(v2).pdf)\n" }); From 876e3dba4de2bf12c9bf69e5238dd94ab5ad08d5 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:00:40 -0400 Subject: [PATCH 48/73] fix(knowledge): validate every relative definition and treat backslash-newline as no escape Reference definitions are link syntax, so the layout check now validates every relative definition destination, extensionless ones included; the PATH_LIKE gate stays for backticked code spans only. A backslash escapes only ASCII punctuation in angle destinations, bare destinations, titles and labels, so a backslash before a line ending no longer hides the line ending. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 33 ++++++++++++------- .../extraction/watch/relayout-slice.test.js | 26 +++++++++++++++ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index d2a849677aa..aad223e5b69 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -132,17 +132,27 @@ const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /<[^<>]*>|\$\{[^}]*\}|\{[^{}]*\}|(?:^|\/)[^/]*\*/; /** `:12`, `:12-20`, `:12,15` and `:12:5` (line and column) suffixes. */ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; +/** ASCII punctuation: the only characters a CommonMark backslash escapes. */ +const ESCAPABLE = /[!-/:-@[-`{-~]/; +/** + * A backslash and the punctuation it escapes, or a lone literal backslash: + * before anything else, a line ending included, it escapes nothing. + */ +const BACKSLASH = String.raw`\\[!-/:-@[-\x60{-~]|\\(?![!-/:-@[-\x60{-~])`; /** * The `[label]:` opening of a reference-style link definition, with * CommonMark's label rules: up to 999 characters, backslash escapes allowed * (`[a\]b]`), no unescaped bracket, line endings allowed but not a blank line. * The destination may follow one line ending; parseDestination reads it. */ -const REFERENCE_DEFINITION_LABEL = - /^ {0,3}\[((?:\\[^\n]|[^[\]\\\n]|\r?\n(?![ \t]*\r?\n)){1,999})\]:[ \t]*(?:\r?\n[ \t]*)?/gm; +const REFERENCE_DEFINITION_LABEL = new RegExp( + String.raw`^ {0,3}\[((?:${BACKSLASH}|[^[\]\\\r\n]|\r?\n(?![ \t]*\r?\n)){1,999})\]:[ \t]*(?:\r?\n[ \t]*)?`, + "gm", +); /** What may follow a definition's destination: a line ending, or a title then one. */ -const DEFINITION_TAIL = - /^(?:[ \t]+(?:"(?:\\[^]|[^"\\])*"|'(?:\\[^]|[^'\\])*'|\((?:\\[^]|[^()\\])*\)))?[ \t]*(?:\r?\n|$)/; +const DEFINITION_TAIL = new RegExp( + String.raw`^(?:[ \t]+(?:"(?:${BACKSLASH}|[^"\\])*"|'(?:${BACKSLASH}|[^'\\])*'|\((?:${BACKSLASH}|[^()\\])*\)))?[ \t]*(?:\r?\n|$)`, +); /** * Every reference-style link definition in `text` whose label holds a @@ -694,11 +704,11 @@ export function parseDestination(text, at) { if (text[at] === "<") { for (let scan = at + 1; scan < text.length; scan += 1) { const char = text[scan]; - if (char === "\\") { + if (char === "\\" && ESCAPABLE.test(text[scan + 1] ?? "")) { scan += 1; continue; } - if (char === "\n" || char === "<") return null; + if (char === "\n" || char === "\r" || char === "<") return null; if (char === ">") return { raw: text.slice(at + 1, scan), angle: true, end: scan + 1 }; } return null; @@ -707,7 +717,7 @@ export function parseDestination(text, at) { let depth = 0; while (end < text.length) { const char = text[end]; - if (char === "\\" && /[!-/:-@[-`{-~]/.test(text[end + 1] ?? "")) { + if (char === "\\" && ESCAPABLE.test(text[end + 1] ?? "")) { end += 2; continue; } @@ -758,7 +768,7 @@ export function inlineLinks(text) { const closer = quote === "(" ? ")" : quote; let closeQuote = -1; for (let scan = at + 1; scan < text.length; scan += 1) { - if (text[scan] === "\\") { + if (text[scan] === "\\" && ESCAPABLE.test(text[scan + 1] ?? "")) { scan += 1; continue; } @@ -1112,11 +1122,10 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } }; const { fenceMasked, linkMasked, tagText } = maskCode(text); for (const link of inlineLinks(linkMasked)) check(link.dest, true); + // A definition is unambiguous link syntax, so every relative destination is + // checked, an extensionless one (`[guide]: README`) included. for (const { raw } of referenceDefinitions(linkMasked)) { - // A line such as `[0:33]: implementing ...` parses as a definition but is - // prose; only a destination that reads as a path is checked. - const destination = unescapeDestination(raw).dest; - if (PATH_LIKE.test(destination)) check(destination, true); + check(unescapeDestination(raw).dest, true); } for (const { from, to } of htmlAttributeValues(tagText)) { check(decodeHtmlReferences(text.slice(from, to)), true); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 21301d98f57..f8c6a71542b 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1887,6 +1887,32 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it("checks an extensionless definition destination", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "analysis/README": "readme", + "analysis/notes.md": "[guide]: missing-thing\n[readme]: README\n", + }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "analysis/notes.md", ref: "missing-thing" }]); + }); + + it("reads no angle destination broken by a backslash before a line ending", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "frames/visual-frames.md": [ + "[example]()", + "", + "[d]: ", + "", + ].join("\n"), + }); + + expect(checkLayoutLinks(root)).toEqual([]); + }); + it("flags a reference-style definition to a path that does not exist", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "research/claim-inventory.md": "[gone][g]\n\n[g]: findings/gone.md\n" }, From 4259d2b318d023002f3c65dbe6dc999a3361d83d Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:11:45 -0400 Subject: [PATCH 49/73] fix(knowledge): whole-name-first for every reference kind, literal tilde names, regular-file frame keys Inline links, reference definitions and HTML src/href now share the code spans' reading order (whole name, then without a line suffix, then as a fragment) in both the rewrite and the link check; an earlier reading counts only on an exact path-map hit. Only `~`, `~/x` and `~user/x` are skipped as home paths, so `~notes.md` is validated. Every frame-times.json key must name a regular file in the frames dir. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 80 ++++++++++++------- .../extraction/watch/relayout-slice.test.js | 59 ++++++++++++++ 2 files changed, 111 insertions(+), 28 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index aad223e5b69..6a098a68a68 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -132,6 +132,8 @@ const URL_SCHEME = /^[a-z][a-z0-9+.-]*:/i; const PLACEHOLDER = /<[^<>]*>|\$\{[^}]*\}|\{[^{}]*\}|(?:^|\/)[^/]*\*/; /** `:12`, `:12-20`, `:12,15` and `:12:5` (line and column) suffixes. */ const LINE_SUFFIX = /(?::\d+(?:[-,]\d+)*)+$/; +/** A home path: `~` alone, `~/x` or `~user/x`; `~notes.md` is a relative name. */ +const HOME_PATH = /^~(?:[^/]*\/|$)/; /** ASCII punctuation: the only characters a CommonMark backslash escapes. */ const ESCAPABLE = /[!-/:-@[-`{-~]/; /** @@ -444,7 +446,7 @@ function deriveDirectoryMap(fileMap) { * @property {string} targetAbs - absolute final target directory */ -/** @typedef {{ target: string, isDir: boolean } | { retired: string, note: string } | { outside: string }} Lookup */ +/** @typedef {{ target: string, isDir: boolean, guessed?: boolean } | { retired: string, note: string } | { outside: string }} Lookup */ /** Stand-ins for a backslash-escaped `#` or `?`, which are part of the path. */ const LITERAL_HASH = ""; @@ -644,7 +646,7 @@ function lookUpReference(refPath, sourceFile, map) { for (const key of bare.includes("/") ? keys : []) { const dir = key === null ? undefined : map.dirs.get(path.posix.dirname(key)); if (dir !== undefined && key !== null) { - return { target: `${dir}/${path.posix.basename(key)}`, isDir: false }; + return { target: `${dir}/${path.posix.basename(key)}`, isDir: false, guessed: true }; } } if (bare.includes("/")) return null; @@ -656,6 +658,33 @@ function lookUpReference(refPath, sourceFile, map) { const NOT_RETAINED = "not retained in this copy"; +/** + * How a reference is read, in order: the whole name, a raw `#` or `?` and a + * `:12` ending included (`notes#1.md`, `slide.pdf:12`), then without its line + * suffix, and only then with the rest read as a fragment. + */ +const WHOLE_NAME_FIRST = [{ fragments: false, lines: false }, { fragments: false }, {}]; + +/** + * What a reference names under the first reading that names anything (an + * earlier reading must name it exactly, not as a guess into a copied + * directory), and that reading's split options (the last when none does). + * + * @param {string} ref + * @param {string} sourceFile + * @param {PathMap} map + * @returns {{ found: Lookup|null, split: { fragments?: boolean, lines?: boolean } }} + */ +function findReference(ref, sourceFile, map) { + for (const split of WHOLE_NAME_FIRST) { + const found = lookUpReference(splitReference(ref, split).refPath, sourceFile, map); + if (found && !("guessed" in found && split !== WHOLE_NAME_FIRST.at(-1))) { + return { found, split }; + } + } + return { found: null, split: {} }; +} + /** * @typedef {Object} InlineLink * @property {string} dest - the destination, angle brackets removed and backslash escapes decoded @@ -888,7 +917,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { // `](dest)` becomes the note, so an image nested in the label keeps its // own rewrite. Any other destination takes the path rewrite. for (const link of inlineLinks(linkMasked)) { - const found = lookUpReference(splitReference(link.dest).refPath, sourceFile, map); + const { found, split } = findReference(link.dest, sourceFile, map); if (found && "retired" in found && link.labelStart !== -1) { rewrites += 1; const bracket = text[link.linkStart] === "!" ? link.linkStart + 2 : link.linkStart + 1; @@ -900,7 +929,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { }); continue; } - const next = rewrite(link.dest, {}, link.escapedChars, link.references); + const next = rewrite(link.dest, split, link.escapedChars, link.references); if (next === link.dest) continue; const written = next; edits.push({ @@ -915,7 +944,8 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { for (const { from, to, angle } of referenceDefinitions(linkMasked)) { const raw = text.slice(angle ? from + 1 : from, angle ? to - 1 : to); const { dest, escapedChars, references } = unescapeDestination(raw); - const rewritten = rewrite(dest, {}, escapedChars, references); + const { split } = findReference(dest, sourceFile, map); + const rewritten = rewrite(dest, split, escapedChars, references); if (rewritten === dest) continue; edits.push({ from, to, text: angle ? `<${rewritten}>` : rewritten }); } @@ -926,7 +956,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { // quotes gets double quotes. const value = text.slice(from, to); const decoded = decodeHtmlReferences(value); - const next = rewrite(decoded); + const next = rewrite(decoded, findReference(decoded, sourceFile, map).split); if (next === decoded) continue; let escaped = next.replaceAll("&", "&"); if (group === 2) escaped = escaped.replaceAll('"', """); @@ -940,17 +970,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const { ticks, content } = span; const { pad, quote, core, spaced } = spanPath(content); if (core === null) continue; - // The whole span, a raw `#` or `?` and a `:12` ending included - // (`notes#1.md`, `slide.pdf:12`), is tried first, then without its line - // suffix; only when neither names anything is the rest read as a fragment. - /** @type {{ fragments?: boolean, lines?: boolean }} */ - let split = {}; - let found = null; - for (const option of [{ fragments: false, lines: false }, { fragments: false }, {}]) { - split = option; - found = lookUpReference(splitReference(core, option).refPath, sourceFile, map); - if (found) break; - } + const { found, split } = findReference(core, sourceFile, map); // A span with spaces is a path only when it resolves as one whole path. if (spaced && !found) continue; let next; @@ -1098,18 +1118,16 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } /** @type {boolean} */ isLink, /** @type {boolean} */ knownPath = isLink, ) => { - // A backticked path may hold a raw `#` or `?`, or a `:12` ending, in its - // name: if the whole span resolves, it is fine; then without its line - // suffix; otherwise the rest is a fragment. - if (!isLink) { - for (const option of [{ fragments: false, lines: false }, { fragments: false }]) { - const whole = splitReference(ref, option).refPath; - if (whole && fs.existsSync(path.resolve(fileDir, whole))) return; - } + // Any reference may hold a raw `#` or `?`, or a `:12` ending, in its + // name: read as WHOLE_NAME_FIRST orders it, the first reading that + // resolves passes; the last one is checked below. + for (const option of WHOLE_NAME_FIRST.slice(0, -1)) { + const whole = splitReference(ref, option).refPath; + if (whole && fs.existsSync(path.resolve(fileDir, whole))) return; } const { refPath } = splitReference(ref); if (!refPath || URL_SCHEME.test(refPath)) return; - if (refPath.startsWith("/") || refPath.startsWith("~")) return; + if (refPath.startsWith("/") || HOME_PATH.test(refPath)) return; if (isAllowedMiss(refPath)) return; if (!knownPath && !looksSliceInternal(refPath, fileDir, topLevel, sliceDirs)) return; const resolved = path.resolve(fileDir, refPath); @@ -1337,8 +1355,14 @@ function missingFrames(framesDir) { `(malformed: ${malformed.sort().join(", ")})`, ); } - const gone = listed.filter((name) => !names.has(name)); - if (gone.length > 0) missing.push(`the frames frame-times.json lists: ${gone.join(", ")}`); + // Every key, whatever its extension, must name a regular file in the + // frames dir itself: not a directory, a symlink or a missing name. + const gone = listed.filter( + (name) => !names.has(name) || !isRegularFile(path.join(framesDir, name)), + ); + if (gone.length > 0) { + missing.push(`regular files for the frame-times.json keys: ${gone.sort().join(", ")}`); + } // Scene detection records every scene and interval frame it writes; an // anchor frame carries its time in its own name and is never listed. const untimed = [...names].filter( diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index f8c6a71542b..c645deda0dc 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1048,6 +1048,24 @@ describe("relayoutSlice", () => { expect(fs.existsSync(targetDir)).toBe(false); }); + it("refuses, naming it, a frame-times.json key that is a directory", async () => { + const { sliceDir, targetDir, framesDir } = makeFixture(); + fs.writeFileSync( + path.join(framesDir, "frame-times.json"), + JSON.stringify({ "scene_0001.png": { timestampSec: 1 }, scene_0002: { timestampSec: 2 } }), + ); + fs.mkdirSync(path.join(framesDir, "scene_0002")); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + + const result = await relayoutSlice({ sliceDir, targetDir }); + const message = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); + stderr.mockRestore(); + + expect(result.exitCode).toBe(1); + expect(message).toContain("scene_0002"); + expect(fs.existsSync(targetDir)).toBe(false); + }); + it("refuses an empty video file", async () => { const { sliceDir, targetDir, workDir } = makeFixture(); fs.writeFileSync(path.join(workDir, `${VIDEO_ID}.mp4`), ""); @@ -1318,6 +1336,38 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( }, ); + // `:` is not allowed in a Windows file name. + it.skipIf(process.platform === "win32")( + "maps a linked path whose real name ends in :12 before reading it as a line suffix", + async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/slide.pdf:12": "pdf", + "key-frames/vision-plan.md": [ + "Deck: [deck](source/decks/slide.pdf:12).", + 'deck', + "", + "[d]: source/decks/slide.pdf:12", + "", + ].join("\n"), + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/vision-plan.md")).toBe( + [ + "Deck: [deck](../metadata/decks/slide.pdf:12).", + 'deck', + "", + "[d]: ../metadata/decks/slide.pdf:12", + "", + ].join("\n"), + ); + }, + ); + // Creating a symlink on Windows needs a privilege a test run may lack. it.skipIf(process.platform === "win32")( "refuses, naming it, a recorded temp dir that is itself a symlink", @@ -2081,6 +2131,15 @@ describe("checkLayoutLinks", () => { expect(checkLayoutLinks(root)).toEqual([]); }); + it("checks a ~-prefixed relative name and skips only home paths", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "analysis/RESEARCH.md": "[notes](~notes.md), [home](~/x.md), [user](~someone/x.md).\n", + }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "analysis/RESEARCH.md", ref: "~notes.md" }]); + }); + it("flags an unresolved path under a slice lane whose directory the slice had", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From 0f697fd1e77452c606a36306e473f7f61c52a2c5 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:21:18 -0400 Subject: [PATCH 50/73] fix(knowledge): reject titles across blank lines and apply HTML numeric reference replacements A link title, inline or in a reference definition, no longer spans a blank line; such a construct is not a link. Numeric character references follow HTML's replacements: 0, surrogates and values past U+10FFFF become U+FFFD, and 0x80-0x9F map through windows-1252. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 25 +++++++- .../extraction/watch/relayout-slice.test.js | 60 +++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 6a098a68a68..8fd32cd1385 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -151,9 +151,11 @@ const REFERENCE_DEFINITION_LABEL = new RegExp( String.raw`^ {0,3}\[((?:${BACKSLASH}|[^[\]\\\r\n]|\r?\n(?![ \t]*\r?\n)){1,999})\]:[ \t]*(?:\r?\n[ \t]*)?`, "gm", ); +/** A line ending inside a title, which never spans a blank line (only spaces or tabs). */ +const TITLE_LINE_END = String.raw`\n(?![ \t]*\r?\n)`; /** What may follow a definition's destination: a line ending, or a title then one. */ const DEFINITION_TAIL = new RegExp( - String.raw`^(?:[ \t]+(?:"(?:${BACKSLASH}|[^"\\])*"|'(?:${BACKSLASH}|[^'\\])*'|\((?:${BACKSLASH}|[^()\\])*\)))?[ \t]*(?:\r?\n|$)`, + String.raw`^(?:[ \t]+(?:"(?:${BACKSLASH}|${TITLE_LINE_END}|[^"\\\n])*"|'(?:${BACKSLASH}|${TITLE_LINE_END}|[^'\\\n])*'|\((?:${BACKSLASH}|${TITLE_LINE_END}|[^()\\\n])*\)))?[ \t]*(?:\r?\n|$)`, ); /** @@ -245,10 +247,24 @@ const HTML_NAMED_REFERENCES = /** @type {Record} */ ({ apos: "'", }); +/** + * HTML's replacements for numeric references to 0x80-0x9F (windows-1252), + * indexed from 0x80; a 0 entry keeps the code point. + * https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state + */ +const WINDOWS_1252 = [ + 0x20ac, 0, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0x017d, 0, + 0, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0, 0x017e, 0x0178, +]; + /** * An HTML attribute value with its character references decoded: the named * ones a path can hold (`&` `<` `>` `"` `'`) and numeric - * ones (`'`, `'`). Anything else stays as written. + * ones (`'`, `'`) under HTML's replacements: 0, a surrogate or a + * value past U+10FFFF is U+FFFD, and 0x80-0x9F map through windows-1252. + * Anything else stays as written. * * @param {string} value */ @@ -256,7 +272,8 @@ function decodeHtmlReferences(value) { return value.replace(/&(?:#(\d+)|#[xX]([0-9A-Fa-f]+)|([A-Za-z]+));/g, (match, dec, hex, name) => { if (dec !== undefined || hex !== undefined) { const code = dec !== undefined ? Number(dec) : Number.parseInt(hex, 16); - return code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match; + const invalid = code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff); + return String.fromCodePoint(invalid ? 0xfffd : WINDOWS_1252[code - 0x80] || code); } return HTML_NAMED_REFERENCES[name] ?? match; }); @@ -805,6 +822,8 @@ export function inlineLinks(text) { closeQuote = scan; break; } + // A title never spans a blank line (one holding only spaces or tabs). + if (text[scan] === "\n" && /^[ \t]*\r?\n/.test(text.slice(scan + 1))) break; } if (closeQuote === -1) continue; at = closeQuote + 1; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index c645deda0dc..d1cc974dc66 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1283,6 +1283,38 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ); }); + it("keeps a link whose destination names a retained file by a windows-1252 reference", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/€.pdf": "pdf", + "key-frames/visual-frames.md": "[deck](../source/decks/€.pdf)\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "metadata/decks/€.pdf")).toBe("pdf"); + }); + + // HTML numeric character reference replacements: 0x80 is windows-1252's + // euro sign (U+20AC), and 0 becomes U+FFFD. + it("applies HTML's numeric character reference replacements", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/decks/€.pdf": "pdf", + "metadata/deck-inventory.md": "[deck](decks/€.pdf)\n", + }); + + expect(checkLayoutLinks(root)).toEqual([]); + const replacement = String.fromCodePoint(0xfffd); + expect( + inlineLinks("[a](a�b.md) [b](�.md) [c](�.md) [d](.md)").map( + ({ dest }) => dest, + ), + ).toEqual([`a${replacement}b.md`, `${replacement}.md`, `${replacement}.md`, "\u0081.md"]); + }); + it("resolves a destination holding a character reference in the link check", () => { const root = makeDir("relayout-check-"); writeTree(root, { @@ -1405,6 +1437,34 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ).toEqual(["a.md", "b.md", "c.md", "d.md"]); }); + // CommonMark: a link title may span lines but not a blank line (one holding + // only spaces or tabs); with such a title the construct is not a link. + it("reads no link or definition whose title spans a blank line", () => { + expect(inlineLinks('[example](missing.md "title\n\ncontinued")')).toEqual([]); + expect(inlineLinks("[example](missing.md (title\n \t\ncontinued))")).toEqual([]); + expect(inlineLinks('[ok](kept.md "title\ncontinued")').map(({ dest }) => dest)).toEqual([ + "kept.md", + ]); + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/notes.md": [ + '[example](missing.md "title', + "", + 'continued")', + "", + "[d]: gone.md 'title", + " ", + "continued'", + "", + '[e]: lost.md "title', + 'continued"', + "", + ].join("\n"), + }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "lost.md" }]); + }); + it("reads a link with a parenthesized title and rewrites its destination", async () => { expect(inlineLinks("[x](a.md (Title)) and [y](b.md)").map(({ dest }) => dest)).toEqual([ "a.md", From 606d7b04c056ca0836e5952b8e84ca1e9dd34e6c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:28:03 -0400 Subject: [PATCH 51/73] fix(knowledge): never exempt a retired real file from the link check as a placeholder The path lookup now checks the slice's own entries before treating a placeholder-looking name as a placeholder, so a dropped real file such as run-state/report-{draft}.md is identified as retired. The rewrite records each definition, src/href or inline link it leaves naming a retired file, and the link check never applies the placeholder skip to those. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 76 ++++++++++++------- .../extraction/watch/relayout-slice.test.js | 18 +++++ 2 files changed, 68 insertions(+), 26 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 8fd32cd1385..f66e744edb0 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -625,22 +625,6 @@ function lookUpReference(refPath, sourceFile, map) { const found = lookup(key); if (found) return found; } - // A placeholder (`/x`, `${ROOT}/x`, `{tmp}/x`, `research/*.md`) - // names no real file. It counts as one only now, after the path map found - // nothing, so a real `report-{draft}.pdf` is still mapped. - if (PLACEHOLDER.test(bare)) return null; - // A lane folder the layout knows but this slice never produced (prose such - // as "no deck in `source/decks/`") names nothing to point at. - // A path below such a folder (`source/decks/slides.pdf`) names nothing either. - const absentFolder = keys.find( - (key) => - key !== null && - [...KNOWN_LANE_FOLDERS].some( - (folder) => - (key === folder || key.startsWith(`${folder}/`)) && !map.sliceEntries.has(folder), - ), - ); - if (absentFolder) return { retired: absentFolder, note: "none in this slice" }; const retired = keys.find((key) => key !== null && map.sliceEntries.has(key)); if (retired) { // A directory whose kept files went to several places (`source/`) has no @@ -658,6 +642,23 @@ function lookUpReference(refPath, sourceFile, map) { splitInto.length > 0 ? `split across ${splitInto.join(", ")} in this copy` : NOT_RETAINED, }; } + // A placeholder (`/x`, `${ROOT}/x`, `{tmp}/x`, `research/*.md`) + // names no real file. It counts as one only now, after neither the path map + // nor the slice's own entries named it, so a real `report-{draft}.pdf` is + // still mapped or retired. + if (PLACEHOLDER.test(bare)) return null; + // A lane folder the layout knows but this slice never produced (prose such + // as "no deck in `source/decks/`") names nothing to point at. + // A path below such a folder (`source/decks/slides.pdf`) names nothing either. + const absentFolder = keys.find( + (key) => + key !== null && + [...KNOWN_LANE_FOLDERS].some( + (folder) => + (key === folder || key.startsWith(`${folder}/`)) && !map.sliceEntries.has(folder), + ), + ); + if (absentFolder) return { retired: absentFolder, note: "none in this slice" }; // An unknown file in a copied directory moves with the directory, so the // link check flags it at its new location. for (const key of bare.includes("/") ? keys : []) { @@ -886,13 +887,17 @@ export function inlineLinks(text) { * @param {string} sourceFile - slice-relative path of the file * @param {string} targetFile - target-relative path the file is copied to * @param {PathMap} map - * @returns {{ text: string, rewrites: number, spacedRefs: string[] }} - `spacedRefs`, the - * rewritten code spans holding spaces, for the link check to verify + * @returns {{ text: string, rewrites: number, spacedRefs: string[], retiredRefs: string[] }} - + * `spacedRefs`, the rewritten code spans holding spaces, for the link check to verify; + * `retiredRefs`, the references left unchanged because they name a slice path the + * layout drops, which the link check must flag whatever their name looks like */ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { let rewrites = 0; /** @type {string[]} */ const spacedRefs = []; + /** @type {string[]} */ + const retiredRefs = []; const rewrite = ( /** @type {string} */ ref, /** @type {{ fragments?: boolean, lines?: boolean }} */ split = {}, @@ -948,6 +953,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { }); continue; } + if (found && "retired" in found) retiredRefs.push(link.dest); const next = rewrite(link.dest, split, link.escapedChars, link.references); if (next === link.dest) continue; const written = next; @@ -963,7 +969,8 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { for (const { from, to, angle } of referenceDefinitions(linkMasked)) { const raw = text.slice(angle ? from + 1 : from, angle ? to - 1 : to); const { dest, escapedChars, references } = unescapeDestination(raw); - const { split } = findReference(dest, sourceFile, map); + const { found, split } = findReference(dest, sourceFile, map); + if (found && "retired" in found) retiredRefs.push(dest); const rewritten = rewrite(dest, split, escapedChars, references); if (rewritten === dest) continue; edits.push({ from, to, text: angle ? `<${rewritten}>` : rewritten }); @@ -975,7 +982,9 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { // quotes gets double quotes. const value = text.slice(from, to); const decoded = decodeHtmlReferences(value); - const next = rewrite(decoded, findReference(decoded, sourceFile, map).split); + const { found, split } = findReference(decoded, sourceFile, map); + if (found && "retired" in found) retiredRefs.push(decoded); + const next = rewrite(decoded, split); if (next === decoded) continue; let escaped = next.replaceAll("&", "&"); if (group === 2) escaped = escaped.replaceAll('"', """); @@ -1015,7 +1024,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { for (const edit of kept.reverse()) { rewritten = rewritten.slice(0, edit.from) + edit.text + rewritten.slice(edit.to); } - return { text: rewritten, rewrites, spacedRefs }; + return { text: rewritten, rewrites, spacedRefs, retiredRefs }; } /** @@ -1113,14 +1122,23 @@ function looksSliceInternal(refPath, fileDir, topLevel, sliceDirs) { * slice (another repository) and the allowed-miss prefixes are skipped. * * @param {string} rootDir - * @param {{ sliceDirs?: Set, spacedRefs?: Map }} [options] + * @param {{ + * sliceDirs?: Set, + * spacedRefs?: Map, + * retiredRefs?: Map, + * }} [options] * `sliceDirs`, the source slice's directories, narrows which lane-named paths * count as the slice's; * `spacedRefs` maps a target-relative file to the spaced code-span paths a - * rewrite placed in it, which are checked like links + * rewrite placed in it, which are checked like links; + * `retiredRefs` maps a target-relative file to the references a rewrite left + * naming a dropped slice file, never skipped as placeholders * @returns {{ file: string, ref: string }[]} unresolved references */ -export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } = {}) { +export function checkLayoutLinks( + rootDir, + { sliceDirs, spacedRefs = new Map(), retiredRefs = new Map() } = {}, +) { const root = path.resolve(rootDir); const topLevel = new Set(fs.existsSync(root) ? fs.readdirSync(root) : []); /** @type {{ file: string, ref: string }[]} */ @@ -1132,6 +1150,7 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } for (const file of markdown) { const fileDir = path.dirname(path.join(root, file)); const text = fs.readFileSync(path.join(root, file), "utf8"); + const retired = new Set(retiredRefs.get(file)); const check = ( /** @type {string} */ ref, /** @type {boolean} */ isLink, @@ -1153,8 +1172,9 @@ export function checkLayoutLinks(rootDir, { sliceDirs, spacedRefs = new Map() } const rel = path.relative(root, resolved); if (rel.startsWith("..") || path.isAbsolute(rel)) return; if (fs.existsSync(resolved)) return; - // A placeholder that names no real file is not a dangling path. - if (PLACEHOLDER.test(refPath)) return; + // A placeholder that names no real file is not a dangling path; one the + // rewrite found naming a dropped slice file is. + if (PLACEHOLDER.test(refPath) && !retired.has(ref)) return; unresolved.push({ file, ref: literalText(ref) }); }; const { fenceMasked, linkMasked, tagText } = maskCode(text); @@ -1817,6 +1837,8 @@ export async function relayoutSlice({ let rewrites = 0; /** @type {Map} */ const spacedRefs = new Map(); + /** @type {Map} */ + const retiredRefs = new Map(); let copied = 0; /** @type {{ file: string, ref: string }[]} */ let unresolved; @@ -1837,6 +1859,7 @@ export async function relayoutSlice({ ); rewrites += result.rewrites; if (result.spacedRefs.length > 0) spacedRefs.set(entry.to, result.spacedRefs); + if (result.retiredRefs.length > 0) retiredRefs.set(entry.to, result.retiredRefs); write(entry.to, result.text); } else { fs.mkdirSync(path.dirname(path.join(staging, entry.to)), { recursive: true }); @@ -1875,6 +1898,7 @@ export async function relayoutSlice({ unresolved = checkLayoutLinks(staging, { sliceDirs: new Set([...sliceEntries].filter((entry) => !sliceFiles.includes(entry))), spacedRefs, + retiredRefs, }); } catch (error) { discard(staging); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index d1cc974dc66..6295a31189c 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1634,6 +1634,24 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( }, ); + it("flags a definition or src naming a dropped real file whose name looks like a placeholder", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "run-state/report-{draft}.md": "draft", + "research/claim-inventory.md": + '[state]: ../run-state/report-{draft}.md\n\nSee [state] and .\n', + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/research/claim-inventory.md", ref: "../run-state/report-{draft}.md" }, + { file: "analysis/research/claim-inventory.md", ref: "../run-state/report-{draft}.md" }, + ]); + }); + it("matches the AGENTS.md allowed miss exactly and the directories by whole segment", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From c9b68dd9f316203136618ffb87a6c76436e7781f Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:35:57 -0400 Subject: [PATCH 52/73] fix(knowledge): end bare link destinations only at the ASCII space or a control Co-Authored-By: Claude Opus 5.5 --- .../video-digest/extraction/watch/relayout-slice.js | 3 ++- .../video-digest/extraction/watch/relayout-slice.test.js | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index f66e744edb0..cf0dac8dd4a 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -768,7 +768,8 @@ export function parseDestination(text, at) { end += 2; continue; } - if (/\s/.test(char)) break; + // CommonMark ends a bare destination only at the ASCII space or an ASCII control. + if (/[\x00-\x20\x7f]/.test(char)) break; if (char === "(") depth += 1; else if (char === ")") { if (depth === 0) break; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 6295a31189c..6199ab3f94b 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1437,6 +1437,14 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ).toEqual(["a.md", "b.md", "c.md", "d.md"]); }); + // CommonMark ends a bare destination only at the ASCII space or a control. + it("keeps non-ASCII whitespace inside a bare destination", () => { + const nbsp = String.fromCodePoint(0xa0); + expect(inlineLinks(`[deck](../source/decks/a${nbsp}b.pdf)`).map(({ dest }) => dest)).toEqual([ + `../source/decks/a${nbsp}b.pdf`, + ]); + }); + // CommonMark: a link title may span lines but not a blank line (one holding // only spaces or tabs); with such a title the construct is not a link. it("reads no link or definition whose title spans a blank line", () => { From ef264ed1c8cbe386ad86a8f4d8c863615f7263bc Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:45:49 -0400 Subject: [PATCH 53/73] fix(knowledge): encode rewritten link destinations so they parse back to the new path A rewritten angle destination wrote a `>` or `<` decoded from a character reference raw, so `` became `b.pdf>` and closed early. encodeDestination now writes every Markdown destination rewrite (inline links and reference definitions): angle form escapes `<`, `>` and `\`; bare form escapes `\` and unbalanced parentheses and switches to angle form for a space or control character; the original's escapes are kept. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 77 +++++++++++++++---- .../extraction/watch/relayout-slice.test.js | 67 +++++++++++++++- 2 files changed, 126 insertions(+), 18 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index cf0dac8dd4a..1ae17d753b1 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -501,6 +501,55 @@ function unescapeDestination(raw) { }; } +/** Text a CommonMark destination would decode as a character reference. */ +const REFERENCE_START = /^&(?:#\d+|#[xX][0-9A-Fa-f]+|[A-Za-z]+);/; + +/** + * Write `newPath` as a Markdown destination that parses back to exactly + * `newPath`, in the original's form. A literal `#` or `?` arrives as its + * stand-in and is written backslash-escaped. Angle form escapes `<`, `>` and + * `\`, and writes a line ending as a numeric reference; bare form escapes `\`, + * a leading `<` and every unbalanced parenthesis, and switches to angle form + * for a space or a control character. Punctuation the original escaped stays + * escaped; an `&` is written `&` where the original used references, and + * escaped where it would otherwise start one. + * + * @param {string} newPath + * @param {{ angle: boolean, escapedChars: string, references: boolean }} original + * @returns {string} the destination as written, angle brackets included + */ +export function encodeDestination(newPath, { angle, escapedChars, references }) { + const chars = [...newPath]; + const asAngle = angle || chars.some((char) => /[\x00-\x20\x7f]/.test(char)); + /** @type {Set} */ + const unbalanced = new Set(); + if (!asAngle) { + const open = []; + chars.forEach((char, at) => { + if (char === "(") open.push(at); + else if (char === ")" && open.pop() === undefined) unbalanced.add(at); + }); + for (const at of open) unbalanced.add(at); + } + const written = chars.map((char, at) => { + if (char === LITERAL_HASH) return "\\#"; + if (char === LITERAL_QUERY) return "\\?"; + if (char === "&") { + if (references) return "&"; + const startsReference = REFERENCE_START.test(chars.slice(at).join("")); + return escapedChars.includes("&") || startsReference ? "\\&" : "&"; + } + if (asAngle && (char === "\n" || char === "\r")) return `&#${char.charCodeAt(0)};`; + const structural = + char === "\\" || + (asAngle ? char === "<" || char === ">" : unbalanced.has(at) || (at === 0 && char === "<")); + // A raw `#` or `?` is a fragment or query; only its stand-in was escaped. + const kept = escapedChars.includes(char) && char !== "#" && char !== "?"; + return structural || kept ? `\\${char}` : char; + }); + return asAngle ? `<${written.join("")}>` : written.join(""); +} + /** @param {string} value */ function literalText(value) { return value.replaceAll(LITERAL_HASH, "#").replaceAll(LITERAL_QUERY, "?"); @@ -903,7 +952,6 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { /** @type {string} */ ref, /** @type {{ fragments?: boolean, lines?: boolean }} */ split = {}, escapedChars = "", - references = false, ) => { const { refPath, suffix, encoded } = splitReference(ref, split); const found = lookUpReference(refPath, sourceFile, map); @@ -919,15 +967,14 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { trailing = found.isDir && refPath.endsWith("/") ? "/" : ""; } let written = encoded ? relative.split("/").map(encodeURIComponent).join("/") : relative; - // Re-escape what the original destination backslash-escaped; an `&` in a - // destination that held character references is written `&` instead. - for (const char of escapedChars) { - if (!(references && char === "&")) written = written.replaceAll(char, `\\${char}`); - } + // A `#` or `?` the original escaped stays a literal path character: + // its stand-in, which encodeDestination writes escaped. + if (escapedChars.includes("#")) written = written.replaceAll("#", LITERAL_HASH); + if (escapedChars.includes("?")) written = written.replaceAll("?", LITERAL_QUERY); const next = `${written}${trailing}${suffix}`; if (next === ref) return ref; rewrites += 1; - return references ? next.replaceAll("&", "&") : next; + return next; }; const { fenceMasked, linkMasked, tagText } = maskCode(text); // Every pass records edits by offset into the original text; the scans run @@ -955,26 +1002,22 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { continue; } if (found && "retired" in found) retiredRefs.push(link.dest); - const next = rewrite(link.dest, split, link.escapedChars, link.references); + const next = rewrite(link.dest, split, link.escapedChars); if (next === link.dest) continue; - const written = next; - edits.push({ - from: link.destStart, - to: link.destEnd, - text: link.angle ? `<${written}>` : written, - }); + edits.push({ from: link.destStart, to: link.destEnd, text: encodeDestination(next, link) }); } // Reference-style definitions and HTML src/href values take the same path // rewrite. One naming a file the layout drops is left for the link check to // flag: a definition or attribute has no prose to rewrite into. for (const { from, to, angle } of referenceDefinitions(linkMasked)) { const raw = text.slice(angle ? from + 1 : from, angle ? to - 1 : to); - const { dest, escapedChars, references } = unescapeDestination(raw); + const original = { ...unescapeDestination(raw), angle }; + const { dest } = original; const { found, split } = findReference(dest, sourceFile, map); if (found && "retired" in found) retiredRefs.push(dest); - const rewritten = rewrite(dest, split, escapedChars, references); + const rewritten = rewrite(dest, split, original.escapedChars); if (rewritten === dest) continue; - edits.push({ from, to, text: angle ? `<${rewritten}>` : rewritten }); + edits.push({ from, to, text: encodeDestination(rewritten, original) }); } for (const { from, to, group } of htmlAttributeValues(tagText)) { // An attribute value may hold character references (`&`): map the diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 6199ab3f94b..4e655ae80a8 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -4,7 +4,12 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { checkLayoutLinks, inlineLinks, relayoutSlice } from "./relayout-slice.js"; +import { + checkLayoutLinks, + encodeDestination, + inlineLinks, + relayoutSlice, +} from "./relayout-slice.js"; /** @type {string[]} */ const scratch = []; @@ -1715,6 +1720,66 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( }, ); + // CommonMark: `>` decodes to `>`, which written raw would close the + // angle destination early. + it.skipIf(process.platform === "win32")( + "escapes a > decoded from a character reference in a rewritten angle destination", + async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/a>b.pdf": "pdf", + "key-frames/vision-plan.md": "[deck](<../source/decks/a>b.pdf>)\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/vision-plan.md")).toBe( + "[deck](<../metadata/decks/a\\>b.pdf>)\n", + ); + }, + ); + + it("escapes an unbalanced parenthesis in a rewritten bare destination", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "source/decks/a)b.pdf": "pdf", + "key-frames/vision-plan.md": + "[deck](<../source/decks/a)b.pdf>) [bare](../source/decks/a\\)b.pdf)\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/vision-plan.md")).toBe( + "[deck](<../metadata/decks/a)b.pdf>) [bare](../metadata/decks/a\\)b.pdf)\n", + ); + }); + + // Round trip against the CommonMark reading inlineLinks applies: the written + // destination spans exactly what was written and decodes to the new path. + it.each( + ["a>b.pdf", "a [false, true].map((angle) => [name, angle])) + .flatMap((row) => [false, true].map((references) => [...row, references])), + )( + "writes %s (angle %s, references %s) so it parses back unchanged", + (name, angle, references) => { + const newPath = `../metadata/decks/${name}`; + const written = encodeDestination(newPath, { + angle: Boolean(angle), + escapedChars: "", + references: Boolean(references), + }); + const [link] = inlineLinks(`[x](${written})`); + + expect(link?.dest).toBe(newPath); + expect(link?.destEnd).toBe(4 + written.length); + }, + ); + it("link-checks a spaced code-span path with a raw # as one whole path", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From dc55515b79a287bd7035e9eacd209f5520c01957 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:53:50 -0400 Subject: [PATCH 54/73] fix(knowledge): retired refs override allowed misses; treat ..name as inside the target checkLayoutLinks now flags a reference the rewrite recorded as naming a dropped slice file before any skip, so `AGENTS.md` or `docs/...` text no longer hides it. The outside-the-target tests use isSameOrInside, so a name such as `..notes.md` is checked instead of skipped as outside. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 20 ++++++++-------- .../extraction/watch/relayout-slice.test.js | 23 +++++++++++++++++++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 1ae17d753b1..26de18c45a6 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -661,9 +661,7 @@ function lookUpReference(refPath, sourceFile, map) { const absolute = path.resolve(map.sliceAbs, ...fromFile.split("/")); const inside = path.relative(map.sliceAbs, absolute); if (inside === "") return sliceRootLookup(map); - if (inside.startsWith("..") || path.isAbsolute(inside)) { - return { outside: absolute }; - } + if (!isSameOrInside(absolute, map.sliceAbs)) return { outside: absolute }; keys = [inside.split(path.sep).join("/")]; } else { keys = [fromFile, bare.startsWith(".") ? null : path.posix.normalize(bare)]; @@ -1176,7 +1174,7 @@ function looksSliceInternal(refPath, fileDir, topLevel, sliceDirs) { * `spacedRefs` maps a target-relative file to the spaced code-span paths a * rewrite placed in it, which are checked like links; * `retiredRefs` maps a target-relative file to the references a rewrite left - * naming a dropped slice file, never skipped as placeholders + * naming a dropped slice file, always flagged * @returns {{ file: string, ref: string }[]} unresolved references */ export function checkLayoutLinks( @@ -1200,6 +1198,12 @@ export function checkLayoutLinks( /** @type {boolean} */ isLink, /** @type {boolean} */ knownPath = isLink, ) => { + // A reference the rewrite found naming a dropped slice file dangles, whatever + // its text looks like (an allowed miss, a placeholder). + if (retired.has(ref)) { + unresolved.push({ file, ref: literalText(ref) }); + return; + } // Any reference may hold a raw `#` or `?`, or a `:12` ending, in its // name: read as WHOLE_NAME_FIRST orders it, the first reading that // resolves passes; the last one is checked below. @@ -1213,12 +1217,10 @@ export function checkLayoutLinks( if (isAllowedMiss(refPath)) return; if (!knownPath && !looksSliceInternal(refPath, fileDir, topLevel, sliceDirs)) return; const resolved = path.resolve(fileDir, refPath); - const rel = path.relative(root, resolved); - if (rel.startsWith("..") || path.isAbsolute(rel)) return; + if (!isSameOrInside(resolved, root)) return; if (fs.existsSync(resolved)) return; - // A placeholder that names no real file is not a dangling path; one the - // rewrite found naming a dropped slice file is. - if (PLACEHOLDER.test(refPath) && !retired.has(ref)) return; + // A placeholder that names no real file is not a dangling path. + if (PLACEHOLDER.test(refPath)) return; unresolved.push({ file, ref: literalText(ref) }); }; const { fenceMasked, linkMasked, tagText } = maskCode(text); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 4e655ae80a8..44867bb2158 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1665,6 +1665,22 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it("flags a definition naming a dropped slice file that matches an allowed miss", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "AGENTS.md": "slice notes", + "research/claim-inventory.md": "[rules]: AGENTS.md\n\nSee [rules].\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/research/claim-inventory.md", ref: "AGENTS.md" }, + ]); + }); + it("matches the AGENTS.md allowed miss exactly and the directories by whole segment", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { @@ -2291,6 +2307,13 @@ describe("checkLayoutLinks", () => { expect(checkLayoutLinks(root)).toEqual([{ file: "analysis/RESEARCH.md", ref: "~notes.md" }]); }); + it("checks a ..-prefixed name as inside the target", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { "README.md": "[notes](..notes.md)\n" }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "README.md", ref: "..notes.md" }]); + }); + it("flags an unresolved path under a slice lane whose directory the slice had", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From 860151370731284f7576450aca3a8a8820bf8675 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 08:03:00 -0400 Subject: [PATCH 55/73] fix(knowledge): keep angle link destinations out of HTML masking htmlRanges() read a tag-like angle destination such as [sources]() or [s]: as an HTML start tag and masked it, dropping the link from both the rewrite and the link check. A < right after ]( or a definition label's ]: (plus optional whitespace and one line ending) now opens a destination; real inline HTML stays masked. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 18 ++++++++-- .../extraction/watch/relayout-slice.test.js | 33 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 26de18c45a6..a021c38b708 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -196,6 +196,12 @@ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:\?[^#]*)?(?:#.*)? const HTML_START_TAG = /<([A-Za-z][A-Za-z0-9-]*)(?:\s(?:"[^"]*"|'[^']*'|[^<>"'])*)?\/?>/g; /** An HTML comment. */ const HTML_COMMENT = //g; +/** + * A `<` opening a link or definition destination: after `](` or a definition + * label's `]:`, plus optional whitespace with at most one line ending. + */ +const DESTINATION_OPENER = + /(?<=(?:\]\(|(?:^|\n) {0,3}\[(?:[^\\[\]]|\\[\s\S])+\]:)[ \t]*(?:\r?\n[ \t]*)?)` is a destination, not a tag. * * @param {string} text * @returns {[number, number][]} */ function htmlRanges(text) { - return [...text.matchAll(HTML_COMMENT), ...text.matchAll(HTML_START_TAG)].map((match) => { + const isDestination = (/** @type {number} */ at) => { + DESTINATION_OPENER.lastIndex = at; + return DESTINATION_OPENER.test(text); + }; + const tags = [...text.matchAll(HTML_START_TAG)].filter( + (tag) => !isDestination(Number(tag.index)), + ); + return [...text.matchAll(HTML_COMMENT), ...tags].map((match) => { const from = /** @type {number} */ (match.index); return /** @type {[number, number]} */ ([from, from + match[0].length]); }); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 44867bb2158..e1edf2e2029 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -2204,6 +2204,39 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(read(targetDir, "analysis/RESEARCH.md")).toBe("[t](../transcript/transcript.txt)\n"); }); + it("rewrites a tag-like angle link destination rather than masking it as HTML", async () => { + const { sliceDir, targetDir } = makeFixture(); + fs.writeFileSync( + path.join(sliceDir, "RESEARCH.md"), + "[sources]() beside tag.\n", + ); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "analysis/RESEARCH.md")).toBe( + "sources (source, split across analysis/, metadata/, transcript/ in this copy)" + + " beside tag.\n", + ); + }); + + it("checks tag-like angle link and definition destinations", async () => { + const { sliceDir, targetDir } = makeFixture(); + fs.writeFileSync( + path.join(sliceDir, "RESEARCH.md"), + "[x]() beside tag.\n\n[s]: \n", + ); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + // A definition has no prose to take the split text, so the check flags it. + expect(result.exitCode).toBe(1); + expect(result.unresolved).toEqual([ + { file: "analysis/RESEARCH.md", ref: "missing" }, + { file: "analysis/RESEARCH.md", ref: "source" }, + ]); + }); + it("reads the .work// form only with the slice's own work path", async () => { const { sliceDir: original, targetDir } = makeFixture(); const slug = path.basename(original); From 1f280fdeb67b2e6ab6226dd420697fba2049be57 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 08:11:59 -0400 Subject: [PATCH 56/73] fix(knowledge): decode semicolonless numeric references in HTML attributes Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 12 ++++++++---- .../extraction/watch/relayout-slice.test.js | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index a021c38b708..4b785a2a92f 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -282,8 +282,12 @@ const WINDOWS_1252 = [ * * @param {string} value */ -function decodeHtmlReferences(value) { - return value.replace(/&(?:#(\d+)|#[xX]([0-9A-Fa-f]+)|([A-Za-z]+));/g, (match, dec, hex, name) => { +function decodeHtmlReferences(value, { attribute = false } = {}) { + // HTML attributes also accept a numeric reference without its semicolon; Markdown does not. + const pattern = attribute + ? /&(?:#(\d+);?|#[xX]([0-9A-Fa-f]+);?|([A-Za-z]+);)/g + : /&(?:#(\d+)|#[xX]([0-9A-Fa-f]+)|([A-Za-z]+));/g; + return value.replace(pattern, (match, dec, hex, name) => { if (dec !== undefined || hex !== undefined) { const code = dec !== undefined ? Number(dec) : Number.parseInt(hex, 16); const invalid = code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff); @@ -1037,7 +1041,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { // the enclosing quote character, and an unquoted value that now needs // quotes gets double quotes. const value = text.slice(from, to); - const decoded = decodeHtmlReferences(value); + const decoded = decodeHtmlReferences(value, { attribute: true }); const { found, split } = findReference(decoded, sourceFile, map); if (found && "retired" in found) retiredRefs.push(decoded); const next = rewrite(decoded, split); @@ -1245,7 +1249,7 @@ export function checkLayoutLinks( check(unescapeDestination(raw).dest, true); } for (const { from, to } of htmlAttributeValues(tagText)) { - check(decodeHtmlReferences(text.slice(from, to)), true); + check(decodeHtmlReferences(text.slice(from, to), { attribute: true }), true); } for (const span of codeSpans(fenceMasked)) { // A span with spaces may be a command (`node run.mjs x.js --flag`): only diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index e1edf2e2029..a6c6c9dbfab 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1302,6 +1302,21 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(read(targetDir, "metadata/decks/€.pdf")).toBe("pdf"); }); + // HTML attributes accept a numeric reference without its semicolon; Markdown + // destinations do not, so only the attribute is decoded. + it("decodes semicolonless numeric references in HTML attributes only", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/decks/a&b.pdf": "pdf", + "metadata/deck-inventory.md": + 'deck\n[md](decks/c&d.pdf)\n', + }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "metadata/deck-inventory.md", ref: "decks/c&d.pdf" }, + ]); + }); + // HTML numeric character reference replacements: 0x80 is windows-1252's // euro sign (U+20AC), and 0 becomes U+FFFD. it("applies HTML's numeric character reference replacements", () => { From 7bdf4b3cdde868de6cf30e7c7ad13c35acf7e41e Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 08:19:45 -0400 Subject: [PATCH 57/73] fix(knowledge): keep word-distinguishing punctuation in caption dedupe; decode legacy named refs in attributes The boundary carry-over dedupe now strips only leading and trailing sentence punctuation, so C++. and C# no longer compare equal. HTML attribute decoding also accepts amp, lt, gt and quot without a semicolon unless a letter, digit or = follows. Co-Authored-By: Claude Opus 5.5 --- .../extraction/transcript/write-transcript.js | 9 +++++++-- .../transcript/write-transcript.test.js | 16 ++++++++++++++++ .../extraction/watch/relayout-slice.js | 9 ++++++--- .../extraction/watch/relayout-slice.test.js | 16 ++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index 6b07e548542..e5ba6440f24 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -46,9 +46,14 @@ function tokens(text) { /** @param {string} word */ function comparableWord(word) { // Apostrophes in any form (', ’, ʼ) drop out, so don't, don’t and dont match. - return word.toLowerCase().replace(/\u02BC|[^\p{L}\p{N}]/gu, ""); + // Only sentence punctuation at either end goes, so C++. and C# stay distinct. + return word.toLowerCase().replace(/['\u2019\u02BC]/g, "").replace(EDGE_PUNCTUATION, ""); } +/** Sentence punctuation (quotes, brackets, ellipses, dashes) at a word's ends. */ +const EDGE_PUNCTUATION = + /^[.,;:!?"()[\]{}\u2026\u2010-\u2015-]+|[.,;:!?"()[\]{}\u2026\u2010-\u2015-]+$/g; + /** * The lexical words of `text`: punctuation-only tokens such as `>>` are not * words, so they never count toward a run's length. @@ -56,7 +61,7 @@ function comparableWord(word) { * @param {string} text */ function words(text) { - return tokens(text).filter((token) => comparableWord(token) !== ""); + return tokens(text).filter((token) => /[\p{L}\p{N}]/u.test(token)); } /** @param {string[]} run */ diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index 84a7530afac..c0d558a8c93 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -138,6 +138,22 @@ Really ${variant} care about it. expect(result.transcript).toBe("[0:01] so we really don't care.\n\n[0:05] about it."); }); + it("keeps an overlapping cue whose words differ only by symbols, like C++ and C#", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +these days we + +00:00:03.000 --> 00:00:05.500 +use C++. + +00:00:05.000 --> 00:00:07.000 +use C# today +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe("[0:01] these days we use C++.\n\n[0:05] use C# today"); + }); + it("keeps a three-word repeat the speaker said when no cue carries it over", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 4b785a2a92f..3b15615e2d4 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -283,11 +283,14 @@ const WINDOWS_1252 = [ * @param {string} value */ function decodeHtmlReferences(value, { attribute = false } = {}) { - // HTML attributes also accept a numeric reference without its semicolon; Markdown does not. + // HTML attributes also accept a numeric reference without its semicolon, and a + // legacy named one (amp, lt, gt, quot) unless a letter, digit or `=` follows; + // Markdown accepts neither. const pattern = attribute - ? /&(?:#(\d+);?|#[xX]([0-9A-Fa-f]+);?|([A-Za-z]+);)/g + ? /&(?:#(\d+);?|#[xX]([0-9A-Fa-f]+);?|([A-Za-z]+);|(amp|lt|gt|quot)(?![A-Za-z0-9=]))/g : /&(?:#(\d+)|#[xX]([0-9A-Fa-f]+)|([A-Za-z]+));/g; - return value.replace(pattern, (match, dec, hex, name) => { + return value.replace(pattern, (match, dec, hex, named, legacy) => { + const name = named ?? legacy; if (dec !== undefined || hex !== undefined) { const code = dec !== undefined ? Number(dec) : Number.parseInt(hex, 16); const invalid = code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index a6c6c9dbfab..1e271f2cb3f 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1317,6 +1317,22 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + // HTML attributes decode a legacy named reference without its semicolon, + // except before a letter, digit or `=`: `&.` is `&`, `&x` stays. + it("decodes a semicolonless legacy named reference in an HTML attribute", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/decks/a&.pdf": "pdf", + "metadata/decks/a&x.pdf": "pdf", + "metadata/deck-inventory.md": + 'deck\nx\n', + }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "metadata/deck-inventory.md", ref: "decks/a&x.pdf" }, + ]); + }); + // HTML numeric character reference replacements: 0x80 is windows-1252's // euro sign (U+20AC), and 0 becomes U+FFFD. it("applies HTML's numeric character reference replacements", () => { From fe69da9939f0f83ed3c380f74760692df4356ef1 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 08:26:39 -0400 Subject: [PATCH 58/73] fix(knowledge): an outer link whose label holds a link is literal text CommonMark links cannot contain links, so in [outer [real](ok.md)](missing.md) only the inner link is a link. inlineLinks now drops a non-image link whose label holds another non-image link; an image in the label still leaves it a link. The rewrite and the link check both read through inlineLinks, so neither touches the literal outer text. Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 14 ++++++++++- .../extraction/watch/relayout-slice.test.js | 24 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 3b15615e2d4..3f9f3dd9c3a 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -864,7 +864,19 @@ export function inlineLinks(text) { let from = 0; for (;;) { const open = text.indexOf("](", from); - if (open === -1) return links; + // Links cannot contain links: one whose label holds another (non-image) + // link is literal text. An image in the label leaves it a link. + if (open === -1) { + const isLink = (/** @type {InlineLink} */ link) => text[link.linkStart] !== "!"; + return links.filter( + (outer) => + !isLink(outer) || + !links.some( + (inner) => + isLink(inner) && inner.linkStart > outer.linkStart && inner.linkEnd <= outer.labelEnd, + ), + ); + } from = open + 2; let at = open + 2; at = skipLinkSpace(text, at); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 1e271f2cb3f..99974facaaf 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1568,6 +1568,30 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it("reads an outer link whose label holds a link as literal text", () => { + expect(inlineLinks("[outer [real](ok.md)](missing.md)").map(({ dest }) => dest)).toEqual([ + "ok.md", + ]); + }); + + it("leaves the literal outer text of a link holding a link untouched", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "key-frames/visual-frames.md": + "[outer [t](../source/transcript.txt)](missing.md)\n" + + "[outer [t](../source/transcript.txt)](../run-state/watch.json)\n", + }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "[outer [t](../transcript/transcript.txt)](missing.md)\n" + + "[outer [t](../transcript/transcript.txt)](../run-state/watch.json)\n", + ); + }); + it("rewrites single-quoted and unquoted HTML src and href", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { From 4267d49855b42210a290663692b00d8095067bb4 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 08:36:48 -0400 Subject: [PATCH 59/73] fix(knowledge): mask autolinks, reject paren titles with bare parens, count only real words in caption runs Co-Authored-By: Claude Opus 5.5 --- .../extraction/transcript/write-transcript.js | 9 +++++++-- .../transcript/write-transcript.test.js | 18 ++++++++++++++++++ .../extraction/watch/relayout-slice.js | 18 +++++++++++++----- .../extraction/watch/relayout-slice.test.js | 15 +++++++++++++++ 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index e5ba6440f24..6026cb4b540 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -61,7 +61,12 @@ const EDGE_PUNCTUATION = * @param {string} text */ function words(text) { - return tokens(text).filter((token) => /[\p{L}\p{N}]/u.test(token)); + return tokens(text).filter(isWord); +} + +/** @param {string} token - a word holds at least one letter or digit */ +function isWord(token) { + return /[\p{L}\p{N}]/u.test(token); } /** @param {string[]} run */ @@ -120,7 +125,7 @@ export function dropParagraphBoundaryRepeats(transcript, cues) { if (!current || !previous) continue; const previousWords = words(previous[2]); const currentTokens = tokens(current[2]); - const wordAt = currentTokens.flatMap((token, at) => (comparableWord(token) ? [at] : [])); + const wordAt = currentTokens.flatMap((token, at) => (isWord(token) ? [at] : [])); const currentWords = wordAt.map((at) => currentTokens[at]); const longest = Math.min( BOUNDARY_REPEAT_MAX_WORDS, diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index c0d558a8c93..fa0d10a3bdd 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -188,6 +188,24 @@ that answer was ); }); + it("drops overlapping cues' carried-over words after a >> marker, keeping the marker", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +that answer was + +00:00:03.000 --> 00:00:05.500 +>> Very good. + +00:00:05.000 --> 00:00:07.000 +>> Very good question, thanks. +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe( + "[0:01] that answer was >> Very good.\n\n[0:05] >> question, thanks.", + ); + }); + it("keeps two words the speaker genuinely repeats across a paragraph boundary", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 3f9f3dd9c3a..de477b801e7 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -196,6 +196,11 @@ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:\?[^#]*)?(?:#.*)? const HTML_START_TAG = /<([A-Za-z][A-Za-z0-9-]*)(?:\s(?:"[^"]*"|'[^']*'|[^<>"'])*)?\/?>/g; /** An HTML comment. */ const HTML_COMMENT = //g; +/** A CommonMark URI or email autolink: https://spec.commonmark.org/0.31.2/#autolinks */ +const URI_AUTOLINK = /[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\x00-\x20<>]*/.source; +const EMAIL_LABEL = /[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?/.source; +const EMAIL_AUTOLINK = `${/[\w.!#$%&'*+/=?^`{|}~-]+@/.source}${EMAIL_LABEL}(?:\\.${EMAIL_LABEL})*`; +const AUTOLINK = new RegExp(`<(?:${URI_AUTOLINK}|${EMAIL_AUTOLINK})>`, "g"); /** * A `<` opening a link or definition destination: after `](` or a definition * label's `]:`, plus optional whitespace with at most one line ending. @@ -231,10 +236,11 @@ function htmlAttributeValues(text) { } /** - * Ranges of `text` holding HTML start tags and comments: attribute and - * comment text is data, so the link, definition and backticked-path passes - * skip it; only htmlAttributeValues reads a tag. An angle-bracket link or - * definition destination such as `` is a destination, not a tag. + * Ranges of `text` holding HTML start tags, comments and autolinks: attribute, + * comment and autolink text is data, so the link, definition and + * backticked-path passes skip it (an autolink's `](` is no link); only + * htmlAttributeValues reads a tag. An angle-bracket link or definition + * destination such as `` is a destination, not a tag or autolink. * * @param {string} text * @returns {[number, number][]} @@ -244,7 +250,7 @@ function htmlRanges(text) { DESTINATION_OPENER.lastIndex = at; return DESTINATION_OPENER.test(text); }; - const tags = [...text.matchAll(HTML_START_TAG)].filter( + const tags = [...text.matchAll(HTML_START_TAG), ...text.matchAll(AUTOLINK)].filter( (tag) => !isDestination(Number(tag.index)), ); return [...text.matchAll(HTML_COMMENT), ...tags].map((match) => { @@ -904,6 +910,8 @@ export function inlineLinks(text) { closeQuote = scan; break; } + // A (...) title holds no unescaped `(`; with one, this is no link. + if (quote === "(" && text[scan] === "(") break; // A title never spans a blank line (one holding only spaces or tabs). if (text[scan] === "\n" && /^[ \t]*\r?\n/.test(text.slice(scan + 1))) break; } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 99974facaaf..db5c80427ad 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1509,6 +1509,21 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "lost.md" }]); }); + // CommonMark: a (...) title holds no unescaped parenthesis, so this is no link. + it("reads no link whose parenthesized title holds a bare parenthesis", () => { + expect(inlineLinks("[example](missing.md (title(with paren))")).toEqual([]); + }); + + // CommonMark: an autolink binds before link grouping, so its `](` is no link. + it("flags no link inside a URI autolink", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/notes.md": "[foo\n", + }); + + expect(checkLayoutLinks(root)).toEqual([]); + }); + it("reads a link with a parenthesized title and rewrites its destination", async () => { expect(inlineLinks("[x](a.md (Title)) and [y](b.md)").map(({ dest }) => dest)).toEqual([ "a.md", From 0e5d4d049f7d9964e66d61fd32a9be7ae0581ea5 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 08:45:46 -0400 Subject: [PATCH 60/73] fix(knowledge): CRLF blank lines, raw fragment delimiters in links; document the literal-entity limit A code span search now stops at a CRLF blank line, so a lone backtick no longer hides a later dangling link. Markdown links and HTML src/href read a raw # or ? as a fragment or query delimiter; only code spans try it as part of the file name. The named-reference known limit now notes that a link to a retained file whose name literally holds such a reference is left as written. Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 2 + .../extraction/watch/relayout-slice.js | 37 ++++++++++------ .../extraction/watch/relayout-slice.test.js | 43 +++++++++++++++++++ 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 1e083559642..955f1340971 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -156,6 +156,8 @@ It also does not decode named character references beyond `& < > " (numeric references are decoded). A destination holding another one (`a@b.pdf` for the file `a@b.pdf`) is not mapped to its file, and the link check refuses it, naming the path, so nothing passes silently. +When a retained file's name literally contains such a reference and the layout leaves its path +unchanged, the link is left as written and is not flagged. ## Agent-written artifacts share the same root diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index de477b801e7..5889d46eb11 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -152,7 +152,7 @@ const REFERENCE_DEFINITION_LABEL = new RegExp( "gm", ); /** A line ending inside a title, which never spans a blank line (only spaces or tabs). */ -const TITLE_LINE_END = String.raw`\n(?![ \t]*\r?\n)`; +const TITLE_LINE_END = String.raw`\r?\n(?![ \t]*\r?\n)`; /** What may follow a definition's destination: a line ending, or a title then one. */ const DEFINITION_TAIL = new RegExp( String.raw`^(?:[ \t]+(?:"(?:${BACKSLASH}|${TITLE_LINE_END}|[^"\\\n])*"|'(?:${BACKSLASH}|${TITLE_LINE_END}|[^'\\\n])*'|\((?:${BACKSLASH}|${TITLE_LINE_END}|[^()\\\n])*\)))?[ \t]*(?:\r?\n|$)`, @@ -374,7 +374,7 @@ export function codeSpans(text) { while (scan < text.length) { const next = text.indexOf("`", scan); if (next === -1) break; - if (/\n[ \t]*\n/.test(text.slice(runEnd, next))) break; + if (/\n[ \t]*\r?\n/.test(text.slice(runEnd, next))) break; let nextEnd = next; while (text[nextEnd] === "`") nextEnd += 1; if (nextEnd - next === ticks.length) { @@ -751,26 +751,35 @@ function lookUpReference(refPath, sourceFile, map) { const NOT_RETAINED = "not retained in this copy"; /** - * How a reference is read, in order: the whole name, a raw `#` or `?` and a + * How a code span is read, in order: the whole name, a raw `#` or `?` and a * `:12` ending included (`notes#1.md`, `slide.pdf:12`), then without its line - * suffix, and only then with the rest read as a fragment. + * suffix, and only then with the rest read as a fragment. A code span names a + * file literally. */ const WHOLE_NAME_FIRST = [{ fragments: false, lines: false }, { fragments: false }, {}]; /** - * What a reference names under the first reading that names anything (an - * earlier reading must name it exactly, not as a guess into a copied + * How a link destination (Markdown or HTML) is read: a raw `#` or `?` always + * starts its fragment or query (an escaped `\#` or `%23` is a name + * character), and the name is tried with its `:12` ending, then without it. + */ +const LINK_READINGS = [{ lines: false }, {}]; + +/** + * What a reference names under the first of `readings` that names anything + * (an earlier reading must name it exactly, not as a guess into a copied * directory), and that reading's split options (the last when none does). * * @param {string} ref * @param {string} sourceFile * @param {PathMap} map + * @param {{ fragments?: boolean, lines?: boolean }[]} [readings] * @returns {{ found: Lookup|null, split: { fragments?: boolean, lines?: boolean } }} */ -function findReference(ref, sourceFile, map) { - for (const split of WHOLE_NAME_FIRST) { +function findReference(ref, sourceFile, map, readings = LINK_READINGS) { + for (const split of readings) { const found = lookUpReference(splitReference(ref, split).refPath, sourceFile, map); - if (found && !("guessed" in found && split !== WHOLE_NAME_FIRST.at(-1))) { + if (found && !("guessed" in found && split !== readings.at(-1))) { return { found, split }; } } @@ -1081,7 +1090,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { const { ticks, content } = span; const { pad, quote, core, spaced } = spanPath(content); if (core === null) continue; - const { found, split } = findReference(core, sourceFile, map); + const { found, split } = findReference(core, sourceFile, map, WHOLE_NAME_FIRST); // A span with spaces is a path only when it resolves as one whole path. if (spaced && !found) continue; let next; @@ -1245,10 +1254,10 @@ export function checkLayoutLinks( unresolved.push({ file, ref: literalText(ref) }); return; } - // Any reference may hold a raw `#` or `?`, or a `:12` ending, in its - // name: read as WHOLE_NAME_FIRST orders it, the first reading that - // resolves passes; the last one is checked below. - for (const option of WHOLE_NAME_FIRST.slice(0, -1)) { + // A code span may hold a raw `#` or `?` in its name, and any reference a + // `:12` ending: read as WHOLE_NAME_FIRST or LINK_READINGS orders it, the + // first reading that resolves passes; the last one is checked below. + for (const option of (isLink ? LINK_READINGS : WHOLE_NAME_FIRST).slice(0, -1)) { const whole = splitReference(ref, option).refPath; if (whole && fs.existsSync(path.resolve(fileDir, whole))) return; } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index db5c80427ad..2956641787d 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -837,6 +837,15 @@ describe("relayoutSlice", () => { ); }); + it("ends a code span search at a CRLF blank line, so a later link is still checked", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "analysis/notes.md": "A lone ` backtick.\r\n\r\n[gone](missing.md) then ` again.\r\n", + }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "analysis/notes.md", ref: "missing.md" }]); + }); + it("reads \\![x](dest) as a plain link and keeps the \\! when retiring it", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { @@ -1023,6 +1032,40 @@ describe("relayoutSlice", () => { ); }); + it("reads a raw # in a link as a fragment, unlike the same backticked path", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "analysis/research/findings/notes#1.md": "n\n", + "analysis/notes.md": + "[notes](research/findings/notes#1.md) `research/findings/notes#1.md`\n", + }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "analysis/notes.md", ref: "research/findings/notes#1.md" }, + ]); + writeTree(root, { "analysis/research/findings/notes": "n\n" }); + expect(checkLayoutLinks(root)).toEqual([]); + }); + + it("refuses a link whose raw # names a retained file only as part of the name", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { + "research/findings/notes#1.md": "n\n", + "recommendations/questions.md": "[notes](research/findings/notes#1.md)\n", + }, + }); + const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + try { + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(result.unresolved).toHaveLength(1); + expect(result.unresolved[0].ref).toMatch(/research\/findings\/notes#1\.md$/); + } finally { + stderr.mockRestore(); + } + }); + it.each([ [ "a frame name that is a directory", From 95ecca9848ac76c7be33e7872e3da35c9f78d951 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 08:52:51 -0400 Subject: [PATCH 61/73] fix(knowledge): keep edge dashes in caption word matching; read wrapped HTML URLs Caption dedupe no longer strips hyphens and dashes from a word's ends, so C-- and C stay distinct. HTML src/href values drop ASCII tab, LF and CR before decoding, as the URL parser does, so a value wrapped across lines is rewritten and link-checked instead of skipped. Co-Authored-By: Claude Opus 5.5 --- .../extraction/transcript/write-transcript.js | 8 +++++--- .../transcript/write-transcript.test.js | 16 ++++++++++++++++ .../extraction/watch/relayout-slice.js | 18 +++++++++++++++--- .../extraction/watch/relayout-slice.test.js | 18 ++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index 6026cb4b540..2fa5fd097b5 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -50,9 +50,11 @@ function comparableWord(word) { return word.toLowerCase().replace(/['\u2019\u02BC]/g, "").replace(EDGE_PUNCTUATION, ""); } -/** Sentence punctuation (quotes, brackets, ellipses, dashes) at a word's ends. */ -const EDGE_PUNCTUATION = - /^[.,;:!?"()[\]{}\u2026\u2010-\u2015-]+|[.,;:!?"()[\]{}\u2026\u2010-\u2015-]+$/g; +/** + * Sentence punctuation (quotes, brackets, ellipses) at a word's ends. Hyphens + * and dashes stay, so C-- and C never compare equal. + */ +const EDGE_PUNCTUATION = /^[.,;:!?"()[\]{}\u2026]+|[.,;:!?"()[\]{}\u2026]+$/g; /** * The lexical words of `text`: punctuation-only tokens such as `>>` are not diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index fa0d10a3bdd..70453011212 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -154,6 +154,22 @@ use C# today expect(result.transcript).toBe("[0:01] these days we use C++.\n\n[0:05] use C# today"); }); + it("keeps an overlapping cue whose word differs only by trailing dashes, like C-- and C", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +these days we + +00:00:03.000 --> 00:00:05.500 +use C--. + +00:00:05.000 --> 00:00:07.000 +use C today +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe("[0:01] these days we use C--.\n\n[0:05] use C today"); + }); + it("keeps a three-word repeat the speaker said when no cue carries it over", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 5889d46eb11..4dbaa7a17ef 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -228,13 +228,25 @@ function htmlAttributeValues(text) { if (!/^(?:src|href)$/i.test(match[1])) continue; const group = [2, 3, 4].find((at) => match?.[at] !== undefined) ?? 2; const span = match.indices?.[group]; - if (!span || span[0] === span[1] || tag[0].slice(span[0], span[1]).includes("\n")) continue; + if (!span || span[0] === span[1]) continue; values.push({ from: base + span[0], to: base + span[1], group }); } } return values; } +/** + * The URL an HTML `src`/`href` value names: ASCII tab, LF and CR removed, as + * the URL parser removes them, so a value wrapped across lines still names its + * path, then character references decoded. + * https://url.spec.whatwg.org/#concept-basic-url-parser + * + * @param {string} value + */ +function htmlAttributeUrl(value) { + return decodeHtmlReferences(value.replace(/[\t\n\r]/g, ""), { attribute: true }); +} + /** * Ranges of `text` holding HTML start tags, comments and autolinks: attribute, * comment and autolink text is data, so the link, definition and @@ -1073,7 +1085,7 @@ export function rewriteSliceReferences(text, sourceFile, targetFile, map) { // the enclosing quote character, and an unquoted value that now needs // quotes gets double quotes. const value = text.slice(from, to); - const decoded = decodeHtmlReferences(value, { attribute: true }); + const decoded = htmlAttributeUrl(value); const { found, split } = findReference(decoded, sourceFile, map); if (found && "retired" in found) retiredRefs.push(decoded); const next = rewrite(decoded, split); @@ -1281,7 +1293,7 @@ export function checkLayoutLinks( check(unescapeDestination(raw).dest, true); } for (const { from, to } of htmlAttributeValues(tagText)) { - check(decodeHtmlReferences(text.slice(from, to), { attribute: true }), true); + check(htmlAttributeUrl(text.slice(from, to)), true); } for (const span of codeSpans(fenceMasked)) { // A span with spaces may be a command (`node run.mjs x.js --flag`): only diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 2956641787d..8c379144ef2 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -2011,6 +2011,24 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it("reads an HTML href wrapped across lines without its tab, LF and CR", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": 't\n' }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + 't\n', + ); + const root = makeDir("relayout-check-"); + writeTree(root, { "frames/visual-frames.md": '\n' }); + expect(checkLayoutLinks(root)).toEqual([ + { file: "frames/visual-frames.md", ref: "key/gone.png" }, + ]); + }); + it("flags a single-quoted HTML src that does not resolve", () => { const root = makeDir("relayout-check-"); writeTree(root, { "frames/visual-frames.md": "\n" }); From 7ca97e3d7cbb18b4dc2732df29179dfb2637dcde Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 08:53:51 -0400 Subject: [PATCH 62/73] fix(knowledge): decode HTML URL references before dropping tab and line breaks Co-Authored-By: Claude Opus 5.5 --- .../video-digest/extraction/watch/relayout-slice.js | 3 ++- .../extraction/watch/relayout-slice.test.js | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 4dbaa7a17ef..944b60b66fb 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -244,7 +244,8 @@ function htmlAttributeValues(text) { * @param {string} value */ function htmlAttributeUrl(value) { - return decodeHtmlReferences(value.replace(/[\t\n\r]/g, ""), { attribute: true }); + // The HTML parser decodes references first; the URL parser then drops tab, LF and CR. + return decodeHtmlReferences(value, { attribute: true }).replace(/[\t\n\r]/g, ""); } /** diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 8c379144ef2..5b0ac4538ed 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1345,6 +1345,18 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(read(targetDir, "metadata/decks/€.pdf")).toBe("pdf"); }); + // The HTML parser decodes references before the URL parser drops tab, LF and CR, + // so an encoded line break in an attribute is removed too. + it("drops a line break written as a reference in an HTML URL", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "transcript/transcript.txt": "[0:01] hi\n", + "metadata/notes.md": 't\n', + }); + + expect(checkLayoutLinks(root)).toEqual([]); + }); + // HTML attributes accept a numeric reference without its semicolon; Markdown // destinations do not, so only the attribute is decoded. it("decodes semicolonless numeric references in HTML attributes only", () => { From bc0be2239e7fcbdbd8b1c438d1740fc9d4873303 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:01:08 -0400 Subject: [PATCH 63/73] fix(knowledge): trim HTML URL edge whitespace; document link text in definition titles Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 3 +++ .../extraction/watch/relayout-slice.js | 7 ++++-- .../extraction/watch/relayout-slice.test.js | 23 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 955f1340971..b2c7e152e3f 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -159,6 +159,9 @@ passes silently. When a retained file's name literally contains such a reference and the layout leaves its path unchanged, the link is left as written and is not flagged. +Link-shaped text inside a reference definition's title is read as a link: the rewrite may update +it, and if it names a missing path the link check refuses the run. + ## Agent-written artifacts share the same root **Do not split the slice.** Every `.work//…` path in this skill and its `context/` diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 944b60b66fb..cf230aee11d 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -244,8 +244,11 @@ function htmlAttributeValues(text) { * @param {string} value */ function htmlAttributeUrl(value) { - // The HTML parser decodes references first; the URL parser then drops tab, LF and CR. - return decodeHtmlReferences(value, { attribute: true }).replace(/[\t\n\r]/g, ""); + // The HTML parser decodes references first; the URL parser then trims leading and + // trailing C0 controls and spaces and drops tab, LF and CR. + return decodeHtmlReferences(value, { attribute: true }) + .replace(/^[\x00-\x20]+|[\x00-\x20]+$/g, "") + .replace(/[\t\n\r]/g, ""); } /** diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 5b0ac4538ed..ffb5b313564 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1345,6 +1345,29 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(read(targetDir, "metadata/decks/€.pdf")).toBe("pdf"); }); + // The URL parser trims leading and trailing spaces and controls from an attribute URL. + it("trims edge whitespace from an HTML URL", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/ok.md": "ok\n", + "metadata/notes.md": 'ok\n', + }); + + expect(checkLayoutLinks(root)).toEqual([]); + }); + + // Known limit: a definition title is not masked, so link-shaped text in it is checked + // and a missing path there makes the run refuse rather than pass. + it("refuses a missing path written as a link inside a definition title", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/ok.md": "ok\n", + "metadata/notes.md": '[real]: ok.md "see [fake](missing.md)"\n', + }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "missing.md" }]); + }); + // The HTML parser decodes references before the URL parser drops tab, LF and CR, // so an encoded line break in an attribute is removed too. it("drops a line break written as a reference in an HTML URL", () => { From e0bd11d4c2a5a7f61e23fb1b86818883d0148677 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:09:46 -0400 Subject: [PATCH 64/73] fix(knowledge): match CommonMark start tags only; treat curly quotes as edge punctuation Co-Authored-By: Claude Opus 5.5 --- .../extraction/transcript/write-transcript.js | 3 ++- .../transcript/write-transcript.test.js | 16 ++++++++++++++++ .../extraction/watch/relayout-slice.js | 3 ++- .../extraction/watch/relayout-slice.test.js | 13 +++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index 2fa5fd097b5..3ce1fd484ce 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -54,7 +54,8 @@ function comparableWord(word) { * Sentence punctuation (quotes, brackets, ellipses) at a word's ends. Hyphens * and dashes stay, so C-- and C never compare equal. */ -const EDGE_PUNCTUATION = /^[.,;:!?"()[\]{}\u2026]+|[.,;:!?"()[\]{}\u2026]+$/g; +const EDGE_PUNCTUATION = + /^[.,;:!?"\u201c\u201d\u201e\u00ab\u00bb()[\]{}\u2026]+|[.,;:!?"\u201c\u201d\u201e\u00ab\u00bb()[\]{}\u2026]+$/g; /** * The lexical words of `text`: punctuation-only tokens such as `>>` are not diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index 70453011212..cb68bb6c0b6 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -170,6 +170,22 @@ use C today expect(result.transcript).toBe("[0:01] these days we use C--.\n\n[0:05] use C today"); }); + it("drops an overlapping carry-over that differs only by curly quotes", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +she said + +00:00:03.000 --> 00:00:05.500 +“very good”. + +00:00:05.000 --> 00:00:07.000 +Very good question +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe("[0:01] she said “very good”.\n\n[0:05] question"); + }); + it("keeps a three-word repeat the speaker said when no cue carries it over", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index cf230aee11d..47eb6261099 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -193,7 +193,8 @@ const HTML_ATTRIBUTE = /[\s/]*([^\s"'>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:\?[^#]*)?(?:#.*)?$/; /** An HTML start tag: ``. */ -const HTML_START_TAG = /<([A-Za-z][A-Za-z0-9-]*)(?:\s(?:"[^"]*"|'[^']*'|[^<>"'])*)?\/?>/g; +const HTML_START_TAG = + /<([A-Za-z][A-Za-z0-9-]*)(?:\s+[A-Za-z_:][A-Za-z0-9_.:-]*(?:\s*=\s*(?:[^\s"'=<>`]+|'[^']*'|"[^"]*"))?)*\s*\/?>/g; /** An HTML comment. */ const HTML_COMMENT = //g; /** A CommonMark URI or email autolink: https://spec.commonmark.org/0.31.2/#autolinks */ diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index ffb5b313564..c5f9de64b89 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -2030,6 +2030,19 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ); }); + it("does not treat an invalid HTML-looking wrapper as a tag around a real link", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "key-frames/visual-frames.md": "\n" }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(0); + expect(read(targetDir, "frames/visual-frames.md")).toBe( + "\n", + ); + }); + it("rewrites and checks upper-case HTML attribute names", async () => { const { sliceDir, targetDir } = makeFixture({ extraFiles: { "key-frames/visual-frames.md": '\n' }, From 359e1a62db780e82c0390a74d0ec35330c596b04 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:17:21 -0400 Subject: [PATCH 65/73] fix(knowledge): mask every raw inline HTML form; ignore escaped link brackets Co-Authored-By: Claude Opus 5.5 --- .../video-digest/context/output-contract.md | 3 +++ .../extraction/watch/relayout-slice.js | 7 +++++-- .../extraction/watch/relayout-slice.test.js | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index b2c7e152e3f..04a9c5ec139 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -162,6 +162,9 @@ unchanged, the link is left as written and is not flagged. Link-shaped text inside a reference definition's title is read as a link: the rewrite may update it, and if it names a missing path the link check refuses the run. +A `<` right after an unescaped `](` is read as a link destination even when the `]` has no +opening label; an HTML tag there is not masked, and a missing path inside it makes the run refuse. + ## Agent-written artifacts share the same root **Do not split the slice.** Every `.work//…` path in this skill and its `context/` diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 47eb6261099..6c29cad1146 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -197,6 +197,9 @@ const HTML_START_TAG = /<([A-Za-z][A-Za-z0-9-]*)(?:\s+[A-Za-z_:][A-Za-z0-9_.:-]*(?:\s*=\s*(?:[^\s"'=<>`]+|'[^']*'|"[^"]*"))?)*\s*\/?>/g; /** An HTML comment. */ const HTML_COMMENT = //g; +/** Other raw inline HTML: a closing tag, processing instruction, declaration or CDATA section. */ +const HTML_OTHER = + /<\/[A-Za-z][A-Za-z0-9-]*\s*>|<\?[\s\S]*?\?>|]*>|/g; /** A CommonMark URI or email autolink: https://spec.commonmark.org/0.31.2/#autolinks */ const URI_AUTOLINK = /[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\x00-\x20<>]*/.source; const EMAIL_LABEL = /[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?/.source; @@ -207,7 +210,7 @@ const AUTOLINK = new RegExp(`<(?:${URI_AUTOLINK}|${EMAIL_AUTOLINK})>`, "g"); * label's `]:`, plus optional whitespace with at most one line ending. */ const DESTINATION_OPENER = - /(?<=(?:\]\(|(?:^|\n) {0,3}\[(?:[^\\[\]]|\\[\s\S])+\]:)[ \t]*(?:\r?\n[ \t]*)?) !isDestination(Number(tag.index)), ); - return [...text.matchAll(HTML_COMMENT), ...tags].map((match) => { + return [...text.matchAll(HTML_COMMENT), ...text.matchAll(HTML_OTHER), ...tags].map((match) => { const from = /** @type {number} */ (match.index); return /** @type {[number, number]} */ ([from, from + match[0].length]); }); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index c5f9de64b89..13f8b40c869 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1345,6 +1345,23 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(read(targetDir, "metadata/decks/€.pdf")).toBe("pdf"); }); + // Processing instructions, declarations, CDATA and closing tags are raw inline HTML, + // and a tag after an escaped `](` is a tag, not a link destination. + it("treats link-shaped text inside other raw HTML forms as data", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/notes.md": [ + "before after", + " ", + '\\]()', + "[real](gone.md)", + "", + ].join("\n"), + }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "gone.md" }]); + }); + // The URL parser trims leading and trailing spaces and controls from an attribute URL. it("trims edge whitespace from an HTML URL", () => { const root = makeDir("relayout-check-"); From 06ce2ae018fb0871c140576deeb21331fff3f26b Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:22:25 -0400 Subject: [PATCH 66/73] fix(knowledge): hide CDATA, declarations and processing instructions from the attribute scan Co-Authored-By: Claude Opus 5.5 --- .../video-digest/extraction/watch/relayout-slice.js | 4 +++- .../extraction/watch/relayout-slice.test.js | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 6c29cad1146..5a0dddb623f 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -1192,7 +1192,9 @@ export function maskCode(text) { return { fenceMasked: blank(fenceMasked, html), linkMasked: blank(codeMasked, html), - tagText: codeMasked.replace(HTML_COMMENT, (comment) => comment.replace(/[^\n]/g, " ")), + tagText: codeMasked + .replace(HTML_COMMENT, (comment) => comment.replace(/[^\n]/g, " ")) + .replace(HTML_OTHER, (raw) => raw.replace(/[^\n]/g, " ")), }; } diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index 13f8b40c869..b215bf093e2 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1362,6 +1362,16 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "gone.md" }]); }); + it("does not read attributes inside CDATA, declarations or processing instructions", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/notes.md": + ']]> ?> x\n', + }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "lost.md" }]); + }); + // The URL parser trims leading and trailing spaces and controls from an attribute URL. it("trims edge whitespace from an HTML URL", () => { const root = makeDir("relayout-check-"); From 4e5901c4a93d5d881de5ce589ae407ddea65eb03 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:29:43 -0400 Subject: [PATCH 67/73] fix(knowledge): refuse a non-directory --replace target and bare-CR Markdown Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 11 ++++++++++ .../extraction/watch/relayout-slice.test.js | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 5a0dddb623f..a17adc3074f 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -1709,6 +1709,14 @@ export async function relayoutSlice({ if (symlinks.length > 0) { return refuse(`the slice holds symlinks, which a copy would follow: ${symlinks.join(", ")}`); } + // The reference scanner reads LF and CRLF line endings only; refuse rather than misread. + const bareCr = sliceFiles.filter( + (file) => + MARKDOWN_FILE.test(file) && /\r(?!\n)/.test(fs.readFileSync(path.join(slice, file), "utf8")), + ); + if (bareCr.length > 0) { + return refuse(`Markdown with bare CR line endings is not supported: ${bareCr.join(", ")}`); + } // An interrupted `--replace` swap leaves the previous target at this name. if (fs.existsSync(backup)) { @@ -1770,6 +1778,9 @@ export async function relayoutSlice({ if (fs.existsSync(target) && !replace) { return refuse(`${target} exists; pass --replace to replace it`); } + if (fs.existsSync(target) && !fs.lstatSync(target).isDirectory()) { + return refuse(`${target} exists and is not a directory; --replace only replaces a directory`); + } // `--no-media` takes nothing from the temp session, even when it still exists. // `close` deletes every temp dir watch.json records, so in every mode, diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index b215bf093e2..aca6c142267 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -231,6 +231,28 @@ describe("relayoutSlice", () => { expect(fs.readdirSync(path.dirname(targetDir))).toEqual(["out"]); }); + it("refuses --replace on a target that is a regular file and keeps the file", async () => { + const { sliceDir, targetDir } = makeFixture(); + fs.mkdirSync(path.dirname(targetDir), { recursive: true }); + fs.writeFileSync(targetDir, "precious\n"); + + const result = await relayoutSlice({ sliceDir, targetDir, replace: true }); + + expect(result.exitCode).toBe(1); + expect(fs.readFileSync(targetDir, "utf8")).toBe("precious\n"); + }); + + it("refuses a slice whose Markdown uses bare CR line endings", async () => { + const { sliceDir, targetDir } = makeFixture({ + extraFiles: { "research/findings/cr.md": "[deck]: ../../source/transcript.txt\rnext\r" }, + }); + + const result = await relayoutSlice({ sliceDir, targetDir }); + + expect(result.exitCode).toBe(1); + expect(fs.existsSync(targetDir)).toBe(false); + }); + it("refuses an existing target without --replace and leaves it untouched", async () => { const { sliceDir, targetDir } = makeFixture(); writeTree(targetDir, { "stale.md": "old\n" }); From 857edfbfbf67164c65283e6fa1c200e55065b986 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:36:35 -0400 Subject: [PATCH 68/73] fix(knowledge): follow CommonMark raw-HTML whitespace, short comments and reference digit limits Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 8 ++++---- .../extraction/watch/relayout-slice.test.js | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index a17adc3074f..6a52b25e056 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -194,9 +194,9 @@ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:\?[^#]*)?(?:#.*)? /** An HTML start tag: ``. */ const HTML_START_TAG = - /<([A-Za-z][A-Za-z0-9-]*)(?:\s+[A-Za-z_:][A-Za-z0-9_.:-]*(?:\s*=\s*(?:[^\s"'=<>`]+|'[^']*'|"[^"]*"))?)*\s*\/?>/g; + /<([A-Za-z][A-Za-z0-9-]*)(?:[ \t\r\n]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \t\r\n]*=[ \t\r\n]*(?:[^ \t\r\n"'=<>`]+|'[^']*'|"[^"]*"))?)*[ \t\r\n]*\/?>/g; /** An HTML comment. */ -const HTML_COMMENT = //g; +const HTML_COMMENT = /||/g; /** Other raw inline HTML: a closing tag, processing instruction, declaration or CDATA section. */ const HTML_OTHER = /<\/[A-Za-z][A-Za-z0-9-]*\s*>|<\?[\s\S]*?\?>|]*>|/g; @@ -313,8 +313,8 @@ function decodeHtmlReferences(value, { attribute = false } = {}) { // legacy named one (amp, lt, gt, quot) unless a letter, digit or `=` follows; // Markdown accepts neither. const pattern = attribute - ? /&(?:#(\d+);?|#[xX]([0-9A-Fa-f]+);?|([A-Za-z]+);|(amp|lt|gt|quot)(?![A-Za-z0-9=]))/g - : /&(?:#(\d+)|#[xX]([0-9A-Fa-f]+)|([A-Za-z]+));/g; + ? /&(?:#(\d{1,7});?|#[xX]([0-9A-Fa-f]{1,6});?|([A-Za-z]+);|(amp|lt|gt|quot)(?![A-Za-z0-9=]))/g + : /&(?:#(\d{1,7})|#[xX]([0-9A-Fa-f]{1,6})|([A-Za-z]+));/g; return value.replace(pattern, (match, dec, hex, named, legacy) => { const name = named ?? legacy; if (dec !== undefined || hex !== undefined) { diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index aca6c142267..ccb153bbe97 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1384,6 +1384,25 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "gone.md" }]); }); + it("follows CommonMark's raw HTML whitespace, short comments and reference digit limits", () => { + const nbsp = String.fromCodePoint(0xa0); + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/notes.md": [ + ``, + "[t](gone-2.md)-->", + "[t](aA.md)", + "", + ].join("\n"), + }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "metadata/notes.md", ref: "gone-1.md" }, + { file: "metadata/notes.md", ref: "gone-2.md" }, + { file: "metadata/notes.md", ref: "aA.md" }, + ]); + }); + it("does not read attributes inside CDATA, declarations or processing instructions", () => { const root = makeDir("relayout-check-"); writeTree(root, { From b5136cb796af467066a2aa47720b116b33326300 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:43:15 -0400 Subject: [PATCH 69/73] fix(knowledge): honor escaped HTML openers; keep HTML attribute references uncapped Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 18 +++++++++++++----- .../extraction/watch/relayout-slice.test.js | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 6a52b25e056..cb810321012 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -270,13 +270,21 @@ function htmlRanges(text) { DESTINATION_OPENER.lastIndex = at; return DESTINATION_OPENER.test(text); }; + // A `<` after an odd run of backslashes is escaped, so it opens no HTML. + const escaped = (/** @type {number} */ at) => { + let run = 0; + while (text[at - 1 - run] === "\\") run += 1; + return run % 2 === 1; + }; const tags = [...text.matchAll(HTML_START_TAG), ...text.matchAll(AUTOLINK)].filter( (tag) => !isDestination(Number(tag.index)), ); - return [...text.matchAll(HTML_COMMENT), ...text.matchAll(HTML_OTHER), ...tags].map((match) => { - const from = /** @type {number} */ (match.index); - return /** @type {[number, number]} */ ([from, from + match[0].length]); - }); + return [...text.matchAll(HTML_COMMENT), ...text.matchAll(HTML_OTHER), ...tags] + .filter((match) => !escaped(Number(match.index))) + .map((match) => { + const from = /** @type {number} */ (match.index); + return /** @type {[number, number]} */ ([from, from + match[0].length]); + }); } const HTML_NAMED_REFERENCES = /** @type {Record} */ ({ @@ -313,7 +321,7 @@ function decodeHtmlReferences(value, { attribute = false } = {}) { // legacy named one (amp, lt, gt, quot) unless a letter, digit or `=` follows; // Markdown accepts neither. const pattern = attribute - ? /&(?:#(\d{1,7});?|#[xX]([0-9A-Fa-f]{1,6});?|([A-Za-z]+);|(amp|lt|gt|quot)(?![A-Za-z0-9=]))/g + ? /&(?:#(\d+);?|#[xX]([0-9A-Fa-f]+);?|([A-Za-z]+);|(amp|lt|gt|quot)(?![A-Za-z0-9=]))/g : /&(?:#(\d{1,7})|#[xX]([0-9A-Fa-f]{1,6})|([A-Za-z]+));/g; return value.replace(pattern, (match, dec, hex, named, legacy) => { const name = named ?? legacy; diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index ccb153bbe97..aeb74e66070 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1403,6 +1403,20 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( ]); }); + it("reads an escaped HTML opener as text and uncapped digits in attributes", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/decks/a&b.pdf": "pdf", + "metadata/notes.md": [ + '\\', + 'deck', + "", + ].join("\n"), + }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "gone.md" }]); + }); + it("does not read attributes inside CDATA, declarations or processing instructions", () => { const root = makeDir("relayout-check-"); writeTree(root, { From 204799e2bf08b072d5b9f0e02fafbd3282ca1781 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:51:08 -0400 Subject: [PATCH 70/73] fix(knowledge): one line ending per tag separator, escaped tags skip the attribute pass, curly single quotes Co-Authored-By: Claude Opus 5.5 --- .../extraction/transcript/write-transcript.js | 2 +- .../transcript/write-transcript.test.js | 16 +++++++++++++ .../extraction/watch/relayout-slice.js | 23 ++++++++++++------- .../extraction/watch/relayout-slice.test.js | 14 +++++++++++ 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js index 3ce1fd484ce..6e3bd5cac0b 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.js @@ -55,7 +55,7 @@ function comparableWord(word) { * and dashes stay, so C-- and C never compare equal. */ const EDGE_PUNCTUATION = - /^[.,;:!?"\u201c\u201d\u201e\u00ab\u00bb()[\]{}\u2026]+|[.,;:!?"\u201c\u201d\u201e\u00ab\u00bb()[\]{}\u2026]+$/g; + /^[.,;:!?"\u2018\u201a\u201c\u201d\u201e\u2039\u203a\u00ab\u00bb()[\]{}\u2026]+|[.,;:!?"\u2018\u201a\u201c\u201d\u201e\u2039\u203a\u00ab\u00bb()[\]{}\u2026]+$/g; /** * The lexical words of `text`: punctuation-only tokens such as `>>` are not diff --git a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js index cb68bb6c0b6..bb2600c46a9 100644 --- a/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/transcript/write-transcript.test.js @@ -170,6 +170,22 @@ use C today expect(result.transcript).toBe("[0:01] these days we use C--.\n\n[0:05] use C today"); }); + it("drops an overlapping carry-over that differs only by curly single quotes", () => { + const autoVtt = `WEBVTT + +00:00:01.000 --> 00:00:03.000 +she said + +00:00:03.000 --> 00:00:05.500 +‘very good’. + +00:00:05.000 --> 00:00:07.000 +Very good question +`; + const result = buildTranscriptText(autoVtt, true); + expect(result.transcript).toBe("[0:01] she said ‘very good’.\n\n[0:05] question"); + }); + it("drops an overlapping carry-over that differs only by curly quotes", () => { const autoVtt = `WEBVTT diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index cb810321012..12ae743e04a 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -194,7 +194,7 @@ const PATH_LIKE = /[/\\]|\.[A-Za-z0-9]+(?::\d+(?:[-,]\d+)*)*(?:\?[^#]*)?(?:#.*)? /** An HTML start tag: ``. */ const HTML_START_TAG = - /<([A-Za-z][A-Za-z0-9-]*)(?:[ \t\r\n]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \t\r\n]*=[ \t\r\n]*(?:[^ \t\r\n"'=<>`]+|'[^']*'|"[^"]*"))?)*[ \t\r\n]*\/?>/g; + /<([A-Za-z][A-Za-z0-9-]*)(?:(?:[ \t]+(?:\r?\n[ \t]*)?|\r?\n[ \t]*)[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \t]*(?:\r?\n[ \t]*)?=[ \t]*(?:\r?\n[ \t]*)?(?:[^ \t\r\n"'=<>`]+|'[^']*'|"[^"]*"))?)*[ \t]*(?:\r?\n[ \t]*)?\/?>/g; /** An HTML comment. */ const HTML_COMMENT = /||/g; /** Other raw inline HTML: a closing tag, processing instruction, declaration or CDATA section. */ @@ -222,11 +222,23 @@ const DESTINATION_OPENER = * @param {string} text * @returns {{ from: number, to: number, group: number }[]} */ +/** + * Whether the character at `at` follows an odd run of backslashes, so it is escaped. + * @param {string} text + * @param {number} at + */ +function isEscapedAt(text, at) { + let run = 0; + while (text[at - 1 - run] === "\\") run += 1; + return run % 2 === 1; +} + function htmlAttributeValues(text) { /** @type {{ from: number, to: number, group: number }[]} */ const values = []; for (const tag of text.matchAll(HTML_START_TAG)) { const base = /** @type {number} */ (tag.index); + if (isEscapedAt(text, base)) continue; HTML_ATTRIBUTE.lastIndex = 1 + tag[1].length; for (let match; (match = HTML_ATTRIBUTE.exec(tag[0])) !== null; ) { if (!/^(?:src|href)$/i.test(match[1])) continue; @@ -270,17 +282,12 @@ function htmlRanges(text) { DESTINATION_OPENER.lastIndex = at; return DESTINATION_OPENER.test(text); }; - // A `<` after an odd run of backslashes is escaped, so it opens no HTML. - const escaped = (/** @type {number} */ at) => { - let run = 0; - while (text[at - 1 - run] === "\\") run += 1; - return run % 2 === 1; - }; const tags = [...text.matchAll(HTML_START_TAG), ...text.matchAll(AUTOLINK)].filter( (tag) => !isDestination(Number(tag.index)), ); return [...text.matchAll(HTML_COMMENT), ...text.matchAll(HTML_OTHER), ...tags] - .filter((match) => !escaped(Number(match.index))) + // A `<` after an odd run of backslashes is escaped, so it opens no HTML. + .filter((match) => !isEscapedAt(text, Number(match.index))) .map((match) => { const from = /** @type {number} */ (match.index); return /** @type {[number, number]} */ ([from, from + match[0].length]); diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index aeb74e66070..6a253875172 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1417,6 +1417,20 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "gone.md" }]); }); + it("ignores escaped tags in the attribute pass and tags that span a blank line", () => { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/notes.md": [ + '\\', + "", + '', + "", + ].join("\n"), + }); + + expect(checkLayoutLinks(root)).toEqual([{ file: "metadata/notes.md", ref: "gone-2.md" }]); + }); + it("does not read attributes inside CDATA, declarations or processing instructions", () => { const root = makeDir("relayout-check-"); writeTree(root, { From 4d1d723d4af8ed67695ffff7580b04cccbf3f88c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:59:24 -0400 Subject: [PATCH 71/73] docs(knowledge): record the code-span-before-HTML limit of the relayout scanner Co-Authored-By: Claude Opus 5.5 --- .../knowledge/skills/video-digest/context/output-contract.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/knowledge/skills/video-digest/context/output-contract.md b/plugins/knowledge/skills/video-digest/context/output-contract.md index 04a9c5ec139..c7af5d6103c 100644 --- a/plugins/knowledge/skills/video-digest/context/output-contract.md +++ b/plugins/knowledge/skills/video-digest/context/output-contract.md @@ -165,6 +165,9 @@ it, and if it names a missing path the link check refuses the run. A `<` right after an unescaped `](` is read as a link destination even when the `]` has no opening label; an HTML tag there is not masked, and a missing path inside it makes the run refuse. +Code spans are found before raw HTML, so a backtick inside a quoted HTML attribute can pair with a +later backtick and hide the text between them from the rewrite and the link check. + ## Agent-written artifacts share the same root **Do not split the slice.** Every `.work//…` path in this skill and its `context/` From 82ad3c0d58864f09ae434957a5d72f47171f0721 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:06:47 -0400 Subject: [PATCH 72/73] fix(knowledge): stop code spans at paragraph-interrupting block starts Co-Authored-By: Claude Opus 5.5 --- .../extraction/watch/relayout-slice.js | 28 +++++++++++-------- .../extraction/watch/relayout-slice.test.js | 21 ++++++++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 12ae743e04a..697d8802886 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -123,6 +123,12 @@ const TRANSCRIPT_TEXT = /^transcript(?:-\d+)?\.txt$/; /** Markdown files, by extension, in any case. */ const MARKDOWN_FILE = /\.(?:md|markdown)$/i; +/** + * A paragraph boundary a code span cannot cross: a blank line, or a line that + * interrupts a paragraph (ATX heading, block quote, fence, thematic break, list item). + */ +const BLOCK_BREAK = + /\n(?:[ \t]*\r?\n| {0,3}(?:#{1,6}(?:[ \t]|\r?$)|>|`{3,}|~{3,}|(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})\r?$|[-*+][ \t]+\S|1[.)][ \t]+\S))/m; const VIDEO_FILE = /\.(?:mp4|mkv|webm)$/; const FRAME_IMAGE = /\.(?:png|jpe?g)$/i; /** The slice lane directories, plus the `SLICE/` root alias. */ @@ -212,16 +218,6 @@ const AUTOLINK = new RegExp(`<(?:${URI_AUTOLINK}|${EMAIL_AUTOLINK})>`, "g"); const DESTINATION_OPENER = /(?<=(?:(? { + const root = makeDir("relayout-check-"); + writeTree(root, { + "metadata/notes.md": [ + "dangling `", + "# [h](gone-1.md) `", + "again `", + "> [q](gone-2.md) `", + "once more `", + "- [l](gone-3.md) `", + "", + ].join("\n"), + }); + + expect(checkLayoutLinks(root)).toEqual([ + { file: "metadata/notes.md", ref: "gone-1.md" }, + { file: "metadata/notes.md", ref: "gone-2.md" }, + { file: "metadata/notes.md", ref: "gone-3.md" }, + ]); + }); + it("ignores escaped tags in the attribute pass and tags that span a blank line", () => { const root = makeDir("relayout-check-"); writeTree(root, { From 46e31deb27552cd8a28b29a6fd1c639226b70c14 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:13:25 -0400 Subject: [PATCH 73/73] fix(knowledge): stop code spans at setext heading underlines Co-Authored-By: Claude Opus 5.5 --- .../skills/video-digest/extraction/watch/relayout-slice.js | 5 +++-- .../video-digest/extraction/watch/relayout-slice.test.js | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js index 697d8802886..44a1b1eeaad 100755 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.js @@ -125,10 +125,11 @@ const TRANSCRIPT_TEXT = /^transcript(?:-\d+)?\.txt$/; const MARKDOWN_FILE = /\.(?:md|markdown)$/i; /** * A paragraph boundary a code span cannot cross: a blank line, or a line that - * interrupts a paragraph (ATX heading, block quote, fence, thematic break, list item). + * interrupts or ends a paragraph (ATX or setext heading, block quote, fence, thematic + * break, list item). */ const BLOCK_BREAK = - /\n(?:[ \t]*\r?\n| {0,3}(?:#{1,6}(?:[ \t]|\r?$)|>|`{3,}|~{3,}|(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})\r?$|[-*+][ \t]+\S|1[.)][ \t]+\S))/m; + /\n(?:[ \t]*\r?\n| {0,3}(?:#{1,6}(?:[ \t]|\r?$)|>|`{3,}|~{3,}|(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})\r?$|(?:=+|-+)[ \t]*\r?$|[-*+][ \t]+\S|1[.)][ \t]+\S))/m; const VIDEO_FILE = /\.(?:mp4|mkv|webm)$/; const FRAME_IMAGE = /\.(?:png|jpe?g)$/i; /** The slice lane directories, plus the `SLICE/` root alias. */ diff --git a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js index d7123623325..efc513f2e6d 100644 --- a/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js +++ b/plugins/knowledge/skills/video-digest/extraction/watch/relayout-slice.test.js @@ -1428,6 +1428,10 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( "once more `", "- [l](gone-3.md) `", "", + "setext `", + "===", + "[s](gone-4.md) `", + "", ].join("\n"), }); @@ -1435,6 +1439,7 @@ describe("relayoutSlice input shapes and reference forms (adversarial sweep)", ( { file: "metadata/notes.md", ref: "gone-1.md" }, { file: "metadata/notes.md", ref: "gone-2.md" }, { file: "metadata/notes.md", ref: "gone-3.md" }, + { file: "metadata/notes.md", ref: "gone-4.md" }, ]); });