diff --git a/.claude/skills/docs-audit/references/RUBRIC.md b/.claude/skills/docs-audit/references/RUBRIC.md index 652ab3d4f0e..d53066c5819 100644 --- a/.claude/skills/docs-audit/references/RUBRIC.md +++ b/.claude/skills/docs-audit/references/RUBRIC.md @@ -70,7 +70,7 @@ Check information architecture, page shape, writing, links, components, diagrams Type: deterministic. -Use `scripts/run-checks.sh` for audited-page formatting, Remark, Vale, and repository validation. It wraps `pnpm check --format json` and attributes findings to the page by file path. Report missing tools or ambiguous attribution as `warn`; report proven unrelated failures separately. +Use `scripts/run-checks.sh` for audited-page formatting, Remark, Vale, and repository validation. It wraps `pnpm check --format json` and attributes findings to the page by file path. If `vale-target=fail` comes from `mastra/vale-unavailable`, run `pnpm vale:download` in `docs/` and rerun the script before reporting; don't report it as an environment limitation. Report other missing tools or ambiguous attribution as `warn`; report proven unrelated failures separately. - `pass`: No audited-target error. - `warn`: The audited page has warning-level findings, a check could not run, or repository validation attribution is ambiguous. diff --git a/.claude/skills/docs-audit/scripts/run-checks.sh b/.claude/skills/docs-audit/scripts/run-checks.sh index 57c4ae1424e..c5b0dce50d5 100755 --- a/.claude/skills/docs-audit/scripts/run-checks.sh +++ b/.claude/skills/docs-audit/scripts/run-checks.sh @@ -151,7 +151,11 @@ for (const f of report.findings) { const target = targetFor(f) const where = f.file ? `${f.file}${f.line ? `:${f.line}${f.column ? `:${f.column}` : ''}` : ''}` : '(no file)' const line = `${where} ${f.severity} ${f.ruleId}: ${f.message}${f.suggestion ? ` (${f.suggestion})` : ''}` - if (f.file && targetSet.has(f.file)) { + // Vale didn't run at all, so no audited page was prose-checked. + if (f.ruleId === 'mastra/vale-unavailable') { + raise('vale', 'fail') + diagnostics.vale.push(line) + } else if (f.file && targetSet.has(f.file)) { raise(target, f.severity === 'error' ? 'fail' : 'warn') diagnostics[target].push(line) } else if (f.severity === 'error') { @@ -166,13 +170,8 @@ for (const f of report.findings) { } for (const skipped of report.summary.skipped) { - if (skipped.check.startsWith('mastra/vale')) { - raise('vale', 'warn') - diagnostics.vale.push(`Vale skipped: ${skipped.reason}`) - } else { - raise('validate', 'warn') - diagnostics.validate.push(`${skipped.check} skipped: ${skipped.reason}`) - } + raise('validate', 'warn') + diagnostics.validate.push(`${skipped.check} skipped: ${skipped.reason}`) } for (const [target, lines] of Object.entries(diagnostics)) { diff --git a/.claude/skills/mastra-docs/references/CHECKS.md b/.claude/skills/mastra-docs/references/CHECKS.md index 36265f99684..4b3b26711de 100644 --- a/.claude/skills/mastra-docs/references/CHECKS.md +++ b/.claude/skills/mastra-docs/references/CHECKS.md @@ -15,11 +15,11 @@ pnpm check --no-vale # skip Vale (CI runs Vale separately The command exits `1` when any error is reported and `0` otherwise. Warnings and info findings never fail it. New rules start as warnings and become errors once existing content is clean. -With file arguments, file-level checks (frontmatter, structure, links, prose, format) only look at those files. Sidebar and redirect checks always run repo-wide. Vale is skipped with a note when its binary isn't installed; run `pnpm vale:download` first. +With file arguments, file-level checks (frontmatter, structure, links, prose, format) only look at those files. Sidebar and redirect checks always run repo-wide. Vale needs its binary, which is gitignored, so run `pnpm vale:download` once per checkout. When the binary is missing or Vale fails to start, `pnpm check` reports a `mastra/vale-unavailable` error and exits `1`. Pass `--no-vale` only when you mean to skip Vale. ## JSON output -`--format json` (or `--json`) prints one object. `status` is `partial` when no errors were found but a check, usually Vale, was skipped: +`--format json` (or `--json`) prints one object. `status` is `partial` when no errors were found but a check was skipped, for example a file argument that is not a content page: ```json { @@ -100,6 +100,7 @@ Vale also honors its own `{/* vale off */}` and `{/* vale on */}` comments. | `mastra/redirected-link` | links | warn | Yes | An internal link points to a redirect source. The fix rewrites it to the destination. | | `mastra/no-self-host-url` | links | warn | Yes | A link uses `https://mastra.ai/...` for a docs route. The fix rewrites it to a root-relative path. | | `mastra/bare-url` | prose | warn | No | A URL appears as plain text instead of a Markdown link. | +| `mastra/vale-unavailable` | prose | error | No | Vale is not installed or failed to start, so prose was not checked. Run `pnpm vale:download`, or pass `--no-vale` to skip Vale on purpose. | | `mastra/format` | format | error | Yes | The file is not formatted with oxfmt (MDX via `@mastra/lint-docs/oxfmt-mdx`). | | `mastra/redirect-duplicate` | redirects | error | No | `vercel.redirects.json` has two redirects with the same source. | | `mastra/redirect-chain` | redirects | error | No | A redirect points to the source of another redirect. | diff --git a/docs/scripts/check/__tests__/check.test.ts b/docs/scripts/check/__tests__/check.test.ts index 67c7e2d39ac..ab147dd7c02 100644 --- a/docs/scripts/check/__tests__/check.test.ts +++ b/docs/scripts/check/__tests__/check.test.ts @@ -1,7 +1,7 @@ import fs from 'fs/promises' import os from 'os' import path from 'path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { checkFrontmatter } from '../checks/frontmatter' import { checkLinks, remarkFindings } from '../checks/mdx' import { checkRedirectList } from '../checks/redirects' @@ -385,6 +385,31 @@ describe('runChecks', () => { await fs.rm(rootDir, { recursive: true, force: true }) }) + it('fails when Vale is not installed unless --no-vale is passed', async () => { + // The temp root has no scripts/vale/bin/vale; an empty PATH hides any global vale. + vi.stubEnv('PATH', '') + try { + const report = await runChecks({ rootDir, rules: ['mastra/vale-*'] }) + expect(report.summary).toMatchObject({ ok: false, status: 'failed', errors: 1 }) + expect(report.findings).toEqual([ + expect.objectContaining({ + ruleId: 'mastra/vale-unavailable', + severity: 'error', + suggestion: expect.stringContaining('pnpm vale:download'), + }), + ]) + + const single = await runChecks({ rootDir, rules: ['mastra/vale-mastra-headings'] }) + expect(single.summary).toMatchObject({ ok: false, errors: 1 }) + expect(single.findings.map(f => f.ruleId)).toEqual(['mastra/vale-unavailable']) + + const skipped = await runChecks({ rootDir, categories: ['prose'], skipVale: true }) + expect(skipped.findings.map(f => f.ruleId)).not.toContain('mastra/vale-unavailable') + } finally { + vi.unstubAllEnvs() + } + }) + it('reports normalized findings and a summary', async () => { const report = await runChecks({ rootDir, categories: ['frontmatter', 'structure', 'links'] }) expect(report.version).toBe(1) diff --git a/docs/scripts/check/checks/vale.ts b/docs/scripts/check/checks/vale.ts index 243860d08f9..abe9d77f143 100644 --- a/docs/scripts/check/checks/vale.ts +++ b/docs/scripts/check/checks/vale.ts @@ -64,13 +64,22 @@ export const valeCheck: Check = { const files = ctx.files.filter(isFormattedFile).map(file => file.path) if (files.length === 0) return { findings: [] } + // A missing or broken Vale binary is an error, not a skip, so a run without Vale never looks green. + const unavailable = (message: string): CheckResult => ({ + findings: [ + finding( + 'mastra/vale-unavailable', + message, + {}, + { + suggestion: 'Run `pnpm vale:download`, or pass `--no-vale` to skip Vale on purpose', + }, + ), + ], + }) + const vale = await findVale(ctx.rootDir) - if (!vale) { - return { - findings: [], - skipped: [{ check: 'mastra/vale-*', reason: 'Vale is not installed; run `pnpm vale:download`' }], - } - } + if (!vale) return unavailable('Vale is not installed, so prose was not checked') let stdout: string try { @@ -82,12 +91,7 @@ export const valeCheck: Check = { // Vale exits 1 when it reports errors; the JSON is still on stdout. const err = error as { code?: number; stdout?: string; stderr?: string } if (err.code !== 1 || !err.stdout) { - return { - findings: [], - skipped: [ - { check: 'mastra/vale-*', reason: `Vale failed: ${(err.stderr || String(error)).trim().split('\n')[0]}` }, - ], - } + return unavailable(`Vale failed to run: ${(err.stderr || String(error)).trim().split('\n')[0]}`) } stdout = err.stdout } diff --git a/docs/scripts/check/rules.ts b/docs/scripts/check/rules.ts index 0106074ba36..dea9eb6b777 100644 --- a/docs/scripts/check/rules.ts +++ b/docs/scripts/check/rules.ts @@ -58,6 +58,7 @@ const RULE_ROWS: RuleRow[] = [ ['mastra/no-self-host-url', 'links', 'warn', 'A link uses `https://mastra.ai/...` for a docs route. The fix rewrites it to a root-relative path.', true], // prose ['mastra/bare-url', 'prose', 'warn', 'A URL appears as plain text instead of a Markdown link.'], + ['mastra/vale-unavailable', 'prose', 'error', 'Vale is not installed or failed to start, so prose was not checked. Run `pnpm vale:download`, or pass `--no-vale` to skip Vale on purpose.'], // format ['mastra/format', 'format', 'error', 'The file is not formatted with oxfmt (MDX via `@mastra/lint-docs/oxfmt-mdx`).', true], // redirects diff --git a/docs/scripts/check/runner.ts b/docs/scripts/check/runner.ts index a82f578a9ba..e005e596a86 100644 --- a/docs/scripts/check/runner.ts +++ b/docs/scripts/check/runner.ts @@ -139,7 +139,9 @@ export async function runChecks(options: RunOptions): Promise { fixed += result.fixed ?? 0 skipped.push(...(result.skipped ?? [])) for (const item of result.findings) { - if (!categories.includes(item.category) || !ruleSelected(item.ruleId)) continue + // Vale only runs when a filter can select a Vale rule, and when it can't start, none of those rules were checked. + const selected = item.ruleId === 'mastra/vale-unavailable' || ruleSelected(item.ruleId) + if (!categories.includes(item.category) || !selected) continue if (suppressed(item.ruleId, item.file, item.line)) continue findings.push(item) }