From 2604d00d841a60f50bebbd68ae31c1fd6b39883b Mon Sep 17 00:00:00 2001 From: Paul Gottschling Date: Tue, 19 May 2026 08:51:23 -0400 Subject: [PATCH] Add an outcome statement linter Add outcome statement helps human and AI agent readers determine the scope of a docs page within a single paragraph so they can make a decision as to whether to continue reading. While there is currently a check for opening paragraphs in docs pages, the outcome statement check goes a step further and requires a specific sentence structure depending on the kind of guide, which it obtains from the `page_type` frontmatter field. This helps ensure that docs authors include the most appropriate level of detail in each page's outcome statement. For example, while an LLM can determine the scope of a reference page based on the tables, lists, and so on, this linter enforces the presence of an explicit description of the kinds of items a page lists: "This page lists the...". To implement the outcome statement linter, this change adds a check to lint-page-structure. --- package.json | 3 +- server/lint-page-structure.test.ts | 164 +++++++++++++++++++++++++++++ server/lint-page-structure.ts | 113 ++++++++++++++++---- 3 files changed, 257 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index ea798159..7d9c8e86 100644 --- a/package.json +++ b/package.json @@ -53,10 +53,10 @@ "@docusaurus/theme-classic": "^3.10.1", "@docusaurus/theme-mermaid": "^3.10.1", "@docusaurus/tsconfig": "^3.10.1", - "@inkeep/cxkit-react": "^0.5.117", "@iconify-json/carbon": "^1.2.20", "@iconify-json/logos": "^1.2.11", "@iconify/tools": "^5.0.11", + "@inkeep/cxkit-react": "^0.5.117", "@mdx-js/react": "^3.1.1", "@signalwire/docusaurus-plugin-llms-txt": "2.0.0-alpha.7", "classnames": "^2.3.1", @@ -90,6 +90,7 @@ "@storybook/test": "^8.6.15", "@storybook/test-runner": "^0.24.3", "@svgr/webpack": "^8.1.0", + "@types/json-schema": "^7.0.15", "@types/react": "^19.2.14", "ajv": "^8.20.0", "concurrently": "9.2.1", diff --git a/server/lint-page-structure.test.ts b/server/lint-page-structure.test.ts index 4fa04e98..d6ab01ef 100644 --- a/server/lint-page-structure.test.ts +++ b/server/lint-page-structure.test.ts @@ -386,6 +386,170 @@ This is another intro paragraph. ## Concepts Here's some conceptual information. +`, + expected: [], + }, + ]; + + test.each(testCases)("$description", (tc) => { + expect(getReasons(tc.input)).toEqual(tc.expected); + }); + }); + + describe("linting outcome statements in guides", () => { + interface testCase { + description: string; + input: string; + expected: Array; + } + + const testCases: Array = [ + { + description: "introductory how-to section with no outcome statement", + input: `--- +title: My Page +description: This is a page. +page_type: how-to +--- + +This is the introduction of this how-to guide. + +## Prerequisites + +- A computer +`, + expected: [ + `In a how-to guide, the introductory section must include an outcome statement with one of the following prefixes: "After following this guide, you will", "This guide shows you how to", "In this guide, you will". This must be a concrete objective you expect the reader to have achieved. Disable this warning by adding {/* lint ignore page-structure remark-lint */} before this line.`, + ], + }, + { + description: + 'no violations with "After following this guide, you will"', + input: `--- +title: My Page +description: This is a page. +page_type: how-to +--- + +This is the introduction of this how-to guide. + +After following this guide, you will have a working implementation of our +product. + +## Prerequisites + +- A computer +`, + expected: [], + }, + { + description: "outcome statement with code and link in paragraph", + input: `--- +title: My Page +description: This is a page. +page_type: how-to +--- + +This is the introduction of this how-to guide. + +After following this guide, you will have a working implementation of our +product, \`example\`, which you can read about on the [main +site](https://example.com). + +## Prerequisites + +- A computer +`, + expected: [], + }, + { + description: 'no violations with "This guide shows you how to"', + input: `--- +title: My Page +description: This is a page. +page_type: how-to +--- + +This is the introduction of this how-to guide. + +This guide shows you how to enable our product. + +## Prerequisites + +- A computer +`, + expected: [], + }, + { + description: 'no violations with "In this guide, you will"', + input: `--- +title: My Page +description: This is a page. +page_type: how-to +--- + +This is the introduction of this how-to guide. + +In this guide, you will enable our product. + +## Prerequisites + +- A computer +`, + expected: [], + }, + { + description: 'no violations with "In this guide, you will"', + input: `--- +title: My Page +description: This is a page. +page_type: how-to +--- + +This is the introduction of this how-to guide. + +In this guide, you will: +- Install our product. +- Roll it out across your infrastructure. +- Enroll all of the users in your organization. + +## Prerequisites + +- A computer +`, + expected: [], + }, + { + description: "introductory reference section with no outcome statement", + input: `--- +title: My Page +description: This is a page. +page_type: reference +--- + +This is the introduction of this guide. + +## Supported fields + +`, + expected: [ + `In a reference page, the introductory section must include an outcome statement with the prefix, "This page lists the", plus the kinds of items that this page provides a reference for. Disable this warning by adding {/* lint ignore page-structure remark-lint */} before this line.`, + ], + }, + { + description: + "introductory troubleshooting outcome statement in multiple lines", + input: `--- +title: Troubleshooting Apps +description: This is a page. +page_type: troubleshooting +--- + +This page describes common issues that you might encounter in managing access to applications +with Teleport and how to work around or resolve them. + +## Supported fields + `, expected: [], }, diff --git a/server/lint-page-structure.ts b/server/lint-page-structure.ts index b180e079..45b29d64 100644 --- a/server/lint-page-structure.ts +++ b/server/lint-page-structure.ts @@ -1,15 +1,9 @@ import { lintRule } from "unified-lint-rule"; import { visit } from "unist-util-visit"; -import type { Heading, Code, Paragraph, Text } from "mdast"; -import type { - MdxJsxFlowElement, - EsmNode, - MdxAnyElement, - MdxastNode, -} from "./types-unist"; +import type { Code, Heading, Literal, Paragraph, Text } from "mdast"; +import type { MdxJsxFlowElement, MdxAnyElement } from "./types-unist"; import type { Node, Parent, Position } from "unist"; - -const mdxNodeTypes = new Set(["mdxJsxFlowElement", "mdxJsxTextElement"]); +import { parse } from "yaml"; interface stepNumber { numerator: number; @@ -30,11 +24,54 @@ interface paragraphWithIndex { const stepNumberPattern = `^Step ([0-9]+)/([0-9]+)`; const messageSuffix = `Disable this warning by adding {/* lint ignore page-structure remark-lint */} before this line.`; +interface OutcomePatternCheck { + pattern: string; + error: string; +} + +// allowedOutcomePatterns maps values of the page_type frontmatter field to +// rules for the content of an outcome statement. The opening paragraphs of a +// docs page must include a paragraph that matches the pattern, providing a +// concise description of the scope of the page. +const allowedOutcomePatterns: Record = { + "how-to": { + pattern: `(After following this guide, you will|This guide shows you how to|In this guide, you will)`, + error: `In a how-to guide, the introductory section must include an outcome statement with one of the following prefixes: "After following this guide, you will", "This guide shows you how to", "In this guide, you will". This must be a concrete objective you expect the reader to have achieved.`, + }, + troubleshooting: { + pattern: `This (page|guide) describes common issues.+how to work around or resolve them`, + error: `In a troubleshooting guide, the introductory section must include an outcome statement with the pattern, "This page describes common issues...how to work around or resolve them". An example is, "This page describes common issues with connecting self-hosted MySQL databases to Teleport and how to work around or resolve them.`, + }, + conceptual: { + pattern: `This (page|guide) describes (how|what|the)`, + error: `In a conceptual guide, the introductory section must include an outcome statement with one of the following prefixes: "This page describes how", "This page describes what", or "This page describes the". The rest of the outcome statement consists of the concept this page sets out to explain.`, + }, + faq: { + pattern: `This (page|guide) provides answers to frequently asked questions about`, + error: `In a FAQ page, the introductory section must include an outcome statement with the prefix, "This page provides answers to frequently asked questions about", plus the subject of the FAQ page.`, + }, + reference: { + pattern: `This (page|guide) lists the`, + error: `In a reference page, the introductory section must include an outcome statement with the prefix, "This page lists the", plus the kinds of items that this page provides a reference for.`, + }, +}; + export const remarkLintPageStructure = lintRule( "remark-lint:page-structure", (root: Node, vfile) => { const h2s: Array = []; const paras: Array = []; + let frontmatter: Record = {}; + visit(root, "yaml", (node: Node) => { + try { + frontmatter = parse((node as Literal).value) ?? {}; + } catch (err) { + vfile.message( + `page has invalid YAML in frontmatter: ${(err as Error).message}`, + ); + return; + } + }); // Collect paragraphs and headings from first-level children of root. (root as Parent).children.forEach((node, idx) => { @@ -79,22 +116,54 @@ export const remarkLintPageStructure = lintRule( } }); - // See if there is a paragraph that comes before the first H2 in root's - // children. We compare indices instead of line numbers because - // remark-includes preserves the line numbers of any partials it includes. - if ( - h2s.length > 0 && - !paras.some((para) => { + if (h2s.length > 0) { + const introParas = paras.filter((para) => { return para.rootIndex < h2s[0].rootIndex; - }) - ) { - vfile.message( - "This guide is missing at least one introductory paragraph before the first H2. Use introductory paragraphs to explain the purpose and scope of this guide. " + - messageSuffix, - h2s[0].node.position, - ); + }); + + // See if there is a paragraph that comes before the first H2 in root's + // children. We compare indices instead of line numbers because + // remark-includes preserves the line numbers of any partials it includes. + if (introParas.length === 0) { + vfile.message( + "This guide is missing at least one introductory paragraph before the first H2. Use introductory paragraphs to explain the purpose and scope of this guide. " + + messageSuffix, + h2s[0].node.position, + ); + } + + const outcome = allowedOutcomePatterns[frontmatter.page_type]; + // If there is no intro paragraph, the error message above will have + // alerted the author. Otherwise, dig in further to see if there is an + // outcome statement. + if (outcome && introParas.length > 0) { + const outcomeRE = new RegExp(outcome.pattern); + const hasOutcomeParagraph = introParas.some((pwi) => { + const para = pwi.node; + // Collect text nodes from the paragraph. We expect the outcome + // statement pattern to match text, rather than HTML elements, so we + // only perform a first-level search for text nodes. + const txt = para.children.reduce((accum, current) => { + if (current.type !== "text") { + return accum; + } + // Put all paragraph text on a single line for evaluation. + return accum + current.value.replaceAll(`\n`, " "); + }, ""); + return outcomeRE.test(txt); + }); + if (!hasOutcomeParagraph) { + vfile.message( + outcome.error + " " + messageSuffix, + introParas[0].node.position, + ); + } + } } + // We are using the presence of a "## Step" section as a proxy for a how-to + // guide until we can roll out the page_type frontmatter field to all + // how-to guides. const hasStep = h2s.some((h) => h.node.value.match(/^Step [0-9]/) !== null); if (hasStep && h2s[0].node.value !== "How it works") { vfile.message(