perf: reduce build memory and time

.changeset/build-memory.md

@@ -0,0 +1,7 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+'@node-core/doc-kit-legacy': minor
+---
+
+Builds take about half the memory and a third of the time (measured on the Node.js API docs): Shiki registers each language once code in it is highlighted, highlighted code reaches pages as static markup rather than a tree per token, `all.html` is rendered by the worker pool, HTML is no longer minified (serve it compressed instead), idle workers end after 250ms, and each worker's heap is limited to 512MB, which Node.js's `--max-old-space-size` raises. `@doc-kit/core/utils/html-minifier.mjs` is removed.

.github/dependabot.yml

@@ -73,7 +73,6 @@ updates:
           - 'recma-*'
       compiling:
         patterns:
-          - '@swc/html-wasm'
           - '@rollup/*'
           - 'rolldown'
           - 'lightningcss-wasm'

docs/configuration.md

@@ -109,7 +109,10 @@ Everything under the `global` key applies to every generator:
   logos, and templated text. Defaults to the `name` in your `package.json`.
 - `version` {string|SemVer} Documentation version. **Default:**
   `process.version`.
-- `minify` {boolean} Whether to minify output. **Default:** `true`.
+- `minify` {boolean} Whether to minify JSON, scripts, and styles. HTML is
+  written as is: serve it compressed instead (see
+  [Compress the pages](./publishing.md#compress-the-pages)). **Default:**
+  `true`.
 - `repository` {string} GitHub repository in `owner/repo` format; without
   one, repository UI is omitted.
 - `ref` {string} Git reference (branch, tag, or commit SHA). **Default:**
@@ -134,7 +137,8 @@ these for that generator alone, except `markdown`, which it adds to.
 Top-level, alongside `target` and `global`:
 
 - `threads` {number} Worker threads used for generation. Defaults to your
-  CPU count.
+  CPU count. Each worker's heap is limited to 512MB; to raise the limit, run
+  Node.js with `--max-old-space-size` (directly or through `NODE_OPTIONS`).
 - `chunkSize` {number} Items processed per worker thread. **Default:** `10`.
 
 ## Generator options
@@ -196,6 +200,7 @@ own.
 - `rehypePlugins` {Array} Run on the HTML of the generators rendering
   Markdown, such as `jsx-ast`, before code is highlighted.
 - `recmaPlugins` {Array} Run on the JavaScript `jsx-ast` compiles the pages to.
+  Highlighted code appears in it as HTML strings, not as JSX elements.
 
 A generator only takes the plugins its pipeline has a place for: `jsx-ast`
 takes all three kinds, `ast`, `metadata`, and `json` take remark plugins, and
@@ -218,6 +223,12 @@ bundles. To configure it, list it with options in
 dependencies, so your configuration file can resolve it):
 
 - `langs` {Array} More languages: grammars, or modules default-exporting them.
+  Shiki's bundled languages are registered the first time code in them is
+  highlighted, so one that injects into other languages (such as `ts-tags`,
+  for tagged templates in JavaScript) only applies to its own code blocks.
+  List it here to register it up front (e.g.
+  `import.meta.resolve('shiki/langs/ts-tags.mjs')`), along with any language a
+  transformer highlights.
 - `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`.
 - `themes` {Object} The `light` and `dark` themes: names of themes Shiki
   bundles, modules default-exporting themes, or themes.

docs/creating-generators.md

@@ -524,7 +524,9 @@ A plugin needing asynchronous setup can export an async `load(options)`
 returning the plugin, instead of a default export. The Shiki plugin does, and
 `getHighlighter(name)` (from `@doc-kit/core/plugins/shiki/highlighter.mjs`)
 returns the highlighter of a generator's Shiki plugin, to highlight code as its
-pipeline does.
+pipeline does. It registers Shiki's bundled languages as code in them is
+highlighted, so before using its `shiki` instance directly, pass the language
+through its `resolveLanguage`.
 
 ## File Output
 

docs/publishing.md

@@ -56,6 +56,24 @@ costs a request per asset each time a new tab opens the site. On Vercel:
 Within a visit, moving between pages loads no assets at all: the site
 navigates client-side, swapping in the next page's content.
 
+## Compress the pages
+
+doc-kit writes HTML as rendered, without minifying it. Served compressed (gzip
+or brotli), as hosts do, pages shrink to a fraction of their size, which
+minifying them would barely improve on.
+
+Cloudflare, Vercel, Netlify, and GitHub Pages compress HTML on their own, with
+nothing to configure, and so does `npx serve out`. If you serve the output
+yourself, turn compression on:
+
+- **nginx:** `gzip on;` compresses HTML; add the other types with
+  `gzip_types text/css application/javascript application/json;`.
+- **Caddy:** `encode zstd gzip`.
+- **Apache:** with `mod_deflate`,
+  `AddOutputFilterByType DEFLATE text/html text/css application/javascript application/json`.
+- **Amazon S3:** serve the bucket through CloudFront, with **Compress objects
+  automatically** on.
+
 ## Tell doc-kit its public URL
 
 Set `baseURL` to where the site will live. Generators that emit absolute

packages/core/package.json

@@ -52,9 +52,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@actions/core": "^3.0.0",
-    "@node-core/rehype-shiki": "^1.5.0",
-    "@swc/html-wasm": "^1.16.2",
+    "@node-core/rehype-shiki": "^1.5.1",
     "@swc/wasm": "^1.15.46",
     "acorn": "^8.17.0",
     "cosmiconfig": "^9.0.2",

packages/core/src/generators/metadata/generate.mjs

@@ -3,15 +3,17 @@
 import getConfig from '#utils/configuration/index.mjs';
 import { loadFromURL } from '#utils/loaders.mjs';
 
-import { parseApiDoc } from './utils/parse.mjs';
-
 /**
  * Process a chunk of API doc files in a worker thread.
  * Called by chunk-worker.mjs for parallel processing.
  *
  * @type {import('./types').Generator['processChunk']}
  */
 export async function processChunk(fullInput, itemIndices, typeMap) {
+  // Loaded on first use rather than with the generator, so the main thread,
+  // which never parses, does not load the TypeScript parser (~40MB of WASM)
+  const { parseApiDoc } = await import('./utils/parse.mjs');
+
   const results = [];
 
   for (const idx of itemIndices) {

packages/core/src/logger/__tests__/transports/github.test.mjs

@@ -148,6 +148,43 @@ describe('github', () => {
     );
   });
 
+  it('should escape messages and files', t => {
+    process.env.FORCE_COLOR = '1';
+
+    t.mock.timers.enable({ apis: ['Date'] });
+
+    const fn = t.mock.method(process.stdout, 'write');
+
+    // noop
+    fn.mock.mockImplementation(() => {});
+
+    github({
+      level: LogLevel.info,
+      message: '100%: a, b\r\nc',
+      metadata: {
+        file: {
+          path: 'a,b:c%.md',
+          position: {
+            start: { line: 1 },
+            end: { line: 2 },
+          },
+        },
+      },
+      timestamp: Date.now(),
+    });
+
+    const callsArgs = process.stdout.write.mock.calls.map(
+      call => call.arguments[0]
+    );
+
+    strictEqual(process.stdout.write.mock.callCount(), 1);
+    // The message keeps its `,` and `:`, which only separate properties
+    strictEqual(
+      callsArgs[0].trim(),
+      '::notice file=a%2Cb%3Ac%25.md,line=1,endLine=2::[00:00:00.000] \x1B[32mINFO\x1B[39m: 100%25: a, b%0D%0Ac'
+    );
+  });
+
   it('should print child logger name', t => {
     process.env.FORCE_COLOR = '1';
 

packages/core/src/logger/constants.mjs

@@ -32,3 +32,38 @@ export const levelToColorMap = {
   [LogLevel.error]: 'magenta',
   [LogLevel.fatal]: 'red',
 };
+
+/**
+ * Maps log level numbers to the GitHub Actions workflow commands reporting
+ * them.
+ */
+export const levelToGitHubCommand = {
+  [LogLevel.debug]: 'debug',
+  [LogLevel.info]: 'notice',
+  [LogLevel.warn]: 'warning',
+  [LogLevel.error]: 'error',
+  [LogLevel.fatal]: 'error',
+};
+
+/**
+ * Maps the characters a workflow command's message can't hold to their
+ * escapes. A command is a line, `::command key=value,key=value::message`,
+ * so its message, which runs to the end of the line, can't break it, nor hold
+ * the `%` an escape starts with.
+ */
+export const commandDataEscapes = {
+  '%': '%25',
+  '\r': '%0D',
+  '\n': '%0A',
+};
+
+/**
+ * Maps the characters a workflow command's property can't hold to their
+ * escapes: those its message can't, and the `,` separating properties and
+ * the `:` of the `::` ending them.
+ */
+export const commandPropertyEscapes = {
+  ...commandDataEscapes,
+  ':': '%3A',
+  ',': '%2C',
+};

packages/core/src/logger/transports/github.mjs

@@ -1,23 +1,35 @@
 'use strict';
 
-import { debug, notice, warning, error } from '@actions/core';
+import { EOL } from 'node:os';
 
-import { LogLevel } from '#logger/constants.mjs';
+import {
+  commandDataEscapes,
+  commandPropertyEscapes,
+  levelToGitHubCommand,
+} from '#logger/constants.mjs';
 import { prettifyLevel } from '#logger/utils/colors.mjs';
 import { prettifyTimestamp } from '#logger/utils/time.mjs';
 
-const actions = {
-  [LogLevel.debug]: debug,
-  [LogLevel.info]: notice,
-  [LogLevel.warn]: warning,
-  [LogLevel.error]: error,
-  [LogLevel.fatal]: error,
-};
+/**
+ * Escapes a value of a workflow command: its message, with
+ * `commandDataEscapes`, or one of its properties, with
+ * `commandPropertyEscapes`.
+ *
+ * @param {string | number} value - The value
+ * @param {Record<string, string>} escapes - The escapes of the characters it can't hold
+ * @returns {string}
+ */
+const escapeCommandValue = (value, escapes) =>
+  String(value).replace(
+    /[%\r\n:,]/g,
+    character => escapes[character] ?? character
+  );
 
 /**
- * Logs messages to GitHub Actions with formatted output and file info with
- * appropriate gh actions based on level.
+ * Logs messages to GitHub Actions as workflow commands, with the file and
+ * lines they're about, for them to annotate.
  *
+ * @see https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands
  * @param {import('../types').TransportContext} context
  * @returns {void}
  */
@@ -30,13 +42,34 @@ const github = ({ level, message, timestamp, metadata = {}, module }) => {
 
   const logMessage = `[${time}] ${prettyLevel}${module ? ` (${module})` : ''}: ${message}`;
 
-  const logFn = actions[level] ?? notice;
+  const command = levelToGitHubCommand[level] ?? 'notice';
+
+  const data = escapeCommandValue(logMessage, commandDataEscapes);
 
-  logFn(logMessage, {
+  // Debug messages take no properties
+  if (command === 'debug') {
+    process.stdout.write(`::debug::${data}${EOL}`);
+
+    return;
+  }
+
+  const properties = {
     file: file?.path,
-    startLine: file?.position?.start.line,
+    line: file?.position?.start.line,
     endLine: file?.position?.end.line,
-  });
+  };
+
+  const annotations = [];
+
+  for (const [key, value] of Object.entries(properties)) {
+    if (value) {
+      annotations.push(
+        `${key}=${escapeCommandValue(value, commandPropertyEscapes)}`
+      );
+    }
+  }
+
+  process.stdout.write(`::${command} ${annotations.join(',')}::${data}${EOL}`);
 };
 
 export default github;

packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs

@@ -86,6 +86,57 @@ describe('createHighlighter', () => {
     );
   });
 
+  it('registers a bundled language once code in it is highlighted', async () => {
+    const highlighter = await createHighlighter({
+      langAlias: { py: 'python' },
+    });
+    const loaded = () => highlighter.shiki.getLoadedLanguages();
+
+    assert.ok(!loaded().includes('python'));
+    assert.ok(!loaded().includes('javascript'));
+
+    // By its name, an alias of its own, or one of the options
+    assert.equal(highlighter.resolveLanguage('py'), 'py');
+    assert.equal(highlighter.resolveLanguage('mjs'), 'mjs');
+
+    assert.ok(loaded().includes('python'));
+    assert.ok(loaded().includes('javascript'));
+    assert.ok(loaded().includes('cjs'));
+
+    assert.equal(highlighter.resolveLanguage(undefined), 'text');
+    assert.equal(highlighter.resolveLanguage('plaintext'), 'plaintext');
+  });
+
+  it('lists the bundled languages without registering them', async () => {
+    const highlighter = await createHighlighter({ langs: [grammar] });
+
+    assert.deepStrictEqual(
+      highlighter.langs.find(({ name }) => name === 'rust'),
+      { name: 'rust', displayName: 'Rust', aliases: ['rs'] }
+    );
+    assert.equal(highlighter.langs.at(-1), grammar);
+    assert.ok(!highlighter.shiki.getLoadedLanguages().includes('rust'));
+  });
+
+  it('registers the bundled languages a language embeds', async () => {
+    const highlighter = await createHighlighter({
+      langs: [
+        {
+          ...grammar,
+          name: 'oxcscript',
+          scopeName: 'source.oxcscript',
+          embeddedLangs: ['javascript'],
+          patterns: [{ include: 'source.js' }],
+        },
+      ],
+    });
+
+    assert.match(
+      highlighter.highlightToHtml('const on = 1', 'oxcscript'),
+      /--shiki-dark/
+    );
+  });
+
   it('gives the same highlighter for the same options', async () => {
     const highlighter = await createHighlighter({ langs: [grammar] });