perf: lazy load shiki languages (#4326)

Author: brc-dd

package.json

@@ -185,6 +185,7 @@
     "sirv": "^3.0.0",
     "sitemap": "^8.0.0",
     "supports-color": "^9.4.0",
+    "synckit": "^0.9.2",
     "tinyglobby": "^0.2.10",
     "typescript": "^5.6.3",
     "vitest": "^2.1.4",

pnpm-lock.yaml

@@ -1,7 +1,7 @@
-import { promises as fs } from 'fs'
-import { builtinModules, createRequire } from 'module'
-import { resolve } from 'path'
-import { fileURLToPath } from 'url'
+import * as fs from 'node:fs/promises'
+import { builtinModules, createRequire } from 'node:module'
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
 import { type RollupOptions, defineConfig } from 'rollup'
 import { nodeResolve } from '@rollup/plugin-node-resolve'
 import commonjs from '@rollup/plugin-commonjs'
@@ -10,6 +10,7 @@ import json from '@rollup/plugin-json'
 import replace from '@rollup/plugin-replace'
 import alias from '@rollup/plugin-alias'
 import dts from 'rollup-plugin-dts'
+import { globSync } from 'tinyglobby'
 
 const ROOT = fileURLToPath(import.meta.url)
 const r = (p: string) => resolve(ROOT, '..', p)
@@ -43,11 +44,15 @@ const plugins = [
 ]
 
 const esmBuild: RollupOptions = {
-  input: [r('src/node/index.ts'), r('src/node/cli.ts')],
+  input: [
+    r('src/node/index.ts'),
+    r('src/node/cli.ts'),
+    ...globSync(r('src/node/worker_*.ts'))
+  ],
   output: {
     format: 'esm',
     entryFileNames: `[name].js`,
-    chunkFileNames: 'serve-[hash].js',
+    chunkFileNames: 'chunk-[hash].js',
     dir: r('dist/node'),
     sourcemap: DEV
   },

rollup.config.ts

@@ -1,7 +1,3 @@
-import { customAlphabet } from 'nanoid'
-import c from 'picocolors'
-import type { ShikiTransformer } from 'shiki'
-import { bundledLanguages, createHighlighter, isSpecialLang } from 'shiki'
 import {
   transformerCompactLineOptions,
   transformerNotationDiff,
@@ -10,9 +6,21 @@ import {
   transformerNotationHighlight,
   type TransformerCompactLineOption
 } from '@shikijs/transformers'
+import { customAlphabet } from 'nanoid'
+import { createRequire } from 'node:module'
+import c from 'picocolors'
+import type { ShikiTransformer } from 'shiki'
+import { createHighlighter, isSpecialLang } from 'shiki'
+import { createSyncFn } from 'synckit'
 import type { Logger } from 'vite'
+import type { ShikiResolveLang } from 'worker_shikiResolveLang'
 import type { MarkdownOptions, ThemeOptions } from '../markdown'
 
+const require = createRequire(import.meta.url)
+
+const resolveLangSync = createSyncFn<ShikiResolveLang>(
+  require.resolve('vitepress/dist/node/worker_shikiResolveLang.js')
+)
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)
 
 /**
@@ -62,7 +70,10 @@ export async function highlight(
       typeof theme === 'object' && 'light' in theme && 'dark' in theme
         ? [theme.light, theme.dark]
         : [theme],
-    langs: [...Object.keys(bundledLanguages), ...(options.languages || [])],
+    langs: [
+      ...(options.languages || []),
+      ...Object.values(options.languageAlias || {})
+    ],
     langAlias: options.languageAlias
   })
 
@@ -108,14 +119,19 @@ export async function highlight(
       if (lang) {
         const langLoaded = highlighter.getLoadedLanguages().includes(lang)
         if (!langLoaded && !isSpecialLang(lang)) {
-          logger.warn(
-            c.yellow(
-              `\nThe language '${lang}' is not loaded, falling back to '${
-                defaultLang || 'txt'
-              }' for syntax highlighting.`
+          const resolvedLang = resolveLangSync(lang)
+          if (!resolvedLang) {
+            logger.warn(
+              c.yellow(
+                `\nThe language '${lang}' is not loaded, falling back to '${
+                  defaultLang || 'txt'
+                }' for syntax highlighting.`
+              )
             )
-          )
-          lang = defaultLang
+            lang = defaultLang
+          } else {
+            highlighter.loadLanguageSync(resolvedLang)
+          }
         }
       }
 

src/node/markdown/plugins/highlight.ts

@@ -0,0 +1,14 @@
+import { bundledLanguages, type DynamicImportLanguageRegistration } from 'shiki'
+import { runAsWorker } from 'synckit'
+
+async function resolveLang(lang: string) {
+  return (
+    (bundledLanguages as Record<string, DynamicImportLanguageRegistration>)
+      [lang]?.()
+      .then((m) => m.default) || []
+  )
+}
+
+runAsWorker(resolveLang)
+
+export type ShikiResolveLang = typeof resolveLang