Skip to content

Commit 0ede395

Browse files
Rich-HarrisRich-Harris
authored
move logic into walkTokens (#287)
* do async transformation stuff in walkTokens * move transformation into walkTokens, so it can be async * simplify * tidy up
1 parent 2d2e6f6 commit 0ede395

File tree

2 files changed

+125
-146
lines changed

2 files changed

+125
-146
lines changed

‎packages/site-kit/src/lib/markdown/renderer.ts

Lines changed: 114 additions & 143 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import MagicString from 'magic-string';
22
import { createHash } from 'node:crypto';
3-
import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises';
3+
import fs from 'node:fs';
44
import path from 'node:path';
55
import ts from 'typescript';
66
import * as prettier from 'prettier';
@@ -31,8 +31,6 @@ interface RenderContentOptions {
3131
const METADATA_REGEX =
3232
/(?:<!---\s*|\/\/\/\s*|###\s*)(?<key>file|link|copy):\s*(?<value>.*?)(?:\s*--->|$)\n/gm;
3333

34-
const CACHE_MAP = new Map<string, string>();
35-
3634
let twoslash_module: typeof import('shiki-twoslash');
3735

3836
/**
@@ -133,130 +131,121 @@ export async function render_content_markdown(
133131
const highlighter = await twoslash_module.createShikiHighlighter({ theme: 'css-variables' });
134132

135133
const { type_links, type_regex } = create_type_links(modules, resolveTypeLinks);
136-
const SNIPPET_CACHE = await create_snippet_cache(cacheCodeSnippets);
134+
const snippets = await create_snippet_cache(cacheCodeSnippets);
137135

138136
body = await replace_export_type_placeholders(body, modules);
139137

140-
const conversions = new Map<string, string>();
141-
142-
for (const [_, language, code] of body.matchAll(/```(js|svelte)\n([\s\S]+?)\n```/g)) {
143-
let { source, options } = parse_options(code, language);
144-
145-
const converted = await generate_ts_from_js(source, language as 'js' | 'svelte', options);
146-
if (converted) {
147-
conversions.set(source, converted);
148-
}
149-
}
150-
151138
const headings: string[] = [];
152139

153140
// this is a bit hacky, but it allows us to prevent type declarations
154141
// from linking to themselves
155142
let current = '';
156143

157144
return await transform(body, {
158-
text(token) {
159-
// @ts-expect-error I think this is a bug in marked — some text tokens have children,
160-
// but that's not reflected in the types. In these cases we can't just use `token.tokens`
161-
// because that will result in e.g. `<code>` elements not being generated
162-
if (token.tokens) {
163-
// @ts-expect-error
164-
return this.parser!.parseInline(token.tokens);
145+
async walkTokens(token) {
146+
if (token.type === 'heading') {
147+
current = token.text;
165148
}
166149

167-
return smart_quotes(token.text, true);
168-
},
169-
heading({ tokens, depth, raw }) {
170-
const text = this.parser!.parseInline(tokens);
171-
172-
const title = text
173-
.replace(/<\/?code>/g, '')
174-
.replace(/&quot;/g, '"')
175-
.replace(/&lt;/g, '<')
176-
.replace(/&gt;/g, '>');
177-
current = title;
178-
const normalized = normalizeSlugify(raw);
179-
headings[depth - 1] = normalized;
180-
headings.length = depth;
181-
const slug = headings.filter(Boolean).join('-');
182-
return `<h${depth} id="${slug}">${text.replace(
183-
/<\/?code>/g,
184-
''
185-
)}<a href="#${slug}" class="permalink"><span class="visually-hidden">permalink</span></a></h${depth}>`;
186-
},
187-
code({ text, lang = 'js' }) {
188-
const cached_snippet = SNIPPET_CACHE.get(text + lang + current);
189-
if (cached_snippet.code) return cached_snippet.code;
190-
191-
let { source, options } = parse_options(text, lang);
192-
source = adjust_tab_indentation(source, lang);
150+
if (token.type === 'code') {
151+
if (snippets.get(token.text)) return;
193152

194-
const converted = conversions.get(source);
153+
let { source, options } = parse_options(token.text, token.lang);
154+
source = adjust_tab_indentation(source, token.lang);
195155

196-
let html = '<div class="code-block"><div class="controls">';
156+
const converted =
157+
token.lang === 'js' || token.lang === 'svelte'
158+
? await generate_ts_from_js(source, token.lang, options)
159+
: undefined;
197160

198-
if (options.file) {
199-
const ext = options.file.slice(options.file.lastIndexOf('.'));
200-
if (!ext) throw new Error(`Missing file extension: ${options.file}`);
161+
let html = '<div class="code-block"><div class="controls">';
201162

202-
html += `<span class="filename" data-ext="${ext}">${options.file.slice(0, -ext.length)}</span>`;
203-
}
163+
if (options.file) {
164+
const ext = options.file.slice(options.file.lastIndexOf('.'));
165+
if (!ext) throw new Error(`Missing file extension: ${options.file}`);
204166

205-
if (converted) {
206-
html += `<input class="ts-toggle raised" checked title="Toggle language" type="checkbox" aria-label="Toggle JS/TS">`;
207-
}
167+
html += `<span class="filename" data-ext="${ext}">${options.file.slice(0, -ext.length)}</span>`;
168+
}
208169

209-
if (options.copy) {
210-
html += `<button class="copy-to-clipboard raised" title="Copy to clipboard" aria-label="Copy to clipboard"></button>`;
211-
}
170+
if (converted) {
171+
html += `<input class="ts-toggle raised" checked title="Toggle language" type="checkbox" aria-label="Toggle JS/TS">`;
172+
}
212173

213-
html += '</div>';
174+
if (options.copy) {
175+
html += `<button class="copy-to-clipboard raised" title="Copy to clipboard" aria-label="Copy to clipboard"></button>`;
176+
}
214177

215-
html += syntax_highlight({
216-
filename,
217-
highlighter,
218-
language: lang,
219-
source,
220-
twoslashBanner,
221-
options
222-
});
178+
html += '</div>';
223179

224-
if (converted) {
225180
html += syntax_highlight({
226181
filename,
227182
highlighter,
228-
language: lang === 'js' ? 'ts' : lang,
229-
source: converted,
183+
language: token.lang,
184+
source,
230185
twoslashBanner,
231186
options
232187
});
233-
}
234188

235-
html += '</div>';
189+
if (converted) {
190+
html += syntax_highlight({
191+
filename,
192+
highlighter,
193+
language: token.lang === 'js' ? 'ts' : token.lang,
194+
source: converted,
195+
twoslashBanner,
196+
options
197+
});
198+
}
236199

237-
// TODO this is currently disabled, we don't have access to `modules`
238-
if (type_regex) {
239-
type_regex.lastIndex = 0;
200+
html += '</div>';
240201

241-
html = html.replace(type_regex, (match, prefix, name, pos, str) => {
242-
const char_after = str.slice(pos + match.length, pos + match.length + 1);
202+
// TODO this is currently disabled, we don't have access to `modules`
203+
if (type_regex) {
204+
type_regex.lastIndex = 0;
243205

244-
if (!options.link || name === current || /(\$|\d|\w)/.test(char_after)) {
245-
// we don't want e.g. RequestHandler to link to RequestHandler
246-
return match;
247-
}
206+
html = html.replace(type_regex, (match, prefix, name, pos, str) => {
207+
const char_after = str.slice(pos + match.length, pos + match.length + 1);
248208

249-
const link = type_links?.get(name)
250-
? `<a href="${type_links.get(name)?.relativeURL}">${name}</a>`
251-
: '';
252-
return `${prefix || ''}${link}`;
253-
});
209+
if (!options.link || name === current || /(\$|\d|\w)/.test(char_after)) {
210+
// we don't want e.g. RequestHandler to link to RequestHandler
211+
return match;
212+
}
213+
214+
const link = type_links?.get(name)
215+
? `<a href="${type_links.get(name)?.relativeURL}">${name}</a>`
216+
: '';
217+
return `${prefix || ''}${link}`;
218+
});
219+
}
220+
221+
// Save everything locally now
222+
snippets.save(token.text, html);
223+
}
224+
},
225+
text(token) {
226+
// @ts-expect-error I think this is a bug in marked — some text tokens have children,
227+
// but that's not reflected in the types. In these cases we can't just use `token.tokens`
228+
// because that will result in e.g. `<code>` elements not being generated
229+
if (token.tokens) {
230+
// @ts-expect-error
231+
return this.parser!.parseInline(token.tokens);
254232
}
255233

256-
// Save everything locally now
257-
SNIPPET_CACHE.save(cached_snippet?.uid, html);
234+
return smart_quotes(token.text, true);
235+
},
236+
heading({ tokens, depth, raw }) {
237+
const text = this.parser!.parseInline(tokens);
258238

259-
return html;
239+
headings[depth - 1] = normalizeSlugify(raw);
240+
headings.length = depth;
241+
const slug = headings.filter(Boolean).join('-');
242+
return `<h${depth} id="${slug}">${text.replace(
243+
/<\/?code>/g,
244+
''
245+
)}<a href="#${slug}" class="permalink"><span class="visually-hidden">permalink</span></a></h${depth}>`;
246+
},
247+
code({ text }) {
248+
return snippets.get(text);
260249
},
261250
codespan({ text }) {
262251
return (
@@ -799,17 +788,12 @@ function stringify(member: TypeElement, lang: keyof typeof SHIKI_LANGUAGE_MAP =
799788
);
800789
}
801790

802-
async function find_nearest_node_modules(start_path: string): Promise<string | null> {
803-
try {
804-
if (await stat(path.join(start_path, 'node_modules'))) {
805-
return path.resolve(start_path, 'node_modules');
806-
}
807-
} catch {
808-
const parentDir = path.dirname(start_path);
791+
function find_nearest_node_modules(file: string): string | null {
792+
let current = file;
809793

810-
if (start_path === parentDir) return null;
811-
812-
return find_nearest_node_modules(parentDir);
794+
while (current !== (current = path.dirname(current))) {
795+
const resolved = path.join(current, 'node_modules');
796+
if (fs.existsSync(resolved)) return resolved;
813797
}
814798

815799
return null;
@@ -830,57 +814,44 @@ async function find_nearest_node_modules(start_path: string): Promise<string | n
830814
* ```
831815
*/
832816
async function create_snippet_cache(should: boolean) {
833-
const snippet_cache = (await find_nearest_node_modules(import.meta.url)) + '/.snippets';
817+
const cache = new Map();
818+
const directory = find_nearest_node_modules(import.meta.url) + '/.snippets';
834819

835-
// No local cache exists yet
836-
if (!CACHE_MAP.size && should) {
837-
try {
838-
await mkdir(snippet_cache, { recursive: true });
839-
} catch {}
840-
841-
// Read all the cache files and populate the CACHE_MAP
842-
try {
843-
const files = await readdir(snippet_cache);
844-
845-
const file_contents = await Promise.all(
846-
files.map(async (file) => ({
847-
file,
848-
content: await readFile(`${snippet_cache}/${file}`, 'utf-8')
849-
}))
850-
);
820+
function get_file(source: string) {
821+
const hash = createHash('sha256');
822+
hash.update(source);
823+
const digest = hash.digest().toString('base64').replace(/\//g, '-');
851824

852-
for (const { file, content } of file_contents) {
853-
const uid = file.replace(/\.html$/, '');
854-
CACHE_MAP.set(uid, content);
855-
}
856-
} catch {}
825+
return `${directory}/${digest}.html`;
857826
}
858827

859-
function get(source: string) {
860-
if (!should) return { uid: null, code: null };
828+
return {
829+
get(source: string) {
830+
if (!should) return;
861831

862-
const hash = createHash('sha256');
863-
hash.update(source);
864-
const digest = hash.digest().toString('base64').replace(/\//g, '-');
832+
let snippet = cache.get(source);
865833

866-
try {
867-
return {
868-
uid: digest,
869-
code: CACHE_MAP.get(digest)
870-
};
871-
} catch {}
834+
if (snippet === undefined) {
835+
const file = get_file(source);
872836

873-
return { uid: digest, code: null };
874-
}
837+
if (fs.existsSync(file)) {
838+
snippet = fs.readFileSync(file, 'utf-8');
839+
cache.set(source, snippet);
840+
}
841+
}
875842

876-
function save(uid: string | null, content: string) {
877-
if (!should || !uid) return;
843+
return snippet;
844+
},
845+
save(source: string, html: string) {
846+
cache.set(source, html);
878847

879-
CACHE_MAP.set(uid, content);
880-
writeFile(`${snippet_cache}/${uid}.html`, content);
881-
}
848+
try {
849+
fs.mkdirSync(directory);
850+
} catch {}
882851

883-
return { get, save };
852+
fs.writeFileSync(get_file(source), html);
853+
}
854+
};
884855
}
885856

886857
function create_type_links(

‎packages/site-kit/src/lib/markdown/utils.ts

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Marked, Renderer, type TokenizerObject } from 'marked';
1+
import { Marked, Renderer, type TokenizerObject, type MarkedExtension } from 'marked';
22
import json5 from 'json5';
33

44
const escapeTest = /[&<>"']/;
@@ -114,10 +114,18 @@ const tokenizer: TokenizerObject = {
114114
}
115115
};
116116

117-
export async function transform(markdown: string, renderer: Partial<Renderer> = {}) {
117+
export async function transform(
118+
markdown: string,
119+
{
120+
walkTokens,
121+
...renderer
122+
}: Partial<Renderer> & { walkTokens?: MarkedExtension['walkTokens'] } = {}
123+
) {
118124
const marked = new Marked({
125+
async: true,
119126
renderer,
120-
tokenizer
127+
tokenizer,
128+
walkTokens
121129
});
122130

123131
return (await marked.parse(markdown)) ?? '';

0 commit comments

Comments
 (0)