Skip to content

invalidate snippet cache when module contents change #315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 10, 2024
Merged

invalidate snippet cache when module contents change #315

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b2d0003
use module contents to determine cache validity
Rich-Harris Oct 10, 2024
74d6a80
share cache
Rich-Harris Oct 10, 2024
6c7a084
remove some unused code
Rich-Harris Oct 10, 2024
ce48b12
empty
Rich-Harris Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
198 changes: 81 additions & 117 deletions packages/site-kit/src/lib/markdown/renderer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import MagicString from 'magic-string';
import { createHash } from 'node:crypto';
import { createHash, Hash } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import ts from 'typescript';
Expand Down Expand Up @@ -34,6 +34,81 @@ const theme = createCssVariablesTheme({
fontStyle: true
});

const hash = createHash('sha256');
hash.update(fs.readFileSync('../../pnpm-lock.yaml', 'utf-8'));
hash_graph(hash, fileURLToPath(import.meta.url));
const digest = hash.digest().toString('base64').replace(/\//g, '-');

/**
* Utility function to work with code snippet caching.
*
* @example
*
* ```js
* const SNIPPETS_CACHE = create_snippet_cache(true);
*
* const { uid, code } = SNIPPETS_CACHE.get(source);
*
* // Later to save the code to the cache
* SNIPPETS_CACHE.save(uid, processed_code);
* ```
*/
async function create_snippet_cache() {
const cache = new Map();
const directory = find_nearest_node_modules(import.meta.url) + '/.snippets';
const current = `${directory}/${digest}`;

if (fs.existsSync(directory)) {
for (const dir of fs.readdirSync(directory)) {
if (dir !== digest) {
fs.rmSync(`${directory}/${dir}`, { force: true, recursive: true });
}
}
} else {
fs.mkdirSync(directory);
}

try {
fs.mkdirSync(`${directory}/${digest}`);
} catch {}

function get_file(source: string) {
const hash = createHash('sha256');
hash.update(source);
const digest = hash.digest().toString('base64').replace(/\//g, '-');

return `${current}/${digest}.html`;
}

return {
get(source: string) {
let snippet = cache.get(source);

if (snippet === undefined) {
const file = get_file(source);

if (fs.existsSync(file)) {
snippet = fs.readFileSync(file, 'utf-8');
cache.set(source, snippet);
}
}

return snippet;
},
save(source: string, html: string) {
cache.set(source, html);

try {
fs.mkdirSync(directory);
} catch {}

fs.writeFileSync(get_file(source), html);
}
};
}

const snippets = await create_snippet_cache();

/**
* A super markdown renderer function. Renders svelte and kit docs specific specific markdown code to html.
*
Expand Down Expand Up @@ -119,8 +194,6 @@ export async function render_content_markdown(
body: string,
{ twoslashBanner }: { twoslashBanner?: TwoslashBanner } = {}
) {
const snippets = await create_snippet_cache(true);

const headings: string[] = [];

return await transform(body, {
Expand Down Expand Up @@ -453,14 +526,13 @@ function find_nearest_node_modules(file: string): string | null {
}

/**
* Get the `mtime` of the most recently modified file in a dependency graph,
* Get the hash of a dependency graph,
* excluding imports from `node_modules`
*/
function get_mtime(file: string, seen = new Set<string>()) {
if (seen.has(file)) return -1;
function hash_graph(hash: Hash, file: string, seen = new Set<string>()) {
if (seen.has(file)) return;
seen.add(file);

let mtime = fs.statSync(file).mtimeMs;
const content = fs.readFileSync(file, 'utf-8');

for (const [_, source] of content.matchAll(/^import(?:.+?\s+from\s+)?['"](.+)['"];?$/gm)) {
Expand All @@ -471,118 +543,10 @@ function get_mtime(file: string, seen = new Set<string>()) {
if (!fs.existsSync(resolved))
throw new Error(`Could not resolve ${source} relative to ${file}`);

mtime = Math.max(mtime, get_mtime(resolved, seen));
}

return mtime;
}

const mtime = Math.max(
get_mtime(fileURLToPath(import.meta.url)),
fs.statSync('node_modules').mtimeMs,
fs.statSync('../../pnpm-lock.yaml').mtimeMs
);

/**
* Utility function to work with code snippet caching.
*
* @example
*
* ```js
* const SNIPPETS_CACHE = create_snippet_cache(true);
*
* const { uid, code } = SNIPPETS_CACHE.get(source);
*
* // Later to save the code to the cache
* SNIPPETS_CACHE.save(uid, processed_code);
* ```
*/
async function create_snippet_cache(should: boolean) {
const cache = new Map();
const directory = find_nearest_node_modules(import.meta.url) + '/.snippets';

if (fs.existsSync(directory)) {
for (const dir of fs.readdirSync(directory)) {
if (!fs.statSync(`${directory}/${dir}`).isDirectory() || +dir < mtime) {
fs.rmSync(`${directory}/${dir}`, { force: true, recursive: true });
}
}
} else {
fs.mkdirSync(directory);
}

try {
fs.mkdirSync(`${directory}/${mtime}`);
} catch {}

function get_file(source: string) {
const hash = createHash('sha256');
hash.update(source);
const digest = hash.digest().toString('base64').replace(/\//g, '-');

return `${directory}/${mtime}/${digest}.html`;
}

return {
get(source: string) {
if (!should) return;

let snippet = cache.get(source);

if (snippet === undefined) {
const file = get_file(source);

if (fs.existsSync(file)) {
snippet = fs.readFileSync(file, 'utf-8');
cache.set(source, snippet);
}
}

return snippet;
},
save(source: string, html: string) {
cache.set(source, html);

try {
fs.mkdirSync(directory);
} catch {}

fs.writeFileSync(get_file(source), html);
}
};
}

function create_type_links(
modules: Modules | undefined,
resolve_link?: (module_name: string, type_name: string) => { slug: string; page: string }
): {
type_regex: RegExp | null;
type_links: Map<string, { slug: string; page: string; relativeURL: string }> | null;
} {
if (!modules || modules.length === 0 || !resolve_link)
return { type_regex: null, type_links: null };

const type_regex = new RegExp(
`(import\\(&apos;(?:svelte|@sveltejs\\/kit)&apos;\\)\\.)?\\b(${modules
.flatMap((module) => module.types)
.map((type) => type?.name)
.join('|')})\\b`,
'g'
);

/** @type {Map<string, { slug: string; page: string; relativeURL: string; }>} */
const type_links = new Map();

for (const module of modules) {
if (!module || !module.name) continue;

for (const type of module.types ?? []) {
const link = resolve_link(module.name, type.name);
type_links.set(type.name, { ...link, relativeURL: link.page + '#' + link.slug });
}
hash_graph(hash, resolved, seen);
}

return { type_regex, type_links };
hash.update(content);
}

function parse_options(source: string, language: string) {
Expand Down
Loading