Ensure we don't index when language doesn't match

Previously we pulled the language off what we indexed which could cause
an issue when the search index was updated for a new language but the
content had not been reloaded. Instead an index now knows its language.

Additionally, ensure we wait for the languages of all content to match
before indexing.

Author: microbit-matt-hillsdon

src/documentation/ApiArea.tsx

@@ -9,7 +9,7 @@ import { useDocumentation } from "./documentation-hooks";
 
 const ApiArea = () => {
   const { api } = useDocumentation();
-  return api ? <ApiDocumentation docs={api} /> : <Spinner />;
+  return api ? <ApiDocumentation docs={api.content} /> : <Spinner />;
 };
 
 export default ApiArea;

src/documentation/api/apidocs-util.test.ts

@@ -3,41 +3,44 @@
  *
  * SPDX-License-Identifier: MIT
  */
-import { ApiDocsResponse } from "../../language-server/apidocs";
+import { ApiDocsContent } from "../../language-server/apidocs";
 import { moduleAndApiFromId, pullModulesToTop } from "./apidocs-util";
 
 describe("pullModulesToTop", () => {
   it("pulls modules up", () => {
-    const mutated: ApiDocsResponse = {
-      microbit: {
-        kind: "module",
-        fullName: "microbit",
-        id: "microbit",
-        name: "microbit",
-        children: [
-          {
-            kind: "module",
-            fullName: "microbit.compass",
-            id: "microbit.compass",
-            name: "compass",
-          },
-          {
-            kind: "module",
-            fullName: "microbit.display",
-            id: "microbit.display",
-            name: "display",
-          },
-          {
-            kind: "variable",
-            fullName: "microbit.display.foo",
-            id: "foo",
-            name: "foo",
-          },
-        ],
+    const mutated: ApiDocsContent = {
+      languageId: "en",
+      content: {
+        microbit: {
+          kind: "module",
+          fullName: "microbit",
+          id: "microbit",
+          name: "microbit",
+          children: [
+            {
+              kind: "module",
+              fullName: "microbit.compass",
+              id: "microbit.compass",
+              name: "compass",
+            },
+            {
+              kind: "module",
+              fullName: "microbit.display",
+              id: "microbit.display",
+              name: "display",
+            },
+            {
+              kind: "variable",
+              fullName: "microbit.display.foo",
+              id: "foo",
+              name: "foo",
+            },
+          ],
+        },
       },
     };
     pullModulesToTop(mutated);
-    expect(mutated).toEqual({
+    expect(mutated.content).toEqual({
       microbit: {
         kind: "module",
         fullName: "microbit",

src/documentation/api/apidocs-util.ts

@@ -3,14 +3,18 @@
  *
  * SPDX-License-Identifier: MIT
  */
-import { ApiDocsEntry, ApiDocsResponse } from "../../language-server/apidocs";
+import {
+  ApiDocsContent,
+  ApiDocsEntry,
+  ApiDocsResponse,
+} from "../../language-server/apidocs";
 
-export const pullModulesToTop = (input: ApiDocsResponse) => {
+export const pullModulesToTop = (input: ApiDocsContent) => {
   const recurse = (docs: ApiDocsEntry[], topLevel: boolean) => {
     let removedSoFar = 0;
     [...docs].forEach((d, index) => {
       if (d.kind === "module" && !topLevel) {
-        input[d.fullName] = d;
+        input.content[d.fullName] = d;
         docs.splice(index - removedSoFar, 1);
         removedSoFar++;
       }
@@ -19,7 +23,7 @@ export const pullModulesToTop = (input: ApiDocsResponse) => {
       }
     });
   };
-  recurse(Object.values(input), true);
+  recurse(Object.values(input.content), true);
 };
 
 export const resolveModule = (

src/documentation/documentation-hooks.tsx

@@ -13,7 +13,7 @@ import {
   useRef,
   useState,
 } from "react";
-import { apiDocs, ApiDocsResponse } from "../language-server/apidocs";
+import { apiDocs, ApiDocsContent } from "../language-server/apidocs";
 import { useLanguageServerClient } from "../language-server/language-server-hooks";
 import { useLogging } from "../logging/logging-hooks";
 import { useSettings } from "../settings/settings";
@@ -26,7 +26,7 @@ import { fetchReferenceToolkit } from "./reference/content";
 import { Toolkit } from "./reference/model";
 
 export type ContentState<T> =
-  | { status: "ok"; content: T }
+  | { status: "ok"; content: T; languageId: string }
   | { status: "error" }
   | { status: "loading" };
 
@@ -44,7 +44,7 @@ const useContent = <T,>(
       try {
         const content = await fetchContent(languageId);
         if (!ignore) {
-          setState({ status: "ok", content });
+          setState({ status: "ok", content, languageId });
         }
       } catch (e) {
         logging.error(e);
@@ -63,9 +63,9 @@ const useContent = <T,>(
   return state;
 };
 
-const useApiDocumentation = (): ApiDocsResponse | undefined => {
+const useApiDocumentation = (): ApiDocsContent | undefined => {
   const client = useLanguageServerClient();
-  const [apidocs, setApiDocs] = useState<ApiDocsResponse | undefined>();
+  const [apidocs, setApiDocs] = useState<ApiDocsContent | undefined>();
   useEffect(() => {
     let ignore = false;
     const load = async () => {
@@ -86,7 +86,7 @@ const useApiDocumentation = (): ApiDocsResponse | undefined => {
 };
 
 export interface DocumentationContextValue {
-  api: ApiDocsResponse | undefined;
+  api: ApiDocsContent | undefined;
   ideas: ContentState<Idea[]>;
   reference: ContentState<Toolkit>;
   apiReferenceMap: ContentState<ApiReferenceMap>;

src/documentation/search/search-hooks.tsx

@@ -51,9 +51,13 @@ const SearchProvider = ({ children }: { children: ReactNode }) => {
       search.current = new WorkerSearch(languageId);
       setQuery("");
     }
-    // Wait for both, no reason to index with just one then redo with both.
-    if (reference.status === "ok" && api) {
-      search.current.index(reference.content, api);
+    // Wait for everything to be loaded and in the right language
+    if (
+      reference.status === "ok" &&
+      reference.languageId === languageId &&
+      api?.languageId === languageId
+    ) {
+      search.current.index(reference.content, api.content);
     }
   }, [languageId, reference, api]);
 

src/documentation/search/search.worker.de.ts

@@ -6,4 +6,4 @@
 import { SearchWorker } from "./search.worker";
 import languageSupport from "@microbit/lunr-languages/lunr.de";
 
-new SearchWorker(self as DedicatedWorkerGlobalScope, languageSupport);
+new SearchWorker(self as DedicatedWorkerGlobalScope, "de", languageSupport);

src/documentation/search/search.worker.en.ts

@@ -5,4 +5,4 @@
  */
 import { SearchWorker } from "./search.worker";
 
-new SearchWorker(self as DedicatedWorkerGlobalScope, undefined);
+new SearchWorker(self as DedicatedWorkerGlobalScope, undefined, undefined);

src/documentation/search/search.worker.es.ts

@@ -6,4 +6,5 @@
 import { SearchWorker } from "./search.worker";
 import languageSupport from "@microbit/lunr-languages/lunr.es";
 
-new SearchWorker(self as DedicatedWorkerGlobalScope, languageSupport);
+// Note the language code is different to the app
+new SearchWorker(self as DedicatedWorkerGlobalScope, "es", languageSupport);

src/documentation/search/search.worker.fr.ts

@@ -6,4 +6,4 @@
 import { SearchWorker } from "./search.worker";
 import languageSupport from "@microbit/lunr-languages/lunr.fr";
 
-new SearchWorker(self as DedicatedWorkerGlobalScope, languageSupport);
+new SearchWorker(self as DedicatedWorkerGlobalScope, "fr", languageSupport);

src/documentation/search/search.worker.ja.ts

@@ -6,4 +6,4 @@
 import { SearchWorker } from "./search.worker";
 import languageSupport from "@microbit/lunr-languages/lunr.ja";
 
-new SearchWorker(self as DedicatedWorkerGlobalScope, languageSupport);
+new SearchWorker(self as DedicatedWorkerGlobalScope, "ja", languageSupport);

src/documentation/search/search.worker.ko.ts

@@ -6,4 +6,4 @@
 import { SearchWorker } from "./search.worker";
 import languageSupport from "@microbit/lunr-languages/lunr.ko";
 
-new SearchWorker(self as DedicatedWorkerGlobalScope, languageSupport);
+new SearchWorker(self as DedicatedWorkerGlobalScope, "ko", languageSupport);

src/documentation/search/search.worker.nl.ts

@@ -5,5 +5,7 @@
  */
 import { SearchWorker } from "./search.worker";
 import languageSupport from "@microbit/lunr-languages/lunr.nl";
-
-new SearchWorker(self as DedicatedWorkerGlobalScope, languageSupport);
+if (!languageSupport) {
+  throw new Error("Whoops!");
+}
+new SearchWorker(self as DedicatedWorkerGlobalScope, "nl", languageSupport);

src/documentation/search/search.worker.test.ts

@@ -122,12 +122,12 @@ describe("buildReferenceIndex", () => {
       ...referenceEn,
       language: "fr",
     };
-    const enIndex = await buildIndex(referenceEn, api, undefined);
+    const enIndex = await buildIndex(referenceEn, api, undefined, undefined);
     expect(enIndex.search("topic").reference.length).toEqual(1);
     // "that" is an English stopword
     expect(enIndex.search("that").reference.length).toEqual(0);
 
-    const frIndex = await buildIndex(referenceFr, api, frLanguageSupport);
+    const frIndex = await buildIndex(referenceFr, api, "fr", frLanguageSupport);
     expect(frIndex.search("topic").reference.length).toEqual(1);
     // "that" is not a French stopword
     expect(frIndex.search("that").reference.length).toEqual(1);
@@ -141,7 +141,7 @@ describe("SearchWorker", () => {
       postMessage,
     } as unknown as DedicatedWorkerGlobalScope;
 
-    new SearchWorker(ctx, undefined);
+    new SearchWorker(ctx, undefined, undefined);
 
     ctx.onmessage!(
       new MessageEvent("message", {
@@ -184,7 +184,7 @@ describe("SearchWorker", () => {
       postMessage,
     } as unknown as DedicatedWorkerGlobalScope;
 
-    new SearchWorker(ctx, undefined);
+    new SearchWorker(ctx, undefined, undefined);
 
     const emptyIndex: IndexMessage = {
       kind: "index",

src/documentation/search/search.worker.ts

@@ -268,34 +268,34 @@ export const buildSearchIndex = (
 export const buildIndex = async (
   reference: Toolkit,
   api: ApiDocsResponse,
+  lunrLanguage: LunrLanguage | undefined,
   languageSupport: ((l: typeof lunr) => void) | undefined
 ): Promise<LunrSearch> => {
-  const language = convertLangToLunrParam(reference.language);
   const plugins: lunr.Builder.Plugin[] = [];
-  if (languageSupport && language) {
+  if (languageSupport && lunrLanguage) {
     languageSupport(lunr);
-    plugins.push(lunr[language]);
+    plugins.push(lunr[lunrLanguage]);
   }
 
   // There is always some degree of English content.
   const multiLanguages = ["en"];
-  if (language) {
-    multiLanguages.push(language);
+  if (lunrLanguage) {
+    multiLanguages.push(lunrLanguage);
   }
   const languagePlugin = lunr.multiLanguage(...multiLanguages);
 
   return new LunrSearch(
     buildSearchIndex(
       referenceSearchableContent(reference),
       "reference",
-      language,
+      lunrLanguage,
       languagePlugin,
       ...plugins
     ),
     buildSearchIndex(
       apiSearchableContent(api),
       "api",
-      language,
+      lunrLanguage,
       languagePlugin,
       ...plugins
     )
@@ -304,27 +304,6 @@ export const buildIndex = async (
 
 type LunrLanguage = "de" | "es" | "fr" | "ja" | "nl" | "ko";
 
-function convertLangToLunrParam(language: string): LunrLanguage | undefined {
-  // See also workerForLanguage
-  switch (language.toLowerCase()) {
-    case "de":
-      return "de";
-    case "fr":
-      return "fr";
-    case "es-es":
-      return "es";
-    case "ja":
-      return "ja";
-    case "ko":
-      return "ko";
-    case "nl":
-      return "nl";
-    default:
-      // No search support for the language, default to lunr's built-in English support.
-      return undefined;
-  }
-}
-
 export class SearchWorker {
   private search: LunrSearch | undefined;
   // We block queries on indexing.
@@ -333,6 +312,7 @@ export class SearchWorker {
 
   constructor(
     private ctx: DedicatedWorkerGlobalScope,
+    private languageId: LunrLanguage | undefined,
     private languageSupport: ((l: typeof lunr) => void) | undefined
   ) {
     // We return Promises here just to allow for easy testing.
@@ -358,6 +338,7 @@ export class SearchWorker {
     this.search = await buildIndex(
       message.reference,
       message.api,
+      this.languageId,
       this.languageSupport
     );
     this.recordInitialization!();

src/language-server/apidocs.ts

@@ -44,6 +44,11 @@ export interface ApiDocsEntry {
   params?: ApiDocsFunctionParameter[];
 }
 
+export interface ApiDocsContent {
+  languageId: string;
+  content: ApiDocsResponse;
+}
+
 export interface ApiDocsResponse extends Record<string, ApiDocsEntry> {}
 
 export const apiDocsRequestType = new ProtocolRequestType<
@@ -56,10 +61,10 @@ export const apiDocsRequestType = new ProtocolRequestType<
 
 export const apiDocs = async (
   client: LanguageServerClient
-): Promise<ApiDocsResponse> => {
+): Promise<ApiDocsContent> => {
   // This is a non-standard LSP call that we've added support for to Pyright.
   try {
-    const result = await client.connection.sendRequest(apiDocsRequestType, {
+    const content = await client.connection.sendRequest(apiDocsRequestType, {
       path: client.rootUri,
       documentationFormat: [MarkupKind.Markdown],
       modules: [
@@ -84,10 +89,10 @@ export const apiDocs = async (
         "time",
       ],
     });
-    return result;
+    return { content, languageId: client.locale };
   } catch (e) {
     if (isErrorDueToDispose(e)) {
-      return {};
+      return { content: {}, languageId: client.locale };
     }
     throw e;
   }

src/language-server/client.ts

@@ -57,7 +57,7 @@ export class LanguageServerClient extends EventEmitter {
 
   constructor(
     public connection: MessageConnection,
-    private locale: string,
+    public locale: string,
     public rootUri: string
   ) {
     super();