Search: enable for ko; fix ja; hide for zh-*

package.json

@@ -22,6 +22,7 @@
     "@codemirror/view": "^0.19.44",
     "@emotion/react": "^11.7.1",
     "@emotion/styled": "^11.6.0",
+    "@microbit/lunr-languages": "^1.9.0-microbit.1",
     "@microbit/microbit-fs": "^0.9.2",
     "@react-three/fiber": "^7.0.27",
     "@sanity/block-content-to-react": "^3.0.0",
@@ -46,7 +47,6 @@
     "lodash.debounce": "^4.0.8",
     "lodash.sortby": "^4.7.0",
     "lunr": "^2.3.9",
-    "lunr-languages": "^1.9.0",
     "lzma": "^2.3.2",
     "marked": "^4.0.15",
     "mobile-drag-drop": "^2.3.0-rc.2",

src/documentation/search/lunr-languages.d.ts

@@ -3,24 +3,31 @@
  *
  * SPDX-License-Identifier: MIT
  */
-declare module "lunr-languages/lunr.stemmer.support";
-declare module "lunr-languages/tinyseg";
-declare module "lunr-languages/lunr.multi";
+declare module "@microbit/lunr-languages/lunr.stemmer.support";
+declare module "@microbit/lunr-languages/tinyseg";
+declare module "@microbit/lunr-languages/lunr.multi";
 
-declare module "lunr-languages/lunr.*" {
+declare module "@microbit/lunr-languages/lunr.*" {
   import lunr from "lunr";
   function register(l: typeof lunr): void;
   export = register;
 }
 
 declare namespace lunr {
   import { Builder } from "lunr";
+
+  interface LanguagePlugin extends Builder.Plugin {
+    tokenizer?: (
+      obj?: string | object | object[] | null | undefined
+    ) => lunr.Token[];
+  }
+
   export function multiLanguage(...lang: string[]): Builder.Plugin;
 
   // Add more here.
   // I don't think we can use module augmentation—lunr is a namespace.
-  export const es: Builder.Plugin;
-  export const fr: Builder.Plugin;
-  export const ja: Builder.Plugin;
-  export const zh: Builder.Plugin;
+  export const es: LanguagePlugin;
+  export const fr: LanguagePlugin;
+  export const ja: LanguagePlugin;
+  export const ko: LanguagePlugin;
 }

src/documentation/search/search.test.ts

@@ -3,14 +3,16 @@
  *
  * SPDX-License-Identifier: MIT
  */
+import lunr from "lunr";
 import { ApiDocsResponse } from "../../language-server/apidocs";
 import { Toolkit } from "../reference/model";
 import { IndexMessage } from "./common";
+import lunrJa from "@microbit/lunr-languages/lunr.ja";
 import {
-  SearchableContent,
+  buildReferenceIndex,
   buildSearchIndex,
+  SearchableContent,
   SearchWorker,
-  buildReferenceIndex,
 } from "./search";
 
 const searchableReferenceContent: SearchableContent[] = [
@@ -37,11 +39,22 @@ const searchableReferenceContent: SearchableContent[] = [
   },
 ];
 
+const searchableReferenceContentJa: SearchableContent[] = [
+  {
+    id: "",
+    title: "",
+    containerTitle: "",
+    content: "この文章は日本語で書かれています",
+  },
+];
+
 describe("Search", () => {
+  const languagePlugin = lunr.multiLanguage("en");
   const search = buildSearchIndex(
     searchableReferenceContent,
     "reference",
-    "en"
+    undefined,
+    languagePlugin
   );
 
   it("finds stuff", () => {
@@ -227,3 +240,30 @@ describe("SearchWorker", () => {
     expect(postMessage.mock.calls[1][0].reference.length).toEqual(1);
   });
 });
+
+describe("Search in Japanese", () => {
+  const plugins: lunr.Builder.Plugin[] = [];
+  lunrJa(lunr);
+  plugins.push((lunr as any)["ja"]);
+  const languagePluginJa = lunr.multiLanguage("en", "ja");
+
+  const search = buildSearchIndex(
+    searchableReferenceContentJa,
+    "reference",
+    "ja",
+    languagePluginJa,
+    ...plugins
+  );
+
+  it("matches Japanese characters", () => {
+    expect(search.search("書か").length).toEqual(1);
+  });
+
+  it("matches Japanese characters with spaces between", () => {
+    expect(search.search("書か れ").length).toEqual(1);
+  });
+
+  it("matches Japanese characters with no spaces between", () => {
+    expect(search.search("この文章は日本語").length).toEqual(1);
+  });
+});