add lunr search

.changeset/beige-suits-arrive.md

@@ -0,0 +1,8 @@
+---
+"hyperbook": minor
+"@hyperbook/markdown": minor
+"hyperbook-studio": minor
+"@hyperbook/types": minor
+---
+
+Add option to enable search. Just set the search key to true in your hyperbook config and a search icon will be visible in the top right hand corner.

packages/hyperbook/build.ts

@@ -12,9 +12,10 @@ import {
   HyperbookContext,
   Navigation,
 } from "@hyperbook/types";
+import lunr from "lunr";
 import { process as hyperbookProcess } from "@hyperbook/markdown";
 
-const ASSETS_FOLDER = "__hyperbook_assets";
+export const ASSETS_FOLDER = "__hyperbook_assets";
 
 export async function runBuildProject(
   project: Hyperproject,
@@ -129,6 +130,7 @@ async function runBuild(
     pagesAndSections.sections,
     pagesAndSections.pages,
   );
+  const searchDocuments: any[] = [];
 
   let bookFiles = await vfile.listForFolder(root, "book");
   if (filter) {
@@ -147,6 +149,7 @@ async function runBuild(
       navigation,
     };
     const result = await hyperbookProcess(file.markdown.content, ctx);
+    searchDocuments.push(...(result.data.searchDocuments || []));
     for (let directive of Object.keys(result.data.directives || {})) {
       directives.add(directive);
     }
@@ -204,6 +207,7 @@ async function runBuild(
       navigation,
     };
     const result = await hyperbookProcess(file.markdown.content, ctx);
+    searchDocuments.push(...(result.data.searchDocuments || []));
     for (let directive of Object.keys(result.data.directives || {})) {
       directives.add(directive);
     }
@@ -292,9 +296,22 @@ async function runBuild(
   for (let asset of mainAssets) {
     const assetPath = path.join(assetsPath, asset);
     const assetOut = path.join(assetsOut, asset);
-    const stat = await fs.stat(assetPath);
-    if (stat.isFile()) {
-      await cp(assetPath, assetOut, { recursive: true });
+    if (!asset.startsWith("directive-")) {
+      await cp(assetPath, assetOut, {
+        recursive: true,
+        filter: (src) => {
+          if (src.includes("lunr-languages")) {
+            return (
+              hyperbookJson.language !== undefined &&
+              hyperbookJson.language !== "en" &&
+              (src.endsWith("lunr-languages") ||
+                src.endsWith(`lunr.${hyperbookJson.language}.min.js`) ||
+                src.endsWith(`lunr.stemmer.support.min.js`))
+            );
+          }
+          return true;
+        },
+      });
     }
     if (!process.env.CI) {
       readline.clearLine(process.stdout, 0);
@@ -309,5 +326,52 @@ async function runBuild(
   }
   process.stdout.write("\n");
 
+  if (hyperbookJson.search) {
+    const documents: Record<string, any> = {};
+    console.log(`${chalk.blue(`[${prefix}]`)} Building search index`);
+
+    let foundLanguage = false;
+    if (hyperbookJson.language && hyperbookJson.language !== "en") {
+      try {
+        require("lunr-languages/lunr.stemmer.support.js")(lunr);
+        require(`lunr-languages/lunr.${hyperbookJson.language}.js`)(lunr);
+        foundLanguage = true;
+      } catch (e) {
+        console.log(
+          `${chalk.yellow(`[${prefix}]`)} ${hyperbookJson.language} is no valid value for the lanuage key. See https://github.com/MihaiValentin/lunr-languages for possible values. Falling back to English.`,
+        );
+      }
+    }
+    const idx = lunr(function () {
+      if (foundLanguage) {
+        // @ts-ignore
+        this.use(lunr[hyperbookJson.language]);
+      }
+      this.ref("href");
+      this.field("description");
+      this.field("keywords");
+      this.field("heading");
+      this.field("content");
+      this.metadataWhitelist = ["position"];
+
+      searchDocuments.forEach((doc) => {
+        const href = baseCtx.makeUrl(doc.href, "book");
+        const docWithBase = {
+          ...doc,
+          href,
+        };
+        this.add(docWithBase);
+        documents[href] = docWithBase;
+      });
+    });
+
+    const js = `
+const LUNR_INDEX = ${JSON.stringify(idx)};
+const SEARCH_DOCUMENTS = ${JSON.stringify(documents)};
+`;
+
+    await fs.writeFile(path.join(rootOut, ASSETS_FOLDER, "search.js"), js);
+  }
+
   console.log(`${chalk.green(`[${prefix}]`)} Build success: ${rootOut}`);
 }

packages/hyperbook/index.ts

@@ -9,7 +9,6 @@ import { getPkgManager } from "./helpers/get-pkg-manager";
 import { hyperproject } from "@hyperbook/fs";
 import { runNew } from "./new";
 import packageJson from "./package.json";
-import { deprecate } from "util";
 
 const program = new Command();
 

packages/hyperbook/package.json

@@ -36,23 +36,28 @@
     "@types/archiver": "6.0.2",
     "@types/async-retry": "1.4.8",
     "@types/cross-spawn": "6.0.6",
+    "@types/lunr": "^2.3.7",
     "@types/prompts": "2.4.9",
     "@types/tar": "6.1.13",
     "@types/ws": "^8.5.12",
     "@vercel/ncc": "0.38.1",
     "archiver": "7.0.1",
     "async-retry": "1.3.3",
     "chalk": "5.2.0",
+    "chokidar": "3.6.0",
     "commander": "12.1.0",
     "cpy": "11.0.1",
     "cross-spawn": "7.0.3",
+    "domutils": "^3.1.0",
     "got": "12.6.0",
+    "htmlparser2": "^9.1.0",
+    "lunr": "^2.3.9",
+    "lunr-languages": "^1.14.0",
+    "mime": "^4.0.4",
     "prompts": "2.4.2",
     "rimraf": "5.0.7",
     "tar": "6.2.1",
     "update-check": "1.5.4",
-    "chokidar": "3.6.0",
-    "mime": "^4.0.4",
     "ws": "^8.18.0"
   },
   "dependencies": {

packages/markdown/assets/client.js

@@ -33,13 +33,81 @@ var hyperbook = (function () {
     const tocDrawerEl = document.getElementById("toc-drawer");
     tocDrawerEl.open = !tocDrawerEl.open;
   }
+  // search
+
+  const searchInputEl = document.getElementById("search-input");
+  searchInputEl.addEventListener("keypress", (event) => {
+    if (event.key === "Enter") {
+      event.preventDefault();
+      search();
+    }
+  });
+
+  function searchToggle() {
+    const searchDrawerEl = document.getElementById("search-drawer");
+    searchDrawerEl.open = !searchDrawerEl.open;
+  }
+
+  function search() {
+    const resultsEl = document.getElementById("search-results");
+    resultsEl.innerHTML = "";
+    const query = searchInputEl.value;
+    const idx = window.lunr.Index.load(LUNR_INDEX);
+    const documents = SEARCH_DOCUMENTS;
+    const results = idx.search(query);
+    for (let result of results) {
+      const doc = documents[result.ref];
+
+      const container = document.createElement("a");
+      container.href = doc.href;
+      container.classList.add("search-result");
+      const heading = document.createElement("div");
+      heading.textContent = doc.heading;
+      heading.classList.add("search-result-heading");
+      const content = document.createElement("div");
+      content.classList.add("search-result-content");
+      const href = document.createElement("div");
+      href.classList.add("search-result-href");
+      href.textContent = doc.href;
+
+      let contentHTML = "";
+      const terms = Object.keys(result.matchData.metadata);
+      const term = terms[0];
+      if (result?.matchData?.metadata?.[term]?.content?.position?.length > 0) {
+        const pos = result.matchData.metadata[term].content.position[0];
+        const start = pos[0];
+        const len = pos[1];
+        let cutoffBefore = start - 50;
+        if (cutoffBefore < 0) {
+          cutoffBefore = 0;
+        } else {
+          contentHTML += "...";
+        }
+        contentHTML += doc.content.slice(cutoffBefore, start);
+
+        contentHTML += `<mark>${doc.content.slice(start, start + len)}</mark>`;
+        let cutoffAfter = start + len + 50;
+
+        contentHTML += doc.content.slice(start + len, cutoffAfter);
+        if (cutoffAfter < doc.content.length) {
+          contentHTML += "...";
+        }
+      }
+
+      content.innerHTML = contentHTML;
+
+      container.appendChild(heading);
+      container.appendChild(content);
+      container.appendChild(href);
+      resultsEl.appendChild(container);
+    }
+  }
 
   function qrcodeOpen() {
     const qrCodeDialog = document.getElementById("qrcode-dialog");
     const qrcodeEls = qrCodeDialog.getElementsByClassName("make-qrcode");
     const urlEls = qrCodeDialog.getElementsByClassName("url");
     const qrcodeEl = qrcodeEls[0];
-    const urlEl = urlEls[0];
     const qrcode = new window.QRCode({
       content: window.location.href,
       padding: 0,
@@ -113,6 +181,8 @@ var hyperbook = (function () {
     toggleBookmark,
     navToggle,
     tocToggle,
+    searchToggle,
+    search,
     qrcodeOpen,
     qrcodeClose,
   };