Implement user-configurable search completion providers

background_scripts/bg_utils.js

@@ -1,3 +1,5 @@
+import {BaseEngine} from "./completion_engines.js";
+
 const TIME_DELTA = 500; // Milliseconds.
 
 // TabRecency associates a logical timestamp with each tab id. These are used to provide an initial
@@ -124,12 +126,51 @@ const SearchEngines = {
     if ((this.previousSearchEngines == null) || (searchEngines !== this.previousSearchEngines)) {
       this.previousSearchEngines = searchEngines;
       this.searchEngines = new AsyncDataFetcher(function (callback) {
+        CompletionSearch.clearCache();
         const engines = {};
+        CompletionEngines = [...BuiltinCompletionEngines];
         for (let line of BgUtils.parseLines(searchEngines)) {
           const tokens = line.split(/\s+/);
           if (2 <= tokens.length) {
             const keyword = tokens[0].split(":")[0];
             const searchUrl = tokens[1];
+            // Build and register a new completion engine using the @URL
+            if (3 <= tokens.length && tokens[2].startsWith("@")) {
+              const [_, engineUrl, jsonPath] = tokens[2].split("@");
+              const engineRegexp = "^" + searchUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+              class NewCompletionEngine extends BaseEngine {
+                constructor() {
+                  super({
+                    engineUrl: engineUrl,
+                    regexps: [engineRegexp],
+                    example: {
+                      searchUrl: searchUrl,
+                      keyword: keyword,
+                    },
+                  });
+                }
+                parse(text) {
+                  if (!jsonPath) return super.parse(text);
+                  let data = JSON.parse(text);
+                  let star = false;
+                  for (const key of jsonPath.split(".")) {
+                    if (key === "*") {
+                      // TODO when ES2019: Replace with data=data.flat(1)
+                      if (star) data = [].concat.apply([], data);
+                      star = true;
+                    } else if (star && data instanceof Array) {
+                      star = false;
+                      data = data.map(val => val[key]);
+                    } else {
+                      data = data[key];
+                    }
+                  }
+                  return data;
+                }
+              }
+              CompletionEngines.unshift(NewCompletionEngine);
+              tokens.shift();
+            }
             const description = tokens.slice(2).join(" ") || `search (${keyword})`;
             if (Utils.hasFullUrlPrefix(searchUrl) || Utils.hasJavascriptPrefix(searchUrl)) {
               engines[keyword] = { keyword, searchUrl, description };

background_scripts/completion_engines.js

@@ -27,7 +27,7 @@
 
 // A base class for common regexp-based matching engines. "options" must define:
 //   options.engineUrl: the URL to use for the completion engine. This must be a string.
-//   options.regexps: one or regular expressions.  This may either a single string or a list of strings.
+//   options.regexps: Regexes matching search URLs this engine handles. List of strings or RegExps.
 //   options.example: an example object containing at least "keyword" and "searchUrl", and optional "description".
 // TODO(philc): This base class is doing very little. We should remove it and use composition.
 class BaseEngine {
@@ -42,6 +42,9 @@ class BaseEngine {
   getUrl(queryTerms) {
     return Utils.createSearchUrl(queryTerms, this.engineUrl);
   }
+  parse(text) {
+    return JSON.parse(text)[1];
+  }
 }
 
 class Google extends BaseEngine {
@@ -55,10 +58,6 @@ class Google extends BaseEngine {
       },
     });
   }
-
-  parse(text) {
-    return JSON.parse(text)[1];
-  }
 }
 
 const googleMapsPrefix = "map of ";
@@ -99,10 +98,6 @@ class Youtube extends BaseEngine {
       },
     });
   }
-
-  parse(text) {
-    return JSON.parse(text)[1];
-  }
 }
 
 class Wikipedia extends BaseEngine {
@@ -116,10 +111,6 @@ class Wikipedia extends BaseEngine {
       },
     });
   }
-
-  parse(text) {
-    return JSON.parse(text)[1];
-  }
 }
 
 class Bing extends BaseEngine {
@@ -133,10 +124,6 @@ class Bing extends BaseEngine {
       },
     });
   }
-
-  parse(text) {
-    return JSON.parse(text)[1];
-  }
 }
 
 class Amazon extends BaseEngine {
@@ -237,6 +224,6 @@ const CompletionEngines = [
   DummyCompletionEngine,
 ];
 
-globalThis.CompletionEngines = CompletionEngines;
+globalThis.CompletionEngines = globalThis.BuiltinCompletionEngines = CompletionEngines;
 
-export { Amazon, DuckDuckGo, Qwant, Webster };
+export { Amazon, BaseEngine, DuckDuckGo, Qwant, Webster };

background_scripts/completion_search.js

@@ -45,8 +45,13 @@ class EnginePrefixWrapper {
 const CompletionSearch = {
   debug: false,
   inTransit: {},
-  completionCache: new SimpleCache(2 * 60 * 60 * 1000, 5000), // Two hours, 5000 entries.
-  engineCache: new SimpleCache(1000 * 60 * 60 * 1000), // 1000 hours.
+  completionCache: undefined,
+  engineCache: undefined,
+
+  clearCache() {
+    this.completionCache = new SimpleCache(2 * 60 * 60 * 1000, 5000); // Two hours, 5000 entries.
+    this.engineCache = new SimpleCache(1000 * 60 * 60 * 1000); // 1000 hours.
+  },
 
   // The amount of time to wait for new requests before launching the current request (for example,
   // if the user is still typing).
@@ -221,7 +226,8 @@ const CompletionSearch = {
                   () => this.completionCache.set(completionCacheKey, null),
                 );
                 if (this.debug) {
-                  console.log("fail", url);
+                  console.error("completion failed:", url);
+                  console.error(error);
                 }
               }
 
@@ -255,4 +261,5 @@ const CompletionSearch = {
   },
 };
 
+CompletionSearch.clearCache();
 globalThis.CompletionSearch = CompletionSearch;

lib/settings.js

@@ -51,7 +51,7 @@ w: https://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedi
 
 # More examples.
 #
-# (Vimium supports search completion Wikipedia, as
+# (Vimium supports search completion for Wikipedia, as
 # above, and for these.)
 #
 # g: https://www.google.com/search?q=%s Google
@@ -62,6 +62,15 @@ w: https://www.wikipedia.org/w/index.php?title=Special:Search&search=%s Wikipedi
 # d: https://duckduckgo.com/?q=%s DuckDuckGo
 # az: https://www.amazon.com/s/?field-keywords=%s
 # qw: https://www.qwant.com/?q=%s Qwant\
+#
+# You can also configure a custom search completion provider.
+#   shortcut: search-url?search=%s @autocomplete-url?search=%s Description
+# By default data[1] is used, but you can also set a path:
+#   shortcut: search-url?search=%s @autocomplete-url?search=%s@path.2.*.list Description
+# Examples:
+# rs: https://runescape.wiki/?search=%s @https://runescape.wiki/api.php?action=opensearch&format=json&search=%s Runescape Wiki
+# os: https://oldschool.runescape.wiki/?search=%s @https://oldschool.runescape.wiki/api.php?action=opensearch&format=json&search=%s Old School Runescape wiki
+# man: https://www.mankier.com/?q=%s @https://www.mankier.com/api/v2/suggest/?q=%s@results.*.text ManKier Man Pages
 `,
   newTabUrl: "about:newtab", // Equal to the value of Utils.chromeNewTabUrl.
   grabBackFocus: false,