From 45e89e8fc25ca82393c4dc152a773878695b0068 Mon Sep 17 00:00:00 2001
From: KShivendu <shivendu@iitbhilai.ac.in>
Date: Sun, 27 Mar 2022 17:00:30 +0530
Subject: [PATCH] origin-search: Autocomplete feature for search QL
Summary:
Autocomplete for the search query langauge using the swh_ql.wasm file
generated from swh-search
Reviewers: #reviewers
Differential Revision: https://forge.softwareheritage.org/D6091
---
assets/config/webpack.config.development.js | 52 ++--
assets/src/bundles/browse/origin-search.js | 13 +-
assets/src/utils/autocomplete.css | 44 ++++
assets/src/utils/autocomplete.js | 140 +++++++++++
assets/src/utils/search-ql-autocomplete.js | 224 ++++++++++++++++++
package.json | 1 +
swh/__init__.py | 2 +-
swh/web/__init__.py | 2 +-
swh/web/config.py | 3 +-
swh/web/settings/common.py | 11 +-
.../includes/origin-search-form.html | 5 +-
yarn.lock | 5 +
12 files changed, 481 insertions(+), 21 deletions(-)
create mode 100644 assets/src/utils/autocomplete.css
create mode 100644 assets/src/utils/autocomplete.js
create mode 100644 assets/src/utils/search-ql-autocomplete.js
diff --git a/assets/config/webpack.config.development.js b/assets/config/webpack.config.development.js
index 1b64e368d..7047da21c 100644
--- a/assets/config/webpack.config.development.js
+++ b/assets/config/webpack.config.development.js
@@ -26,6 +26,7 @@ const ESLintPlugin = require('eslint-webpack-plugin');
// are we running webpack-dev-server ?
const isDevServer = process.argv.find(v => v.includes('serve')) !== undefined;
// webpack-dev-server configuration
+const host = '0.0.0.0';
const devServerPort = 3000;
const devServerPublicPath = 'http://localhost:' + devServerPort + '/static/';
// set publicPath according if we are using webpack-dev-server to serve
@@ -97,19 +98,8 @@ module.exports = {
devtool: isDevServer ? 'eval' : 'source-map',
// webpack-dev-server configuration
devServer: {
- client: {
- logging: 'warn',
- overlay: {
- warnings: true,
- errors: true
- },
- progress: true
- },
- devMiddleware: {
- publicPath: devServerPublicPath,
- stats: 'errors-only'
- },
- host: '0.0.0.0',
+ clientLogLevel: 'warning',
+ host: host,
port: devServerPort,
// enable to serve static assets not managed by webpack
static: {
@@ -145,10 +135,16 @@ module.exports = {
alias: {
'pdfjs-dist': 'pdfjs-dist/build/pdf.min.js'
},
+ // for web-tree-sitter
+ fallback: {
+ 'path': false,
+ 'fs': false
+ },
// configure base paths for resolving modules with webpack
modules: [
'node_modules',
- path.resolve(__dirname, '../src')
+ path.resolve(__dirname, '../src'),
+ path.resolve(__dirname, '../../../swh-search/query_language/')
]
},
stats: 'errors-warnings',
@@ -240,6 +236,34 @@ module.exports = {
}
}]
},
+ {
+ test: require.resolve('js-cookie'),
+ use: [{
+ loader: 'expose-loader',
+ options: {
+ exposes: {
+ globalName: 'Cookies',
+ override: true
+ }
+ }
+ }]
+ },
+ // import .wasm files (for web-tree-sitter)
+ // web-tree-sitter tries to load static/js/tree-sitter.wasm
+ // so don't create new path 'wasm/' for .wasm resources.
+ // otherwise, web-tree-sitter will have to be patched
+ // in order to use the new path.
+ {
+ test: /\.wasm$/,
+ type: 'javascript/auto',
+ use: [{
+ loader: 'file-loader',
+ options: {
+ name: '[name].[ext]',
+ outputPath: 'js/'
+ }
+ }]
+ },
// css import configuration:
// - first process it with postcss
// - then extract it to a dedicated file associated to each bundle
diff --git a/assets/src/bundles/browse/origin-search.js b/assets/src/bundles/browse/origin-search.js
index 2f571f851..99e687989 100644
--- a/assets/src/bundles/browse/origin-search.js
+++ b/assets/src/bundles/browse/origin-search.js
@@ -6,7 +6,7 @@
*/
import {handleFetchError, isArchivedOrigin} from 'utils/functions';
-
+import {initAutocomplete} from 'utils/search-ql-autocomplete';
const limit = 100;
const linksPrev = [];
let linkNext = null;
@@ -193,6 +193,17 @@ async function doSearch() {
export function initOriginSearch() {
$(document).ready(() => {
+ const inputBox = document.querySelector('#swh-origins-url-patterns');
+ const submitBtn = document.querySelector('#swh-search-submit');
+ const validQueryCallback = (isValid) => {
+ submitBtn.disabled = !isValid;
+ // if (!isValid)
+ // inputBox.classList.add('invalid');
+ // else
+ // inputBox.classList.remove('invalid');
+ };
+ initAutocomplete(inputBox, validQueryCallback);
+
$('#swh-search-origins').submit(event => {
event.preventDefault();
if (event.target.checkValidity()) {
diff --git a/assets/src/utils/autocomplete.css b/assets/src/utils/autocomplete.css
new file mode 100644
index 000000000..4ebd48357
--- /dev/null
+++ b/assets/src/utils/autocomplete.css
@@ -0,0 +1,44 @@
+/* the container must be positioned relative: */
+.autocomplete {
+ position: relative;
+ display: inline-block;
+}
+
+input.invalid {
+ outline: none !important;
+ /* border: 2px solid red; */
+}
+
+/* position the autocomplete items to be the same width as the container: */
+.autocomplete-items {
+ position: absolute;
+ border: 1px solid #d4d4d4;
+ width: 200px;
+ /* overflow-y: scroll; */
+ border-top: none;
+ z-index: 99998;
+ /* z-index: 99999; is taken by swh-top-bar */
+ top: 100%;
+ left: 0;
+ right: 0;
+}
+
+.autocomplete-items div {
+ padding: 3px;
+ padding-left: 5px;
+ /* font-size: 15px; */
+ cursor: pointer;
+ background-color: #fff;
+ /* border-bottom: 1px solid #d4d4d4; */
+}
+
+/* when hovering an item: */
+.autocomplete-items div:hover {
+ background-color: #e9e9e9;
+}
+
+/* when navigating through the items using the arrow keys: */
+.autocomplete-active {
+ background-color: #E20026 !important;
+ color: #fff;
+}
diff --git a/assets/src/utils/autocomplete.js b/assets/src/utils/autocomplete.js
new file mode 100644
index 000000000..194345605
--- /dev/null
+++ b/assets/src/utils/autocomplete.js
@@ -0,0 +1,140 @@
+import 'utils/autocomplete.css';
+
+export class Autocomplete {
+ constructor(params) {
+ const {inputBox, suggestions} = params;
+ this.inputBox = inputBox;
+ this.suggestions = suggestions;
+ this.currentIndex = -1;
+
+ this.autocompleteList = document.createElement('div');
+ this.autocompleteList.setAttribute('class', 'autocomplete-items');
+ this.inputBox.parentNode.appendChild(this.autocompleteList);
+
+ this.initListeners();
+ }
+
+ initListeners() {
+ this.inputBox.addEventListener('focus', this.updateLists.bind(this));
+ this.inputBox.addEventListener('input', this.updateLists.bind(this));
+
+ this.inputBox.addEventListener('keydown', (e) => {
+ if (e.keyCode === 40) { // down
+ this.currentIndex++;
+ this.addActive();
+ } else if (e.keyCode === 38) { // up
+ this.currentIndex--;
+ this.addActive();
+ } else if (e.keyCode === 13) { // enter
+ e.preventDefault();
+ if (this.currentIndex > -1) {
+ // Simulate a click on the "active" item:
+ if (this.autocompleteList) this.autocompleteList.children[this.currentIndex].click();
+ }
+ } else if (e.keyCode === 27) { // escape
+ this.removeAllItems(e.target);
+ }
+ });
+
+ document.addEventListener('click', (e) => { this.removeAllItems(e.target); });
+ }
+
+ updateLists() {
+ const inputValue = this.inputBox.value;
+
+ const tokens = inputValue.split();
+ const lastToken = tokens[tokens.length - 1];
+ const lastChar = lastToken[lastToken.length - 1];
+
+ /* close any already open lists of autocompleted values */
+ this.removeAllItems();
+
+ this.currentIndex = -1;
+
+ const suggestions = this.suggestions.filter(s => (s.indexOf(lastToken) >= 0 || lastChar === ' '));
+
+ suggestions.slice(0, 10).forEach(suggestion => {
+ const itemDiv = document.createElement('div');
+ if (lastChar === ' ') {
+ itemDiv.innerHTML = suggestion;
+ } else {
+ const indexOfLastToken = suggestion.indexOf(lastToken);
+
+ itemDiv.innerHTML = suggestion.substr(0, indexOfLastToken) +
+ '<strong>' +
+ suggestion.substr(indexOfLastToken, lastToken.length) +
+ '</strong>' +
+ suggestion.substr(
+ indexOfLastToken + lastToken.length, suggestion.length - (lastToken.length - 2)
+ );
+
+ }
+
+ itemDiv.setAttribute('data-value', suggestion);
+
+ const suggestionClick = (e) => {
+ const toInsert = e.target.getAttribute('data-value');
+
+ const oldValue = this.inputBox.value;
+ const tokens = oldValue.split();
+ const lastToken = tokens[tokens.length - 1];
+ const lastChar = lastToken[lastToken.length - 1];
+
+ let newValue = '';
+
+ if (lastChar === ' ' || oldValue === '') {
+ newValue = oldValue + toInsert;
+ } else {
+ // const position = this.inputBox.selectionStart;
+ const queryWithoutLastToken = tokens.slice(0, tokens.length - 2).join(' ');
+ newValue = queryWithoutLastToken + ((queryWithoutLastToken !== '') ? ' ' : '') + toInsert;
+ }
+
+ this.inputBox.value = newValue;
+ this.inputBox.blur();
+ this.inputBox.focus();
+ // this.inputBox.dispatchEvent(new Event('input'))
+ };
+
+ itemDiv.addEventListener('click', suggestionClick.bind(this));
+
+ this.autocompleteList.appendChild(itemDiv);
+ });
+
+ if (suggestions?.length) {
+ // Select first element on each update
+ this.currentIndex = 0;
+ this.addActive();
+ }
+ }
+
+ addActive() {
+ // a function to classify an item as "active":
+ if (!this.autocompleteList) return false;
+ // start by removing the "active" class on all items:
+ const n = this.autocompleteList.childElementCount;
+ this.removeActive();
+ if (this.currentIndex >= n) this.currentIndex = 0;
+ if (this.currentIndex < 0) this.currentIndex = (n - 1);
+ // add class "autocomplete-active":
+ this.autocompleteList.children[this.currentIndex].classList.add('autocomplete-active');
+ }
+
+ removeActive() {
+ /* a function to remove the "active" class from all autocomplete items */
+ Array.from(this.autocompleteList.children).forEach(autocompleteItem => {
+ autocompleteItem.classList.remove('autocomplete-active');
+ });
+ }
+
+ removeAllItems(element) {
+ /*
+ close all autocomplete lists in the document,
+ except the one passed as an argument
+ */
+ if (element !== this.inputBox && this.autocompleteList) {
+ this.autocompleteList.innerHTML = '';
+ }
+ }
+
+}
diff --git a/assets/src/utils/search-ql-autocomplete.js b/assets/src/utils/search-ql-autocomplete.js
new file mode 100644
index 000000000..07a0a72b1
--- /dev/null
+++ b/assets/src/utils/search-ql-autocomplete.js
@@ -0,0 +1,224 @@
+import '../../../node_modules/web-tree-sitter/tree-sitter.wasm';
+import {Parser} from 'web-tree-sitter';
+import {Autocomplete} from 'utils/autocomplete.js';
+import {
+ fields, limitField, sortByField, // fields
+ sortByOptions, visitTypeOptions, // options
+ equalOp, rangeOp, choiceOp, // operators
+ AND, OR, TRUE, FALSE // special tokens
+} from '../../../../swh-search/query_language/tokens.js';
+
+const filterNames = fields.concat(sortByField, limitField);
+
+const languageSyntax = [
+ {
+ category: 'patternFilter',
+ field: 'patternField',
+ operator: 'equalOp',
+ value: 'patternVal',
+ suggestion: ['string', '"string"']
+ },
+ {
+ category: 'booleanFilter',
+ field: 'booleanField',
+ operator: 'equalOp',
+ value: 'booleanVal',
+ suggestion: [TRUE, FALSE]
+ },
+ {
+ category: 'numericFilter',
+ field: 'numericField',
+ operator: 'rangeOp',
+ value: 'numberVal',
+ suggestion: ['15']
+ },
+ {
+ category: 'boundedListFilter',
+ field: 'visitTypeField',
+ operator: 'equalOp',
+ value: 'visitTypeVal',
+ options: visitTypeOptions,
+ suggestion: ['[']
+ },
+ {
+ category: 'unboundedListFilter',
+ field: 'listField',
+ operator: 'choiceOp',
+ value: 'listVal',
+ options: ['string', '"string"'],
+ suggestion: ['[']
+ },
+ {
+ category: 'dateFilter',
+ field: 'dateField',
+ operator: 'rangeOp',
+ value: 'dateVal',
+ suggestion: ['2000-01-01', '2000-01-01T00:00Z']
+ },
+ {
+ category: 'sortBy',
+ field: 'sortByField',
+ operator: 'equalOp',
+ value: 'sortByVal',
+ options: sortByOptions,
+ suggestion: ['[']
+ },
+ {
+ category: 'limit',
+ field: 'limit',
+ operator: 'equalOp',
+ value: 'number',
+ suggestion: ['50']
+ }
+];
+
+const filterOperators = {equalOp, choiceOp, rangeOp};
+
+const findMissingNode = (node) => {
+ if (node.isMissing()) {
+ return node;
+ }
+ if (node.children.length > 0) {
+ for (let i = 0; i < node.children.length; i++) {
+ const missingNode = findMissingNode(node.children[i]);
+ if (missingNode !== null) { return missingNode; }
+ }
+ }
+
+ return null;
+};
+
+const isWrapperNode = (child, parent) => {
+ if (!child || !parent) return false;
+ if (parent.namedChildren.length === 1 && parent.type !== 'ERROR') return true;
+ return (
+ (child.startPosition.column === parent.startPosition.column) &&
+ (child.endPosition.column === parent.endPosition.column)
+ );
+};
+
+const isCategoryNode = (node) => {
+ if (!node || node === null) return false;
+ if (node.type === 'ERROR' || languageSyntax.filter(f => f.category === node.type).length > 0) { return true; }
+
+ return false;
+};
+
+const suggestNextNode = (tree, inputBox) => {
+ const cursor = inputBox.selectionStart - 1;
+ const query = inputBox.value;
+
+ let lastTokenIndex = cursor;
+ // let distFromLastToken = 0;
+ while (query[lastTokenIndex] === ' ') {
+ lastTokenIndex--;
+ // distFromLastToken++;
+ }
+
+ // if(query === "visit_type = []") debugger;
+
+ const lastTokenPosition = {row: 0, column: lastTokenIndex};
+ const lastTokenNode = tree.rootNode.descendantForPosition(lastTokenPosition, lastTokenPosition);
+
+ const missingNode = findMissingNode(tree.rootNode);
+
+ // Find last token node wrapper
+ let lastTokenNodeWrapper = lastTokenNode;
+ while (isWrapperNode(lastTokenNodeWrapper, lastTokenNodeWrapper.parent)) {
+ lastTokenNodeWrapper = lastTokenNodeWrapper.parent;
+ }
+
+ // Find last token node wrapper sibling
+ const lastTokenNodeWrapperSibling = lastTokenNodeWrapper.previousSibling;
+
+ // Find current filter category
+ let currentFilterCategory = lastTokenNode;
+ while (!isCategoryNode(currentFilterCategory)) {
+ currentFilterCategory = currentFilterCategory.parent;
+ }
+
+ console.log(lastTokenNode);
+ console.log(`LAST NODE: ${lastTokenNode.type}`);
+ console.log(`LAST NODE ANCESTOR: ${lastTokenNodeWrapper.type}`);
+ console.log(`LAST NODE ANCESTOR SIBLING: ${lastTokenNodeWrapperSibling?.type}`);
+ console.log(`LAST CATEGORY: ${currentFilterCategory.type}`);
+
+ // Suggest options for array valued filters
+ if ((lastTokenNode.type === ',' && lastTokenNodeWrapper.type.indexOf('Val') > 0) ||
+ (lastTokenNode.type === '[' && currentFilterCategory)
+ ) {
+ const filter = languageSyntax.filter(f => f.category === currentFilterCategory.type)[0];
+ console.log(filter.options);
+ return filter.options ?? [];
+ }
+ if (
+ (!tree.rootNode.hasError() && (lastTokenNodeWrapper.type.indexOf('Val') > 0)) ||
+ (lastTokenNode.type === ')' || lastTokenNode.type === ']')
+ ) {
+ // Suggest AND/OR
+ return [AND, OR];
+ }
+ if (missingNode && missingNode !== null) {
+ // Suggest missing nodes (Automatically suggested by Tree-sitter)
+ if (missingNode.type === ')') {
+ return [AND, OR, ')'];
+ } else if (missingNode.type === ']') {
+ return [',', ']'];
+ }
+ }
+
+ if (lastTokenNode.type === 'ERROR' ||
+ (lastTokenNode.type === '(') ||
+ ((lastTokenNode.type === AND || lastTokenNode.type === OR))
+ ) {
+ // Suggest field names
+ return filterNames.concat('(');
+ } else if (languageSyntax.map(f => f.field).includes(lastTokenNode.type)) {
+ // Suggest operators
+ const filter = languageSyntax.filter(f => f.field === lastTokenNode.type)[0];
+ return filterOperators[filter.operator];
+ } else if (lastTokenNode.type in filterOperators) {
+ // Suggest values
+ const filter = languageSyntax.filter(f => (
+ f.field === lastTokenNodeWrapperSibling.type
+ ))[0];
+ return filter.suggestion;
+ }
+
+ return [];
+};
+
+export const initAutocomplete = (inputBox, validQueryCallback) => {
+ Parser.init().then(async() => {
+ const parser = new Parser();
+
+ const swhSearchQL = await Parser.Language.load(`${window.location.origin}/static/swh_ql.wasm`);
+ parser.setLanguage(swhSearchQL);
+
+ const autocomplete = new Autocomplete(
+ {inputBox, suggestions: ['('].concat(filterNames)}
+ );
+
+ const getSuggestions = (e) => {
+ // if (e.keycode !== 32) // space
+ // return;
+ const tree = parser.parse(inputBox.value);
+
+ if (tree.rootNode.hasError()) {
+ validQueryCallback(false);
+ // inputBox.classList.add('invalid');
+ } else {
+ validQueryCallback(true);
+ // inputBox.classList.remove('invalid');
+ }
+
+ console.log(`input(${inputBox.value}) => ${tree.rootNode.toString()}`);
+
+ const suggestions = suggestNextNode(tree, inputBox);
+ // if (suggestions)
+ autocomplete.suggestions = suggestions; // .map(item => `${item} `);
+ };
+
+ inputBox.addEventListener('keydown', getSuggestions.bind(this));
+ });
+};
diff --git a/package.json b/package.json
index 6284a74bb..ad8fede7e 100644
--- a/package.json
+++ b/package.json
@@ -90,6 +90,7 @@
"typeface-alegreya": "^1.1.13",
"typeface-alegreya-sans": "^1.1.13",
"waypoints": "^4.0.1",
+ "web-tree-sitter": "^0.19.4",
"whatwg-fetch": "^3.6.2"
},
"devDependencies": {
diff --git a/swh/__init__.py b/swh/__init__.py
index b36383a61..976d068d2 100644
--- a/swh/__init__.py
+++ b/swh/__init__.py
@@ -1,3 +1,3 @@
from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
+__path__ = extend_path(__path__, __name__) # type: ignore
diff --git a/swh/web/__init__.py b/swh/web/__init__.py
index b36383a61..976d068d2 100644
--- a/swh/web/__init__.py
+++ b/swh/web/__init__.py
@@ -1,3 +1,3 @@
from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
+__path__ = extend_path(__path__, __name__) # type: ignore
diff --git a/swh/web/config.py b/swh/web/config.py
index 8b62d0096..e6e8a23c8 100644
--- a/swh/web/config.py
+++ b/swh/web/config.py
@@ -44,7 +44,8 @@ DEFAULT_CONFIG = {
),
"search_config": (
"dict",
- {"metadata_backend": "swh-indexer-storage",}, # or "swh-search"
+ {"metadata_backend": "swh-indexer-storage", "enable_ql": True},
+ # or {"metadata_backend" : "swh-search", ...}
),
"log_dir": ("string", "/tmp/swh/log"),
"debug": ("bool", False),
diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py
index e8ea4c11a..111d338b2 100644
--- a/swh/web/settings/common.py
+++ b/swh/web/settings/common.py
@@ -12,6 +12,7 @@ import os
import sys
from typing import Any, Dict
+from swh import search
from swh.web.auth.utils import OIDC_SWH_WEB_CLIENT_ID
from swh.web.config import get_config
@@ -135,7 +136,15 @@ STATIC_DIR = os.path.join(sys.prefix, "share/swh/web/static")
if not os.path.exists(STATIC_DIR):
# static folder location when developping swh-web
STATIC_DIR = os.path.join(PROJECT_DIR, "../../../static")
-STATICFILES_DIRS = [STATIC_DIR]
+
+SEARCH_DIR = os.path.dirname(search.__file__)
+# static folder location when swh-search has been installed with pip
+SEARCH_STATIC_DIR = os.path.join(SEARCH_DIR, "static")
+if not os.path.exists(SEARCH_STATIC_DIR):
+ # static folder location when developping swh-search
+ SEARCH_STATIC_DIR = os.path.join(SEARCH_DIR, "../../static")
+STATICFILES_DIRS = [STATIC_DIR, SEARCH_STATIC_DIR]
+
INTERNAL_IPS = ["127.0.0.1"]
diff --git a/swh/web/templates/includes/origin-search-form.html b/swh/web/templates/includes/origin-search-form.html
index 3356bba68..ddf61384b 100644
--- a/swh/web/templates/includes/origin-search-form.html
+++ b/swh/web/templates/includes/origin-search-form.html
@@ -10,9 +10,10 @@ See top-level LICENSE file for more information
<input class="form-control"
placeholder="Enter a SWHID to resolve or string pattern(s) to search for in origin urls"
type="text" id="swh-origins-url-patterns"
- oninput="swh.webapp.validateSWHIDInput(this)" autofocus required>
+ oninput="swh.webapp.validateSWHIDInput(this)"
+ autofocus required autocomplete="off">
<div class="input-group-append">
- <button class="btn btn-primary" type="submit"><i class="swh-search-icon mdi mdi-24px mdi-magnify" aria-hidden="true"></i></button>
+ <button class="btn btn-primary" type="submit" id="swh-search-submit"><i class="swh-search-icon mdi mdi-24px mdi-magnify" aria-hidden="true"></i></button>
</div>
<div class="invalid-feedback"></div>
</div>
diff --git a/yarn.lock b/yarn.lock
index cde753421..f17b77346 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -11184,6 +11184,11 @@ wbuf@^1.1.0, wbuf@^1.7.3:
dependencies:
minimalistic-assert "^1.0.0"
+web-tree-sitter@^0.19.4:
+ version "0.19.4"
+ resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.19.4.tgz#975076e233204de9063e7a7bda1138c4b454b424"
+ integrity sha512-8G0xBj05hqZybCqBtW7RPZ/hWEtP3DiLTauQzGJZuZYfVRgw7qj7iaZ+8djNqJ4VPrdOO+pS2dR1JsTbsLxdYg==
+
webidl-conversions@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff"
--
GitLab