SimpleSearch: Add com.zms.catalog.lunr (experimental)

Products/zms/conf/metaobj_manager/com.zms.catalog.lunr/__init__.py

@@ -0,0 +1,28 @@
+class com_zms_catalog_lunr:
+	"""
+	python-representation of com.zms.catalog.lunr
+	"""
+
+	# Access
+	access = {"delete_custom":""
+		,"delete_deny":[]
+		,"insert_custom":""
+		,"insert_deny":[]}
+
+	# Enabled
+	enabled = 0
+
+	# Id
+	id = "com.zms.catalog.lunr"
+
+	# Name
+	name = "com.zms.catalog.lunr"
+
+	# Package
+	package = ""
+
+	# Revision
+	revision = "1.0.0"
+
+	# Type
+	type = "ZMSPackage"

Products/zms/conf/metaobj_manager/com.zms.catalog.lunr/lunr_connector/__init__.py

@@ -0,0 +1,62 @@
+class lunr_connector:
+	"""
+	python-representation of lunr_connector
+	"""
+
+	# Access
+	access = {"delete_custom":""
+		,"delete_deny":[]
+		,"insert_custom":""
+		,"insert_deny":[]}
+
+	# Enabled
+	enabled = 0
+
+	# Id
+	id = "lunr_connector"
+
+	# Lang_dict
+	lang_dict = {"lunr_connector.BTN_BUILD":{"eng":"Build Index"
+			,"ger":"Index erzeugen"}}
+
+	# Name
+	name = "Lunr-Connector"
+
+	# Package
+	package = "com.zms.catalog.lunr"
+
+	# Revision
+	revision = "1.0.0"
+
+	# Type
+	type = "ZMSLibrary"
+
+	# Attrs
+	class Attrs:
+		properties = {"custom":"[]"
+			,"default":""
+			,"id":"properties"
+			,"keys":[]
+			,"mandatory":0
+			,"multilang":1
+			,"name":"Property-Definitions (JSON)"
+			,"repetitive":0
+			,"type":"constant"}
+
+		manage_lunr_build_index = {"default":""
+			,"id":"manage_lunr_build_index"
+			,"keys":[]
+			,"mandatory":0
+			,"multilang":0
+			,"name":"Build: search_index.json"
+			,"repetitive":0
+			,"type":"External Method"}
+
+		readme = {"default":""
+			,"id":"readme"
+			,"keys":[]
+			,"mandatory":0
+			,"multilang":0
+			,"name":"readme.md"
+			,"repetitive":0
+			,"type":"resource"}

Products/zms/conf/metaobj_manager/com.zms.catalog.lunr/lunr_connector/manage_lunr_build_index.py

@@ -0,0 +1,77 @@
+from lunr import lunr
+from bs4 import BeautifulSoup
+from typing import List
+from OFS.Image import manage_addFile
+import json
+import requests
+
+
+# Taken from https://scrapfly.io/blog/search-engine-using-web-scraping/#building-index
+def build_index(docs: List[dict], fields: List[dict]):
+
+    config = {
+        "lang": ["en"],
+        "min_search_length": 1,
+    }
+
+    page_dicts = {"docs": docs, "config": config}
+
+    if bool(docs[0]):
+        idx = lunr(
+            ref="location",
+            fields=fields,
+            documents=docs,
+        )
+        page_dicts["index"] = idx.serialize()
+        return json.dumps(page_dicts, sort_keys=True, separators=(",", ":"), indent=2)
+    else:
+        return json.dumps({})
+
+
+def manage_lunr_build_index(self):
+
+    documents = []
+    fields = []
+    attr_ids = []
+    root_url = self.getHome().content.absolute_url()
+    adapter = self.getCatalogAdapter()
+    attrs = adapter.getAttrs()
+
+    for attr_id in adapter._getAttrIds():
+        attr = attrs.get(attr_id, {})
+        attr_boost = attr.get('boost')
+        if attr_boost is not None:
+            fields.append(dict(field_name=attr_id, boost=attr_boost))
+            attr_ids.append(attr_id)
+
+    for node in requests.get(f'{root_url}/++rest_api/content/get_tree_nodes').json():
+        if node.get('meta_id') not in adapter.getIds():
+            continue
+
+        field = {"location": node["index_html"].replace("./../..", root_url),
+                 "content": ""}  # TODO: remove this aggregated field as workaround for search frontend
+
+        for attr_id in attr_ids:
+            html = attr_id in node and node[attr_id] or ""
+            field[attr_id] = BeautifulSoup(html).get_text()
+        if "title" not in field:
+            field["title"] = "titlealt" in field and field["titlealt"] or ""
+
+        # aggregate field attributes into "content" to be processed in main.js (L43) and worker.js (L80+L123)
+        # TODO: keep field attributes in python and adapt processing in javascript according to split values
+        for key, val in field.items():
+            if key not in ['location', 'title', 'titlealt']:
+                field["content"] += val
+
+        documents.append(field)
+
+    index_file = 'search_index.json'
+    index_data = build_index(documents, fields)
+    index_path = self.lunr_page
+
+    if index_file in index_path:
+        index_path.manage_delObjects(ids=[index_file])
+
+    manage_addFile(index_path, id=index_file, title=index_file, file=index_data.encode('utf-8'), content_type='application/javascript')
+
+    return index_data

Products/zms/conf/metaobj_manager/com.zms.catalog.lunr/lunr_connector/readme

@@ -0,0 +1,12 @@
+# SimpleSearch powered by Lunr
+
+This is a demo how to easily display [Lunr.js](https://lunrjs.com/) index using javascript and generic styling using [Pico.css](https://picocss.com/). It is inspired by [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) and utilizes [ZMS REST API](swagger_ui) to fetch the content nodes. Intended use cases are self-contained (micro)sites, where complex search backend services are overkill.
+
+It expects documents to be composed of fields `title` and `text`.
+
+To search just Build a `search_index.json` powered by [Lunr.py](https://lunr.readthedocs.io/en/latest/) and go to the [search interface](lunr_page).
+
+**Sources:**
+
+- [https://github.com/Granitosaurus/simple-lunrjs-display](https://github.com/Granitosaurus/simple-lunrjs-display)
+- [https://scrapfly.io/blog/search-engine-using-web-scraping/](https://scrapfly.io/blog/search-engine-using-web-scraping/)

Products/zms/conf/metaobj_manager/com.zms.catalog.lunr/lunr_page/__init__.py

@@ -0,0 +1,85 @@
+class lunr_page:
+	"""
+	python-representation of lunr_page
+	"""
+
+	# Access
+	access = {"delete_custom":""
+		,"delete_deny":[]
+		,"insert_custom":""
+		,"insert_deny":[]}
+
+	# Enabled
+	enabled = 0
+
+	# Id
+	id = "lunr_page"
+
+	# Name
+	name = "Lunr-Page"
+
+	# Package
+	package = "com.zms.catalog.lunr"
+
+	# Revision
+	revision = "1.0.0"
+
+	# Type
+	type = "ZMSLibrary"
+
+	# Attrs
+	class Attrs:
+		icon_clazz = {"custom":"fas fa-moon text-warning"
+			,"default":""
+			,"id":"icon_clazz"
+			,"keys":[]
+			,"mandatory":0
+			,"multilang":1
+			,"name":"Icon (Class)"
+			,"repetitive":0
+			,"type":"constant"}
+
+		lunr_page_css_picomincss = {"default":""
+			,"id":"lunr_page/css/pico.min.css"
+			,"keys":[]
+			,"mandatory":0
+			,"multilang":0
+			,"name":"pico.min.css"
+			,"repetitive":0
+			,"type":"File"}
+
+		lunr_page_js_lunrjs = {"default":""
+			,"id":"lunr_page/js/lunr.js"
+			,"keys":[]
+			,"mandatory":0
+			,"multilang":0
+			,"name":"lunr.js"
+			,"repetitive":0
+			,"type":"File"}
+
+		lunr_page_js_mainjs = {"default":""
+			,"id":"lunr_page/js/main.js"
+			,"keys":[]
+			,"mandatory":0
+			,"multilang":0
+			,"name":"main.js"
+			,"repetitive":0
+			,"type":"File"}
+
+		lunr_page_js_workerjs = {"default":""
+			,"id":"lunr_page/js/worker.js"
+			,"keys":[]
+			,"mandatory":0
+			,"multilang":0
+			,"name":"worker.js"
+			,"repetitive":0
+			,"type":"File"}
+
+		lunr_page_index_html = {"default":""
+			,"id":"lunr_page/index_html"
+			,"keys":[]
+			,"mandatory":0
+			,"multilang":0
+			,"name":"index.html"
+			,"repetitive":0
+			,"type":"Page Template"}

Products/zms/conf/metaobj_manager/com.zms.catalog.lunr/lunr_page/index_html.zpt

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+
+<head>
+  <link rel="stylesheet" href="css/pico.min.css">
+  <script
+     tal:define="base_url python:container.absolute_url()"
+     tal:content="python:'var base_url = \042%s\042'%(base_url)">
+  </script>
+  <script src="js/main.js"></script>
+</head>
+
+<body>
+<main class="container">
+  <input id="search-query" placeholder="enter search query here"></input>
+  <div id="search-results"></div>
+</main>
+</body>