Merge branch 'search-with-redisearch' into e-commerce-refactor

erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json

@@ -44,7 +44,13 @@
   "enable_attribute_filters",
   "filter_attributes",
   "shop_by_category_section",
-  "slideshow"
+  "slideshow",
+  "item_search_settings_section",
+  "redisearch_warning",
+  "search_index_fields",
+  "show_categories_in_search_autocomplete",
+  "show_brand_line",
+  "is_redisearch_loaded"
  ],
  "fields": [
   {
@@ -300,12 +306,53 @@
    "fieldname": "enable_reviews",
    "fieldtype": "Check",
    "label": "Enable Reviews and Ratings"
+  },
+  {
+   "fieldname": "search_index_fields",
+   "fieldtype": "Small Text",
+   "label": "Search Index Fields",
+   "read_only_depends_on": "eval:!doc.is_redisearch_loaded"
+  },
+  {
+   "collapsible": 1,
+   "fieldname": "item_search_settings_section",
+   "fieldtype": "Section Break",
+   "label": "Item Search Settings"
+  },
+  {
+   "default": "1",
+   "fieldname": "show_categories_in_search_autocomplete",
+   "fieldtype": "Check",
+   "label": "Show Categories in Search Autocomplete",
+   "read_only_depends_on": "eval:!doc.is_redisearch_loaded"
+  },
+  {
+   "default": "0",
+   "description": "e.g. \"iPhone 12 by Apple\"",
+   "fieldname": "show_brand_line",
+   "fieldtype": "Check",
+   "label": "Show Brand Line",
+   "read_only_depends_on": "eval:!doc.is_redisearch_loaded"
+  },
+  {
+   "default": "0",
+   "fieldname": "is_redisearch_loaded",
+   "fieldtype": "Check",
+   "hidden": 1,
+   "label": "Is Redisearch Loaded"
+  },
+  {
+   "depends_on": "eval:!doc.is_redisearch_loaded",
+   "fieldname": "redisearch_warning",
+   "fieldtype": "HTML",
+   "label": "Redisearch Warning",
+   "options": "<p class=\"alert alert-warning\">Redisearch module not loaded. If you want to use advanced product search features, refer documentation <a class=\"alert-link\" href=\"https://frappeframework.com/docs/user/en/installation\" target=\"_blank\">here</a>.</p>"
   }
  ],
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2021-03-23 17:15:01.956630",
+ "modified": "2021-05-28 07:09:32.639710",
  "modified_by": "Administrator",
  "module": "E-commerce",
  "name": "E Commerce Settings",

erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py

@@ -7,13 +7,15 @@ import frappe
 from frappe.utils import cint, comma_and
 from frappe import _, msgprint
 from frappe.model.document import Document
-from frappe.utils import get_datetime, get_datetime_str, now_datetime
+from frappe.utils import get_datetime, get_datetime_str, now_datetime, unique
+from erpnext.e_commerce.website_item_indexing import create_website_items_index, ALLOWED_INDEXABLE_FIELDS_SET, is_search_module_loaded
 
 class ShoppingCartSetupError(frappe.ValidationError): pass
 
 class ECommerceSettings(Document):
 	def onload(self):
 		self.get("__onload").quotation_series = frappe.get_meta("Quotation").get_options("naming_series")
+		self.is_redisearch_loaded = is_search_module_loaded()
 
 	def validate(self):
 		if self.home_page_is_products:
@@ -24,6 +26,9 @@ class ECommerceSettings(Document):
 		self.validate_field_filters()
 		self.validate_attribute_filters()
 		self.validate_checkout()
+		self.validate_brand_check()
+		self.validate_search_index_fields()
+
 		if self.enabled:
 			self.validate_exchange_rates_exist()
 
@@ -51,6 +56,33 @@ class ECommerceSettings(Document):
 		if self.enable_checkout and not self.payment_gateway_account:
 			self.enable_checkout = 0
 
+	def validate_search_index_fields(self):
+		if not self.search_index_fields:
+			return 
+		
+		# Clean up
+		# Remove whitespaces
+		fields = self.search_index_fields.replace(' ', '')
+		# Remove extra ',' and remove duplicates
+		fields = unique(fields.strip(',').split(','))
+
+		# All fields should be indexable
+		if not (set(fields).issubset(ALLOWED_INDEXABLE_FIELDS_SET)):
+			invalid_fields = list(set(fields).difference(ALLOWED_INDEXABLE_FIELDS_SET))
+			num_invalid_fields = len(invalid_fields)
+			invalid_fields = comma_and(invalid_fields)
+
+			if num_invalid_fields > 1:
+				frappe.throw(_("{0} are not valid options for Search Index Field.").format(frappe.bold(invalid_fields)))
+			else:
+				frappe.throw(_("{0} is not a valid option for Search Index Field.").format(frappe.bold(invalid_fields)))
+
+		self.search_index_fields = ','.join(fields)
+
+	def validate_brand_check(self):
+		if self.show_brand_line and not ("brand" in self.search_index_fields):
+			self.search_index_fields += ",brand"
+
 	def validate_exchange_rates_exist(self):
 		"""check if exchange rates exist for all Price List currencies (to company's currency)"""
 		company_currency = frappe.get_cached_value('Company',  self.company,  "default_currency")
@@ -102,6 +134,15 @@ class ECommerceSettings(Document):
 	def get_shipping_rules(self, shipping_territory):
 		return self.get_name_from_territory(shipping_territory, "shipping_rules", "shipping_rule")
 
+	def on_change(self):
+		old_doc = self.get_doc_before_save()
+		old_fields = old_doc.search_index_fields
+		new_fields = self.search_index_fields
+		
+		# if search index fields get changed
+		if not (new_fields == old_fields):
+			create_website_items_index()
+
 def validate_cart_settings(doc, method):
 	frappe.get_doc("E Commerce Settings", "E Commerce Settings").run_method("validate")
 

erpnext/e_commerce/doctype/website_item/website_item.py

@@ -16,6 +16,14 @@ from frappe.website.doctype.website_slideshow.website_slideshow import get_slide
 from erpnext.setup.doctype.item_group.item_group import (get_parent_item_groups, invalidate_cache_for)
 from erpnext.e_commerce.doctype.item_review.item_review import get_item_reviews
 
+# SEARCH 
+from erpnext.e_commerce.website_item_indexing import (
+	insert_item_to_index, 
+	update_index_for_item, 
+	delete_item_from_index
+)
+# -----
+
 class WebsiteItem(WebsiteGenerator):
 	website = frappe._dict(
 		page_title_field="web_item_name",
@@ -49,6 +57,8 @@ class WebsiteItem(WebsiteGenerator):
 
 	def on_trash(self):
 		super(WebsiteItem, self).on_trash()
+		# Delete Item from search index
+		delete_item_from_index(self)
 		self.publish_unpublish_desk_item(publish=False)
 
 	def validate_duplicate_website_item(self):
@@ -378,6 +388,9 @@ def invalidate_cache_for_web_item(doc):
 	for item_group in website_item_groups:
 		invalidate_cache_for(doc, item_group)
 
+	# Update Search Cache
+	update_index_for_item(doc)
+
 	invalidate_item_variants_cache_for_website(doc)
 
 @frappe.whitelist()
@@ -404,6 +417,10 @@ def make_website_item(doc, save=True):
 		return website_item
 
 	website_item.save()
+
+	# Add to search cache
+	insert_item_to_index(website_item)
+	
 	return [website_item.name, website_item.web_item_name]
 
 def on_doctype_update():

erpnext/e_commerce/website_item_indexing.py

@@ -0,0 +1,214 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# License: GNU General Public License v3. See license.txt
+
+import frappe
+from frappe.utils.redis_wrapper import RedisWrapper
+
+from redisearch import (
+        Client, AutoCompleter,
+        Suggestion, IndexDefinition, 
+        TextField, TagField
+	)
+
+def is_search_module_loaded():
+    cache = frappe.cache()
+    out = cache.execute_command('MODULE LIST')
+
+    parsed_output = " ".join(
+        (" ".join([s.decode() for s in o if not isinstance(s, int)]) for o in out)
+    )
+
+    return "search" in parsed_output
+
+# Decorator for checking wether Redisearch is there or not
+def redisearch_decorator(function):
+    def wrapper(*args, **kwargs):
+        if is_search_module_loaded():
+            func = function(*args, **kwargs)
+            return func
+        return
+
+    return wrapper 
+
+def make_key(key):
+	return "{0}|{1}".format(frappe.conf.db_name, key).encode('utf-8')
+
+# GLOBAL CONSTANTS
+WEBSITE_ITEM_INDEX = 'website_items_index'
+WEBSITE_ITEM_KEY_PREFIX = 'website_item:'
+WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict'
+WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE = 'website_items_category_dict'
+
+ALLOWED_INDEXABLE_FIELDS_SET = {
+	'item_code',
+	'item_name',
+	'item_group',
+	'brand',
+	'description',
+	'web_long_description'
+}
+
+@redisearch_decorator
+def create_website_items_index():
+	'''Creates Index Definition'''
+	# CREATE index
+	client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache())
+
+	# DROP if already exists
+	try:
+		client.drop_index()
+	except:
+		pass
+
+	
+	idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)])
+
+	# Based on e-commerce settings
+	idx_fields = frappe.db.get_single_value(
+		'E Commerce Settings', 
+		'search_index_fields'
+	).split(',')
+
+	if 'web_item_name' in idx_fields:
+		idx_fields.remove('web_item_name')
+	
+	idx_fields = list(map(to_search_field, idx_fields))
+
+	client.create_index(
+		[TextField("web_item_name", sortable=True)] + idx_fields,
+		definition=idx_def,
+	)
+
+	reindex_all_web_items()
+	define_autocomplete_dictionary()
+
+def to_search_field(field):
+	if field == "tags":
+		return TagField("tags", separator=",")
+
+	return TextField(field)
+
+@redisearch_decorator
+def insert_item_to_index(website_item_doc):
+	# Insert item to index
+	key = get_cache_key(website_item_doc.name)
+	r = frappe.cache()
+	web_item = create_web_item_map(website_item_doc)
+
+	for k, v in web_item.items():
+		super(RedisWrapper, r).hset(make_key(key), k, v)
+
+	insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name)
+
+@redisearch_decorator
+def insert_to_name_ac(web_name, doc_name):
+	ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache())
+	ac.add_suggestions(Suggestion(web_name, payload=doc_name))
+
+def create_web_item_map(website_item_doc):
+	fields_to_index = get_fields_indexed()
+
+	web_item = {}
+
+	for f in fields_to_index:
+		web_item[f] = website_item_doc.get(f) or ''
+
+	return web_item
+
+@redisearch_decorator
+def update_index_for_item(website_item_doc):
+	# Reinsert to Cache
+	insert_item_to_index(website_item_doc)
+	define_autocomplete_dictionary()
+
+@redisearch_decorator
+def delete_item_from_index(website_item_doc):
+	r = frappe.cache()
+	key = get_cache_key(website_item_doc.name)
+	
+	try:
+		r.delete(key)
+	except:
+		return False
+
+	delete_from_ac_dict(website_item_doc)
+
+	return True
+
+@redisearch_decorator
+def delete_from_ac_dict(website_item_doc):
+	'''Removes this items's name from autocomplete dictionary'''
+	r = frappe.cache()
+	name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=r)
+	name_ac.delete(website_item_doc.web_item_name)
+
+@redisearch_decorator
+def define_autocomplete_dictionary():
+	"""Creates an autocomplete search dictionary for `name`.
+	   Also creats autocomplete dictionary for `categories` if 
+	   checked in E Commerce Settings"""
+
+	r = frappe.cache()
+	name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=r)
+	cat_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=r)
+
+	ac_categories = frappe.db.get_single_value(
+		'E Commerce Settings', 
+		'show_categories_in_search_autocomplete'
+	)
+	
+	# Delete both autocomplete dicts
+	try:
+		r.delete(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE))
+		r.delete(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE))
+	except:
+		return False
+	
+	items = frappe.get_all(
+		'Website Item', 
+		fields=['web_item_name', 'item_group'], 
+		filters={"published": True}
+	)
+
+	for item in items:
+		name_ac.add_suggestions(Suggestion(item.web_item_name))
+		if ac_categories and item.item_group:
+			cat_ac.add_suggestions(Suggestion(item.item_group))
+
+	return True
+
+@redisearch_decorator
+def reindex_all_web_items():
+	items = frappe.get_all(
+		'Website Item', 
+		fields=get_fields_indexed(), 
+		filters={"published": True}
+	)
+
+	r = frappe.cache()
+	for item in items:
+		web_item = create_web_item_map(item)
+		key = make_key(get_cache_key(item.name))
+
+		for k, v in web_item.items():
+			super(RedisWrapper, r).hset(key, k, v)
+
+def get_cache_key(name):
+	name = frappe.scrub(name)
+	return f"{WEBSITE_ITEM_KEY_PREFIX}{name}"
+
+def get_fields_indexed():
+	fields_to_index = frappe.db.get_single_value(
+		'E Commerce Settings', 
+		'search_index_fields'
+	).split(',')
+
+	mandatory_fields = ['name', 'web_item_name', 'route', 'thumbnail']
+	fields_to_index = fields_to_index + mandatory_fields
+
+	return fields_to_index
+
+# TODO: Remove later
+# # Figure out a way to run this at startup
+define_autocomplete_dictionary()
+create_website_items_index()

erpnext/templates/pages/product_search.py

@@ -1,21 +1,39 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: GNU General Public License v3. See license.txt
 
-from __future__ import unicode_literals
 import frappe
 from frappe.utils import cstr, nowdate, cint
 from erpnext.setup.doctype.item_group.item_group import get_item_for_list_in_html
 from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website
 
+# For SEARCH -------
+from redisearch import AutoCompleter, Client, Query
+from erpnext.e_commerce.website_item_indexing import (
+	is_search_module_loaded,
+	WEBSITE_ITEM_INDEX, 
+	WEBSITE_ITEM_NAME_AUTOCOMPLETE,
+	WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE,
+	make_key
+)
+# -----------------
+
 no_cache = 1
 
+
 def get_context(context):
 	context.show_search = True
 
 @frappe.whitelist(allow_guest=True)
 def get_product_list(search=None, start=0, limit=12):
-	# limit = 12 because we show 12 items in the grid view
+	data = get_product_data(search, start, limit)
 
+	for item in data:
+		set_product_info_for_website(item)
+
+	return [get_item_for_list_in_html(r) for r in data]
+
+def get_product_data(search=None, start=0, limit=12):
+	# limit = 12 because we show 12 items in the grid view
 	# base query
 	query = """select I.name, I.item_name, I.item_code, I.route, I.image, I.website_image, I.thumbnail, I.item_group,
 			I.description, I.web_long_description as website_description, I.is_stock_item,
@@ -38,13 +56,74 @@ def get_product_list(search=None, start=0, limit=12):
 	# order by
 	query += """ order by I.weightage desc, in_stock desc, I.modified desc limit %s, %s""" % (cint(start), cint(limit))
 
-	data = frappe.db.sql(query, {
+	return frappe.db.sql(query, {
 		"search": search,
 		"today": nowdate()
 	}, as_dict=1)
 
-	for item in data:
-		set_product_info_for_website(item)
+@frappe.whitelist(allow_guest=True)
+def search(query, limit=10, fuzzy_search=True):
+	search_results = {"from_redisearch": True, "results": []}
 
-	return [get_item_for_list_in_html(r) for r in data]
+	if not is_search_module_loaded():
+		# Redisearch module not loaded
+		search_results["from_redisearch"] = False
+		search_results["results"] = get_product_data(query, 0, limit)
+		return search_results
 
+	if not query:
+		return search_results
+
+	red = frappe.cache()
+	query = clean_up_query(query)
+
+	ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red)
+	client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red)
+	suggestions = ac.get_suggestions(
+		query, 
+		num=limit, 
+		fuzzy= fuzzy_search and len(query) > 4 # Fuzzy on length < 3 can be real slow
+	)
+
+	# Build a query
+	query_string = query
+
+	for s in suggestions:
+		query_string += f"|('{clean_up_query(s.string)}')"
+
+	q = Query(query_string)
+
+	print(f"Executing query: {q.query_string()}")
+
+	results = client.search(q)
+	search_results['results'] = list(map(convert_to_dict, results.docs))
+
+	# FOR DEBUGGING
+	print("SEARCH RESULTS ------------------\n ", search_results)
+
+	return search_results
+
+def clean_up_query(query):
+	return ''.join(c for c in query if c.isalnum() or c.isspace())
+
+def convert_to_dict(redis_search_doc):
+	return redis_search_doc.__dict__
+
+@frappe.whitelist(allow_guest=True)
+def get_category_suggestions(query):
+	search_results = {"from_redisearch": True, "results": []}
+
+	if not is_search_module_loaded():
+		# Redisearch module not loaded
+		search_results["from_redisearch"] = False
+		return search_results
+
+	if not query:
+		return search_results
+
+	ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache())
+	suggestions = ac.get_suggestions(query, num=10)
+
+	search_results['results'] = [s.string for s in suggestions]
+	
+	return search_results

erpnext/www/all-products/search.css

@@ -0,0 +1,9 @@
+.item-thumb {
+    height: 50px;
+    width: 50px;
+    object-fit: cover;
+}
+
+.brand-line {
+    color: gray;
+}

erpnext/www/all-products/search.html

@@ -0,0 +1,46 @@
+{% extends "templates/web.html" %}
+
+{% block title %}{{ _('Search') }}{% endblock %}
+
+{%- block head_include %}
+	<link rel="stylesheet" href="search.css">
+{% endblock -%}
+
+{% block header %}
+<div class="mb-6">{{ _('Search Products') }}</div>
+{% endblock header %}
+
+{% block page_content %}
+<div class="input-group mb-3">
+	<input type="text" name="query" id="search-box" class="form-control" placeholder="Search for products..." aria-label="Product" aria-describedby="button-addon2">
+	<div class="input-group-append">
+		<button class="btn btn-outline-secondary" type="button" id="search-button">Search</button>
+	</div>
+</div>
+
+<!-- To show recent searches -->
+<div class="my-2" id="recent-search-chips"></div>
+
+<div class="row mt-2">
+	<!-- Search Results -->
+	<div class="col-sm">
+		<h2>Products</h2>
+		<ul id="results" class="list-group"></ul>
+	</div>
+	
+	{% set show_categories = frappe.db.get_single_value('E Commerce Settings', 'show_categories_in_search_autocomplete') %}
+	{% if show_categories %}
+	<div id="categories" class="col-sm">
+		<h2>Categories</h2>
+		<ul id="category-suggestions">
+		</ul>
+	</div>
+	{% endif %}
+	
+	{% set show_brand_line = frappe.db.get_single_value('E Commerce Settings', 'show_brand_line') %}
+	{% if show_brand_line %}
+	<span id="show-brand-line"></span>
+	{% endif %}
+</div>
+
+{% endblock %}

erpnext/www/all-products/search.js

@@ -0,0 +1,148 @@
+let loading = false;
+
+const MAX_RECENT_SEARCHES = 4;
+
+const searchBox = document.getElementById("search-box");
+const searchButton = document.getElementById("search-button");
+const results = document.getElementById("results");
+const categoryList = document.getElementById("category-suggestions");
+const showBrandLine = document.getElementById("show-brand-line");
+const recentSearchArea = document.getElementById("recent-search-chips");
+
+function getRecentSearches() {
+    return JSON.parse(localStorage.getItem("recent_searches") || "[]");
+}
+
+function attachEventListenersToChips() {
+    const chips = document.getElementsByClassName("recent-chip");
+
+    for (let chip of chips) {
+        chip.addEventListener("click", () => {
+            searchBox.value = chip.innerText;
+
+            // Start search with `recent query`
+            const event = new Event("input");
+            searchBox.dispatchEvent(event);
+            searchBox.focus();
+        });
+    }
+}
+
+function populateRecentSearches() {
+    let recents = getRecentSearches();
+
+    if (!recents.length) {
+        return;
+    }
+
+    html = "Recent Searches: ";
+    for (let query of recents) {
+        html += `<button class="btn btn-secondary btn-sm recent-chip mr-1">${query}</button>`;
+    }
+
+    recentSearchArea.innerHTML = html;
+    attachEventListenersToChips();
+}
+
+function populateResults(data) {
+    if (!data.message.from_redisearch) {
+        // Data not from redisearch
+    }
+
+    if (data.message.results.length === 0) {
+        results.innerHTML = 'No results';
+        return;
+    }
+
+    html = ""
+    search_results = data.message.results
+    for (let res of search_results) {
+        html += `<li class="list-group-item list-group-item-action">
+        <img class="item-thumb" src="${res.thumbnail || 'img/placeholder.png'}" />
+        <a href="/${res.route}">${res.web_item_name} <span class="brand-line">${showBrandLine && res.brand ? "by " + res.brand : ""}</span></a>
+        </li>`
+    }
+    results.innerHTML = html;
+}
+
+function populateCategoriesList(data) {
+    if (!data.message.from_redisearch) {
+        // Data not from redisearch
+        categoryList.innerHTML = "Install Redisearch to enable autocompletions.";
+        return;
+    }
+
+    if (data.message.results.length === 0) {
+        categoryList.innerHTML = 'No results';
+        return;
+    }
+
+    html = ""
+    search_results = data.message.results
+    for (let category of search_results) {
+        html += `<li>${category}</li>`
+    }
+
+    categoryList.innerHTML = html;
+}
+
+function updateLoadingState() {
+    if (loading) {
+        results.innerHTML = `<div class="spinner-border"><span class="sr-only">loading...<span></div>`;
+    }
+}
+
+searchBox.addEventListener("input", (e) => {
+    loading = true;
+    updateLoadingState();
+    frappe.call({
+        method: "erpnext.templates.pages.product_search.search", 
+        args: {
+            query: e.target.value 
+        },
+        callback: (data) => {
+            populateResults(data);
+            loading = false;
+        }
+    });
+
+    // If there is a suggestion list node
+    if (categoryList) {
+        frappe.call({
+            method: "erpnext.templates.pages.product_search.get_category_suggestions",
+            args: {
+                query: e.target.value
+            },
+            callback: (data) => {
+                populateCategoriesList(data);
+            }
+        });
+    }
+});
+
+searchButton.addEventListener("click", (e) => {
+    let query = searchBox.value;
+    if (!query) {
+        return;
+    }
+
+    let recents = getRecentSearches();
+
+    if (recents.length >= MAX_RECENT_SEARCHES) {
+        // Remove the `First` query
+        recents.splice(0, 1);
+    }
+
+    if (recents.indexOf(query) >= 0) {
+        return;
+    }
+
+    recents.push(query);
+
+    localStorage.setItem("recent_searches", JSON.stringify(recents));
+
+    // Refresh recent searches
+    populateRecentSearches();
+});
+
+populateRecentSearches();

requirements.txt

@@ -10,3 +10,4 @@ python-youtube~=0.8.0
 taxjar~=1.9.2
 tweepy~=3.10.0
 Unidecode~=1.2.0
+redisearch==2.0.0