+ ${res.web_item_name} ${showBrandLine && res.brand ? "by " + res.brand : ""}
+ Redisearch module not loaded. If you want to use advanced product search features, refer documentation here.
" } ], "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2021-03-23 17:15:01.956630", + "modified": "2021-05-28 07:09:32.639710", "modified_by": "Administrator", "module": "E-commerce", "name": "E Commerce Settings", diff --git a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py index 9d3ca37abbf..a795a11b19f 100644 --- a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py +++ b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py @@ -7,13 +7,15 @@ import frappe from frappe.utils import cint, comma_and from frappe import _, msgprint from frappe.model.document import Document -from frappe.utils import get_datetime, get_datetime_str, now_datetime +from frappe.utils import get_datetime, get_datetime_str, now_datetime, unique +from erpnext.e_commerce.website_item_indexing import create_website_items_index, ALLOWED_INDEXABLE_FIELDS_SET, is_search_module_loaded class ShoppingCartSetupError(frappe.ValidationError): pass class ECommerceSettings(Document): def onload(self): self.get("__onload").quotation_series = frappe.get_meta("Quotation").get_options("naming_series") + self.is_redisearch_loaded = is_search_module_loaded() def validate(self): if self.home_page_is_products: @@ -24,6 +26,9 @@ class ECommerceSettings(Document): self.validate_field_filters() self.validate_attribute_filters() self.validate_checkout() + self.validate_brand_check() + self.validate_search_index_fields() + if self.enabled: self.validate_exchange_rates_exist() @@ -51,6 +56,33 @@ class ECommerceSettings(Document): if self.enable_checkout and not self.payment_gateway_account: self.enable_checkout = 0 + def validate_search_index_fields(self): + if not self.search_index_fields: + return + + # Clean up + # Remove whitespaces + fields = self.search_index_fields.replace(' ', '') + # Remove extra ',' and remove duplicates + fields = unique(fields.strip(',').split(',')) + + # All fields should be indexable + if not (set(fields).issubset(ALLOWED_INDEXABLE_FIELDS_SET)): + invalid_fields = list(set(fields).difference(ALLOWED_INDEXABLE_FIELDS_SET)) + num_invalid_fields = len(invalid_fields) + invalid_fields = comma_and(invalid_fields) + + if num_invalid_fields > 1: + frappe.throw(_("{0} are not valid options for Search Index Field.").format(frappe.bold(invalid_fields))) + else: + frappe.throw(_("{0} is not a valid option for Search Index Field.").format(frappe.bold(invalid_fields))) + + self.search_index_fields = ','.join(fields) + + def validate_brand_check(self): + if self.show_brand_line and not ("brand" in self.search_index_fields): + self.search_index_fields += ",brand" + def validate_exchange_rates_exist(self): """check if exchange rates exist for all Price List currencies (to company's currency)""" company_currency = frappe.get_cached_value('Company', self.company, "default_currency") @@ -102,6 +134,15 @@ class ECommerceSettings(Document): def get_shipping_rules(self, shipping_territory): return self.get_name_from_territory(shipping_territory, "shipping_rules", "shipping_rule") + def on_change(self): + old_doc = self.get_doc_before_save() + old_fields = old_doc.search_index_fields + new_fields = self.search_index_fields + + # if search index fields get changed + if not (new_fields == old_fields): + create_website_items_index() + def validate_cart_settings(doc, method): frappe.get_doc("E Commerce Settings", "E Commerce Settings").run_method("validate") diff --git a/erpnext/e_commerce/doctype/website_item/website_item.py b/erpnext/e_commerce/doctype/website_item/website_item.py index 504d493a80e..5a0363c2876 100644 --- a/erpnext/e_commerce/doctype/website_item/website_item.py +++ b/erpnext/e_commerce/doctype/website_item/website_item.py @@ -16,6 +16,14 @@ from frappe.website.doctype.website_slideshow.website_slideshow import get_slide from erpnext.setup.doctype.item_group.item_group import (get_parent_item_groups, invalidate_cache_for) from erpnext.e_commerce.doctype.item_review.item_review import get_item_reviews +# SEARCH +from erpnext.e_commerce.website_item_indexing import ( + insert_item_to_index, + update_index_for_item, + delete_item_from_index +) +# ----- + class WebsiteItem(WebsiteGenerator): website = frappe._dict( page_title_field="web_item_name", @@ -49,6 +57,8 @@ class WebsiteItem(WebsiteGenerator): def on_trash(self): super(WebsiteItem, self).on_trash() + # Delete Item from search index + delete_item_from_index(self) self.publish_unpublish_desk_item(publish=False) def validate_duplicate_website_item(self): @@ -378,6 +388,9 @@ def invalidate_cache_for_web_item(doc): for item_group in website_item_groups: invalidate_cache_for(doc, item_group) + # Update Search Cache + update_index_for_item(doc) + invalidate_item_variants_cache_for_website(doc) @frappe.whitelist() @@ -404,6 +417,10 @@ def make_website_item(doc, save=True): return website_item website_item.save() + + # Add to search cache + insert_item_to_index(website_item) + return [website_item.name, website_item.web_item_name] def on_doctype_update(): diff --git a/erpnext/e_commerce/website_item_indexing.py b/erpnext/e_commerce/website_item_indexing.py new file mode 100644 index 00000000000..3b82a324395 --- /dev/null +++ b/erpnext/e_commerce/website_item_indexing.py @@ -0,0 +1,214 @@ +# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# License: GNU General Public License v3. See license.txt + +import frappe +from frappe.utils.redis_wrapper import RedisWrapper + +from redisearch import ( + Client, AutoCompleter, + Suggestion, IndexDefinition, + TextField, TagField + ) + +def is_search_module_loaded(): + cache = frappe.cache() + out = cache.execute_command('MODULE LIST') + + parsed_output = " ".join( + (" ".join([s.decode() for s in o if not isinstance(s, int)]) for o in out) + ) + + return "search" in parsed_output + +# Decorator for checking wether Redisearch is there or not +def redisearch_decorator(function): + def wrapper(*args, **kwargs): + if is_search_module_loaded(): + func = function(*args, **kwargs) + return func + return + + return wrapper + +def make_key(key): + return "{0}|{1}".format(frappe.conf.db_name, key).encode('utf-8') + +# GLOBAL CONSTANTS +WEBSITE_ITEM_INDEX = 'website_items_index' +WEBSITE_ITEM_KEY_PREFIX = 'website_item:' +WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict' +WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE = 'website_items_category_dict' + +ALLOWED_INDEXABLE_FIELDS_SET = { + 'item_code', + 'item_name', + 'item_group', + 'brand', + 'description', + 'web_long_description' +} + +@redisearch_decorator +def create_website_items_index(): + '''Creates Index Definition''' + # CREATE index + client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache()) + + # DROP if already exists + try: + client.drop_index() + except: + pass + + + idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)]) + + # Based on e-commerce settings + idx_fields = frappe.db.get_single_value( + 'E Commerce Settings', + 'search_index_fields' + ).split(',') + + if 'web_item_name' in idx_fields: + idx_fields.remove('web_item_name') + + idx_fields = list(map(to_search_field, idx_fields)) + + client.create_index( + [TextField("web_item_name", sortable=True)] + idx_fields, + definition=idx_def, + ) + + reindex_all_web_items() + define_autocomplete_dictionary() + +def to_search_field(field): + if field == "tags": + return TagField("tags", separator=",") + + return TextField(field) + +@redisearch_decorator +def insert_item_to_index(website_item_doc): + # Insert item to index + key = get_cache_key(website_item_doc.name) + r = frappe.cache() + web_item = create_web_item_map(website_item_doc) + + for k, v in web_item.items(): + super(RedisWrapper, r).hset(make_key(key), k, v) + + insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name) + +@redisearch_decorator +def insert_to_name_ac(web_name, doc_name): + ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache()) + ac.add_suggestions(Suggestion(web_name, payload=doc_name)) + +def create_web_item_map(website_item_doc): + fields_to_index = get_fields_indexed() + + web_item = {} + + for f in fields_to_index: + web_item[f] = website_item_doc.get(f) or '' + + return web_item + +@redisearch_decorator +def update_index_for_item(website_item_doc): + # Reinsert to Cache + insert_item_to_index(website_item_doc) + define_autocomplete_dictionary() + +@redisearch_decorator +def delete_item_from_index(website_item_doc): + r = frappe.cache() + key = get_cache_key(website_item_doc.name) + + try: + r.delete(key) + except: + return False + + delete_from_ac_dict(website_item_doc) + + return True + +@redisearch_decorator +def delete_from_ac_dict(website_item_doc): + '''Removes this items's name from autocomplete dictionary''' + r = frappe.cache() + name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=r) + name_ac.delete(website_item_doc.web_item_name) + +@redisearch_decorator +def define_autocomplete_dictionary(): + """Creates an autocomplete search dictionary for `name`. + Also creats autocomplete dictionary for `categories` if + checked in E Commerce Settings""" + + r = frappe.cache() + name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=r) + cat_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=r) + + ac_categories = frappe.db.get_single_value( + 'E Commerce Settings', + 'show_categories_in_search_autocomplete' + ) + + # Delete both autocomplete dicts + try: + r.delete(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE)) + r.delete(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE)) + except: + return False + + items = frappe.get_all( + 'Website Item', + fields=['web_item_name', 'item_group'], + filters={"published": True} + ) + + for item in items: + name_ac.add_suggestions(Suggestion(item.web_item_name)) + if ac_categories and item.item_group: + cat_ac.add_suggestions(Suggestion(item.item_group)) + + return True + +@redisearch_decorator +def reindex_all_web_items(): + items = frappe.get_all( + 'Website Item', + fields=get_fields_indexed(), + filters={"published": True} + ) + + r = frappe.cache() + for item in items: + web_item = create_web_item_map(item) + key = make_key(get_cache_key(item.name)) + + for k, v in web_item.items(): + super(RedisWrapper, r).hset(key, k, v) + +def get_cache_key(name): + name = frappe.scrub(name) + return f"{WEBSITE_ITEM_KEY_PREFIX}{name}" + +def get_fields_indexed(): + fields_to_index = frappe.db.get_single_value( + 'E Commerce Settings', + 'search_index_fields' + ).split(',') + + mandatory_fields = ['name', 'web_item_name', 'route', 'thumbnail'] + fields_to_index = fields_to_index + mandatory_fields + + return fields_to_index + +# TODO: Remove later +# # Figure out a way to run this at startup +define_autocomplete_dictionary() +create_website_items_index() diff --git a/erpnext/templates/pages/product_search.py b/erpnext/templates/pages/product_search.py index 4f4d95bc473..df8ba075946 100644 --- a/erpnext/templates/pages/product_search.py +++ b/erpnext/templates/pages/product_search.py @@ -1,21 +1,39 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals import frappe from frappe.utils import cstr, nowdate, cint from erpnext.setup.doctype.item_group.item_group import get_item_for_list_in_html from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website +# For SEARCH ------- +from redisearch import AutoCompleter, Client, Query +from erpnext.e_commerce.website_item_indexing import ( + is_search_module_loaded, + WEBSITE_ITEM_INDEX, + WEBSITE_ITEM_NAME_AUTOCOMPLETE, + WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE, + make_key +) +# ----------------- + no_cache = 1 + def get_context(context): context.show_search = True @frappe.whitelist(allow_guest=True) def get_product_list(search=None, start=0, limit=12): - # limit = 12 because we show 12 items in the grid view + data = get_product_data(search, start, limit) + for item in data: + set_product_info_for_website(item) + + return [get_item_for_list_in_html(r) for r in data] + +def get_product_data(search=None, start=0, limit=12): + # limit = 12 because we show 12 items in the grid view # base query query = """select I.name, I.item_name, I.item_code, I.route, I.image, I.website_image, I.thumbnail, I.item_group, I.description, I.web_long_description as website_description, I.is_stock_item, @@ -38,13 +56,74 @@ def get_product_list(search=None, start=0, limit=12): # order by query += """ order by I.weightage desc, in_stock desc, I.modified desc limit %s, %s""" % (cint(start), cint(limit)) - data = frappe.db.sql(query, { + return frappe.db.sql(query, { "search": search, "today": nowdate() }, as_dict=1) - for item in data: - set_product_info_for_website(item) +@frappe.whitelist(allow_guest=True) +def search(query, limit=10, fuzzy_search=True): + search_results = {"from_redisearch": True, "results": []} - return [get_item_for_list_in_html(r) for r in data] + if not is_search_module_loaded(): + # Redisearch module not loaded + search_results["from_redisearch"] = False + search_results["results"] = get_product_data(query, 0, limit) + return search_results + if not query: + return search_results + + red = frappe.cache() + query = clean_up_query(query) + + ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red) + client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red) + suggestions = ac.get_suggestions( + query, + num=limit, + fuzzy= fuzzy_search and len(query) > 4 # Fuzzy on length < 3 can be real slow + ) + + # Build a query + query_string = query + + for s in suggestions: + query_string += f"|('{clean_up_query(s.string)}')" + + q = Query(query_string) + + print(f"Executing query: {q.query_string()}") + + results = client.search(q) + search_results['results'] = list(map(convert_to_dict, results.docs)) + + # FOR DEBUGGING + print("SEARCH RESULTS ------------------\n ", search_results) + + return search_results + +def clean_up_query(query): + return ''.join(c for c in query if c.isalnum() or c.isspace()) + +def convert_to_dict(redis_search_doc): + return redis_search_doc.__dict__ + +@frappe.whitelist(allow_guest=True) +def get_category_suggestions(query): + search_results = {"from_redisearch": True, "results": []} + + if not is_search_module_loaded(): + # Redisearch module not loaded + search_results["from_redisearch"] = False + return search_results + + if not query: + return search_results + + ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache()) + suggestions = ac.get_suggestions(query, num=10) + + search_results['results'] = [s.string for s in suggestions] + + return search_results \ No newline at end of file diff --git a/erpnext/www/all-products/img/placeholder.png b/erpnext/www/all-products/img/placeholder.png new file mode 100644 index 00000000000..9780ad8e057 Binary files /dev/null and b/erpnext/www/all-products/img/placeholder.png differ diff --git a/erpnext/www/all-products/search.css b/erpnext/www/all-products/search.css new file mode 100644 index 00000000000..687532d2eb4 --- /dev/null +++ b/erpnext/www/all-products/search.css @@ -0,0 +1,9 @@ +.item-thumb { + height: 50px; + width: 50px; + object-fit: cover; +} + +.brand-line { + color: gray; +} \ No newline at end of file diff --git a/erpnext/www/all-products/search.html b/erpnext/www/all-products/search.html new file mode 100644 index 00000000000..735822d7659 --- /dev/null +++ b/erpnext/www/all-products/search.html @@ -0,0 +1,46 @@ +{% extends "templates/web.html" %} + +{% block title %}{{ _('Search') }}{% endblock %} + +{%- block head_include %} + +{% endblock -%} + +{% block header %} +
+ ${res.web_item_name} ${showBrandLine && res.brand ? "by " + res.brand : ""}
+