nsinnovations/erpnext
2
0
Fork 0
mirror of https://github.com/frappe/erpnext.git synced 2026-05-19 21:19:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(patch): Use INSERT with ON DUPLICATE KEY UPDATE

Browse Source 
update_total_qty_field patch was using UPDATE statement with CASE

Now use INSERT INTO with ON DUPLICATE KEY UPDATE clause which
allows this query to use indexes and update multiple rows per query.
This commit is contained in:
Aditya Hase
2018-11-14 18:58:30 +05:30
parent e4193030b1
commit d168a71cfb
1 changed files with 23 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
erpnext/patches/v11_0/update_total_qty_field.py
View File

@@ -9,12 +9,12 @@ def execute():
frappe.reload_doc('stock', 'doctype', 'purchase_receipt') frappe.reload_doc('stock', 'doctype', 'purchase_receipt')
frappe.reload_doc('accounts', 'doctype', 'sales_invoice') frappe.reload_doc('accounts', 'doctype', 'sales_invoice')
frappe.reload_doc('accounts', 'doctype', 'purchase_invoice') frappe.reload_doc('accounts', 'doctype', 'purchase_invoice')
doctypes = ["Sales Order", "Sales Invoice", "Delivery Note",\ doctypes = ["Sales Order", "Sales Invoice", "Delivery Note",\
"Purchase Order", "Purchase Invoice", "Purchase Receipt", "Quotation", "Supplier Quotation"] "Purchase Order", "Purchase Invoice", "Purchase Receipt", "Quotation", "Supplier Quotation"]
for doctype in doctypes: for doctype in doctypes:
total_qty = frappe.db.sql(''' total_qty = frappe.db.sql('''
SELECT SELECT
parent, SUM(qty) as qty parent, SUM(qty) as qty
FROM FROM
@@ -22,14 +22,25 @@ def execute():
GROUP BY parent GROUP BY parent
''' % (doctype), as_dict = True) ''' % (doctype), as_dict = True)
when_then = [] # Query to update total_qty might become too big, Update in batches
for d in total_qty: # batch_size is chosen arbitrarily, Don't try too hard to reason about it
when_then.append(""" batch_size = 100000
when dt.name = '{0}' then {1} for i in range(0, len(total_qty), batch_size):
""".format(frappe.db.escape(d.get("parent")), d.get("qty"))) batch_transactions = total_qty[i:i + batch_size]
if when_then: # UPDATE with CASE for some reason cannot use PRIMARY INDEX,
frappe.db.sql(''' # causing all rows to be examined, leading to a very slow update
UPDATE
`tab%s` dt SET dt.total_qty = CASE %s ELSE 0.0 END # UPDATE with WHERE clause uses PRIMARY INDEX, but will lead to too many queries
''' % (doctype, " ".join(when_then)))
# INSERT with ON DUPLICATE KEY UPDATE uses PRIMARY INDEX
# and can perform multiple updates per query
# This is probably never used anywhere else as of now, but should be
values = []
for d in batch_transactions:
values.append("('{}', {})".format(d.parent, d.qty))
conditions = ",".join(values)
frappe.db.sql("""
INSERT INTO `tab{}` (name, total_qty) VALUES {}
ON DUPLICATE KEY UPDATE name = VALUES(name), total_qty = VALUES(total_qty)
""".format(doctype, conditions))