perf: new column posting datetime in SLE to optimize stock ledger related queries

(cherry picked from commit d80ca523a4)

erpnext/stock/stock_ledger.py

@@ -7,13 +7,30 @@ from typing import Optional, Set, Tuple
 import frappe
 from frappe import _
 from frappe.model.meta import get_field_precision
+<<<<<<< HEAD
 from frappe.query_builder.functions import CombineDatetime, Sum
 from frappe.utils import cint, cstr, flt, get_link_to_form, getdate, now, nowdate
+=======
+from frappe.query_builder.functions import Sum
+from frappe.utils import (
+	add_to_date,
+	cint,
+	cstr,
+	flt,
+	get_link_to_form,
+	getdate,
+	now,
+	nowdate,
+	nowtime,
+	parse_json,
+)
+>>>>>>> d80ca523a4 (perf: new column posting datetime in SLE to optimize stock ledger related queries)
 
 import erpnext
 from erpnext.stock.doctype.bin.bin import update_qty as update_bin_qty
 from erpnext.stock.doctype.inventory_dimension.inventory_dimension import get_inventory_dimensions
 from erpnext.stock.utils import (
+	get_combine_datetime,
 	get_incoming_outgoing_rate_for_cancel,
 	get_incoming_rate,
 	get_or_make_bin,
@@ -68,7 +85,11 @@ def make_sl_entries(sl_entries, allow_negative_stock=False, via_landed_cost_vouc
 				sle_doc = make_entry(sle, allow_negative_stock, via_landed_cost_voucher)
 
 			args = sle_doc.as_dict()
+<<<<<<< HEAD
 			args["allow_zero_valuation_rate"] = sle.get("allow_zero_valuation_rate") or False
+=======
+			args["posting_datetime"] = get_combine_datetime(args.posting_date, args.posting_time)
+>>>>>>> d80ca523a4 (perf: new column posting datetime in SLE to optimize stock ledger related queries)
 
 			if sle.get("voucher_type") == "Stock Reconciliation":
 				# preserve previous_qty_after_transaction for qty reposting
@@ -431,12 +452,14 @@ class update_entries_after(object):
 			self.process_sle(sle)
 
 	def get_sle_against_current_voucher(self):
-		self.args["time_format"] = "%H:%i:%s"
+		self.args["posting_datetime"] = get_combine_datetime(
+			self.args.posting_date, self.args.posting_time
+		)
 
 		return frappe.db.sql(
 			"""
 			select
-				*, timestamp(posting_date, posting_time) as "timestamp"
+				*, posting_datetime as "timestamp"
 			from
 				`tabStock Ledger Entry`
 			where
@@ -444,11 +467,10 @@ class update_entries_after(object):
 				and warehouse = %(warehouse)s
 				and is_cancelled = 0
 				and (
-					posting_date = %(posting_date)s and
-					time_format(posting_time, %(time_format)s) = time_format(%(posting_time)s, %(time_format)s)
+					posting_datetime = %(posting_datetime)s
 				)
 			order by
-				creation ASC
+				posting_datetime ASC, creation ASC
 			for update
 		""",
 			self.args,
@@ -1188,9 +1210,14 @@ def get_previous_sle_of_current_voucher(args, operator="<", exclude_current_vouc
 
 	args["time_format"] = "%H:%i:%s"
 	if not args.get("posting_date"):
-		args["posting_date"] = "1900-01-01"
-	if not args.get("posting_time"):
-		args["posting_time"] = "00:00"
+		args["posting_datetime"] = "1900-01-01 00:00:00"
+
+	if not args.get("posting_datetime"):
+		args["posting_datetime"] = get_combine_datetime(args["posting_date"], args["posting_time"])
+
+	if operator == "<=":
+		# Add 1 second to handle millisecond for less than and equal to condition
+		args["posting_datetime"] = add_to_date(args["posting_datetime"], seconds=1)
 
 	voucher_condition = ""
 	if exclude_current_voucher:
@@ -1199,23 +1226,20 @@ def get_previous_sle_of_current_voucher(args, operator="<", exclude_current_vouc
 
 	sle = frappe.db.sql(
 		"""
-		select *, timestamp(posting_date, posting_time) as "timestamp"
+		select *, posting_datetime as "timestamp"
 		from `tabStock Ledger Entry`
 		where item_code = %(item_code)s
 			and warehouse = %(warehouse)s
 			and is_cancelled = 0
 			{voucher_condition}
 			and (
-				posting_date < %(posting_date)s or
-				(
-					posting_date = %(posting_date)s and
-					time_format(posting_time, %(time_format)s) {operator} time_format(%(posting_time)s, %(time_format)s)
-				)
+				posting_datetime {operator} %(posting_datetime)s
 			)
-		order by timestamp(posting_date, posting_time) desc, creation desc
+		order by posting_datetime desc, creation desc
 		limit 1
 		for update""".format(
-			operator=operator, voucher_condition=voucher_condition
+			operator=operator,
+			voucher_condition=voucher_condition,
 		),
 		args,
 		as_dict=1,
@@ -1256,9 +1280,7 @@ def get_stock_ledger_entries(
 	extra_cond=None,
 ):
 	"""get stock ledger entries filtered by specific posting datetime conditions"""
-	conditions = " and timestamp(posting_date, posting_time) {0} timestamp(%(posting_date)s, %(posting_time)s)".format(
-		operator
-	)
+	conditions = " and posting_datetime {0} %(posting_datetime)s".format(operator)
 	if previous_sle.get("warehouse"):
 		conditions += " and warehouse = %(warehouse)s"
 	elif previous_sle.get("warehouse_condition"):
@@ -1284,9 +1306,11 @@ def get_stock_ledger_entries(
 		)
 
 	if not previous_sle.get("posting_date"):
-		previous_sle["posting_date"] = "1900-01-01"
-	if not previous_sle.get("posting_time"):
-		previous_sle["posting_time"] = "00:00"
+		previous_sle["posting_datetime"] = "1900-01-01 00:00:00"
+	else:
+		previous_sle["posting_datetime"] = get_combine_datetime(
+			previous_sle["posting_date"], previous_sle["posting_time"]
+		)
 
 	if operator in (">", "<=") and previous_sle.get("name"):
 		conditions += " and name!=%(name)s"
@@ -1299,12 +1323,12 @@ def get_stock_ledger_entries(
 
 	return frappe.db.sql(
 		"""
-		select *, timestamp(posting_date, posting_time) as "timestamp"
+		select *, posting_datetime as "timestamp"
 		from `tabStock Ledger Entry`
 		where item_code = %%(item_code)s
 		and is_cancelled = 0
 		%(conditions)s
-		order by timestamp(posting_date, posting_time) %(order)s, creation %(order)s
+		order by posting_datetime %(order)s, creation %(order)s
 		%(limit)s %(for_update)s"""
 		% {
 			"conditions": conditions,
@@ -1330,7 +1354,7 @@ def get_sle_by_voucher_detail_no(voucher_detail_no, excluded_sle=None):
 			"posting_date",
 			"posting_time",
 			"voucher_detail_no",
-			"timestamp(posting_date, posting_time) as timestamp",
+			"posting_datetime as timestamp",
 		],
 		as_dict=1,
 	)
@@ -1342,13 +1366,10 @@ def get_batch_incoming_rate(
 
 	sle = frappe.qb.DocType("Stock Ledger Entry")
 
-	timestamp_condition = CombineDatetime(sle.posting_date, sle.posting_time) < CombineDatetime(
-		posting_date, posting_time
-	)
+	timestamp_condition = sle.posting_datetime < get_combine_datetime(posting_date, posting_time)
 	if creation:
 		timestamp_condition |= (
-			CombineDatetime(sle.posting_date, sle.posting_time)
-			== CombineDatetime(posting_date, posting_time)
+			sle.posting_datetime == get_combine_datetime(posting_date, posting_time)
 		) & (sle.creation < creation)
 
 	batch_details = (
@@ -1401,6 +1422,7 @@ def get_valuation_rate(
 		)
 
 	# Get valuation rate from last sle for the same item and warehouse
+<<<<<<< HEAD
 	if not last_valuation_rate or last_valuation_rate[0][0] is None:
 		last_valuation_rate = frappe.db.sql(
 			"""select valuation_rate
@@ -1416,6 +1438,20 @@ def get_valuation_rate(
 		)
 
 	if last_valuation_rate:
+=======
+	if last_valuation_rate := frappe.db.sql(
+		"""select valuation_rate
+		from `tabStock Ledger Entry` force index (item_warehouse)
+		where
+			item_code = %s
+			AND warehouse = %s
+			AND valuation_rate >= 0
+			AND is_cancelled = 0
+			AND NOT (voucher_no = %s AND voucher_type = %s)
+		order by posting_datetime desc, name desc limit 1""",
+		(item_code, warehouse, voucher_no, voucher_type),
+	):
+>>>>>>> d80ca523a4 (perf: new column posting datetime in SLE to optimize stock ledger related queries)
 		return flt(last_valuation_rate[0][0])
 
 	# If negative stock allowed, and item delivered without any incoming entry,
@@ -1473,6 +1509,7 @@ def update_qty_in_future_sle(args, allow_negative_stock=False):
 	qty_shift = args.actual_qty
 
 	args["time_format"] = "%H:%i:%s"
+	args["posting_datetime"] = get_combine_datetime(args["posting_date"], args["posting_time"])
 
 	# find difference/shift in qty caused by stock reconciliation
 	if args.voucher_type == "Stock Reconciliation":
@@ -1482,8 +1519,6 @@ def update_qty_in_future_sle(args, allow_negative_stock=False):
 	next_stock_reco_detail = get_next_stock_reco(args)
 	if next_stock_reco_detail:
 		detail = next_stock_reco_detail[0]
-
-		# add condition to update SLEs before this date & time
 		datetime_limit_condition = get_datetime_limit_condition(detail)
 
 	frappe.db.sql(
@@ -1496,13 +1531,9 @@ def update_qty_in_future_sle(args, allow_negative_stock=False):
 			and voucher_no != %(voucher_no)s
 			and is_cancelled = 0
 			and (
-				posting_date > %(posting_date)s or
-				(
-					posting_date = %(posting_date)s and
-					time_format(posting_time, %(time_format)s) > time_format(%(posting_time)s, %(time_format)s)
-				)
+				posting_datetime > %(posting_datetime)s
 			)
-		{datetime_limit_condition}
+			{datetime_limit_condition}
 		""",
 		args,
 	)
@@ -1557,20 +1588,11 @@ def get_next_stock_reco(kwargs):
 			& (sle.voucher_no != kwargs.get("voucher_no"))
 			& (sle.is_cancelled == 0)
 			& (
-				(
-					CombineDatetime(sle.posting_date, sle.posting_time)
-					> CombineDatetime(kwargs.get("posting_date"), kwargs.get("posting_time"))
-				)
-				| (
-					(
-						CombineDatetime(sle.posting_date, sle.posting_time)
-						== CombineDatetime(kwargs.get("posting_date"), kwargs.get("posting_time"))
-					)
-					& (sle.creation > kwargs.get("creation"))
-				)
+				sle.posting_datetime
+				>= get_combine_datetime(kwargs.get("posting_date"), kwargs.get("posting_time"))
 			)
 		)
-		.orderby(CombineDatetime(sle.posting_date, sle.posting_time))
+		.orderby(sle.posting_datetime)
 		.orderby(sle.creation)
 		.limit(1)
 	)
@@ -1582,11 +1604,13 @@ def get_next_stock_reco(kwargs):
 
 
 def get_datetime_limit_condition(detail):
+	posting_datetime = get_combine_datetime(detail.posting_date, detail.posting_time)
+
 	return f"""
 		and
-		(timestamp(posting_date, posting_time) < timestamp('{detail.posting_date}', '{detail.posting_time}')
+		(posting_datetime < '{posting_datetime}'
 			or (
-				timestamp(posting_date, posting_time) = timestamp('{detail.posting_date}', '{detail.posting_time}')
+				posting_datetime = '{posting_datetime}'
 				and creation < '{detail.creation}'
 			)
 		)"""
@@ -1648,6 +1672,7 @@ def is_negative_with_precision(neg_sle, is_batch=False):
 	return qty_deficit < 0 and abs(qty_deficit) > 0.0001
 
 
+<<<<<<< HEAD
 def get_future_sle_with_negative_qty(sle):
 	SLE = frappe.qb.DocType("Stock Ledger Entry")
 	query = (
@@ -1668,6 +1693,27 @@ def get_future_sle_with_negative_qty(sle):
 		)
 		.orderby(CombineDatetime(SLE.posting_date, SLE.posting_time))
 		.limit(1)
+=======
+def get_future_sle_with_negative_qty(args):
+	return frappe.db.sql(
+		"""
+		select
+			qty_after_transaction, posting_date, posting_time,
+			voucher_type, voucher_no
+		from `tabStock Ledger Entry`
+		where
+			item_code = %(item_code)s
+			and warehouse = %(warehouse)s
+			and voucher_no != %(voucher_no)s
+			and posting_datetime >= %(posting_datetime)s
+			and is_cancelled = 0
+			and qty_after_transaction < 0
+		order by posting_datetime asc
+		limit 1
+	""",
+		args,
+		as_dict=1,
+>>>>>>> d80ca523a4 (perf: new column posting datetime in SLE to optimize stock ledger related queries)
 	)
 
 	if sle.voucher_type == "Stock Reconciliation" and sle.batch_no:
@@ -1681,20 +1727,20 @@ def get_future_sle_with_negative_batch_qty(args):
 		"""
 		with batch_ledger as (
 			select
-				posting_date, posting_time, voucher_type, voucher_no,
-				sum(actual_qty) over (order by posting_date, posting_time, creation) as cumulative_total
+				posting_date, posting_time, posting_datetime, voucher_type, voucher_no,
+				sum(actual_qty) over (order by posting_datetime, creation) as cumulative_total
 			from `tabStock Ledger Entry`
 			where
 				item_code = %(item_code)s
 				and warehouse = %(warehouse)s
 				and batch_no=%(batch_no)s
 				and is_cancelled = 0
-			order by posting_date, posting_time, creation
+			order by posting_datetime, creation
 		)
 		select * from batch_ledger
 		where
 			cumulative_total < 0.0
-			and timestamp(posting_date, posting_time) >= timestamp(%(posting_date)s, %(posting_time)s)
+			and posting_datetime >= %(posting_datetime)s
 		limit 1
 	""",
 		args,
@@ -1746,6 +1792,7 @@ def is_internal_transfer(sle):
 
 def get_stock_value_difference(item_code, warehouse, posting_date, posting_time, voucher_no=None):
 	table = frappe.qb.DocType("Stock Ledger Entry")
+	posting_datetime = get_combine_datetime(posting_date, posting_time)
 
 	query = (
 		frappe.qb.from_(table)
@@ -1754,10 +1801,7 @@ def get_stock_value_difference(item_code, warehouse, posting_date, posting_time,
 			(table.is_cancelled == 0)
 			& (table.item_code == item_code)
 			& (table.warehouse == warehouse)
-			& (
-				(table.posting_date < posting_date)
-				| ((table.posting_date == posting_date) & (table.posting_time <= posting_time))
-			)
+			& (table.posting_datetime <= posting_datetime)
 		)
 	)