perf: cache journal entry balances for 5 minutes

If user is repeatedly adding multiple JV lines then everytime the value
will be refetched from database, reading GLE table can be very expensive
on large DBs.

AFAIK this function is only for UX and balances are anyways overidden
when saving the document.

Proper fix: maybe use closing balance here?

erpnext/accounts/doctype/bank_account/bank_account.py

@@ -57,7 +57,9 @@ class BankAccount(Document):
 
 	def validate_account(self):
 		if self.account:
-			if accounts := frappe.db.get_all("Bank Account", filters={"account": self.account}, as_list=1):
+			if accounts := frappe.db.get_all(
+				"Bank Account", filters={"account": self.account, "name": ["!=", self.name]}, as_list=1
+			):
 				frappe.throw(
 					_("'{0}' account is already used by {1}. Use another account.").format(
 						frappe.bold(self.account),

erpnext/accounts/doctype/journal_entry/journal_entry.py

@@ -7,6 +7,7 @@ import json
 import frappe
 from frappe import _, msgprint, scrub
 from frappe.utils import cstr, flt, fmt_money, formatdate, get_link_to_form, nowdate
+from frappe.utils.caching import redis_cache
 
 import erpnext
 from erpnext.accounts.deferred_revenue import get_deferred_booking_accounts
@@ -1478,6 +1479,7 @@ def get_outstanding(args):
 
 
 @frappe.whitelist()
+@redis_cache(ttl=5 * 60, user=True)
 def get_party_account_and_balance(company, party_type, party, cost_center=None):
 	if not frappe.has_permission("Account"):
 		frappe.msgprint(_("No Permission"), raise_exception=1)

erpnext/accounts/doctype/payment_entry/payment_entry.py

@@ -389,7 +389,10 @@ class PaymentEntry(AccountsController):
 		)
 
 	def set_missing_ref_details(
-		self, force: bool = False, update_ref_details_only_for: list | None = None
+		self,
+		force: bool = False,
+		update_ref_details_only_for: list | None = None,
+		ref_exchange_rate: float | None = None,
 	) -> None:
 		for d in self.get("references"):
 			if d.allocated_amount:
@@ -401,6 +404,8 @@ class PaymentEntry(AccountsController):
 				ref_details = get_reference_details(
 					d.reference_doctype, d.reference_name, self.party_account_currency
 				)
+				if ref_exchange_rate:
+					ref_details.update({"exchange_rate": ref_exchange_rate})
 
 				for field, value in ref_details.items():
 					if d.exchange_gain_loss:

erpnext/accounts/doctype/payment_reconciliation/payment_reconciliation.py

@@ -631,7 +631,12 @@ class PaymentReconciliation(Document):
 			journals_map = frappe._dict(
 				frappe.db.get_all(
 					"Journal Entry Account",
-					filters={"parent": ("in", journals), "account": ("in", [self.receivable_payable_account])},
+					filters={
+						"parent": ("in", journals),
+						"account": ("in", [self.receivable_payable_account]),
+						"party_type": self.party_type,
+						"party": self.party,
+					},
 					fields=[
 						"parent as `name`",
 						"exchange_rate",

erpnext/accounts/doctype/payment_reconciliation/test_payment_reconciliation.py

@@ -56,6 +56,7 @@ class TestPaymentReconciliation(FrappeTestCase):
 		self.expense_account = "Cost of Goods Sold - _PR"
 		self.debit_to = "Debtors - _PR"
 		self.creditors = "Creditors - _PR"
+		self.cash = "Cash - _PR"
 
 		# create bank account
 		if frappe.db.exists("Account", "HDFC - _PR"):
@@ -486,6 +487,91 @@ class TestPaymentReconciliation(FrappeTestCase):
 		self.assertEqual(len(pr.get("invoices")), 0)
 		self.assertEqual(len(pr.get("payments")), 0)
 
+	def test_payment_against_foreign_currency_journal(self):
+		transaction_date = nowdate()
+
+		self.supplier = "_Test Supplier USD"
+		self.supplier2 = make_supplier("_Test Supplier2 USD", "USD")
+		amount = 100
+		exc_rate1 = 80
+		exc_rate2 = 83
+
+		je = frappe.new_doc("Journal Entry")
+		je.posting_date = transaction_date
+		je.company = self.company
+		je.user_remark = "test"
+		je.multi_currency = 1
+		je.set(
+			"accounts",
+			[
+				{
+					"account": self.creditors_usd,
+					"party_type": "Supplier",
+					"party": self.supplier,
+					"exchange_rate": exc_rate1,
+					"cost_center": self.cost_center,
+					"credit": amount * exc_rate1,
+					"credit_in_account_currency": amount,
+				},
+				{
+					"account": self.creditors_usd,
+					"party_type": "Supplier",
+					"party": self.supplier2,
+					"exchange_rate": exc_rate2,
+					"cost_center": self.cost_center,
+					"credit": amount * exc_rate2,
+					"credit_in_account_currency": amount,
+				},
+				{
+					"account": self.expense_account,
+					"cost_center": self.cost_center,
+					"debit": (amount * exc_rate1) + (amount * exc_rate2),
+					"debit_in_account_currency": (amount * exc_rate1) + (amount * exc_rate2),
+				},
+			],
+		)
+		je.save().submit()
+
+		pe = self.create_payment_entry(amount=amount, posting_date=transaction_date)
+		pe.payment_type = "Pay"
+		pe.party_type = "Supplier"
+		pe.party = self.supplier
+		pe.paid_to = self.creditors_usd
+		pe.paid_from = self.cash
+		pe.paid_amount = 8000
+		pe.received_amount = 100
+		pe.target_exchange_rate = exc_rate1
+		pe.paid_to_account_currency = "USD"
+		pe.save().submit()
+
+		pr = self.create_payment_reconciliation(party_is_customer=False)
+		pr.receivable_payable_account = self.creditors_usd
+		pr.minimum_invoice_amount = pr.maximum_invoice_amount = amount
+		pr.from_invoice_date = pr.to_invoice_date = transaction_date
+		pr.from_payment_date = pr.to_payment_date = transaction_date
+
+		pr.get_unreconciled_entries()
+		invoices = [x.as_dict() for x in pr.get("invoices")]
+		payments = [x.as_dict() for x in pr.get("payments")]
+		pr.allocate_entries(frappe._dict({"invoices": invoices, "payments": payments}))
+
+		# There should no difference_amount as the Journal and Payment have same exchange rate -  'exc_rate1'
+		for row in pr.allocation:
+			self.assertEqual(flt(row.get("difference_amount")), 0.0)
+
+		pr.reconcile()
+
+		# check PR tool output
+		self.assertEqual(len(pr.get("invoices")), 0)
+		self.assertEqual(len(pr.get("payments")), 0)
+
+		journals = frappe.db.get_all(
+			"Journal Entry Account",
+			filters={"reference_type": je.doctype, "reference_name": je.name, "docstatus": 1},
+			fields=["parent"],
+		)
+		self.assertEqual([], journals)
+
 	def test_journal_against_invoice(self):
 		transaction_date = nowdate()
 		amount = 100
@@ -1248,3 +1334,17 @@ def make_customer(customer_name, currency=None):
 		return customer.name
 	else:
 		return customer_name
+
+
+def make_supplier(supplier_name, currency=None):
+	if not frappe.db.exists("Supplier", supplier_name):
+		supplier = frappe.new_doc("Supplier")
+		supplier.supplier_name = supplier_name
+		supplier.type = "Individual"
+
+		if currency:
+			supplier.default_currency = currency
+		supplier.save()
+		return supplier.name
+	else:
+		return supplier_name

erpnext/accounts/doctype/sales_invoice/test_sales_invoice.py

@@ -3602,6 +3602,33 @@ class TestSalesInvoice(FrappeTestCase):
 		check_gl_entries(self, pe.name, expected_gle, nowdate(), voucher_type="Payment Entry")
 		set_advance_flag(company="_Test Company", flag=0, default_account="")
 
+	def test_pulling_advance_based_on_debit_to(self):
+		from erpnext.accounts.doctype.payment_entry.test_payment_entry import create_payment_entry
+
+		debtors2 = create_account(
+			parent_account="Accounts Receivable - _TC",
+			account_name="Debtors 2",
+			company="_Test Company",
+			account_type="Receivable",
+		)
+		si = create_sales_invoice(do_not_submit=True)
+		si.debit_to = debtors2
+		si.save()
+
+		pe = create_payment_entry(
+			company=si.company,
+			payment_type="Receive",
+			party_type="Customer",
+			party=si.customer,
+			paid_from=debtors2,
+			paid_to="Cash - _TC",
+			paid_amount=1000,
+		)
+		pe.submit()
+		advances = si.get_advance_entries()
+		self.assertEqual(1, len(advances))
+		self.assertEqual(advances[0].reference_name, pe.name)
+
 
 def set_advance_flag(company, flag, default_account):
 	frappe.db.set_value(

erpnext/accounts/utils.py

@@ -714,7 +714,7 @@ def update_reference_in_payment_entry(
 	payment_entry.setup_party_account_field()
 	payment_entry.set_missing_values()
 	if not skip_ref_details_update_for_pe:
-		payment_entry.set_missing_ref_details()
+		payment_entry.set_missing_ref_details(ref_exchange_rate=d.exchange_rate or None)
 	payment_entry.set_amounts()
 	payment_entry.make_exchange_gain_loss_journal(
 		frappe._dict({"difference_posting_date": d.difference_posting_date}), dimensions_dict

erpnext/controllers/accounts_controller.py

@@ -1133,21 +1133,24 @@ class AccountsController(TransactionBase):
 			self.append("advances", advance_row)
 
 	def get_advance_entries(self, include_unallocated=True):
+		party_account = []
 		if self.doctype == "Sales Invoice":
 			party_type = "Customer"
 			party = self.customer
 			amount_field = "credit_in_account_currency"
 			order_field = "sales_order"
 			order_doctype = "Sales Order"
+			party_account.append(self.debit_to)
 		else:
 			party_type = "Supplier"
 			party = self.supplier
 			amount_field = "debit_in_account_currency"
 			order_field = "purchase_order"
 			order_doctype = "Purchase Order"
+			party_account.append(self.credit_to)
 
-		party_account = get_party_account(
-			party_type, party=party, company=self.company, include_advance=True
+		party_account.extend(
+			get_party_account(party_type, party=party, company=self.company, include_advance=True)
 		)
 
 		order_list = list(set(d.get(order_field) for d in self.get("items") if d.get(order_field)))

erpnext/projects/doctype/project/project.js

@@ -35,6 +35,14 @@ frappe.ui.form.on("Project", {
 			};
 		});
 
+		frm.set_query("department", function (doc) {
+			return {
+				filters: {
+					"company": doc.company,
+				}
+			};
+		});
+
 		// sales order
 		frm.set_query('sales_order', function () {
 			var filters = {

erpnext/public/js/controllers/transaction.js

@@ -1163,6 +1163,7 @@ erpnext.TransactionController = class TransactionController extends erpnext.taxe
 				},
 				callback: function(r) {
 					if(!r.exc) {
+						me.apply_price_list(item, true)
 						frappe.model.set_value(cdt, cdn, 'conversion_factor', r.message.conversion_factor);
 					}
 				}
@@ -1513,6 +1514,8 @@ erpnext.TransactionController = class TransactionController extends erpnext.taxe
 	}
 
 	remove_pricing_rule_for_item(item) {
+		// capture pricing rule before removing it to delete free items
+		let removed_pricing_rule = item.pricing_rules;
 		if (item.pricing_rules){
 			let me = this;
 			return this.frm.call({
@@ -1533,7 +1536,7 @@ erpnext.TransactionController = class TransactionController extends erpnext.taxe
 				},
 				callback: function(r) {
 					if (!r.exc && r.message) {
-						me.remove_pricing_rule(r.message);
+						me.remove_pricing_rule(r.message, removed_pricing_rule);
 						me.calculate_taxes_and_totals();
 						if(me.frm.doc.apply_discount_on) me.frm.trigger("apply_discount_on");
 					}
@@ -1791,7 +1794,7 @@ erpnext.TransactionController = class TransactionController extends erpnext.taxe
 		});
 	}
 
-	remove_pricing_rule(item) {
+	remove_pricing_rule(item, removed_pricing_rule) {
 		let me = this;
 		const fields = ["discount_percentage",
 			"discount_amount", "margin_rate_or_amount", "rate_with_margin"];
@@ -1800,7 +1803,8 @@ erpnext.TransactionController = class TransactionController extends erpnext.taxe
 			let items = [];
 
 			me.frm.doc.items.forEach(d => {
-				if(d.item_code != item.remove_free_item || !d.is_free_item) {
+				// if same item was added a free item through a different pricing rule, keep it
+				if(d.item_code != item.remove_free_item || !d.is_free_item || removed_pricing_rule?.includes(d.pricing_rules)) {
 					items.push(d);
 				}
 			});

erpnext/selling/doctype/sales_order/sales_order.py

@@ -249,7 +249,8 @@ class SalesOrder(SellingController):
 					frappe.msgprint(
 						_("Warning: Sales Order {0} already exists against Customer's Purchase Order {1}").format(
 							frappe.bold(so[0][0]), frappe.bold(self.po_no)
-						)
+						),
+						alert=True,
 					)
 				else:
 					frappe.throw(

erpnext/setup/install.py

@@ -20,7 +20,9 @@ default_mail_footer = """<div style="padding: 7px; text-align: right; color: #88
 
 
 def after_install():
-	frappe.get_doc({"doctype": "Role", "role_name": "Analytics"}).insert()
+	if not frappe.db.exists("Role", "Analytics"):
+		frappe.get_doc({"doctype": "Role", "role_name": "Analytics"}).insert()
+
 	set_single_defaults()
 	create_print_setting_custom_fields()
 	add_all_roles_to("Administrator")

erpnext/stock/deprecated_serial_batch.py

@@ -8,9 +8,12 @@ from pypika import Order
 class DeprecatedSerialNoValuation:
 	@deprecated
 	def calculate_stock_value_from_deprecarated_ledgers(self):
-		serial_nos = list(
-			filter(lambda x: x not in self.serial_no_incoming_rate and x, self.get_serial_nos())
-		)
+		if not frappe.db.get_value(
+			"Stock Ledger Entry", {"serial_no": ("is", "set"), "is_cancelled": 0}, "name"
+		):
+			return
+
+		serial_nos = self.get_serial_nos()
 
 		actual_qty = flt(self.sle.actual_qty)
 
@@ -25,23 +28,12 @@ class DeprecatedSerialNoValuation:
 	@deprecated
 	def get_incoming_value_for_serial_nos(self, serial_nos):
 		# get rate from serial nos within same company
-		all_serial_nos = frappe.get_all(
-			"Serial No", fields=["purchase_rate", "name", "company"], filters={"name": ("in", serial_nos)}
-		)
-
 		incoming_values = 0.0
-		for d in all_serial_nos:
-			if d.company == self.sle.company:
-				self.serial_no_incoming_rate[d.name] += flt(d.purchase_rate)
-				incoming_values += flt(d.purchase_rate)
-
-		# Get rate for serial nos which has been transferred to other company
-		invalid_serial_nos = [d.name for d in all_serial_nos if d.company != self.sle.company]
-		for serial_no in invalid_serial_nos:
+		for serial_no in serial_nos:
 			table = frappe.qb.DocType("Stock Ledger Entry")
-			incoming_rate = (
+			stock_ledgers = (
 				frappe.qb.from_(table)
-				.select(table.incoming_rate)
+				.select(table.incoming_rate, table.actual_qty, table.stock_value_difference)
 				.where(
 					(
 						(table.serial_no == serial_no)
@@ -51,15 +43,18 @@ class DeprecatedSerialNoValuation:
 					)
 					& (table.company == self.sle.company)
 					& (table.serial_and_batch_bundle.isnull())
-					& (table.actual_qty > 0)
 					& (table.is_cancelled == 0)
 				)
-				.orderby(table.posting_date, order=Order.desc)
-				.limit(1)
-			).run()
+				.orderby(table.posting_datetime, order=Order.desc)
+			).run(as_dict=1)
 
-			self.serial_no_incoming_rate[serial_no] += flt(incoming_rate[0][0]) if incoming_rate else 0
-			incoming_values += self.serial_no_incoming_rate[serial_no]
+			for sle in stock_ledgers:
+				self.serial_no_incoming_rate[serial_no] += (
+					flt(sle.incoming_rate)
+					if sle.actual_qty > 0
+					else (sle.stock_value_difference / sle.actual_qty) * -1
+				)
+				incoming_values += self.serial_no_incoming_rate[serial_no]
 
 		return incoming_values
 

erpnext/stock/doctype/serial_and_batch_bundle/serial_and_batch_bundle.py

@@ -2118,7 +2118,7 @@ def is_serial_batch_no_exists(item_code, type_of_transaction, serial_no=None, ba
 
 		make_serial_no(serial_no, item_code)
 
-	if batch_no and frappe.db.exists("Batch", batch_no):
+	if batch_no and not frappe.db.exists("Batch", batch_no):
 		if type_of_transaction != "Inward":
 			frappe.throw(_("Batch No {0} does not exists").format(batch_no))
 

erpnext/stock/doctype/serial_and_batch_bundle/test_serial_and_batch_bundle.py

@@ -540,6 +540,110 @@ class TestSerialandBatchBundle(FrappeTestCase):
 
 		self.assertRaises(frappe.exceptions.ValidationError, pr2.save)
 
+	def test_serial_no_valuation_for_legacy_ledgers(self):
+		sn_item = make_item(
+			"Test Serial No Valuation for Legacy Ledgers",
+			properties={"has_serial_no": 1, "serial_no_series": "SNN-TSNVL.-#####"},
+		).name
+
+		serial_nos = []
+		for serial_no in [f"{sn_item}-0001", f"{sn_item}-0002"]:
+			if not frappe.db.exists("Serial No", serial_no):
+				sn_doc = frappe.get_doc(
+					{
+						"doctype": "Serial No",
+						"serial_no": serial_no,
+						"item_code": sn_item,
+					}
+				).insert(ignore_permissions=True)
+				serial_nos.append(serial_no)
+
+		frappe.flags.ignore_serial_batch_bundle_validation = True
+
+		qty_after_transaction = 0.0
+		stock_value = 0.0
+		for row in [{"qty": 2, "rate": 100}, {"qty": -2, "rate": 100}, {"qty": 2, "rate": 200}]:
+			row = frappe._dict(row)
+			qty_after_transaction += row.qty
+			stock_value += row.rate * row.qty
+
+			doc = frappe.get_doc(
+				{
+					"doctype": "Stock Ledger Entry",
+					"posting_date": today(),
+					"posting_time": nowtime(),
+					"incoming_rate": row.rate if row.qty > 0 else 0,
+					"qty_after_transaction": qty_after_transaction,
+					"stock_value_difference": row.rate * row.qty,
+					"stock_value": stock_value,
+					"valuation_rate": row.rate,
+					"actual_qty": row.qty,
+					"item_code": sn_item,
+					"warehouse": "_Test Warehouse - _TC",
+					"serial_no": "\n".join(serial_nos),
+					"company": "_Test Company",
+				}
+			)
+			doc.flags.ignore_permissions = True
+			doc.flags.ignore_mandatory = True
+			doc.flags.ignore_links = True
+			doc.flags.ignore_validate = True
+			doc.submit()
+
+			for sn in serial_nos:
+				sn_doc = frappe.get_doc("Serial No", sn)
+				if row.qty > 0:
+					sn_doc.db_set("warehouse", "_Test Warehouse - _TC")
+				else:
+					sn_doc.db_set("warehouse", "")
+
+		frappe.flags.ignore_serial_batch_bundle_validation = False
+
+		se = make_stock_entry(
+			item_code=sn_item,
+			qty=2,
+			source="_Test Warehouse - _TC",
+			serial_no="\n".join(serial_nos),
+			use_serial_batch_fields=True,
+			do_not_submit=True,
+		)
+
+		se.save()
+		se.submit()
+
+		stock_value_difference = frappe.db.get_value(
+			"Stock Ledger Entry",
+			{"voucher_no": se.name, "is_cancelled": 0, "voucher_type": "Stock Entry"},
+			"stock_value_difference",
+		)
+
+		self.assertEqual(flt(stock_value_difference, 2), 400.0 * -1)
+
+		se = make_stock_entry(
+			item_code=sn_item,
+			qty=1,
+			rate=353,
+			target="_Test Warehouse - _TC",
+		)
+
+		serial_no = get_serial_nos_from_bundle(se.items[0].serial_and_batch_bundle)[0]
+
+		se = make_stock_entry(
+			item_code=sn_item,
+			qty=1,
+			source="_Test Warehouse - _TC",
+			serial_no=serial_no,
+			use_serial_batch_fields=True,
+		)
+
+		stock_value_difference = frappe.db.get_value(
+			"Stock Ledger Entry",
+			{"voucher_no": se.name, "is_cancelled": 0, "voucher_type": "Stock Entry"},
+			"stock_value_difference",
+		)
+
+		self.assertEqual(flt(stock_value_difference, 2), 353.0 * -1)
+
 
 def get_batch_from_bundle(bundle):
 	from erpnext.stock.serial_batch_bundle import get_batch_nos