Merge pull request #47146 from frappe/mergify/copy/version-14-hotfix/pr-47145

refactor: make AR / AP report more memory efficient (copy #47145)

erpnext/accounts/doctype/accounts_settings/accounts_settings.json

@@ -73,9 +73,12 @@
   "reports_tab",
   "remarks_section",
   "general_ledger_remarks_length",
-  "ignore_is_opening_check_for_reporting",
   "column_break_lvjk",
-  "receivable_payable_remarks_length"
+  "receivable_payable_remarks_length",
+  "accounts_receivable_payable_tuning_section",
+  "receivable_payable_fetch_method",
+  "legacy_section",
+  "ignore_is_opening_check_for_reporting"
  ],
  "fields": [
   {
@@ -479,6 +482,23 @@
    "fieldname": "ignore_is_opening_check_for_reporting",
    "fieldtype": "Check",
    "label": "Ignore Is Opening check for reporting"
+  },
+  {
+   "default": "Buffered Cursor",
+   "fieldname": "receivable_payable_fetch_method",
+   "fieldtype": "Select",
+   "label": "Data Fetch Method",
+   "options": "Buffered Cursor\nUnBuffered Cursor"
+  },
+  {
+   "fieldname": "accounts_receivable_payable_tuning_section",
+   "fieldtype": "Section Break",
+   "label": "Accounts Receivable / Payable Tuning"
+  },
+  {
+   "fieldname": "legacy_section",
+   "fieldtype": "Section Break",
+   "label": "Legacy Fields"
   }
  ],
  "icon": "icon-cog",
@@ -486,7 +506,7 @@
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2025-01-23 13:15:44.077853",
+ "modified": "2025-05-05 12:29:38.302027",
  "modified_by": "Administrator",
  "module": "Accounts",
  "name": "Accounts Settings",

erpnext/accounts/report/accounts_receivable/accounts_receivable.py

@@ -49,6 +49,10 @@ class ReceivablePayableReport:
 		self.age_as_on = (
 			getdate(nowdate()) if self.filters.report_date > getdate(nowdate()) else self.filters.report_date
 		)
+		self.ple_fetch_method = (
+			frappe.db.get_single_value("Accounts Settings", "receivable_payable_fetch_method")
+			or "Buffered Cursor"
+		)  # Fail Safe
 
 	def run(self, args):
 		self.filters.update(args)
@@ -85,10 +89,7 @@ class ReceivablePayableReport:
 				self.skip_total_row = 1
 
 	def get_data(self):
-		self.get_ple_entries()
 		self.get_sales_invoices_or_customers_based_on_sales_person()
-		self.voucher_balance = OrderedDict()
-		self.init_voucher_balance()  # invoiced, paid, credit_note, outstanding
 
 		# Build delivery note map against all sales invoices
 		self.build_delivery_note_map()
@@ -105,12 +106,40 @@ class ReceivablePayableReport:
 		# Get Exchange Rate Revaluations
 		self.get_exchange_rate_revaluations()
 
+		self.prepare_ple_query()
 		self.data = []
+		self.voucher_balance = OrderedDict()
 
+		if self.ple_fetch_method == "Buffered Cursor":
+			self.fetch_ple_in_buffered_cursor()
+		elif self.ple_fetch_method == "UnBuffered Cursor":
+			self.fetch_ple_in_unbuffered_cursor()
+
+		self.build_data()
+
+	def fetch_ple_in_buffered_cursor(self):
+		self.ple_entries = frappe.db.sql(self.ple_query.get_sql(), as_dict=True)
+
+		for ple in self.ple_entries:
+			self.init_voucher_balance(ple)  # invoiced, paid, credit_note, outstanding
+
+		# This is unavoidable. Initialization and allocation cannot happen in same loop
 		for ple in self.ple_entries:
 			self.update_voucher_balance(ple)
 
-		self.build_data()
+		delattr(self, "ple_entries")
+
+	def fetch_ple_in_unbuffered_cursor(self):
+		self.ple_entries = []
+		with frappe.db.unbuffered_cursor():
+			for ple in frappe.db.sql(self.ple_query.get_sql(), as_dict=True, as_iterator=True):
+				self.init_voucher_balance(ple)  # invoiced, paid, credit_note, outstanding
+				self.ple_entries.append(ple)
+
+		# This is unavoidable. Initialization and allocation cannot happen in same loop
+		for ple in self.ple_entries:
+			self.update_voucher_balance(ple)
+		delattr(self, "ple_entries")
 
 	def build_voucher_dict(self, ple):
 		return frappe._dict(
@@ -131,26 +160,22 @@ class ReceivablePayableReport:
 			outstanding_in_account_currency=0.0,
 		)
 
-	def init_voucher_balance(self):
+	def init_voucher_balance(self, ple):
-		# build all keys, since we want to exclude vouchers beyond the report date
+		if self.filters.get("ignore_accounts"):
-		for ple in self.ple_entries:
+			key = (ple.voucher_type, ple.voucher_no, ple.party)
-			# get the balance object for voucher_type
+		else:
+			key = (ple.account, ple.voucher_type, ple.voucher_no, ple.party)
 
-			if self.filters.get("ignore_accounts"):
+		if key not in self.voucher_balance:
-				key = (ple.voucher_type, ple.voucher_no, ple.party)
+			self.voucher_balance[key] = self.build_voucher_dict(ple)
-			else:
-				key = (ple.account, ple.voucher_type, ple.voucher_no, ple.party)
 
-			if key not in self.voucher_balance:
+		if ple.voucher_type == ple.against_voucher_type and ple.voucher_no == ple.against_voucher_no:
-				self.voucher_balance[key] = self.build_voucher_dict(ple)
+			self.voucher_balance[key].cost_center = ple.cost_center
 
-			if ple.voucher_type == ple.against_voucher_type and ple.voucher_no == ple.against_voucher_no:
+		self.get_invoices(ple)
-				self.voucher_balance[key].cost_center = ple.cost_center
 
-			self.get_invoices(ple)
+		if self.filters.get("group_by_party"):
-
+			self.init_subtotal_row(ple.party)
-			if self.filters.get("group_by_party"):
-				self.init_subtotal_row(ple.party)
 
 		if self.filters.get("group_by_party") and not self.filters.get("in_party_currency"):
 			self.init_subtotal_row("Total")
@@ -764,7 +789,7 @@ class ReceivablePayableReport:
 			index = 4
 		row["range" + str(index + 1)] = row.outstanding
 
-	def get_ple_entries(self):
+	def prepare_ple_query(self):
 		# get all the GL entries filtered by the given filters
 
 		self.prepare_conditions()
@@ -817,7 +842,7 @@ class ReceivablePayableReport:
 		else:
 			query = query.orderby(self.ple.posting_date, self.ple.party)
 
-		self.ple_entries = query.run(as_dict=True)
+		self.ple_query = query
 
 	def get_sales_invoices_or_customers_based_on_sales_person(self):
 		if self.filters.get("sales_person"):

erpnext/patches.txt

@@ -373,3 +373,4 @@ execute:frappe.db.set_single_value("Accounts Settings", "exchange_gain_loss_post
 erpnext.patches.v14_0.update_posting_datetime
 erpnext.stock.doctype.stock_ledger_entry.patches.ensure_sle_indexes
 erpnext.patches.v14_0.rename_group_by_to_categorize_by
+execute:frappe.db.set_single_value("Accounts Settings", "receivable_payable_fetch_method", "Buffered Cursor")