From 08903459c2fa09b6d60ab8c08b7cc8b5680c070f Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sun, 27 Apr 2025 11:12:55 +0530 Subject: [PATCH 1/4] refactor: use unbuffered cursor for fetching --- .../accounts_receivable.py | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/erpnext/accounts/report/accounts_receivable/accounts_receivable.py b/erpnext/accounts/report/accounts_receivable/accounts_receivable.py index 973b7eede66..09554c334b0 100644 --- a/erpnext/accounts/report/accounts_receivable/accounts_receivable.py +++ b/erpnext/accounts/report/accounts_receivable/accounts_receivable.py @@ -90,10 +90,7 @@ class ReceivablePayableReport: self.skip_total_row = 1 def get_data(self): - self.get_ple_entries() self.get_sales_invoices_or_customers_based_on_sales_person() - self.voucher_balance = OrderedDict() - self.init_voucher_balance() # invoiced, paid, credit_note, outstanding # Build delivery note map against all sales invoices self.build_delivery_note_map() @@ -110,7 +107,15 @@ class ReceivablePayableReport: # Get Exchange Rate Revaluations self.get_exchange_rate_revaluations() + self.prepare_ple_query() self.data = [] + self.voucher_balance = OrderedDict() + self.ple_entries = [] + + with frappe.db.unbuffered_cursor(): + for ple in frappe.db.sql(self.ple_query.get_sql(), as_dict=True, as_iterator=True): + self.init_voucher_balance(ple) # invoiced, paid, credit_note, outstanding + self.ple_entries.append(ple) for ple in self.ple_entries: self.update_voucher_balance(ple) @@ -136,26 +141,22 @@ class ReceivablePayableReport: outstanding_in_account_currency=0.0, ) - def init_voucher_balance(self): - # build all keys, since we want to exclude vouchers beyond the report date - for ple in self.ple_entries: - # get the balance object for voucher_type + def init_voucher_balance(self, ple): + if self.filters.get("ignore_accounts"): + key = (ple.voucher_type, ple.voucher_no, ple.party) + else: + key = (ple.account, ple.voucher_type, ple.voucher_no, ple.party) - if self.filters.get("ignore_accounts"): - key = (ple.voucher_type, ple.voucher_no, ple.party) - else: - key = (ple.account, ple.voucher_type, ple.voucher_no, ple.party) + if key not in self.voucher_balance: + self.voucher_balance[key] = self.build_voucher_dict(ple) - if key not in self.voucher_balance: - self.voucher_balance[key] = self.build_voucher_dict(ple) + if ple.voucher_type == ple.against_voucher_type and ple.voucher_no == ple.against_voucher_no: + self.voucher_balance[key].cost_center = ple.cost_center - if ple.voucher_type == ple.against_voucher_type and ple.voucher_no == ple.against_voucher_no: - self.voucher_balance[key].cost_center = ple.cost_center + self.get_invoices(ple) - self.get_invoices(ple) - - if self.filters.get("group_by_party"): - self.init_subtotal_row(ple.party) + if self.filters.get("group_by_party"): + self.init_subtotal_row(ple.party) if self.filters.get("group_by_party") and not self.filters.get("in_party_currency"): self.init_subtotal_row("Total") @@ -778,7 +779,7 @@ class ReceivablePayableReport: ) row["range" + str(index + 1)] = row.outstanding - def get_ple_entries(self): + def prepare_ple_query(self): # get all the GL entries filtered by the given filters self.prepare_conditions() @@ -831,7 +832,7 @@ class ReceivablePayableReport: else: query = query.orderby(self.ple.posting_date, self.ple.party) - self.ple_entries = query.run(as_dict=True) + self.ple_query = query def get_sales_invoices_or_customers_based_on_sales_person(self): if self.filters.get("sales_person"): From 66fd639b523c8ad21526c611fd37b0559cb7ba79 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Mon, 5 May 2025 12:29:58 +0530 Subject: [PATCH 2/4] refactor: configurable fetch method for AR / AP report --- .../accounts_settings/accounts_settings.json | 24 +++++++++++++++++-- .../accounts_settings/accounts_settings.py | 1 + 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/erpnext/accounts/doctype/accounts_settings/accounts_settings.json b/erpnext/accounts/doctype/accounts_settings/accounts_settings.json index 285a14b40a6..ae4adfa8a67 100644 --- a/erpnext/accounts/doctype/accounts_settings/accounts_settings.json +++ b/erpnext/accounts/doctype/accounts_settings/accounts_settings.json @@ -79,9 +79,12 @@ "reports_tab", "remarks_section", "general_ledger_remarks_length", - "ignore_is_opening_check_for_reporting", "column_break_lvjk", "receivable_payable_remarks_length", + "accounts_receivable_payable_tuning_section", + "receivable_payable_fetch_method", + "legacy_section", + "ignore_is_opening_check_for_reporting", "payment_request_settings", "create_pr_in_draft_status" ], @@ -545,6 +548,23 @@ "fieldname": "use_sales_invoice_in_pos", "fieldtype": "Check", "label": "Use Sales Invoice" + }, + { + "default": "Buffered Cursor", + "fieldname": "receivable_payable_fetch_method", + "fieldtype": "Select", + "label": "Data Fetch Method", + "options": "Buffered Cursor\nUnBuffered Cursor" + }, + { + "fieldname": "accounts_receivable_payable_tuning_section", + "fieldtype": "Section Break", + "label": "Accounts Receivable / Payable Tuning" + }, + { + "fieldname": "legacy_section", + "fieldtype": "Section Break", + "label": "Legacy Fields" } ], "grid_page_length": 50, @@ -553,7 +573,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2025-03-30 20:47:17.954736", + "modified": "2025-05-05 12:29:38.302027", "modified_by": "Administrator", "module": "Accounts", "name": "Accounts Settings", diff --git a/erpnext/accounts/doctype/accounts_settings/accounts_settings.py b/erpnext/accounts/doctype/accounts_settings/accounts_settings.py index 997ba49ed26..bbf1c2ef060 100644 --- a/erpnext/accounts/doctype/accounts_settings/accounts_settings.py +++ b/erpnext/accounts/doctype/accounts_settings/accounts_settings.py @@ -54,6 +54,7 @@ class AccountsSettings(Document): merge_similar_account_heads: DF.Check over_billing_allowance: DF.Currency post_change_gl_entries: DF.Check + receivable_payable_fetch_method: DF.Literal["Buffered Cursor", "UnBuffered Cursor"] receivable_payable_remarks_length: DF.Int reconciliation_queue_size: DF.Int role_allowed_to_over_bill: DF.Link | None From b5bb6f3508953822f0e076184f729b08368153a7 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Mon, 5 May 2025 13:46:15 +0530 Subject: [PATCH 3/4] refactor: use fetch method based on configuration --- .../accounts_receivable.py | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/erpnext/accounts/report/accounts_receivable/accounts_receivable.py b/erpnext/accounts/report/accounts_receivable/accounts_receivable.py index 09554c334b0..6423e8f87ed 100644 --- a/erpnext/accounts/report/accounts_receivable/accounts_receivable.py +++ b/erpnext/accounts/report/accounts_receivable/accounts_receivable.py @@ -54,6 +54,9 @@ class ReceivablePayableReport: self.filters.range = "30, 60, 90, 120" self.ranges = [num.strip() for num in self.filters.range.split(",") if num.strip().isdigit()] self.range_numbers = [num for num in range(1, len(self.ranges) + 2)] + self.ple_fetch_method = frappe.db.get_single_value( + "Accounts Settings", "receivable_payable_fetch_method" + ) def run(self, args): self.filters.update(args) @@ -110,17 +113,37 @@ class ReceivablePayableReport: self.prepare_ple_query() self.data = [] self.voucher_balance = OrderedDict() - self.ple_entries = [] + if self.ple_fetch_method == "Buffered Cursor": + self.fetch_ple_in_buffered_cursor() + elif self.ple_fetch_method == "UnBuffered Cursor": + self.fetch_ple_in_unbuffered_cursor() + + self.build_data() + + def fetch_ple_in_buffered_cursor(self): + self.ple_entries = frappe.db.sql(self.ple_query.get_sql(), as_dict=True) + + for ple in self.ple_entries: + self.init_voucher_balance(ple) # invoiced, paid, credit_note, outstanding + + # This is unavoidable. Initialization and allocation cannot happen in same loop + for ple in self.ple_entries: + self.update_voucher_balance(ple) + + delattr(self, "ple_entries") + + def fetch_ple_in_unbuffered_cursor(self): + self.ple_entries = [] with frappe.db.unbuffered_cursor(): for ple in frappe.db.sql(self.ple_query.get_sql(), as_dict=True, as_iterator=True): self.init_voucher_balance(ple) # invoiced, paid, credit_note, outstanding self.ple_entries.append(ple) + # This is unavoidable. Initialization and allocation cannot happen in same loop for ple in self.ple_entries: self.update_voucher_balance(ple) - - self.build_data() + delattr(self, "ple_entries") def build_voucher_dict(self, ple): return frappe._dict( From ca1e81e1b57036860071e1c148712affbd28fa5e Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Mon, 5 May 2025 13:58:08 +0530 Subject: [PATCH 4/4] refactor: set default for fetch methods --- .../report/accounts_receivable/accounts_receivable.py | 7 ++++--- erpnext/patches.txt | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/erpnext/accounts/report/accounts_receivable/accounts_receivable.py b/erpnext/accounts/report/accounts_receivable/accounts_receivable.py index 6423e8f87ed..5a850f06b98 100644 --- a/erpnext/accounts/report/accounts_receivable/accounts_receivable.py +++ b/erpnext/accounts/report/accounts_receivable/accounts_receivable.py @@ -54,9 +54,10 @@ class ReceivablePayableReport: self.filters.range = "30, 60, 90, 120" self.ranges = [num.strip() for num in self.filters.range.split(",") if num.strip().isdigit()] self.range_numbers = [num for num in range(1, len(self.ranges) + 2)] - self.ple_fetch_method = frappe.db.get_single_value( - "Accounts Settings", "receivable_payable_fetch_method" - ) + self.ple_fetch_method = ( + frappe.db.get_single_value("Accounts Settings", "receivable_payable_fetch_method") + or "Buffered Cursor" + ) # Fail Safe def run(self, args): self.filters.update(args) diff --git a/erpnext/patches.txt b/erpnext/patches.txt index 90d04f52fae..56e1d54d081 100644 --- a/erpnext/patches.txt +++ b/erpnext/patches.txt @@ -409,3 +409,4 @@ erpnext.patches.v15_0.update_query_report erpnext.patches.v15_0.set_purchase_receipt_row_item_to_capitalization_stock_item erpnext.patches.v15_0.update_payment_schedule_fields_in_invoices erpnext.patches.v15_0.rename_group_by_to_categorize_by +execute:frappe.db.set_single_value("Accounts Settings", "receivable_payable_fetch_method", "Buffered Cursor")