From 6d9f5fac767e0232659ce7880b00ebc4bbf94204 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 11:29:18 +0530 Subject: [PATCH 1/5] fix: race condition and repeatable read in process pcv - Update using child table name to avoid scanning whole table, which eventually leads to mariadb 1020 (REPEATABLE READ). - Avoid race condition in final summarization (cherry picked from commit ff6881764b843a3046934fddbbbfd8a907637ef8) --- .../process_period_closing_voucher.py | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py index 3239e6f4a00..966ded8f6fa 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py @@ -100,7 +100,7 @@ def start_pcv_processing(docname: str): ppcvd = qb.DocType("Process Period Closing Voucher Detail") if normal_balances := ( qb.from_(ppcvd) - .select(ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) + .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) .where(ppcvd.parent.eq(docname) & ppcvd.status.eq("Queued")) .orderby(ppcvd.parentfield, ppcvd.idx, ppcvd.processing_date) .limit(4) @@ -111,12 +111,7 @@ def start_pcv_processing(docname: str): for x in normal_balances: frappe.db.set_value( "Process Period Closing Voucher Detail", - { - "processing_date": x.processing_date, - "parent": docname, - "report_type": x.report_type, - "parentfield": x.parentfield, - }, + x.name, "status", "Running", ) @@ -127,10 +122,12 @@ def start_pcv_processing(docname: str): is_async=True, enqueue_after_commit=True, docname=docname, + row_name=x.name, date=x.processing_date, report_type=x.report_type, parentfield=x.parentfield, ) + frappe.db.commit() else: frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -254,7 +251,7 @@ def schedule_next_date(docname: str): ppcvd = qb.DocType("Process Period Closing Voucher Detail") if to_process := ( qb.from_(ppcvd) - .select(ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) + .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) .where(ppcvd.parent.eq(docname) & ppcvd.status.eq("Queued")) .orderby(ppcvd.parentfield, ppcvd.idx, ppcvd.processing_date) .limit(1) @@ -264,15 +261,11 @@ def schedule_next_date(docname: str): if not is_scheduler_inactive(): frappe.db.set_value( "Process Period Closing Voucher Detail", - { - "processing_date": to_process[0].processing_date, - "parent": docname, - "report_type": to_process[0].report_type, - "parentfield": to_process[0].parentfield, - }, + to_process[0].name, "status", "Running", ) + frappe.db.commit() frappe.enqueue( method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", queue="long", @@ -280,6 +273,7 @@ def schedule_next_date(docname: str): is_async=True, enqueue_after_commit=True, docname=docname, + row_name=to_process[0].name, date=to_process[0].processing_date, report_type=to_process[0].report_type, parentfield=to_process[0].parentfield, @@ -444,6 +438,8 @@ def summarize_and_post_ledger_entries(docname): make_closing_entries(closing_entries, pcv.name, pcv.company, pcv.period_end_date) + frappe.db.commit() + frappe.db.set_value("Period Closing Voucher", pcv.name, "gle_processing_status", "Completed") frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -529,10 +525,10 @@ def build_dimension_wise_balance_dict(gl_entries): return dimension_balances -def process_individual_date(docname: str, date, report_type, parentfield): +def process_individual_date(docname: str, row_name, date, report_type, parentfield): current_date_status = frappe.db.get_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "report_type": report_type, "parentfield": parentfield}, + row_name, "status", ) if current_date_status != "Running": @@ -579,17 +575,18 @@ def process_individual_date(docname: str, date, report_type, parentfield): # save results frappe.db.set_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "parent": docname, "report_type": report_type, "parentfield": parentfield}, + row_name, "closing_balance", frappe.json.dumps(res), ) frappe.db.set_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "parent": docname, "report_type": report_type, "parentfield": parentfield}, + row_name, "status", "Completed", ) + frappe.db.commit() # chain call schedule_next_date(docname) From 019b02adcdf95a1f0c31ebff0f0440200160d9c3 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 13:00:00 +0530 Subject: [PATCH 2/5] fix: prevent repeatable read related concurrency errors Process Period Closing Voucher and Process Period Closing Voucher Details are trackers how the jobs are processed. Keep transactions on them very short. (cherry picked from commit 7e4045e8282714928989453529d679ff3bf4b6eb) --- .../process_period_closing_voucher.py | 99 +++++++++++-------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py index 966ded8f6fa..691948cabad 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py @@ -89,47 +89,55 @@ class ProcessPeriodClosingVoucher(Document): cancel_pcv_processing(self.name) +def initialize_parallel_threads(docname: str): + threads = 4 + timeout = frappe.db.get_single_value("Accounts Settings", "pcv_job_timeout") or 3600 + ppcvd = qb.DocType("Process Period Closing Voucher Detail") + + frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Running") + + if normal_balances := ( + qb.from_(ppcvd) + .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) + .where(ppcvd.parent.eq(docname) & ppcvd.status.eq("Queued")) + .orderby(ppcvd.parentfield, ppcvd.idx, ppcvd.processing_date) + .limit(threads) + .for_update(skip_locked=True) + .run(as_dict=True) + ): + if not is_scheduler_inactive(): + for x in normal_balances: + frappe.db.set_value( + "Process Period Closing Voucher Detail", + x.name, + "status", + "Running", + ) + frappe.enqueue( + method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", + queue="long", + timeout=timeout, + is_async=True, + enqueue_after_commit=True, + docname=docname, + row_name=x.name, + date=x.processing_date, + report_type=x.report_type, + parentfield=x.parentfield, + ) + # keep transaction on PPCV and PPCVD short + # prevents concurrency errors - REPEATABLE READ + if not frappe.in_test: + frappe.db.commit() + else: + frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") + + @frappe.whitelist() def start_pcv_processing(docname: str): if frappe.db.get_value("Process Period Closing Voucher", docname, "status") in ["Queued", "Running"]: frappe.has_permission("Process Period Closing Voucher", "write", doc=docname, throw=True) - frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Running") - - timeout = frappe.db.get_single_value("Accounts Settings", "pcv_job_timeout") or 3600 - - ppcvd = qb.DocType("Process Period Closing Voucher Detail") - if normal_balances := ( - qb.from_(ppcvd) - .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) - .where(ppcvd.parent.eq(docname) & ppcvd.status.eq("Queued")) - .orderby(ppcvd.parentfield, ppcvd.idx, ppcvd.processing_date) - .limit(4) - .for_update(skip_locked=True) - .run(as_dict=True) - ): - if not is_scheduler_inactive(): - for x in normal_balances: - frappe.db.set_value( - "Process Period Closing Voucher Detail", - x.name, - "status", - "Running", - ) - frappe.enqueue( - method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", - queue="long", - timeout=timeout, - is_async=True, - enqueue_after_commit=True, - docname=docname, - row_name=x.name, - date=x.processing_date, - report_type=x.report_type, - parentfield=x.parentfield, - ) - frappe.db.commit() - else: - frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") + initialize_parallel_threads(docname) @frappe.whitelist() @@ -247,8 +255,8 @@ def get_gle_for_closing_account(pcv, dimension_balance, dimensions): @frappe.whitelist() def schedule_next_date(docname: str): timeout = frappe.db.get_single_value("Accounts Settings", "pcv_job_timeout") or 3600 - ppcvd = qb.DocType("Process Period Closing Voucher Detail") + if to_process := ( qb.from_(ppcvd) .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) @@ -265,7 +273,11 @@ def schedule_next_date(docname: str): "status", "Running", ) - frappe.db.commit() + # keep transaction on PPCV and PPCVD short + # prevents concurrency errors - REPEATABLE READ + if not frappe.in_test: + frappe.db.commit() + frappe.enqueue( method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", queue="long", @@ -438,7 +450,10 @@ def summarize_and_post_ledger_entries(docname): make_closing_entries(closing_entries, pcv.name, pcv.company, pcv.period_end_date) - frappe.db.commit() + # keep transaction on PPCV and PPCVD short + # prevents concurrency errors - REPEATABLE READ + if not frappe.in_test: + frappe.db.commit() frappe.db.set_value("Period Closing Voucher", pcv.name, "gle_processing_status", "Completed") frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -586,7 +601,9 @@ def process_individual_date(docname: str, row_name, date, report_type, parentfie "status", "Completed", ) - frappe.db.commit() + # commit heavy computation before touching PPCV or PPCVD + if not frappe.in_test: + frappe.db.commit() # chain call schedule_next_date(docname) From d759574f9a5104b2827247bffca2b4375308885a Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 15:51:18 +0530 Subject: [PATCH 3/5] refactor: prevent whole table scan while scheduling next date - helps in concurrency isolation (cherry picked from commit 21f4603144d3ffbe5e1b871dbaa02631e61919c2) --- .../process_period_closing_voucher_detail.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher_detail/process_period_closing_voucher_detail.py b/erpnext/accounts/doctype/process_period_closing_voucher_detail/process_period_closing_voucher_detail.py index f3a8302ac5b..0e0b905c96a 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher_detail/process_period_closing_voucher_detail.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher_detail/process_period_closing_voucher_detail.py @@ -1,7 +1,7 @@ # Copyright (c) 2025, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -# import frappe +import frappe from frappe.model.document import Document @@ -24,3 +24,10 @@ class ProcessPeriodClosingVoucherDetail(Document): # end: auto-generated types pass + + +def on_doctype_update(): + frappe.db.add_index( + "Process Period Closing Voucher Detail", + ["parent", "status", "parentfield", "idx", "processing_date"], + ) From 9cf7f441fb0c0afcce7f211c8461ab0c63b1fa40 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 17:02:32 +0530 Subject: [PATCH 4/5] refactor(test): row name based utility methods (cherry picked from commit dbc409736a4069d66d110da73aacd3bab1fac371) --- .../test_process_period_closing_voucher.py | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher/test_process_period_closing_voucher.py b/erpnext/accounts/doctype/process_period_closing_voucher/test_process_period_closing_voucher.py index f34c1dbedfe..5de93ef1bdd 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher/test_process_period_closing_voucher.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher/test_process_period_closing_voucher.py @@ -48,18 +48,27 @@ class TestProcessPeriodClosingVoucher(ERPNextTestSuite): ppcv.save() return ppcv - def set_processing_date_status(self, date, ppcv, rpt_type, parentfield, status): + def set_processing_date_status(self, row_name, status): frappe.db.set_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "parent": ppcv, "report_type": rpt_type, "parentfield": parentfield}, + row_name, "status", status, ) - def get_processing_date_closing_balance(self, date, ppcv, rpt_type, parentfield): + def get_row_name(self, ppcv_name, rpt_type, parentfield): + return frappe.db.get_all( + "Process Period Closing Voucher Detail", + filters={"parent": ppcv_name, "report_type": rpt_type, "parentfield": parentfield}, + order_by="report_type, idx", + pluck="name", + limit=1, + )[0] + + def get_processing_date_closing_balance(self, row_name): return frappe.db.get_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "parent": ppcv, "report_type": rpt_type, "parentfield": parentfield}, + row_name, "closing_balance", ) @@ -97,11 +106,10 @@ class TestProcessPeriodClosingVoucher(ERPNextTestSuite): parentfield = "normal_balances" rpt_type = "Profit and Loss" # status has to be set to 'Running' for logic to run - self.set_processing_date_status(today(), ppcv.name, rpt_type, parentfield, "Running") - process_individual_date(ppcv.name, today(), rpt_type, parentfield) - bal = frappe.parse_json( - self.get_processing_date_closing_balance(today(), ppcv.name, rpt_type, parentfield) - ) + row_name = self.get_row_name(ppcv.name, rpt_type, parentfield) + self.set_processing_date_status(row_name, "Running") + process_individual_date(ppcv.name, row_name, today(), rpt_type, parentfield) + bal = frappe.parse_json(self.get_processing_date_closing_balance(row_name)) self.assertEqual(len(bal), 1) expected_pl = { "account": "Sales - _TC", @@ -117,11 +125,10 @@ class TestProcessPeriodClosingVoucher(ERPNextTestSuite): # Balance sheet balance rpt_type = "Balance Sheet" - self.set_processing_date_status(today(), ppcv.name, rpt_type, parentfield, "Running") - process_individual_date(ppcv.name, today(), rpt_type, parentfield) - bal = frappe.parse_json( - self.get_processing_date_closing_balance(today(), ppcv.name, rpt_type, parentfield) - ) + row_name = self.get_row_name(ppcv.name, rpt_type, parentfield) + self.set_processing_date_status(row_name, "Running") + process_individual_date(ppcv.name, row_name, today(), rpt_type, parentfield) + bal = frappe.parse_json(self.get_processing_date_closing_balance(row_name)) self.assertEqual(len(bal), 1) expected_bs = { "account": "Debtors - _TC", @@ -138,11 +145,10 @@ class TestProcessPeriodClosingVoucher(ERPNextTestSuite): # Opening balance parentfield = "z_opening_balances" rpt_type = "Balance Sheet" - self.set_processing_date_status(today(), ppcv.name, rpt_type, parentfield, "Running") - process_individual_date(ppcv.name, today(), rpt_type, parentfield) - bal = frappe.parse_json( - self.get_processing_date_closing_balance(today(), ppcv.name, rpt_type, parentfield) - ) + row_name = self.get_row_name(ppcv.name, rpt_type, parentfield) + self.set_processing_date_status(row_name, "Running") + process_individual_date(ppcv.name, row_name, today(), rpt_type, parentfield) + bal = frappe.parse_json(self.get_processing_date_closing_balance(row_name)) self.assertEqual(len(bal), 2) opening_cash = next(x for x in bal if x["account"] == "Cash - _TC") expected_opening_cash = { From 5a9d40ce047c3a02c7ae9cdb5104a69f9d303f1c Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 17:04:51 +0530 Subject: [PATCH 5/5] chore: linter fix (cherry picked from commit a9ffdac8062de9b2e2e68a9c457e9e96e8ced37b) --- .../process_period_closing_voucher.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py index 691948cabad..be6f8ddfbbb 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py @@ -128,7 +128,7 @@ def initialize_parallel_threads(docname: str): # keep transaction on PPCV and PPCVD short # prevents concurrency errors - REPEATABLE READ if not frappe.in_test: - frappe.db.commit() + frappe.db.commit() # nosemgrep else: frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -276,7 +276,7 @@ def schedule_next_date(docname: str): # keep transaction on PPCV and PPCVD short # prevents concurrency errors - REPEATABLE READ if not frappe.in_test: - frappe.db.commit() + frappe.db.commit() # nosemgrep frappe.enqueue( method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", @@ -453,7 +453,7 @@ def summarize_and_post_ledger_entries(docname): # keep transaction on PPCV and PPCVD short # prevents concurrency errors - REPEATABLE READ if not frappe.in_test: - frappe.db.commit() + frappe.db.commit() # nosemgrep frappe.db.set_value("Period Closing Voucher", pcv.name, "gle_processing_status", "Completed") frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -603,7 +603,7 @@ def process_individual_date(docname: str, row_name, date, report_type, parentfie ) # commit heavy computation before touching PPCV or PPCVD if not frappe.in_test: - frappe.db.commit() + frappe.db.commit() # nosemgrep # chain call schedule_next_date(docname)