From ff6881764b843a3046934fddbbbfd8a907637ef8 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 11:29:18 +0530 Subject: [PATCH 1/5] fix: race condition and repeatable read in process pcv - Update using child table name to avoid scanning whole table, which eventually leads to mariadb 1020 (REPEATABLE READ). - Avoid race condition in final summarization --- .../process_period_closing_voucher.py | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py index 264b3dffd5e..39ec51b5e11 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py @@ -100,7 +100,7 @@ def start_pcv_processing(docname: str): ppcvd = qb.DocType("Process Period Closing Voucher Detail") if normal_balances := ( qb.from_(ppcvd) - .select(ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) + .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) .where(ppcvd.parent.eq(docname) & ppcvd.status.eq("Queued")) .orderby(ppcvd.parentfield, ppcvd.idx, ppcvd.processing_date) .limit(4) @@ -111,12 +111,7 @@ def start_pcv_processing(docname: str): for x in normal_balances: frappe.db.set_value( "Process Period Closing Voucher Detail", - { - "processing_date": x.processing_date, - "parent": docname, - "report_type": x.report_type, - "parentfield": x.parentfield, - }, + x.name, "status", "Running", ) @@ -127,10 +122,12 @@ def start_pcv_processing(docname: str): is_async=True, enqueue_after_commit=True, docname=docname, + row_name=x.name, date=x.processing_date, report_type=x.report_type, parentfield=x.parentfield, ) + frappe.db.commit() else: frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -254,7 +251,7 @@ def schedule_next_date(docname: str): ppcvd = qb.DocType("Process Period Closing Voucher Detail") if to_process := ( qb.from_(ppcvd) - .select(ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) + .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) .where(ppcvd.parent.eq(docname) & ppcvd.status.eq("Queued")) .orderby(ppcvd.parentfield, ppcvd.idx, ppcvd.processing_date) .limit(1) @@ -264,15 +261,11 @@ def schedule_next_date(docname: str): if not is_scheduler_inactive(): frappe.db.set_value( "Process Period Closing Voucher Detail", - { - "processing_date": to_process[0].processing_date, - "parent": docname, - "report_type": to_process[0].report_type, - "parentfield": to_process[0].parentfield, - }, + to_process[0].name, "status", "Running", ) + frappe.db.commit() frappe.enqueue( method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", queue="long", @@ -280,6 +273,7 @@ def schedule_next_date(docname: str): is_async=True, enqueue_after_commit=True, docname=docname, + row_name=to_process[0].name, date=to_process[0].processing_date, report_type=to_process[0].report_type, parentfield=to_process[0].parentfield, @@ -444,6 +438,8 @@ def summarize_and_post_ledger_entries(docname): make_closing_entries(closing_entries, pcv.name, pcv.company, pcv.period_end_date) + frappe.db.commit() + frappe.db.set_value("Period Closing Voucher", pcv.name, "gle_processing_status", "Completed") frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -529,10 +525,10 @@ def build_dimension_wise_balance_dict(gl_entries): return dimension_balances -def process_individual_date(docname: str, date, report_type, parentfield): +def process_individual_date(docname: str, row_name, date, report_type, parentfield): current_date_status = frappe.db.get_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "report_type": report_type, "parentfield": parentfield}, + row_name, "status", ) if current_date_status != "Running": @@ -580,17 +576,18 @@ def process_individual_date(docname: str, date, report_type, parentfield): # save results frappe.db.set_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "parent": docname, "report_type": report_type, "parentfield": parentfield}, + row_name, "closing_balance", frappe.json.dumps(res), ) frappe.db.set_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "parent": docname, "report_type": report_type, "parentfield": parentfield}, + row_name, "status", "Completed", ) + frappe.db.commit() # chain call schedule_next_date(docname) From 7e4045e8282714928989453529d679ff3bf4b6eb Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 13:00:00 +0530 Subject: [PATCH 2/5] fix: prevent repeatable read related concurrency errors Process Period Closing Voucher and Process Period Closing Voucher Details are trackers how the jobs are processed. Keep transactions on them very short. --- .../process_period_closing_voucher.py | 99 +++++++++++-------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py index 39ec51b5e11..3a9d43544b0 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py @@ -89,47 +89,55 @@ class ProcessPeriodClosingVoucher(Document): cancel_pcv_processing(self.name) +def initialize_parallel_threads(docname: str): + threads = 4 + timeout = frappe.db.get_single_value("Accounts Settings", "pcv_job_timeout") or 3600 + ppcvd = qb.DocType("Process Period Closing Voucher Detail") + + frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Running") + + if normal_balances := ( + qb.from_(ppcvd) + .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) + .where(ppcvd.parent.eq(docname) & ppcvd.status.eq("Queued")) + .orderby(ppcvd.parentfield, ppcvd.idx, ppcvd.processing_date) + .limit(threads) + .for_update(skip_locked=True) + .run(as_dict=True) + ): + if not is_scheduler_inactive(): + for x in normal_balances: + frappe.db.set_value( + "Process Period Closing Voucher Detail", + x.name, + "status", + "Running", + ) + frappe.enqueue( + method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", + queue="long", + timeout=timeout, + is_async=True, + enqueue_after_commit=True, + docname=docname, + row_name=x.name, + date=x.processing_date, + report_type=x.report_type, + parentfield=x.parentfield, + ) + # keep transaction on PPCV and PPCVD short + # prevents concurrency errors - REPEATABLE READ + if not frappe.in_test: + frappe.db.commit() + else: + frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") + + @frappe.whitelist() def start_pcv_processing(docname: str): if frappe.db.get_value("Process Period Closing Voucher", docname, "status") in ["Queued", "Running"]: frappe.has_permission("Process Period Closing Voucher", "write", doc=docname, throw=True) - frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Running") - - timeout = frappe.db.get_single_value("Accounts Settings", "pcv_job_timeout") or 3600 - - ppcvd = qb.DocType("Process Period Closing Voucher Detail") - if normal_balances := ( - qb.from_(ppcvd) - .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) - .where(ppcvd.parent.eq(docname) & ppcvd.status.eq("Queued")) - .orderby(ppcvd.parentfield, ppcvd.idx, ppcvd.processing_date) - .limit(4) - .for_update(skip_locked=True) - .run(as_dict=True) - ): - if not is_scheduler_inactive(): - for x in normal_balances: - frappe.db.set_value( - "Process Period Closing Voucher Detail", - x.name, - "status", - "Running", - ) - frappe.enqueue( - method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", - queue="long", - timeout=timeout, - is_async=True, - enqueue_after_commit=True, - docname=docname, - row_name=x.name, - date=x.processing_date, - report_type=x.report_type, - parentfield=x.parentfield, - ) - frappe.db.commit() - else: - frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") + initialize_parallel_threads(docname) @frappe.whitelist() @@ -247,8 +255,8 @@ def get_gle_for_closing_account(pcv, dimension_balance, dimensions): @frappe.whitelist() def schedule_next_date(docname: str): timeout = frappe.db.get_single_value("Accounts Settings", "pcv_job_timeout") or 3600 - ppcvd = qb.DocType("Process Period Closing Voucher Detail") + if to_process := ( qb.from_(ppcvd) .select(ppcvd.name, ppcvd.processing_date, ppcvd.report_type, ppcvd.parentfield) @@ -265,7 +273,11 @@ def schedule_next_date(docname: str): "status", "Running", ) - frappe.db.commit() + # keep transaction on PPCV and PPCVD short + # prevents concurrency errors - REPEATABLE READ + if not frappe.in_test: + frappe.db.commit() + frappe.enqueue( method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", queue="long", @@ -438,7 +450,10 @@ def summarize_and_post_ledger_entries(docname): make_closing_entries(closing_entries, pcv.name, pcv.company, pcv.period_end_date) - frappe.db.commit() + # keep transaction on PPCV and PPCVD short + # prevents concurrency errors - REPEATABLE READ + if not frappe.in_test: + frappe.db.commit() frappe.db.set_value("Period Closing Voucher", pcv.name, "gle_processing_status", "Completed") frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -587,7 +602,9 @@ def process_individual_date(docname: str, row_name, date, report_type, parentfie "status", "Completed", ) - frappe.db.commit() + # commit heavy computation before touching PPCV or PPCVD + if not frappe.in_test: + frappe.db.commit() # chain call schedule_next_date(docname) From 21f4603144d3ffbe5e1b871dbaa02631e61919c2 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 15:51:18 +0530 Subject: [PATCH 3/5] refactor: prevent whole table scan while scheduling next date - helps in concurrency isolation --- .../process_period_closing_voucher_detail.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher_detail/process_period_closing_voucher_detail.py b/erpnext/accounts/doctype/process_period_closing_voucher_detail/process_period_closing_voucher_detail.py index f3a8302ac5b..0e0b905c96a 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher_detail/process_period_closing_voucher_detail.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher_detail/process_period_closing_voucher_detail.py @@ -1,7 +1,7 @@ # Copyright (c) 2025, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -# import frappe +import frappe from frappe.model.document import Document @@ -24,3 +24,10 @@ class ProcessPeriodClosingVoucherDetail(Document): # end: auto-generated types pass + + +def on_doctype_update(): + frappe.db.add_index( + "Process Period Closing Voucher Detail", + ["parent", "status", "parentfield", "idx", "processing_date"], + ) From dbc409736a4069d66d110da73aacd3bab1fac371 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 17:02:32 +0530 Subject: [PATCH 4/5] refactor(test): row name based utility methods --- .../test_process_period_closing_voucher.py | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher/test_process_period_closing_voucher.py b/erpnext/accounts/doctype/process_period_closing_voucher/test_process_period_closing_voucher.py index f34c1dbedfe..5de93ef1bdd 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher/test_process_period_closing_voucher.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher/test_process_period_closing_voucher.py @@ -48,18 +48,27 @@ class TestProcessPeriodClosingVoucher(ERPNextTestSuite): ppcv.save() return ppcv - def set_processing_date_status(self, date, ppcv, rpt_type, parentfield, status): + def set_processing_date_status(self, row_name, status): frappe.db.set_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "parent": ppcv, "report_type": rpt_type, "parentfield": parentfield}, + row_name, "status", status, ) - def get_processing_date_closing_balance(self, date, ppcv, rpt_type, parentfield): + def get_row_name(self, ppcv_name, rpt_type, parentfield): + return frappe.db.get_all( + "Process Period Closing Voucher Detail", + filters={"parent": ppcv_name, "report_type": rpt_type, "parentfield": parentfield}, + order_by="report_type, idx", + pluck="name", + limit=1, + )[0] + + def get_processing_date_closing_balance(self, row_name): return frappe.db.get_value( "Process Period Closing Voucher Detail", - {"processing_date": date, "parent": ppcv, "report_type": rpt_type, "parentfield": parentfield}, + row_name, "closing_balance", ) @@ -97,11 +106,10 @@ class TestProcessPeriodClosingVoucher(ERPNextTestSuite): parentfield = "normal_balances" rpt_type = "Profit and Loss" # status has to be set to 'Running' for logic to run - self.set_processing_date_status(today(), ppcv.name, rpt_type, parentfield, "Running") - process_individual_date(ppcv.name, today(), rpt_type, parentfield) - bal = frappe.parse_json( - self.get_processing_date_closing_balance(today(), ppcv.name, rpt_type, parentfield) - ) + row_name = self.get_row_name(ppcv.name, rpt_type, parentfield) + self.set_processing_date_status(row_name, "Running") + process_individual_date(ppcv.name, row_name, today(), rpt_type, parentfield) + bal = frappe.parse_json(self.get_processing_date_closing_balance(row_name)) self.assertEqual(len(bal), 1) expected_pl = { "account": "Sales - _TC", @@ -117,11 +125,10 @@ class TestProcessPeriodClosingVoucher(ERPNextTestSuite): # Balance sheet balance rpt_type = "Balance Sheet" - self.set_processing_date_status(today(), ppcv.name, rpt_type, parentfield, "Running") - process_individual_date(ppcv.name, today(), rpt_type, parentfield) - bal = frappe.parse_json( - self.get_processing_date_closing_balance(today(), ppcv.name, rpt_type, parentfield) - ) + row_name = self.get_row_name(ppcv.name, rpt_type, parentfield) + self.set_processing_date_status(row_name, "Running") + process_individual_date(ppcv.name, row_name, today(), rpt_type, parentfield) + bal = frappe.parse_json(self.get_processing_date_closing_balance(row_name)) self.assertEqual(len(bal), 1) expected_bs = { "account": "Debtors - _TC", @@ -138,11 +145,10 @@ class TestProcessPeriodClosingVoucher(ERPNextTestSuite): # Opening balance parentfield = "z_opening_balances" rpt_type = "Balance Sheet" - self.set_processing_date_status(today(), ppcv.name, rpt_type, parentfield, "Running") - process_individual_date(ppcv.name, today(), rpt_type, parentfield) - bal = frappe.parse_json( - self.get_processing_date_closing_balance(today(), ppcv.name, rpt_type, parentfield) - ) + row_name = self.get_row_name(ppcv.name, rpt_type, parentfield) + self.set_processing_date_status(row_name, "Running") + process_individual_date(ppcv.name, row_name, today(), rpt_type, parentfield) + bal = frappe.parse_json(self.get_processing_date_closing_balance(row_name)) self.assertEqual(len(bal), 2) opening_cash = next(x for x in bal if x["account"] == "Cash - _TC") expected_opening_cash = { From a9ffdac8062de9b2e2e68a9c457e9e96e8ced37b Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 3 Jul 2026 17:04:51 +0530 Subject: [PATCH 5/5] chore: linter fix --- .../process_period_closing_voucher.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py index 3a9d43544b0..d2cea78a8f0 100644 --- a/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py +++ b/erpnext/accounts/doctype/process_period_closing_voucher/process_period_closing_voucher.py @@ -128,7 +128,7 @@ def initialize_parallel_threads(docname: str): # keep transaction on PPCV and PPCVD short # prevents concurrency errors - REPEATABLE READ if not frappe.in_test: - frappe.db.commit() + frappe.db.commit() # nosemgrep else: frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -276,7 +276,7 @@ def schedule_next_date(docname: str): # keep transaction on PPCV and PPCVD short # prevents concurrency errors - REPEATABLE READ if not frappe.in_test: - frappe.db.commit() + frappe.db.commit() # nosemgrep frappe.enqueue( method="erpnext.accounts.doctype.process_period_closing_voucher.process_period_closing_voucher.process_individual_date", @@ -453,7 +453,7 @@ def summarize_and_post_ledger_entries(docname): # keep transaction on PPCV and PPCVD short # prevents concurrency errors - REPEATABLE READ if not frappe.in_test: - frappe.db.commit() + frappe.db.commit() # nosemgrep frappe.db.set_value("Period Closing Voucher", pcv.name, "gle_processing_status", "Completed") frappe.db.set_value("Process Period Closing Voucher", docname, "status", "Completed") @@ -604,7 +604,7 @@ def process_individual_date(docname: str, row_name, date, report_type, parentfie ) # commit heavy computation before touching PPCV or PPCVD if not frappe.in_test: - frappe.db.commit() + frappe.db.commit() # nosemgrep # chain call schedule_next_date(docname)