feat: opening invoice creation tool with background jobs (#23595)

* feat: opening invoice creation tool with background jobs

* fix: tests

* fix: codacy

* fix: sider

* fix: codacy

Co-authored-by: Nabin Hait <nabinhait@gmail.com>

erpnext/accounts/doctype/opening_invoice_creation_tool/opening_invoice_creation_tool.js

@@ -6,7 +6,7 @@ frappe.ui.form.on('Opening Invoice Creation Tool', {
 		frm.set_query('party_type', 'invoices', function(doc, cdt, cdn) {
 			return {
 				filters: {
-					'name': ['in', 'Customer,Supplier']
+					'name': ['in', 'Customer, Supplier']
 				}
 			};
 		});
@@ -14,29 +14,46 @@ frappe.ui.form.on('Opening Invoice Creation Tool', {
 		if (frm.doc.company) {
 			frm.trigger('setup_company_filters');
 		}
+
+		frappe.realtime.on('opening_invoice_creation_progress', data => {
+			if (!frm.doc.import_in_progress) {
+				frm.dashboard.reset();
+				frm.doc.import_in_progress = true;
+			}
+			if (data.user != frappe.session.user) return;
+			if (data.count == data.total) {
+				setTimeout((title) => {
+					frm.doc.import_in_progress = false;
+					frm.clear_table("invoices");
+					frm.refresh_fields();
+					frm.page.clear_indicator();
+					frm.dashboard.hide_progress(title);
+					frappe.msgprint(__("Opening {0} Invoice created", [frm.doc.invoice_type]));
+				}, 1500, data.title);
+				return;
+			}
+
+			frm.dashboard.show_progress(data.title, (data.count / data.total) * 100, data.message);
+			frm.page.set_indicator(__('In Progress'), 'orange');
+		});
 	},
 
 	refresh: function(frm) {
 		frm.disable_save();
-		frm.trigger("make_dashboard");
+		!frm.doc.import_in_progress && frm.trigger("make_dashboard");
 		frm.page.set_primary_action(__('Create Invoices'), () => {
 			let btn_primary = frm.page.btn_primary.get(0);
 			return frm.call({
 				doc: frm.doc,
-				freeze: true,
 				btn: $(btn_primary),
 				method: "make_invoices",
-				freeze_message: __("Creating {0} Invoice", [frm.doc.invoice_type]),
+				freeze_message: __("Creating {0} Invoice", [frm.doc.invoice_type])
-				callback: (r) => {
-					if(!r.exc){
-						frappe.msgprint(__("Opening {0} Invoice created", [frm.doc.invoice_type]));
-						frm.clear_table("invoices");
-						frm.refresh_fields();
-						frm.reload_doc();
-					}
-				}
 			});
 		});
+
+		if (frm.doc.create_missing_party) {
+			frm.set_df_property("party", "fieldtype", "Data", frm.doc.name, "invoices");
+		}
 	},
 
 	setup_company_filters: function(frm) {

erpnext/accounts/doctype/opening_invoice_creation_tool/opening_invoice_creation_tool.py

@@ -4,9 +4,12 @@
 
 from __future__ import unicode_literals
 import frappe
+import traceback
+from json import dumps
 from frappe import _, scrub
 from frappe.utils import flt, nowdate
 from frappe.model.document import Document
+from frappe.utils.background_jobs import enqueue
 from erpnext.accounts.doctype.accounting_dimension.accounting_dimension import get_accounting_dimensions
 
 
@@ -62,66 +65,47 @@ class OpeningInvoiceCreationTool(Document):
 
 		return invoices_summary, max_count
 	
-	def make_invoices(self):
+	def validate_company(self):
-		names = []
-		mandatory_error_msg = _("Row {0}: {1} is required to create the Opening {2} Invoices")
 		if not self.company:
 			frappe.throw(_("Please select the Company"))
 	
-		company_details = frappe.get_cached_value('Company', self.company,
+	def set_missing_values(self, row):
-			["default_currency", "default_letter_head"], as_dict=1) or {}
+		row.qty = row.qty or 1.0
-
+		row.temporary_opening_account = row.temporary_opening_account or get_temporary_opening_account(self.company)
-		for row in self.invoices:
-			if not row.qty:
-				row.qty = 1.0
-
-			# always mandatory fields for the invoices
-			if not row.temporary_opening_account:
-				row.temporary_opening_account = get_temporary_opening_account(self.company)
 		row.party_type = "Customer" if self.invoice_type == "Sales" else "Supplier"
+		row.item_name = row.item_name or _("Opening Invoice Item")
+		row.posting_date = row.posting_date or nowdate()
+		row.due_date = row.due_date or nowdate()
 
-			# Allow to create invoice even if no party present in customer or supplier.
+	def validate_mandatory_invoice_fields(self, row):
 		if not frappe.db.exists(row.party_type, row.party):
 			if self.create_missing_party:
 				self.add_party(row.party_type, row.party)
 			else:
-					frappe.throw(_("{0} {1} does not exist.").format(frappe.bold(row.party_type), frappe.bold(row.party)))
+				frappe.throw(_("Row #{}: {} {} does not exist.").format(row.idx, frappe.bold(row.party_type), frappe.bold(row.party)))
-
-			if not row.item_name:
-				row.item_name = _("Opening Invoice Item")
-			if not row.posting_date:
-				row.posting_date = nowdate()
-			if not row.due_date:
-				row.due_date = nowdate()
 
+		mandatory_error_msg = _("Row #{0}: {1} is required to create the Opening {2} Invoices")
 		for d in ("Party", "Outstanding Amount", "Temporary Opening Account"):
 			if not row.get(scrub(d)):
-					frappe.throw(mandatory_error_msg.format(row.idx, _(d), self.invoice_type))
+				frappe.throw(mandatory_error_msg.format(row.idx, d, self.invoice_type))
 
-			args = self.get_invoice_dict(row=row)
+	def get_invoices(self):
-			if not args:
+		invoices = []
+		for row in self.invoices:
+			if not row:
 				continue
-
+			self.set_missing_values(row)
+			self.validate_mandatory_invoice_fields(row)
+			invoice = self.get_invoice_dict(row)
+			company_details = frappe.get_cached_value('Company', self.company, ["default_currency", "default_letter_head"], as_dict=1) or {}
 			if company_details:
-				args.update({
+				invoice.update({
 					"currency": company_details.get("default_currency"),
 					"letter_head": company_details.get("default_letter_head")
 				})
+			invoices.append(invoice)
 
-			doc = frappe.get_doc(args).insert()
+		return invoices
-			doc.submit()
-			names.append(doc.name)
-
-			if len(self.invoices) > 5:
-				frappe.publish_realtime(
-					"progress", dict(
-						progress=[row.idx, len(self.invoices)],
-						title=_('Creating {0}').format(doc.doctype)
-					),
-					user=frappe.session.user
-				)
-
-		return names
 
 	def add_party(self, party_type, party):
 		party_doc = frappe.new_doc(party_type)
@@ -140,14 +124,12 @@ class OpeningInvoiceCreationTool(Document):
 
 	def get_invoice_dict(self, row=None):
 		def get_item_dict():
-			default_uom = frappe.db.get_single_value("Stock Settings", "stock_uom") or _("Nos")
+			cost_center = row.get('cost_center') or frappe.get_cached_value('Company', self.company,  "cost_center")
-			cost_center = row.get('cost_center') or frappe.get_cached_value('Company',
-				self.company,  "cost_center")
-
 			if not cost_center:
-				frappe.throw(
+				frappe.throw(_("Please set the Default Cost Center in {0} company.").format(frappe.bold(self.company)))
-					_("Please set the Default Cost Center in {0} company.").format(frappe.bold(self.company))
+
-				)
+			income_expense_account_field = "income_account" if row.party_type == "Customer" else "expense_account"
+			default_uom = frappe.db.get_single_value("Stock Settings", "stock_uom") or _("Nos")
 			rate = flt(row.outstanding_amount) / flt(row.qty)
 
 			return frappe._dict({
@@ -161,18 +143,9 @@ class OpeningInvoiceCreationTool(Document):
 				"cost_center": cost_center
 			})
 
-		if not row:
-			return None
-
-		party_type = "Customer"
-		income_expense_account_field = "income_account"
-		if self.invoice_type == "Purchase":
-			party_type = "Supplier"
-			income_expense_account_field = "expense_account"
-
 		item = get_item_dict()
 
-		args = frappe._dict({
+		invoice = frappe._dict({
 			"items": [item],
 			"is_opening": "Yes",
 			"set_posting_time": 1,
@@ -180,21 +153,76 @@ class OpeningInvoiceCreationTool(Document):
 			"cost_center": self.cost_center,
 			"due_date": row.due_date,
 			"posting_date": row.posting_date,
-			frappe.scrub(party_type): row.party,
+			frappe.scrub(row.party_type): row.party,
+			"is_pos": 0,
 			"doctype": "Sales Invoice" if self.invoice_type == "Sales" else "Purchase Invoice"
 		})
 
 		accounting_dimension = get_accounting_dimensions()
-
 		for dimension in accounting_dimension:
-			args.update({
+			invoice.update({
 				dimension: item.get(dimension)
 			})
 
-		if self.invoice_type == "Sales":
+		return invoice
-			args["is_pos"] = 0
 
-		return args
+	def make_invoices(self):
+		self.validate_company()
+		invoices = self.get_invoices()
+		if len(invoices) < 50:
+			return start_import(invoices)
+		else:
+			from frappe.core.page.background_jobs.background_jobs import get_info
+			from frappe.utils.scheduler import is_scheduler_inactive
+
+			if is_scheduler_inactive() and not frappe.flags.in_test:
+				frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive"))
+
+			enqueued_jobs = [d.get("job_name") for d in get_info()]
+			if self.name not in enqueued_jobs:
+				enqueue(
+					start_import,
+					queue="default",
+					timeout=6000,
+					event="opening_invoice_creation",
+					job_name=self.name,
+					invoices=invoices,
+					now=frappe.conf.developer_mode or frappe.flags.in_test
+				)
+
+def start_import(invoices):
+	errors = 0
+	names = []
+	for idx, d in enumerate(invoices):
+		try:
+			publish(idx, len(invoices), d.doctype)
+			doc = frappe.get_doc(d)
+			doc.insert()
+			doc.submit()
+			frappe.db.commit()
+			names.append(doc.name)
+		except Exception:
+			errors += 1
+			frappe.db.rollback()
+			message = "\n".join(["Data:", dumps(d, default=str, indent=4), "--" * 50, "\nException:", traceback.format_exc()])
+			frappe.log_error(title="Error while creating Opening Invoice", message=message)
+			frappe.db.commit()
+	if errors:
+		frappe.msgprint(_("You had {} errors while creating opening invoices. Check {} for more details")
+			.format(errors, "<a href='#List/Error Log' class='variant-click'>Error Log</a>"), indicator="red", title=_("Error Occured"))
+	return names
+
+def publish(index, total, doctype):
+	if total < 5: return
+	frappe.publish_realtime(
+		"opening_invoice_creation_progress",
+		dict(
+			title=_("Opening Invoice Creation In Progress"),
+			message=_('Creating {} out of {} {}').format(index + 1, total, doctype),
+			user=frappe.session.user,
+			count=index+1,
+			total=total
+		))
 
 @frappe.whitelist()
 def get_temporary_opening_account(company=None):

erpnext/accounts/doctype/opening_invoice_creation_tool/test_opening_invoice_creation_tool.py

@@ -44,7 +44,7 @@ class TestOpeningInvoiceCreationTool(unittest.TestCase):
 			0: ["_Test Supplier", 300, "Overdue"],
 			1: ["_Test Supplier 1", 250, "Overdue"],
 		}
-		self.check_expected_values(invoices, expected_value, invoice_type="Purchase", )
+		self.check_expected_values(invoices, expected_value, "Purchase")
 
 def get_opening_invoice_creation_dict(**args):
 	party = "Customer" if args.get("invoice_type", "Sales") == "Sales" else "Supplier"