feat: Bulk Transaction Processing (#28580)

* feat: Bulk Transaction Processing

* fix: add flags to ignore validations and exception handling correction

* fix: remove duplicate code, added logger functionality and improved notifications

* fix: linting and sider issues

* test: added tests

* fix: linter issues

* fix: failing test case

* fix: sider issues and test cases

* refactor: mapping function calls to create order/invoice

* fix: added more test cases to increase coverage

* fix: test cases

* fix: sider issue

* fix: rename doctype, improve formatting and minor refactor

* fix: update doctype name in hooks and sider issues

* fix: entry log test case

* fix: typos, translations and company name in tests

* fix: linter issues and translations

* fix: linter issue

* fix: split into separate function for marking failed transaction

* fix: typos, retry failed transaction logic and make log read only

* fix: hide retry button when no failed transactions and remove test cases not rrelevant

* fix: sider issues and indentation to tabs

Co-authored-by: Ankush Menat <ankush@frappe.io>

erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js

@@ -0,0 +1,34 @@
+// Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Bulk Transaction Log', {
+
+	before_load: function(frm) {
+		query(frm);
+	},
+
+	refresh: function(frm) {
+		frm.disable_save();
+		frm.add_custom_button(__('Retry Failed Transactions'), ()=>{
+			frappe.confirm(__("Retry Failing Transactions ?"), ()=>{
+				query(frm);
+			}
+			);
+		});
+	}
+});
+
+function query(frm) {
+	frappe.call({
+		method: "erpnext.bulk_transaction.doctype.bulk_transaction_log.bulk_transaction_log.retry_failing_transaction",
+		args: {
+			log_date: frm.doc.log_date
+		}
+	}).then((r) => {
+		if (r.message) {
+			frm.remove_custom_button("Retry Failed Transactions");
+		} else {
+			frappe.show_alert(__("Retrying Failed Transactions"), 5);
+		}
+	});
+}

erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json

@@ -0,0 +1,51 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2021-11-30 13:41:16.343827",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "log_date",
+  "logger_data"
+ ],
+ "fields": [
+  {
+   "fieldname": "log_date",
+   "fieldtype": "Date",
+   "label": "Log Date",
+   "read_only": 1
+  },
+  {
+   "fieldname": "logger_data",
+   "fieldtype": "Table",
+   "label": "Logger Data",
+   "options": "Bulk Transaction Log Detail"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2022-02-03 17:23:02.935325",
+ "modified_by": "Administrator",
+ "module": "Bulk Transaction",
+ "name": "Bulk Transaction Log",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}

erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+from datetime import date
+
+import frappe
+from frappe.model.document import Document
+
+from erpnext.utilities.bulk_transaction import task, update_logger
+
+
+class BulkTransactionLog(Document):
+	pass
+
+
+@frappe.whitelist()
+def retry_failing_transaction(log_date=None):
+	btp = frappe.qb.DocType("Bulk Transaction Log Detail")
+	data = (
+		frappe.qb.from_(btp)
+		.select(btp.transaction_name, btp.from_doctype, btp.to_doctype)
+		.distinct()
+		.where(btp.retried != 1)
+		.where(btp.transaction_status == "Failed")
+		.where(btp.date == log_date)
+	).run(as_dict=True)
+
+	if data:
+		if not log_date:
+			log_date = str(date.today())
+		if len(data) > 10:
+			frappe.enqueue(job, queue="long", job_name="bulk_retry", data=data, log_date=log_date)
+		else:
+			job(data, log_date)
+	else:
+		return "No Failed Records"
+
+def job(data, log_date):
+	for d in data:
+		failed = []
+		try:
+			frappe.db.savepoint("before_creation_of_record")
+			task(d.transaction_name, d.from_doctype, d.to_doctype)
+		except Exception as e:
+			frappe.db.rollback(save_point="before_creation_of_record")
+			failed.append(e)
+			update_logger(
+				d.transaction_name,
+				e,
+				d.from_doctype,
+				d.to_doctype,
+				status="Failed",
+				log_date=log_date,
+				restarted=1
+			)
+
+		if not failed:
+			update_logger(
+				d.transaction_name,
+				None,
+				d.from_doctype,
+				d.to_doctype,
+				status="Success",
+				log_date=log_date,
+				restarted=1,
+			)

erpnext/bulk_transaction/doctype/bulk_transaction_log/test_bulk_transaction_log.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+
+import unittest
+from datetime import date
+
+import frappe
+
+from erpnext.utilities.bulk_transaction import transaction_processing
+
+
+class TestBulkTransactionLog(unittest.TestCase):
+
+	def setUp(self):
+		create_company()
+		create_customer()
+		create_item()
+
+	def test_for_single_record(self):
+		so_name = create_so()
+		transaction_processing([{"name": so_name}], "Sales Order", "Sales Invoice")
+		data = frappe.db.get_list("Sales Invoice", filters = {"posting_date": date.today(), "customer": "Bulk Customer"}, fields=["*"])
+		if not data:
+			self.fail("No Sales Invoice Created !")
+
+	def test_entry_in_log(self):
+		so_name = create_so()
+		transaction_processing([{"name": so_name}], "Sales Order", "Sales Invoice")
+		doc = frappe.get_doc("Bulk Transaction Log", str(date.today()))
+		for d in doc.get("logger_data"):
+			if d.transaction_name == so_name:
+				self.assertEqual(d.transaction_name, so_name)
+				self.assertEqual(d.transaction_status, "Success")
+				self.assertEqual(d.from_doctype, "Sales Order")
+				self.assertEqual(d.to_doctype, "Sales Invoice")
+				self.assertEqual(d.retried, 0)
+
+
+
+def create_company():
+	if not frappe.db.exists('Company', '_Test Company'):
+		frappe.get_doc({
+			'doctype': 'Company',
+			'company_name': '_Test Company',
+			'country': 'India',
+			'default_currency': 'INR'
+		}).insert()
+
+def create_customer():
+	if not frappe.db.exists('Customer', 'Bulk Customer'):
+		frappe.get_doc({
+			'doctype': 'Customer',
+			'customer_name': 'Bulk Customer'
+		}).insert()
+
+def create_item():
+	if not frappe.db.exists("Item", "MK"):
+		frappe.get_doc({
+			"doctype": "Item",
+			"item_code": "MK",
+			"item_name": "Milk",
+			"description": "Milk",
+			"item_group": "Products"
+		}).insert()
+
+def create_so(intent=None):
+	so = frappe.new_doc("Sales Order")
+	so.customer = "Bulk Customer"
+	so.company = "_Test Company"
+	so.transaction_date = date.today()
+
+	so.set_warehouse = "Finished Goods - _TC"
+	so.append("items", {
+		"item_code": "MK",
+		"delivery_date": date.today(),
+		"qty": 10,
+		"rate": 80,
+	})
+	so.insert()
+	so.submit()
+	return so.name

erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json

@@ -0,0 +1,86 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2021-11-30 13:38:30.926047",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "transaction_name",
+  "date",
+  "time",
+  "transaction_status",
+  "error_description",
+  "from_doctype",
+  "to_doctype",
+  "retried"
+ ],
+ "fields": [
+  {
+   "fieldname": "transaction_name",
+   "fieldtype": "Dynamic Link",
+   "in_list_view": 1,
+   "label": "Name",
+   "options": "from_doctype"
+  },
+  {
+   "fieldname": "transaction_status",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Status",
+   "read_only": 1
+  },
+  {
+   "fieldname": "error_description",
+   "fieldtype": "Long Text",
+   "label": "Error Description",
+   "read_only": 1
+  },
+  {
+   "fieldname": "from_doctype",
+   "fieldtype": "Link",
+   "label": "From Doctype",
+   "options": "DocType",
+   "read_only": 1
+  },
+  {
+   "fieldname": "to_doctype",
+   "fieldtype": "Link",
+   "label": "To Doctype",
+   "options": "DocType",
+   "read_only": 1
+  },
+  {
+   "fieldname": "date",
+   "fieldtype": "Date",
+   "in_list_view": 1,
+   "label": "Date ",
+   "read_only": 1
+  },
+  {
+   "fieldname": "time",
+   "fieldtype": "Time",
+   "label": "Time",
+   "read_only": 1
+  },
+  {
+   "fieldname": "retried",
+   "fieldtype": "Int",
+   "label": "Retried",
+   "read_only": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2022-02-03 19:57:31.650359",
+ "modified_by": "Administrator",
+ "module": "Bulk Transaction",
+ "name": "Bulk Transaction Log Detail",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}

erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class BulkTransactionLogDetail(Document):
+	pass