Merge pull request #42026 from ljain112/fix-bank-import

fix:  Multiple fixes in Bank Statement Import

erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js

@@ -130,52 +130,66 @@ frappe.ui.form.on("Bank Statement Import", {
 	},
 
 	show_import_status(frm) {
-		let import_log = JSON.parse(frm.doc.statement_import_log || "[]");
-		let successful_records = import_log.filter((log) => log.success);
-		let failed_records = import_log.filter((log) => !log.success);
-		if (successful_records.length === 0) return;
+		if (frm.doc.status == "Pending") return;
 
-		let message;
-		if (failed_records.length === 0) {
-			let message_args = [successful_records.length];
-			if (frm.doc.import_type === "Insert New Records") {
-				message =
-					successful_records.length > 1
-						? __("Successfully imported {0} records.", message_args)
-						: __("Successfully imported {0} record.", message_args);
-			} else {
-				message =
-					successful_records.length > 1
-						? __("Successfully updated {0} records.", message_args)
-						: __("Successfully updated {0} record.", message_args);
-			}
-		} else {
-			let message_args = [successful_records.length, import_log.length];
-			if (frm.doc.import_type === "Insert New Records") {
-				message =
-					successful_records.length > 1
-						? __(
-								"Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-								message_args
-						  )
-						: __(
-								"Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-								message_args
-						  );
-			} else {
-				message =
-					successful_records.length > 1
-						? __(
-								"Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-								message_args
-						  )
-						: __(
-								"Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-								message_args
-						  );
-			}
-		}
-		frm.dashboard.set_headline(message);
+		frappe.call({
+			method: "erpnext.accounts.doctype.bank_statement_import.bank_statement_import.get_import_status",
+			args: {
+				docname: frm.doc.name,
+			},
+			callback: function (r) {
+				let successful_records = cint(r.message.success);
+				let failed_records = cint(r.message.failed);
+				let total_records = cint(r.message.total_records);
+
+				if (!total_records) {
+					return;
+				}
+
+				let message;
+				if (failed_records === 0) {
+					let message_args = [successful_records];
+					if (frm.doc.import_type === "Insert New Records") {
+						message =
+							successful_records > 1
+								? __("Successfully imported {0} records.", message_args)
+								: __("Successfully imported {0} record.", message_args);
+					} else {
+						message =
+							successful_records > 1
+								? __("Successfully updated {0} records.", message_args)
+								: __("Successfully updated {0} record.", message_args);
+					}
+				} else {
+					let message_args = [successful_records, total_records];
+					if (frm.doc.import_type === "Insert New Records") {
+						message =
+							successful_records > 1
+								? __(
+										"Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
+										message_args
+								  )
+								: __(
+										"Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
+										message_args
+								  );
+					} else {
+						message =
+							successful_records > 1
+								? __(
+										"Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
+										message_args
+								  )
+								: __(
+										"Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
+										message_args
+								  );
+					}
+				}
+
+				frm.dashboard.set_headline(message);
+			},
+		});
 	},
 
 	show_report_error_button(frm) {
@@ -297,7 +311,7 @@ frappe.ui.form.on("Bank Statement Import", {
 	// method: 'frappe.core.doctype.data_import.data_import.get_preview_from_template',
 
 	show_import_preview(frm, preview_data) {
-		let import_log = JSON.parse(frm.doc.statement_import_log || "[]");
+		let import_log = preview_data.import_log;
 
 		if (frm.import_preview && frm.import_preview.doctype === frm.doc.reference_doctype) {
 			frm.import_preview.preview_data = preview_data;
@@ -336,6 +350,15 @@ frappe.ui.form.on("Bank Statement Import", {
 		);
 	},
 
+	export_import_log(frm) {
+		open_url_post(
+			"/api/method/erpnext.accounts.doctype.bank_statement_import.bank_statement_import.download_import_log",
+			{
+				data_import_name: frm.doc.name,
+			}
+		);
+	},
+
 	show_import_warnings(frm, preview_data) {
 		let columns = preview_data.columns;
 		let warnings = JSON.parse(frm.doc.template_warnings || "[]");
@@ -411,49 +434,50 @@ frappe.ui.form.on("Bank Statement Import", {
 		frm.trigger("show_import_log");
 	},
 
-	show_import_log(frm) {
-		let import_log = JSON.parse(frm.doc.statement_import_log || "[]");
-		let logs = import_log;
-		frm.toggle_display("import_log", false);
-		frm.toggle_display("import_log_section", logs.length > 0);
+	render_import_log(frm) {
+		frappe.call({
+			method: "erpnext.accounts.doctype.bank_statement_import.bank_statement_import.get_import_logs",
+			args: {
+				docname: frm.doc.name,
+			},
+			callback: function (r) {
+				let logs = r.message;
 
-		if (logs.length === 0) {
-			frm.get_field("import_log_preview").$wrapper.empty();
-			return;
-		}
+				if (logs.length === 0) return;
 
-		let rows = logs
-			.map((log) => {
-				let html = "";
-				if (log.success) {
-					if (frm.doc.import_type === "Insert New Records") {
-						html = __("Successfully imported {0}", [
-							`<span class="underline">${frappe.utils.get_form_link(
-								frm.doc.reference_doctype,
-								log.docname,
-								true
-							)}<span>`,
-						]);
-					} else {
-						html = __("Successfully updated {0}", [
-							`<span class="underline">${frappe.utils.get_form_link(
-								frm.doc.reference_doctype,
-								log.docname,
-								true
-							)}<span>`,
-						]);
-					}
-				} else {
-					let messages = log.messages
-						.map(JSON.parse)
-						.map((m) => {
-							let title = m.title ? `<strong>${m.title}</strong>` : "";
-							let message = m.message ? `<div>${m.message}</div>` : "";
-							return title + message;
-						})
-						.join("");
-					let id = frappe.dom.get_unique_id();
-					html = `${messages}
+				frm.toggle_display("import_log_section", true);
+
+				let rows = logs
+					.map((log) => {
+						let html = "";
+						if (log.success) {
+							if (frm.doc.import_type === "Insert New Records") {
+								html = __("Successfully imported {0}", [
+									`<span class="underline">${frappe.utils.get_form_link(
+										frm.doc.reference_doctype,
+										log.docname,
+										true
+									)}<span>`,
+								]);
+							} else {
+								html = __("Successfully updated {0}", [
+									`<span class="underline">${frappe.utils.get_form_link(
+										frm.doc.reference_doctype,
+										log.docname,
+										true
+									)}<span>`,
+								]);
+							}
+						} else {
+							let messages = JSON.parse(log.messages || "[]")
+								.map((m) => {
+									let title = m.title ? `<strong>${m.title}</strong>` : "";
+									let message = m.message ? `<div>${m.message}</div>` : "";
+									return title + message;
+								})
+								.join("");
+							let id = frappe.dom.get_unique_id();
+							html = `${messages}
 						<button class="btn btn-default btn-xs" type="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false" aria-controls="${id}" style="margin-top: 15px;">
 							${__("Show Traceback")}
 						</button>
@@ -462,16 +486,16 @@ frappe.ui.form.on("Bank Statement Import", {
 								<pre>${log.exception}</pre>
 							</div>
 						</div>`;
-				}
-				let indicator_color = log.success ? "green" : "red";
-				let title = log.success ? __("Success") : __("Failure");
+						}
+						let indicator_color = log.success ? "green" : "red";
+						let title = log.success ? __("Success") : __("Failure");
 
-				if (frm.doc.show_failed_logs && log.success) {
-					return "";
-				}
+						if (frm.doc.show_failed_logs && log.success) {
+							return "";
+						}
 
-				return `<tr>
-					<td>${log.row_indexes.join(", ")}</td>
+						return `<tr>
+					<td>${JSON.parse(log.row_indexes).join(", ")}</td>
 					<td>
 						<div class="indicator ${indicator_color}">${title}</div>
 					</td>
@@ -479,16 +503,16 @@ frappe.ui.form.on("Bank Statement Import", {
 						${html}
 					</td>
 				</tr>`;
-			})
-			.join("");
+					})
+					.join("");
 
-		if (!rows && frm.doc.show_failed_logs) {
-			rows = `<tr><td class="text-center text-muted" colspan=3>
+				if (!rows && frm.doc.show_failed_logs) {
+					rows = `<tr><td class="text-center text-muted" colspan=3>
 				${__("No failed logs")}
 			</td></tr>`;
-		}
+				}
 
-		frm.get_field("import_log_preview").$wrapper.html(`
+				frm.get_field("import_log_preview").$wrapper.html(`
 			<table class="table table-bordered">
 				<tr class="text-muted">
 					<th width="10%">${__("Row Number")}</th>
@@ -498,5 +522,34 @@ frappe.ui.form.on("Bank Statement Import", {
 				${rows}
 			</table>
 		`);
+			},
+		});
+	},
+
+	show_import_log(frm) {
+		frm.toggle_display("import_log_section", false);
+
+		if (frm.is_new() || frm.import_in_progress) {
+			return;
+		}
+
+		frappe.call({
+			method: "frappe.client.get_count",
+			args: {
+				doctype: "Data Import Log",
+				filters: {
+					data_import: frm.doc.name,
+				},
+			},
+			callback: function (r) {
+				let count = r.message;
+				if (count < 5000) {
+					frm.trigger("render_import_log");
+				} else {
+					frm.toggle_display("import_log_section", false);
+					frm.add_custom_button(__("Export Import Log"), () => frm.trigger("export_import_log"));
+				}
+			},
+		});
 	},
 });

erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json

@@ -11,6 +11,8 @@
   "bank_account",
   "bank",
   "column_break_4",
+  "custom_delimiters",
+  "delimiter_options",
   "google_sheets_url",
   "refresh_google_sheet",
   "html_5",
@@ -24,7 +26,6 @@
   "section_import_preview",
   "import_preview",
   "import_log_section",
-  "statement_import_log",
   "show_failed_logs",
   "import_log_preview",
   "reference_doctype",
@@ -194,15 +195,23 @@
    "fieldtype": "Column Break"
   },
   {
-   "fieldname": "statement_import_log",
-   "fieldtype": "Code",
-   "label": "Statement Import Log",
-   "options": "JSON"
+   "default": "0",
+   "fieldname": "custom_delimiters",
+   "fieldtype": "Check",
+   "label": "Custom delimiters"
+  },
+  {
+   "default": ",;\\t|",
+   "depends_on": "custom_delimiters",
+   "description": "If your CSV uses a different delimiter, add that character here, ensuring no spaces or additional characters are included.",
+   "fieldname": "delimiter_options",
+   "fieldtype": "Data",
+   "label": "Delimiter options"
   }
  ],
  "hide_toolbar": 1,
  "links": [],
- "modified": "2024-03-27 13:06:38.098765",
+ "modified": "2024-06-25 17:32:07.658250",
  "modified_by": "Administrator",
  "module": "Accounts",
  "name": "Bank Statement Import",

erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py

@@ -31,13 +31,14 @@ class BankStatementImport(DataImport):
 		bank: DF.Link | None
 		bank_account: DF.Link
 		company: DF.Link
+		custom_delimiters: DF.Check
+		delimiter_options: DF.Data | None
 		google_sheets_url: DF.Data | None
 		import_file: DF.Attach | None
 		import_type: DF.Literal["", "Insert New Records", "Update Existing Records"]
 		mute_emails: DF.Check
 		reference_doctype: DF.Link
 		show_failed_logs: DF.Check
-		statement_import_log: DF.Code | None
 		status: DF.Literal["Pending", "Success", "Partial Success", "Error"]
 		submit_after_import: DF.Check
 		template_options: DF.Code | None
@@ -121,6 +122,11 @@ def download_errored_template(data_import_name):
 	data_import.export_errored_rows()
 
 
+@frappe.whitelist()
+def download_import_log(data_import_name):
+	return frappe.get_doc("Bank Statement Import", data_import_name).download_import_log()
+
+
 def parse_data_from_template(raw_data):
 	data = []
 
@@ -242,6 +248,47 @@ def write_xlsx(data, sheet_name, wb=None, column_widths=None, file_path=None):
 	return True
 
 
+@frappe.whitelist()
+def get_import_status(docname):
+	import_status = {}
+
+	data_import = frappe.get_doc("Bank Statement Import", docname)
+	import_status["status"] = data_import.status
+
+	logs = frappe.get_all(
+		"Data Import Log",
+		fields=["count(*) as count", "success"],
+		filters={"data_import": docname},
+		group_by="success",
+	)
+
+	total_payload_count = 0
+
+	for log in logs:
+		total_payload_count += log.get("count", 0)
+		if log.get("success"):
+			import_status["success"] = log.get("count")
+		else:
+			import_status["failed"] = log.get("count")
+
+	import_status["total_records"] = total_payload_count
+
+	return import_status
+
+
+@frappe.whitelist()
+def get_import_logs(docname: str):
+	frappe.has_permission("Bank Statement Import")
+
+	return frappe.get_all(
+		"Data Import Log",
+		fields=["success", "docname", "messages", "exception", "row_indexes"],
+		filters={"data_import": docname},
+		limit_page_length=5000,
+		order_by="log_index",
+	)
+
+
 @frappe.whitelist()
 def upload_bank_statement(**args):
 	args = frappe._dict(args)