From 13fb560401a89e64b0ff6400b0477314499bdfa8 Mon Sep 17 00:00:00 2001
From: ljain112 <ljain112@gmail.com>
Date: Tue, 25 Jun 2024 13:05:30 +0530
Subject: [PATCH 1/7] fix: custom delimiters

---
 .../bank_statement_import.json                 | 18 +++++++++++++++++-
 .../bank_statement_import.py                   |  2 ++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json
index 45671f35bd7..e39d8e823af 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json
@@ -16,6 +16,8 @@
   "html_5",
   "import_file",
   "download_template",
+  "custom_delimiters",
+  "delimiter_options",
   "status",
   "template_options",
   "import_warnings_section",
@@ -198,11 +200,25 @@
    "fieldtype": "Code",
    "label": "Statement Import Log",
    "options": "JSON"
+  },
+  {
+   "default": "0",
+   "fieldname": "custom_delimiters",
+   "fieldtype": "Check",
+   "label": "Custom delimiters"
+  },
+  {
+   "default": ",;\\t|",
+   "depends_on": "custom_delimiters",
+   "description": "If your CSV uses a different delimiter, add that character here, ensuring no spaces or additional characters are included.",
+   "fieldname": "delimiter_options",
+   "fieldtype": "Data",
+   "label": "Delimiter options"
   }
  ],
  "hide_toolbar": 1,
  "links": [],
- "modified": "2024-03-27 13:06:38.098765",
+ "modified": "2024-06-25 15:34:35.214946",
  "modified_by": "Administrator",
  "module": "Accounts",
  "name": "Bank Statement Import",
diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
index 49684dfcf7a..70475525ad2 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
@@ -31,6 +31,8 @@ class BankStatementImport(DataImport):
 		bank: DF.Link | None
 		bank_account: DF.Link
 		company: DF.Link
+		custom_delimiters: DF.Check
+		delimiter_options: DF.Data | None
 		google_sheets_url: DF.Data | None
 		import_file: DF.Attach | None
 		import_type: DF.Literal["", "Insert New Records", "Update Existing Records"]

From b20240942831d1118d00ca8762ebd4e0ff18f684 Mon Sep 17 00:00:00 2001
From: ljain112 <ljain112@gmail.com>
Date: Tue, 25 Jun 2024 14:27:04 +0530
Subject: [PATCH 2/7] fix: remove deprecated field "statement_import_log"

---
 .../bank_statement_import/bank_statement_import.json     | 9 +--------
 .../bank_statement_import/bank_statement_import.py       | 1 -
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json
index e39d8e823af..8f9538a1ac9 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json
@@ -26,7 +26,6 @@
   "section_import_preview",
   "import_preview",
   "import_log_section",
-  "statement_import_log",
   "show_failed_logs",
   "import_log_preview",
   "reference_doctype",
@@ -195,12 +194,6 @@
    "fieldname": "column_break_4",
    "fieldtype": "Column Break"
   },
-  {
-   "fieldname": "statement_import_log",
-   "fieldtype": "Code",
-   "label": "Statement Import Log",
-   "options": "JSON"
-  },
   {
    "default": "0",
    "fieldname": "custom_delimiters",
@@ -218,7 +211,7 @@
  ],
  "hide_toolbar": 1,
  "links": [],
- "modified": "2024-06-25 15:34:35.214946",
+ "modified": "2024-06-25 16:33:50.675848",
  "modified_by": "Administrator",
  "module": "Accounts",
  "name": "Bank Statement Import",
diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
index 70475525ad2..484416661db 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
@@ -39,7 +39,6 @@ class BankStatementImport(DataImport):
 		mute_emails: DF.Check
 		reference_doctype: DF.Link
 		show_failed_logs: DF.Check
-		statement_import_log: DF.Code | None
 		status: DF.Literal["Pending", "Success", "Partial Success", "Error"]
 		submit_after_import: DF.Check
 		template_options: DF.Code | None

From 07f68884e0c09fdcf769dc7f1d0cc1a03921ef01 Mon Sep 17 00:00:00 2001
From: ljain112 <ljain112@gmail.com>
Date: Tue, 25 Jun 2024 14:27:28 +0530
Subject: [PATCH 3/7] fix: import status

---
 .../bank_statement_import.js                  | 102 ++++++++++--------
 .../bank_statement_import.py                  |  28 +++++
 2 files changed, 85 insertions(+), 45 deletions(-)

diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
index c4b442070d2..7eb2219d4cc 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
@@ -130,52 +130,64 @@ frappe.ui.form.on("Bank Statement Import", {
 	},
 
 	show_import_status(frm) {
-		let import_log = JSON.parse(frm.doc.statement_import_log || "[]");
-		let successful_records = import_log.filter((log) => log.success);
-		let failed_records = import_log.filter((log) => !log.success);
-		if (successful_records.length === 0) return;
+		frappe.call({
+			method: "erpnext.accounts.doctype.bank_statement_import.bank_statement_import.get_import_status",
+			args: {
+				docname: frm.doc.name,
+			},
+			callback: function (r) {
+				let successful_records = cint(r.message.success);
+				let failed_records = cint(r.message.failed);
+				let total_records = cint(r.message.total_records);
 
-		let message;
-		if (failed_records.length === 0) {
-			let message_args = [successful_records.length];
-			if (frm.doc.import_type === "Insert New Records") {
-				message =
-					successful_records.length > 1
-						? __("Successfully imported {0} records.", message_args)
-						: __("Successfully imported {0} record.", message_args);
-			} else {
-				message =
-					successful_records.length > 1
-						? __("Successfully updated {0} records.", message_args)
-						: __("Successfully updated {0} record.", message_args);
-			}
-		} else {
-			let message_args = [successful_records.length, import_log.length];
-			if (frm.doc.import_type === "Insert New Records") {
-				message =
-					successful_records.length > 1
-						? __(
-								"Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-								message_args
-						  )
-						: __(
-								"Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-								message_args
-						  );
-			} else {
-				message =
-					successful_records.length > 1
-						? __(
-								"Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-								message_args
-						  )
-						: __(
-								"Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
-								message_args
-						  );
-			}
-		}
-		frm.dashboard.set_headline(message);
+				if (!total_records) {
+					return;
+				}
+
+				let message;
+				if (failed_records === 0) {
+					let message_args = [successful_records];
+					if (frm.doc.import_type === "Insert New Records") {
+						message =
+							successful_records > 1
+								? __("Successfully imported {0} records.", message_args)
+								: __("Successfully imported {0} record.", message_args);
+					} else {
+						message =
+							successful_records > 1
+								? __("Successfully updated {0} records.", message_args)
+								: __("Successfully updated {0} record.", message_args);
+					}
+				} else {
+					let message_args = [successful_records, total_records];
+					if (frm.doc.import_type === "Insert New Records") {
+						message =
+							successful_records > 1
+								? __(
+										"Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
+										message_args
+								  )
+								: __(
+										"Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
+										message_args
+								  );
+					} else {
+						message =
+							successful_records > 1
+								? __(
+										"Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.",
+										message_args
+								  )
+								: __(
+										"Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.",
+										message_args
+								  );
+					}
+				}
+
+				frm.dashboard.set_headline(message);
+			},
+		});
 	},
 
 	show_report_error_button(frm) {
diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
index 484416661db..b0cea40c889 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
@@ -243,6 +243,34 @@ def write_xlsx(data, sheet_name, wb=None, column_widths=None, file_path=None):
 	return True
 
 
+@frappe.whitelist()
+def get_import_status(docname):
+	import_status = {}
+
+	data_import = frappe.get_doc("Bank Statement Import", docname)
+	import_status["status"] = data_import.status
+
+	logs = frappe.get_all(
+		"Data Import Log",
+		fields=["count(*) as count", "success"],
+		filters={"data_import": docname},
+		group_by="success",
+	)
+
+	total_payload_count = 0
+
+	for log in logs:
+		total_payload_count += log.get("count", 0)
+		if log.get("success"):
+			import_status["success"] = log.get("count")
+		else:
+			import_status["failed"] = log.get("count")
+
+	import_status["total_records"] = total_payload_count
+
+	return import_status
+
+
 @frappe.whitelist()
 def upload_bank_statement(**args):
 	args = frappe._dict(args)

From 1ad264de48216889d1c1a5a83d0cd773070049d5 Mon Sep 17 00:00:00 2001
From: ljain112 <ljain112@gmail.com>
Date: Tue, 25 Jun 2024 14:47:23 +0530
Subject: [PATCH 4/7] fix: import log preview

---
 .../bank_statement_import.js                  | 142 +++++++++++-------
 .../bank_statement_import.py                  |  13 ++
 2 files changed, 99 insertions(+), 56 deletions(-)

diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
index 7eb2219d4cc..e40187fe820 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
@@ -309,7 +309,7 @@ frappe.ui.form.on("Bank Statement Import", {
 	// method: 'frappe.core.doctype.data_import.data_import.get_preview_from_template',
 
 	show_import_preview(frm, preview_data) {
-		let import_log = JSON.parse(frm.doc.statement_import_log || "[]");
+		let import_log = preview_data.import_log;
 
 		if (frm.import_preview && frm.import_preview.doctype === frm.doc.reference_doctype) {
 			frm.import_preview.preview_data = preview_data;
@@ -423,49 +423,50 @@ frappe.ui.form.on("Bank Statement Import", {
 		frm.trigger("show_import_log");
 	},
 
-	show_import_log(frm) {
-		let import_log = JSON.parse(frm.doc.statement_import_log || "[]");
-		let logs = import_log;
-		frm.toggle_display("import_log", false);
-		frm.toggle_display("import_log_section", logs.length > 0);
+	render_import_log(frm) {
+		frappe.call({
+			method: "erpnext.accounts.doctype.bank_statement_import.bank_statement_import.get_import_logs",
+			args: {
+				docname: frm.doc.name,
+			},
+			callback: function (r) {
+				let logs = r.message;
 
-		if (logs.length === 0) {
-			frm.get_field("import_log_preview").$wrapper.empty();
-			return;
-		}
+				if (logs.length === 0) return;
 
-		let rows = logs
-			.map((log) => {
-				let html = "";
-				if (log.success) {
-					if (frm.doc.import_type === "Insert New Records") {
-						html = __("Successfully imported {0}", [
-							`<span class="underline">${frappe.utils.get_form_link(
-								frm.doc.reference_doctype,
-								log.docname,
-								true
-							)}<span>`,
-						]);
-					} else {
-						html = __("Successfully updated {0}", [
-							`<span class="underline">${frappe.utils.get_form_link(
-								frm.doc.reference_doctype,
-								log.docname,
-								true
-							)}<span>`,
-						]);
-					}
-				} else {
-					let messages = log.messages
-						.map(JSON.parse)
-						.map((m) => {
-							let title = m.title ? `<strong>${m.title}</strong>` : "";
-							let message = m.message ? `<div>${m.message}</div>` : "";
-							return title + message;
-						})
-						.join("");
-					let id = frappe.dom.get_unique_id();
-					html = `${messages}
+				frm.toggle_display("import_log_section", true);
+
+				let rows = logs
+					.map((log) => {
+						let html = "";
+						if (log.success) {
+							if (frm.doc.import_type === "Insert New Records") {
+								html = __("Successfully imported {0}", [
+									`<span class="underline">${frappe.utils.get_form_link(
+										frm.doc.reference_doctype,
+										log.docname,
+										true
+									)}<span>`,
+								]);
+							} else {
+								html = __("Successfully updated {0}", [
+									`<span class="underline">${frappe.utils.get_form_link(
+										frm.doc.reference_doctype,
+										log.docname,
+										true
+									)}<span>`,
+								]);
+							}
+						} else {
+							let messages = JSON.parse(log.messages || "[]")
+								.map((m) => {
+									let title = m.title ? `<strong>${m.title}</strong>` : "";
+									let message = m.message ? `<div>${m.message}</div>` : "";
+									return title + message;
+								})
+								.join("");
+							let id = frappe.dom.get_unique_id();
+							html = `${messages}
 						<button class="btn btn-default btn-xs" type="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false" aria-controls="${id}" style="margin-top: 15px;">
 							${__("Show Traceback")}
 						</button>
@@ -474,16 +475,16 @@ frappe.ui.form.on("Bank Statement Import", {
 								<pre>${log.exception}</pre>
 							</div>
 						</div>`;
-				}
-				let indicator_color = log.success ? "green" : "red";
-				let title = log.success ? __("Success") : __("Failure");
+						}
+						let indicator_color = log.success ? "green" : "red";
+						let title = log.success ? __("Success") : __("Failure");
 
-				if (frm.doc.show_failed_logs && log.success) {
-					return "";
-				}
+						if (frm.doc.show_failed_logs && log.success) {
+							return "";
+						}
 
-				return `<tr>
-					<td>${log.row_indexes.join(", ")}</td>
+						return `<tr>
+					<td>${JSON.parse(log.row_indexes).join(", ")}</td>
 					<td>
 						<div class="indicator ${indicator_color}">${title}</div>
 					</td>
@@ -491,16 +492,16 @@ frappe.ui.form.on("Bank Statement Import", {
 						${html}
 					</td>
 				</tr>`;
-			})
-			.join("");
+					})
+					.join("");
 
-		if (!rows && frm.doc.show_failed_logs) {
-			rows = `<tr><td class="text-center text-muted" colspan=3>
+				if (!rows && frm.doc.show_failed_logs) {
+					rows = `<tr><td class="text-center text-muted" colspan=3>
 				${__("No failed logs")}
 			</td></tr>`;
-		}
+				}
 
-		frm.get_field("import_log_preview").$wrapper.html(`
+				frm.get_field("import_log_preview").$wrapper.html(`
 			<table class="table table-bordered">
 				<tr class="text-muted">
 					<th width="10%">${__("Row Number")}</th>
@@ -510,5 +511,34 @@ frappe.ui.form.on("Bank Statement Import", {
 				${rows}
 			</table>
 		`);
+			},
+		});
+	},
+
+	show_import_log(frm) {
+		frm.toggle_display("import_log_section", false);
+
+		if (frm.is_new() || frm.import_in_progress) {
+			return;
+		}
+
+		frappe.call({
+			method: "frappe.client.get_count",
+			args: {
+				doctype: "Data Import Log",
+				filters: {
+					data_import: frm.doc.name,
+				},
+			},
+			callback: function (r) {
+				let count = r.message;
+				if (count < 5000) {
+					frm.trigger("render_import_log");
+				} else {
+					frm.toggle_display("import_log_section", false);
+					frm.add_custom_button(__("Export Import Log"), () => frm.trigger("export_import_log"));
+				}
+			},
+		});
 	},
 });
diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
index b0cea40c889..de8e6734a55 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
@@ -271,6 +271,19 @@ def get_import_status(docname):
 	return import_status
 
 
+@frappe.whitelist()
+def get_import_logs(docname: str):
+	frappe.has_permission("Bank Statement Import")
+
+	return frappe.get_all(
+		"Data Import Log",
+		fields=["success", "docname", "messages", "exception", "row_indexes"],
+		filters={"data_import": docname},
+		limit_page_length=5000,
+		order_by="log_index",
+	)
+
+
 @frappe.whitelist()
 def upload_bank_statement(**args):
 	args = frappe._dict(args)

From 2112d8f772e1536f15073af123211b7b78d02460 Mon Sep 17 00:00:00 2001
From: ljain112 <ljain112@gmail.com>
Date: Tue, 25 Jun 2024 14:55:51 +0530
Subject: [PATCH 5/7] fix: download_import_log if rows are greater than 5000

---
 .../bank_statement_import/bank_statement_import.js       | 9 +++++++++
 .../bank_statement_import/bank_statement_import.py       | 5 +++++
 2 files changed, 14 insertions(+)

diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
index e40187fe820..a8fd6409bdf 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
@@ -348,6 +348,15 @@ frappe.ui.form.on("Bank Statement Import", {
 		);
 	},
 
+	export_import_log(frm) {
+		open_url_post(
+			"/api/method/erpnext.accounts.doctype.bank_statement_import.bank_statement_import.download_import_log",
+			{
+				data_import_name: frm.doc.name,
+			}
+		);
+	},
+
 	show_import_warnings(frm, preview_data) {
 		let columns = preview_data.columns;
 		let warnings = JSON.parse(frm.doc.template_warnings || "[]");
diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
index de8e6734a55..f6db2e06662 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py
@@ -122,6 +122,11 @@ def download_errored_template(data_import_name):
 	data_import.export_errored_rows()
 
 
+@frappe.whitelist()
+def download_import_log(data_import_name):
+	return frappe.get_doc("Bank Statement Import", data_import_name).download_import_log()
+
+
 def parse_data_from_template(raw_data):
 	data = []
 

From 98ad01c7360803435ee0f6f809b4681e2fe01d48 Mon Sep 17 00:00:00 2001
From: ljain112 <ljain112@gmail.com>
Date: Tue, 25 Jun 2024 15:04:21 +0530
Subject: [PATCH 6/7] fix: field position

---
 .../bank_statement_import/bank_statement_import.json        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json
index 8f9538a1ac9..500e36a8782 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json
@@ -11,13 +11,13 @@
   "bank_account",
   "bank",
   "column_break_4",
+  "custom_delimiters",
+  "delimiter_options",
   "google_sheets_url",
   "refresh_google_sheet",
   "html_5",
   "import_file",
   "download_template",
-  "custom_delimiters",
-  "delimiter_options",
   "status",
   "template_options",
   "import_warnings_section",
@@ -211,7 +211,7 @@
  ],
  "hide_toolbar": 1,
  "links": [],
- "modified": "2024-06-25 16:33:50.675848",
+ "modified": "2024-06-25 17:32:07.658250",
  "modified_by": "Administrator",
  "module": "Accounts",
  "name": "Bank Statement Import",

From ba08b2e8e808c154b155995bd93498004c2ff926 Mon Sep 17 00:00:00 2001
From: ljain112 <ljain112@gmail.com>
Date: Tue, 25 Jun 2024 17:48:10 +0530
Subject: [PATCH 7/7] fix: changes as per review

---
 .../doctype/bank_statement_import/bank_statement_import.js      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
index a8fd6409bdf..bdad264d4f0 100644
--- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
+++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js
@@ -130,6 +130,8 @@ frappe.ui.form.on("Bank Statement Import", {
 	},
 
 	show_import_status(frm) {
+		if (frm.doc.status == "Pending") return;
+
 		frappe.call({
 			method: "erpnext.accounts.doctype.bank_statement_import.bank_statement_import.get_import_status",
 			args: {