feat: bank reconciliation and plaid changes (#33986)

fix: plaid link refresh: update account ids
fix: plaid transactions for credit cards & add accounts on link refresh if they don't exist
fix: bank reconciliation amount matching
fix: bank reconciliation dialog usability
feat: rewrite bank transaction reconciliation to allow multiple transactions to reconcile against vouchers before clearance
fix: matching transaction amounts and race condition bug
fix: ensure there is a reference number in plaid transactions and other tweaks
feat: add references to Payroll Entry Bank Journal Entry
feat: only clear Voucher once all Bank GLEs are allocated to Bank Transactions
fix: strange type error
feat: add payment method field to bank and plaid transactions and prepopulate relevant bank reconciliation new voucher fields
feat: bank reconciliation - allow bank transactions to reconcile against themselves for when there are banking amendments
fix: bank transaction self-reconcile bug and tidy
fix: bank reconciliation datatable index update

erpnext/erpnext_integrations/doctype/plaid_settings/plaid_connector.py

@@ -12,7 +12,7 @@ class PlaidConnector:
 	def __init__(self, access_token=None):
 		self.access_token = access_token
 		self.settings = frappe.get_single("Plaid Settings")
-		self.products = ["auth", "transactions"]
+		self.products = ["transactions"]
 		self.client_name = frappe.local.site
 		self.client = plaid.Client(
 			client_id=self.settings.plaid_client_id,

erpnext/erpnext_integrations/doctype/plaid_settings/plaid_settings.js

@@ -47,7 +47,7 @@ erpnext.integrations.plaidLink = class plaidLink {
 	}
 
 	async init_config() {
-		this.product = ["auth", "transactions"];
+		this.product = ["transactions"];
 		this.plaid_env = this.frm.doc.plaid_env;
 		this.client_name = frappe.boot.sitename;
 		this.token = await this.get_link_token();

erpnext/erpnext_integrations/doctype/plaid_settings/plaid_settings.py

@@ -70,7 +70,8 @@ def add_bank_accounts(response, bank, company):
 	except TypeError:
 		pass
 
-	bank = json.loads(bank)
+	if isinstance(bank, str):
+		bank = json.loads(bank)
 	result = []
 
 	default_gl_account = get_default_bank_cash_account(company, "Bank")
@@ -177,16 +178,15 @@ def sync_transactions(bank, bank_account):
 		)
 
 		result = []
-		for transaction in reversed(transactions):
-			result += new_bank_transaction(transaction)
+		if transactions:
+			for transaction in reversed(transactions):
+				result += new_bank_transaction(transaction)
 
 		if result:
 			last_transaction_date = frappe.db.get_value("Bank Transaction", result.pop(), "date")
 
 			frappe.logger().info(
-				"Plaid added {} new Bank Transactions from '{}' between {} and {}".format(
-					len(result), bank_account, start_date, end_date
-				)
+				f"Plaid added {len(result)} new Bank Transactions from '{bank_account}' between {start_date} and {end_date}"
 			)
 
 			frappe.db.set_value(
@@ -230,19 +230,20 @@ def new_bank_transaction(transaction):
 
 	bank_account = frappe.db.get_value("Bank Account", dict(integration_id=transaction["account_id"]))
 
-	if float(transaction["amount"]) >= 0:
-		debit = 0
-		credit = float(transaction["amount"])
+	amount = float(transaction["amount"])
+	if amount >= 0.0:
+		deposit = 0.0
+		withdrawal = amount
 	else:
-		debit = abs(float(transaction["amount"]))
-		credit = 0
+		deposit = abs(amount)
+		withdrawal = 0.0
 
 	status = "Pending" if transaction["pending"] == "True" else "Settled"
 
 	tags = []
 	try:
 		tags += transaction["category"]
-		tags += ["Plaid Cat. {}".format(transaction["category_id"])]
+		tags += [f'Plaid Cat. {transaction["category_id"]}']
 	except KeyError:
 		pass
 
@@ -254,11 +255,18 @@ def new_bank_transaction(transaction):
 					"date": getdate(transaction["date"]),
 					"status": status,
 					"bank_account": bank_account,
-					"deposit": debit,
-					"withdrawal": credit,
+					"deposit": deposit,
+					"withdrawal": withdrawal,
 					"currency": transaction["iso_currency_code"],
 					"transaction_id": transaction["transaction_id"],
-					"reference_number": transaction["payment_meta"]["reference_number"],
+					"transaction_type": (
+						transaction["transaction_code"] or transaction["payment_meta"]["payment_method"]
+					),
+					"reference_number": (
+						transaction["check_number"]
+						or transaction["payment_meta"]["reference_number"]
+						or transaction["name"]
+					),
 					"description": transaction["name"],
 				}
 			)
@@ -271,7 +279,7 @@ def new_bank_transaction(transaction):
 			result.append(new_transaction.name)
 
 		except Exception:
-			frappe.throw(title=_("Bank transaction creation error"))
+			frappe.throw(_("Bank transaction creation error"))
 
 	return result
 
@@ -300,3 +308,26 @@ def enqueue_synchronization():
 def get_link_token_for_update(access_token):
 	plaid = PlaidConnector(access_token)
 	return plaid.get_link_token(update_mode=True)
+
+
+def get_company(bank_account_name):
+	from frappe.defaults import get_user_default
+
+	company_names = frappe.db.get_all("Company", pluck="name")
+	if len(company_names) == 1:
+		return company_names[0]
+	if frappe.db.exists("Bank Account", bank_account_name):
+		return frappe.db.get_value("Bank Account", bank_account_name, "company")
+	company_default = get_user_default("Company")
+	if company_default:
+		return company_default
+	frappe.throw(_("Could not detect the Company for updating Bank Accounts"))
+
+
+@frappe.whitelist()
+def update_bank_account_ids(response):
+	data = json.loads(response)
+	institution_name = data["institution"]["name"]
+	bank = frappe.get_doc("Bank", institution_name).as_dict()
+	bank_account_name = f"{data['account']['name']} - {institution_name}"
+	return add_bank_accounts(response, bank, get_company(bank_account_name))

erpnext/erpnext_integrations/doctype/plaid_settings/test_plaid_settings.py

@@ -125,6 +125,8 @@ class TestPlaidSettings(unittest.TestCase):
 			"unofficial_currency_code": None,
 			"name": "INTRST PYMNT",
 			"transaction_type": "place",
+			"transaction_code": "direct debit",
+			"check_number": "3456789",
 			"amount": -4.22,
 			"location": {
 				"city": None,