fix: map MT940 per-transaction reference from :61: customer_reference

The mt940 library exposes ``transaction_reference`` from the :20: tag,
which is the statement-level reference and identical for every
transaction in a statement. The bank-statement-to-CSV conversion was
using it verbatim, so every imported row ended up with the same
reference, making reconciliation impossible.

Read the per-transaction reference from ``customer_reference`` on the
:61: tag instead. Handle two edge cases:

- **Overflow >16 chars.** When a bank emits a single-line :61: whose
  reference exceeds 16 characters, the mt940 regex splits the tail into
  ``extra_details``. Gate the rejoin to cases where
  ``customer_reference`` is exactly at the 16-char MT940 cap; below
  that, ``extra_details`` is genuine supplementary information and must
  not be appended.
- **``NONREF`` sentinel.** The MT940 standard marker for "no customer
  reference". Check it against the un-concatenated value so that a
  ``NONREF`` customer reference with populated ``extra_details`` still
  falls back to ``bank_reference`` instead of returning a junk
  ``NONREFsomething`` value.

Also switch the Description column to ``transaction_details`` (the :86:
tag content) so rows carry their real narrative instead of the mostly
empty :61: supplementary field.
This commit is contained in:
srujan00123
2026-04-20 16:23:49 +05:30
committed by khushi8112
parent 1614d33e5c
commit 551d709c64
2 changed files with 158 additions and 2 deletions

View File

@@ -142,6 +142,30 @@ def preprocess_mt940_content(content: str) -> str:
return processed_content
MT940_CUSTOMER_REFERENCE_MAX_LEN = 16
def get_transaction_reference(txn_data: dict) -> str:
"""Extract the per-transaction reference from an MT940 :61: tag.
The mt940 library exposes ``transaction_reference`` from the :20: tag, which is the
statement-level reference and identical for every transaction in a statement. The
real per-transaction reference is ``customer_reference`` (with any overflow captured
into ``extra_details`` when a bank emits a single-line :61: longer than 16 chars).
"""
customer_reference = (txn_data.get("customer_reference") or "").strip()
if len(customer_reference) == MT940_CUSTOMER_REFERENCE_MAX_LEN:
customer_reference += (txn_data.get("extra_details") or "").strip()
if customer_reference and customer_reference.upper() != "NONREF":
return customer_reference
return (txn_data.get("bank_reference") or "").strip() or (
txn_data.get("transaction_reference") or ""
).strip()
@frappe.whitelist()
def convert_mt940_to_csv(data_import: str, mt940_file_path: str):
doc = frappe.get_doc("Bank Statement Import", data_import)
@@ -189,8 +213,8 @@ def convert_mt940_to_csv(data_import: str, mt940_file_path: str):
deposit = amount_value if amount_value > 0 else ""
withdrawal = abs(amount_value) if amount_value < 0 else ""
description = txn.data.get("extra_details") or ""
reference = txn.data.get("transaction_reference") or ""
description = txn.data.get("transaction_details") or txn.data.get("extra_details") or ""
reference = get_transaction_reference(txn.data)
currency = txn.data.get("currency", "")
writer.writerow([date_str, deposit, withdrawal, description, reference, doc.bank_account, currency])

View File

@@ -1,7 +1,10 @@
# Copyright (c) 2020, Frappe Technologies and Contributors
# See license.txt
import mt940
from erpnext.accounts.doctype.bank_statement_import.bank_statement_import import (
get_transaction_reference,
is_mt940_format,
preprocess_mt940_content,
)
@@ -188,6 +191,135 @@ class TestBankStatementImport(ERPNextTestSuite):
self.assertIn(":20:STMTREF167619", result) # Reference should remain unchanged
self.assertIn("UPI/TEST USER/123456789/PaidViaTestApp", result)
def test_get_transaction_reference_uses_customer_reference(self):
"""Per-transaction reference must come from :61: customer_reference, not :20:."""
self.assertEqual(
get_transaction_reference(
{"customer_reference": "UPI-100000000001", "transaction_reference": "STMTREF12345"}
),
"UPI-100000000001",
)
def test_get_transaction_reference_rejoins_overflow(self):
"""When a bank emits a single-line :61: with >16-char reference, the regex
splits the tail into extra_details. We must rejoin them."""
self.assertEqual(
get_transaction_reference(
{
"customer_reference": "NEFTINW-12345678",
"extra_details": "90",
"transaction_reference": "STMTREF12345",
}
),
"NEFTINW-1234567890",
)
def test_get_transaction_reference_falls_back_to_bank_reference_on_nonref(self):
"""NONREF is the MT940 'no customer reference' sentinel; prefer bank_reference."""
self.assertEqual(
get_transaction_reference(
{
"customer_reference": "NONREF",
"bank_reference": "1234567890123456",
"transaction_reference": "STMTREF12345",
}
),
"1234567890123456",
)
def test_get_transaction_reference_falls_back_to_bank_reference_on_nonref_with_extra_details(self):
"""NONREF sentinel must trigger the bank_reference fallback even when
extra_details is populated. Without the 16-char gate, the old naive concat
would produce a junk reference like 'NONREFsome info' and bypass the check."""
self.assertEqual(
get_transaction_reference(
{
"customer_reference": "NONREF",
"extra_details": "some info",
"bank_reference": "1234567890123456",
"transaction_reference": "STMTREF12345",
}
),
"1234567890123456",
)
def test_get_transaction_reference_does_not_append_extra_details_below_16_chars(self):
"""When customer_reference is below the 16-char cap, extra_details is a
genuine supplementary-info field from :61: — not overflow — and must not
be appended to the reference."""
self.assertEqual(
get_transaction_reference(
{
"customer_reference": "TBMS-123456789",
"extra_details": "note field",
"transaction_reference": "STMTREF12345",
}
),
"TBMS-123456789",
)
def test_get_transaction_reference_keeps_noref_literal(self):
"""Bare 'NOREF' (without bank_reference) stays as-is; still better than the
statement-level reference which is identical across all transactions."""
self.assertEqual(
get_transaction_reference(
{
"customer_reference": "NOREF",
"bank_reference": None,
"transaction_reference": "STMTREF12345",
}
),
"NOREF",
)
def test_mt940_parse_per_transaction_reference_mapping(self):
"""End-to-end: every transaction in a statement must get its own distinct
reference from :61: customer_reference, never the statement-level :20: reference."""
mt940_content = """{1:F0112345678901X0000000000}{2:I94012345678901XN}{4:
:20:STMTREF12345
:25:1234567890
:28C:12345/1
:60F:C250716INR88123,38
:61:2509280928D5000,00NMSCUPI-100000000001
:86:UPI/TEST PAYEE ONE/111111111111/TestApp
:61:2509190919D2606,00NMSCUPI-100000000002
:86:UPI/TEST PAYEE TWO/222222222222/TestApp
:61:2509190919D900,00NMSCUPI-100000000003
:86:UPI/TEST PAYEE THREE/333333333333/TestApp
:61:2508140814D5000,00NMSCUPI-100000000004
:86:UPI/TEST PAYEE FOUR/444444444444/TestApp
:61:2508060806D2000,00NMSCUPI-100000000005
:86:UPI/TEST PAYEE FIVE/555555555555/TestApp
:61:2508030803D1066,00NMSC123456789012
:86:PCD/1234/TEST MERCHANT/01234567890123/12:00
:61:2507310731D305,62NMSCTBMS-123456789
:86:Chrg: Debit Card Annual Fee 1234 for 2025
:61:2507240724C1,00NMSCNEFTINW-1234567890
:86:NEFT TEST123456789 TEST SERVICES
:61:2507170717C100000,00NMSCNOREF
:86:BY CLG INST 123456/01-01-25/TESTBANK/TESTCITY
:62F:C250930INR100000,00
-}"""
transactions = list(mt940.parse(preprocess_mt940_content(mt940_content)))
references = [get_transaction_reference(t.data) for t in transactions]
self.assertEqual(
references,
[
"UPI-100000000001",
"UPI-100000000002",
"UPI-100000000003",
"UPI-100000000004",
"UPI-100000000005",
"123456789012",
"TBMS-123456789",
"NEFTINW-1234567890",
"NOREF",
],
)
# No transaction should carry the statement-level reference from :20:
self.assertNotIn("STMTREF12345", references)
def test_preprocess_mt940_content_whitespace_variants(self):
"""Test handling of whitespace and different line endings"""
# Test with trailing spaces