fix: Backfill not_applicable on Item Tax Template Details for German companies (backport #54682) (backport #54686) (#58952)

* fix: Backfill `not_applicable` on Item Tax Template Details for German companies (backport #54682) (#54686)

Co-authored-by: Raffael Meyer <14891507+barredterra@users.noreply.github.com>
fix: Backfill `not_applicable` on Item Tax Template Details for German companies (#54682)
(cherry picked from commit a22d773341)

# Conflicts:
#	erpnext/patches.txt

* chore: resolve conflict

* fix: compare zero-rate row count, not just the identifier set

* chore: correct comment on account name uniqueness

* test: cover German `not_applicable` backfill patch

Run the patch against a company seeded from the shipped German defaults:
matching defaults are backfilled, customised templates and templates with a
duplicate account name are left alone, and a rerun changes nothing.

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: barredterra <14891507+barredterra@users.noreply.github.com>

erpnext/accounts/doctype/item_tax_template/test_item_tax_template.py

@@ -3,6 +3,142 @@
 
 import unittest
 
+import frappe
+from frappe.tests.utils import FrappeTestCase
+
+from erpnext.patches.v16_0.set_not_applicable_on_german_item_tax_templates import (
+	execute as backfill_not_applicable,
+)
+
 
 class TestItemTaxTemplate(unittest.TestCase):
 	pass
+
+
+class TestGermanNotApplicableBackfill(FrappeTestCase):
+	"""Run the `not_applicable` backfill patch against a seeded German company.
+
+	The company is created from the shipped German defaults, so the templates the
+	patch has to recognise are the ones a real site got. Each test resets the flag
+	to its pre-patch state (`not_applicable = 0`) and runs the patch.
+	"""
+
+	TITLES = ("19 %", "7 %", "0%")
+
+	@classmethod
+	def setUpClass(cls):
+		super().setUpClass()
+		cls.company = "_Test German Item Tax Templates"
+		if not frappe.db.exists("Company", cls.company):
+			frappe.get_doc(
+				{
+					"doctype": "Company",
+					"company_name": cls.company,
+					"abbr": "_TGITT",
+					"country": "Germany",
+					"default_currency": "EUR",
+					"create_chart_of_accounts_based_on": "Standard Template",
+					"chart_of_accounts": "Standard",
+				}
+			).insert()
+
+		cls.templates = {
+			title: frappe.db.get_value("Item Tax Template", {"company": cls.company, "title": title}, "name")
+			for title in cls.TITLES
+		}
+		assert all(cls.templates.values()), f"German defaults not seeded: {cls.templates}"
+
+	def setUp(self):
+		frappe.db.savepoint("before_backfill_test")
+		self.addCleanup(frappe.db.rollback, save_point="before_backfill_test")
+		self.seeded_flags = self.flagged_rows()
+		# every default template ships not-applicable rows, otherwise the patch
+		# would be tested against effectively empty data
+		for title in self.TITLES:
+			self.assertTrue(self.seeded_flags[title], f"no not-applicable rows seeded in {title}")
+
+	def flagged_rows(self, title=None) -> dict[str, set]:
+		"""Detail rows currently marked as not applicable, per template title."""
+		return {
+			t: {
+				d.name
+				for d in frappe.get_all(
+					"Item Tax Template Detail",
+					filters={"parent": name, "not_applicable": 1},
+					fields=["name"],
+				)
+			}
+			for t, name in self.templates.items()
+			if title in (None, t)
+		}
+
+	def clear_flags(self):
+		"""Restore the pre-patch state: zero rate, no flag."""
+		for name in self.templates.values():
+			frappe.db.set_value(
+				"Item Tax Template Detail",
+				{"parent": name},
+				"not_applicable",
+				0,
+				update_modified=False,
+			)
+		self.assertEqual(self.flagged_rows(), {t: set() for t in self.TITLES})
+
+	def add_zero_rate_row(self, title, account_name, account_number):
+		"""Add a user-defined zero-rate row, as a customised site would have."""
+		like_account = frappe.db.get_value(
+			"Account", {"company": self.company, "account_name": "Umsatzsteuer 19 %"}, "name"
+		)
+		account = frappe.get_doc(
+			{
+				"doctype": "Account",
+				"company": self.company,
+				"account_name": account_name,
+				"account_number": account_number,
+				"account_type": "Tax",
+				"parent_account": frappe.db.get_value("Account", like_account, "parent_account"),
+			}
+		).insert()
+
+		template = frappe.get_doc("Item Tax Template", self.templates[title])
+		template.append("taxes", {"tax_type": account.name, "tax_rate": 0})
+		template.save()
+
+	def test_backfills_unmodified_defaults(self):
+		self.clear_flags()
+		backfill_not_applicable()
+		self.assertEqual(self.flagged_rows(), self.seeded_flags)
+
+	def test_keeps_customised_template_untouched(self):
+		self.clear_flags()
+		self.add_zero_rate_row("19 %", "Sonstige Umsatzsteuer", "9998")
+		backfill_not_applicable()
+
+		self.assertEqual(self.flagged_rows("19 %"), {"19 %": set()})
+		self.assertEqual(self.flagged_rows("7 %"), {"7 %": self.seeded_flags["7 %"]})
+
+	def test_keeps_duplicate_account_name_untouched(self):
+		"""A numbered account can share `account_name` with a default one.
+
+		Its identifier collapses onto the default's, so only the row count tells
+		the customised template apart from an untouched one.
+		"""
+		self.clear_flags()
+		self.add_zero_rate_row("7 %", "Umsatzsteuer 19 %", "9999")
+		backfill_not_applicable()
+
+		self.assertEqual(self.flagged_rows("7 %"), {"7 %": set()})
+		self.assertEqual(self.flagged_rows("19 %"), {"19 %": self.seeded_flags["19 %"]})
+
+	def test_rerun_changes_nothing(self):
+		def snapshot():
+			return frappe.get_all(
+				"Item Tax Template Detail",
+				filters={"parent": ("in", tuple(self.templates.values()))},
+				fields=["name", "not_applicable", "tax_rate", "modified"],
+				order_by="name",
+			)
+
+		before = snapshot()
+		backfill_not_applicable()
+		self.assertEqual(snapshot(), before)

erpnext/patches.txt

@@ -451,3 +451,4 @@ erpnext.patches.v16_0.remove_frappe_crm_custom_fields
 erpnext.patches.v16_0.append_fieldname_to_pos_search_fields
 erpnext.patches.v16_0.add_transaction_roles_to_sms_settings
 erpnext.patches.v16_0.recalculate_returned_delivery_note_billing_status
+erpnext.patches.v16_0.set_not_applicable_on_german_item_tax_templates

erpnext/patches/v16_0/set_not_applicable_on_german_item_tax_templates.py

@@ -0,0 +1,226 @@
+import frappe
+
+# Snapshot of the relevant German defaults when this migration was written.
+# Migration patches must not read mutable setup data, otherwise future edits to
+# country_wise_tax.json would change what this patch does on sites that have not
+# run it yet.
+#
+# For numbered charts, compare account_number + root_type because Account.account_name
+# is not unique within a company.
+SKR04_NOT_APPLICABLE_7_PERCENT_ACCOUNT_IDS = frozenset(
+	{
+		("3801", "Liability"),
+		("3802", "Liability"),
+		("3835", "Liability"),
+		("1401", "Asset"),
+		("1402", "Asset"),
+		("1541", "Asset"),
+	}
+)
+
+SKR04_NOT_APPLICABLE_19_PERCENT_ACCOUNT_IDS = frozenset(
+	{
+		("3806", "Liability"),
+		("3804", "Liability"),
+		("3837", "Liability"),
+		("1406", "Asset"),
+		("1404", "Asset"),
+		("1540", "Asset"),
+	}
+)
+
+SKR03_NOT_APPLICABLE_7_PERCENT_ACCOUNT_IDS = frozenset(
+	{
+		("1771", "Liability"),
+		("1772", "Liability"),
+		("1785", "Liability"),
+		("1571", "Asset"),
+		("1572", "Asset"),
+		("1541", "Asset"),
+	}
+)
+
+SKR03_NOT_APPLICABLE_19_PERCENT_ACCOUNT_IDS = frozenset(
+	{
+		("1776", "Liability"),
+		("1774", "Liability"),
+		("1787", "Liability"),
+		("1576", "Asset"),
+		("1574", "Asset"),
+		("1540", "Asset"),
+	}
+)
+
+STANDARD_NOT_APPLICABLE_7_PERCENT_ACCOUNT_LABELS = frozenset(
+	{
+		("Umsatzsteuer 7 %", "Liability"),
+		("Umsatzsteuer aus innergemeinschaftlichem Erwerb", "Liability"),
+		("Umsatzsteuer nach § 13b UStG", "Liability"),
+		("Abziehbare Vorsteuer 7 %", "Asset"),
+		("Abziehbare Vorsteuer aus innergemeinschaftlichem Erwerb", "Asset"),
+		("Abziehbare Vorsteuer nach § 13b UStG", "Asset"),
+	}
+)
+
+STANDARD_NOT_APPLICABLE_19_PERCENT_ACCOUNT_LABELS = frozenset(
+	{
+		("Umsatzsteuer 19 %", "Liability"),
+		("Umsatzsteuer aus innergemeinschaftlichem Erwerb 19 %", "Liability"),
+		("Umsatzsteuer nach § 13b UStG 19 %", "Liability"),
+		("Abziehbare Vorsteuer 19 %", "Asset"),
+		("Abziehbare Vorsteuer aus innergemeinschaftlichem Erwerb 19 %", "Asset"),
+		("Abziehbare Vorsteuer nach § 13b UStG 19 %", "Asset"),
+	}
+)
+
+STANDARD_WITH_NUMBERS_NOT_APPLICABLE_7_PERCENT_ACCOUNT_IDS = frozenset(
+	{
+		("2321", "Liability"),
+		("2331", "Liability"),
+		("2341", "Liability"),
+		("1521", "Asset"),
+		("1531", "Asset"),
+		("1541", "Asset"),
+	}
+)
+
+STANDARD_WITH_NUMBERS_NOT_APPLICABLE_19_PERCENT_ACCOUNT_IDS = frozenset(
+	{
+		("2320", "Liability"),
+		("2330", "Liability"),
+		("2340", "Liability"),
+		("1520", "Asset"),
+		("1530", "Asset"),
+		("1540", "Asset"),
+	}
+)
+
+GERMAN_ITEM_TAX_TEMPLATE_NOT_APPLICABLE_ACCOUNTS = {
+	"SKR03 mit Kontonummern": {
+		"identifier_field": "account_number",
+		"templates": {
+			"19 %": SKR03_NOT_APPLICABLE_7_PERCENT_ACCOUNT_IDS,
+			"7 %": SKR03_NOT_APPLICABLE_19_PERCENT_ACCOUNT_IDS,
+			"0 %": SKR03_NOT_APPLICABLE_7_PERCENT_ACCOUNT_IDS
+			| SKR03_NOT_APPLICABLE_19_PERCENT_ACCOUNT_IDS
+			| frozenset({("1588", "Asset")}),
+		},
+	},
+	"SKR04 mit Kontonummern": {
+		"identifier_field": "account_number",
+		"templates": {
+			"19 %": SKR04_NOT_APPLICABLE_7_PERCENT_ACCOUNT_IDS,
+			"7 %": SKR04_NOT_APPLICABLE_19_PERCENT_ACCOUNT_IDS,
+			"0 %": SKR04_NOT_APPLICABLE_7_PERCENT_ACCOUNT_IDS
+			| SKR04_NOT_APPLICABLE_19_PERCENT_ACCOUNT_IDS
+			| frozenset({("1433", "Asset")}),
+		},
+	},
+	"Standard": {
+		"identifier_field": "account_name",
+		"templates": {
+			"19 %": STANDARD_NOT_APPLICABLE_7_PERCENT_ACCOUNT_LABELS,
+			"7 %": STANDARD_NOT_APPLICABLE_19_PERCENT_ACCOUNT_LABELS,
+			"0%": STANDARD_NOT_APPLICABLE_7_PERCENT_ACCOUNT_LABELS
+			| STANDARD_NOT_APPLICABLE_19_PERCENT_ACCOUNT_LABELS
+			| frozenset({("Entstandene Einfuhrumsatzsteuer", "Asset")}),
+		},
+	},
+	"Standard with Numbers": {
+		"identifier_field": "account_number",
+		"templates": {
+			"19%": STANDARD_WITH_NUMBERS_NOT_APPLICABLE_7_PERCENT_ACCOUNT_IDS,
+			"7%": STANDARD_WITH_NUMBERS_NOT_APPLICABLE_19_PERCENT_ACCOUNT_IDS,
+			"0 %": STANDARD_WITH_NUMBERS_NOT_APPLICABLE_7_PERCENT_ACCOUNT_IDS
+			| STANDARD_WITH_NUMBERS_NOT_APPLICABLE_19_PERCENT_ACCOUNT_IDS
+			| frozenset({("1550", "Asset")}),
+		},
+	},
+}
+
+
+def update_account_cache(accounts, account_cache):
+	missing_accounts = set(accounts) - set(account_cache)
+	if not missing_accounts:
+		return
+
+	for account in frappe.get_all(
+		"Account",
+		filters={"name": ("in", tuple(sorted(missing_accounts)))},
+		fields=["name", "account_name", "account_number", "root_type"],
+	):
+		account_cache[account.name] = account
+
+
+def get_account_identifier(account, identifier_field, account_cache):
+	cached_account = account_cache.get(account)
+	if not cached_account:
+		return None
+
+	return cached_account.get(identifier_field), cached_account.root_type
+
+
+def execute():
+	"""Backfill `not_applicable` on Item Tax Template Details for German companies.
+
+	Before the `not_applicable` flag existed, German default templates used
+	`tax_rate: 0` to mean "this tax does not apply to the item" (as opposed to
+	an explicit 0% rate). For each German company, this patch looks up the
+	historical defaults for its Chart of Accounts and sets
+	`not_applicable = 1` on detail rows that still match those defaults
+	(same template title, same zero-rate tax account identifier set, flag still unset),
+	leaving any user-customised rows untouched.
+	"""
+	companies = frappe.get_all(
+		"Company",
+		filters={"country": "Germany"},
+		fields=["name", "chart_of_accounts"],
+	)
+	account_cache = {}
+
+	for company in companies:
+		chart = GERMAN_ITEM_TAX_TEMPLATE_NOT_APPLICABLE_ACCOUNTS.get(company.chart_of_accounts)
+		if not chart:
+			continue
+
+		identifier_field = chart["identifier_field"]
+		for template_title, target_accounts in chart["templates"].items():
+			itt_names = frappe.get_all(
+				"Item Tax Template",
+				filters={"company": company.name, "title": template_title},
+				pluck="name",
+			)
+			for itt_name in itt_names:
+				zero_rate_details = frappe.get_all(
+					"Item Tax Template Detail",
+					filters={"parent": itt_name, "tax_rate": 0},
+					fields=["name", "tax_type", "not_applicable"],
+				)
+				update_account_cache((d.tax_type for d in zero_rate_details), account_cache)
+				zero_rate_accounts_by_detail = {
+					d.name: get_account_identifier(d.tax_type, identifier_field, account_cache)
+					for d in zero_rate_details
+				}
+				if any(identifier is None for identifier in zero_rate_accounts_by_detail.values()):
+					continue
+
+				# Compare the row count as well. Account names are only implicitly unique
+				# among number-less accounts (`Account.name` is `[number - ]account_name - abbr`),
+				# so on a mixed chart a numbered account can share `account_name` with a
+				# default one. Without this, such a user-added zero-rate row collapses onto a
+				# default identifier and makes a customised template look untouched.
+				if len(zero_rate_accounts_by_detail) != len(target_accounts):
+					continue
+
+				if set(zero_rate_accounts_by_detail.values()) != target_accounts:
+					continue
+
+				for d in zero_rate_details:
+					if not d.not_applicable:
+						frappe.db.set_value(
+							"Item Tax Template Detail",
+							d.name,
+							"not_applicable",
+							1,
+							update_modified=False,
+						)