Merge pull request #56051 from mihir-kandoi/pg-accounts-statements

refactor(postgres): port Accounts statement & ledger report queries to the query builder
This commit is contained in:
Mihir Kandoi
2026-06-17 17:55:47 +05:30
committed by GitHub
14 changed files with 295 additions and 199 deletions

View File

@@ -15,10 +15,7 @@ def execute(filters=None):
def get_data(filters):
data = []
depreciation_accounts = frappe.db.sql_list(
""" select name from tabAccount
where ifnull(account_type, '') = 'Depreciation' """
)
depreciation_accounts = frappe.get_all("Account", filters={"account_type": "Depreciation"}, pluck="name")
filters_data = [
["company", "=", filters.get("company")],
@@ -33,10 +30,8 @@ def get_data(filters):
filters_data.append(["against_voucher", "=", filters.get("asset")])
if filters.get("asset_category"):
assets = frappe.db.sql_list(
"""select name from tabAsset
where asset_category = %s and docstatus=1""",
filters.get("asset_category"),
assets = frappe.get_all(
"Asset", filters={"asset_category": filters.get("asset_category"), "docstatus": 1}, pluck="name"
)
filters_data.append(["against_voucher", "in", assets])

View File

@@ -0,0 +1,18 @@
# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
import frappe
from erpnext.accounts.report.asset_depreciation_ledger.asset_depreciation_ledger import execute
from erpnext.tests.utils import ERPNextTestSuite
class TestAssetDepreciationLedger(ERPNextTestSuite):
def test_report_executes(self):
# Smoke-guards the raw-SQL -> query-builder port: the report query must compile and run on
# both MariaDB and postgres.
company = frappe.db.get_value("Company", {}, "name")
columns, *_rest = execute(
frappe._dict({"company": company, "from_date": "2020-01-01", "to_date": "2030-12-31"})
)
self.assertTrue(columns)

View File

@@ -3,7 +3,7 @@
import frappe
from frappe import _
from frappe.query_builder import CustomFunction
from frappe.query_builder.custom import MonthName
from frappe.utils import add_months, flt, formatdate
from erpnext.accounts.doctype.accounting_dimension.accounting_dimension import get_dimensions
@@ -113,7 +113,6 @@ def build_budget_map(budget_records, filters):
def get_actual_transactions(dimension_name, filters):
budget_against = frappe.scrub(filters.get("budget_against"))
monthname = CustomFunction("MONTHNAME", ["date"])
gle = frappe.qb.DocType("GL Entry")
budget = frappe.qb.DocType("Budget")
@@ -126,7 +125,7 @@ def get_actual_transactions(dimension_name, filters):
gle.debit,
gle.credit,
gle.fiscal_year,
monthname(gle.posting_date).as_("month_name"),
MonthName(gle.posting_date).as_("month_name"),
budget[budget_against].as_("budget_against"),
)
.where(
@@ -137,7 +136,10 @@ def get_actual_transactions(dimension_name, filters):
& (gle.is_cancelled == 0)
& (budget[budget_against] == dimension_name)
)
.groupby(gle.name)
# budget[budget_against] is selected from the Budget table, which is not functionally
# dependent on the grouped GL Entry PK, so postgres requires it in the GROUP BY. The WHERE
# pins it to dimension_name (a constant), so grouping by it does not change the result.
.groupby(gle.name, budget[budget_against])
.orderby(gle.fiscal_year)
)
@@ -157,15 +159,11 @@ def get_actual_transactions(dimension_name, filters):
def get_budget_distributions(budget):
return frappe.db.sql(
"""
SELECT start_date, end_date, amount, percent
FROM `tabBudget Distribution`
WHERE parent = %s
ORDER BY start_date ASC
""",
(budget.name,),
as_dict=True,
return frappe.get_all(
"Budget Distribution",
filters={"parent": budget.name},
fields=["start_date", "end_date", "amount", "percent"],
order_by="start_date asc",
)
@@ -351,20 +349,16 @@ def get_columns(filters):
def get_fiscal_years(filters):
fiscal_year = frappe.db.sql(
"""
select
name
from
`tabFiscal Year`
where
name between %(from_fiscal_year)s and %(to_fiscal_year)s
""",
{"from_fiscal_year": filters["from_fiscal_year"], "to_fiscal_year": filters["to_fiscal_year"]},
return frappe.get_all(
"Fiscal Year",
filters={"name": ["between", [filters["from_fiscal_year"], filters["to_fiscal_year"]]]},
fields=["name"],
# the raw query had no ORDER BY (de-facto oldest-first); get_all would otherwise apply the
# Fiscal Year doctype default (name DESC) and reverse column order / cumulative-mode values.
order_by="name asc",
as_list=True,
)
return fiscal_year
def get_cost_center_with_children(cost_centers):
"""Expand each cost center to include itself and all its descendants."""

View File

@@ -0,0 +1,27 @@
# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
import frappe
from erpnext.accounts.report.budget_variance_report.budget_variance_report import execute
from erpnext.tests.utils import ERPNextTestSuite
class TestBudgetVarianceReport(ERPNextTestSuite):
def test_report_executes(self):
# Smoke-guards the raw-SQL -> query-builder port: the report query must compile and run on
# both MariaDB and postgres.
company = frappe.db.get_value("Company", {}, "name")
fy = frappe.db.get_value("Fiscal Year", {}, "name", order_by="year_start_date desc")
columns, *_rest = execute(
frappe._dict(
{
"company": company,
"from_fiscal_year": fy,
"to_fiscal_year": fy,
"period": "Yearly",
"budget_against": "Cost Center",
}
)
)
self.assertTrue(columns)

View File

@@ -7,6 +7,7 @@ from datetime import timedelta
import frappe
from frappe import _
from frappe.query_builder import DocType
from frappe.query_builder.functions import Sum
from frappe.utils import cstr, flt
from pypika import Order
@@ -213,37 +214,43 @@ def get_account_type_based_data(company, account_type, period_list, accumulated_
def get_account_type_based_gl_data(company, filters=None):
cond = ""
filters = frappe._dict(filters or {})
gle = frappe.qb.DocType("GL Entry")
account = frappe.qb.DocType("Account")
query = (
frappe.qb.from_(gle)
.select(Sum(gle.credit) - Sum(gle.debit))
.where(
(gle.company == company)
& (gle.posting_date >= filters.start_date)
& (gle.posting_date <= filters.end_date)
& (gle.voucher_type != "Period Closing Voucher")
& gle.account.isin(
frappe.qb.from_(account)
.select(account.name)
.where(account.account_type == filters.account_type)
)
)
)
if filters.include_default_book_entries:
company_fb = frappe.get_cached_value("Company", company, "default_finance_book")
cond = """ AND (finance_book in ({}, {}, '') OR finance_book IS NULL)
""".format(
frappe.db.escape(filters.finance_book),
frappe.db.escape(company_fb),
query = query.where(
gle.finance_book.isin([filters.finance_book, company_fb, ""]) | gle.finance_book.isnull()
)
else:
cond = " AND (finance_book in (%s, '') OR finance_book IS NULL)" % (
frappe.db.escape(cstr(filters.finance_book))
query = query.where(
gle.finance_book.isin([cstr(filters.finance_book), ""]) | gle.finance_book.isnull()
)
if filters.get("cost_center"):
filters.cost_center = get_cost_centers_with_children(filters.cost_center)
cond += " and cost_center in %(cost_center)s"
cost_centers = get_cost_centers_with_children(filters.cost_center)
query = query.where(gle.cost_center.isin(cost_centers))
gl_sum = frappe.db.sql_list(
f"""
select sum(credit) - sum(debit)
from `tabGL Entry`
where company=%(company)s and posting_date >= %(start_date)s and posting_date <= %(end_date)s
and voucher_type != 'Period Closing Voucher'
and account in ( SELECT name FROM tabAccount WHERE account_type = %(account_type)s) {cond}
""",
filters,
)
return gl_sum[0] if gl_sum and gl_sum[0] else 0
gl_sum = query.run()
return gl_sum[0][0] if gl_sum and gl_sum[0][0] else 0
def get_start_date(period, accumulated_values, company):
@@ -367,11 +374,10 @@ def get_net_income(company, period_list, filters):
from_date, to_date = get_opening_range_using_fiscal_year(company, period_list)
for root_type in ["Income", "Expense"]:
for root in frappe.db.sql(
"""select lft, rgt from tabAccount
where root_type=%s and ifnull(parent_account, '') = ''""",
root_type,
as_dict=1,
for root in frappe.get_all(
"Account",
filters={"root_type": root_type, "parent_account": ["is", "not set"]},
fields=["lft", "rgt"],
):
set_gl_entries_by_account(
company,

View File

@@ -0,0 +1,27 @@
# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
import frappe
from erpnext.accounts.report.cash_flow.cash_flow import execute
from erpnext.tests.utils import ERPNextTestSuite
class TestCashFlow(ERPNextTestSuite):
def test_report_executes(self):
# Smoke-guards the raw-SQL -> query-builder port: the report query must compile and run on
# both MariaDB and postgres.
company = frappe.db.get_value("Company", {}, "name")
fy = frappe.db.get_value("Fiscal Year", {}, "name", order_by="year_start_date desc")
columns, *_rest = execute(
frappe._dict(
{
"company": company,
"from_fiscal_year": fy,
"to_fiscal_year": fy,
"filter_based_on": "Fiscal Year",
"periodicity": "Yearly",
}
)
)
self.assertTrue(columns)

View File

@@ -347,11 +347,10 @@ def get_data(companies, root_type, balance_must_be, fiscal_year, filters=None, i
filters.end_date = end_date
gl_entries_by_account = {}
for root in frappe.db.sql(
"""select lft, rgt from tabAccount
where root_type=%s and ifnull(parent_account, '') = ''""",
root_type,
as_dict=1,
for root in frappe.get_all(
"Account",
filters={"root_type": root_type, "parent_account": ["is", "not set"]},
fields=["lft", "rgt"],
):
set_gl_entries_by_account(
start_date,
@@ -512,9 +511,11 @@ def get_companies(filters):
def get_subsidiary_companies(company):
lft, rgt = frappe.get_cached_value("Company", company, ["lft", "rgt"])
return frappe.db.sql_list(
f"""select name from `tabCompany`
where lft >= {lft} and rgt <= {rgt} order by lft, rgt"""
return frappe.get_all(
"Company",
filters={"lft": [">=", lft], "rgt": ["<=", rgt]},
pluck="name",
order_by="lft, rgt",
)
@@ -604,14 +605,10 @@ def set_gl_entries_by_account(
company_lft, company_rgt = frappe.get_cached_value("Company", filters.get("company"), ["lft", "rgt"])
companies = frappe.db.sql(
""" select name, default_currency from `tabCompany`
where lft >= %(company_lft)s and rgt <= %(company_rgt)s""",
{
"company_lft": company_lft,
"company_rgt": company_rgt,
},
as_dict=1,
companies = frappe.get_all(
"Company",
filters={"lft": [">=", company_lft], "rgt": ["<=", company_rgt]},
fields=["name", "default_currency"],
)
currency_info = frappe._dict(

View File

@@ -126,12 +126,22 @@ def get_data(filters) -> list[list]:
def get_company_wise_tb_data(filters, reporting_currency, ignore_reporting_currency):
accounts = frappe.db.sql(
"""select name, account_number, parent_account, account_name, root_type, report_type, account_type, is_group, lft, rgt
from `tabAccount` where company=%s order by lft""",
filters.company,
as_dict=True,
accounts = frappe.get_all(
"Account",
filters={"company": filters.company},
fields=[
"name",
"account_number",
"parent_account",
"account_name",
"root_type",
"report_type",
"account_type",
"is_group",
"lft",
"rgt",
],
order_by="lft",
)
ignore_is_opening = frappe.get_single_value("Accounts Settings", "ignore_is_opening_check_for_reporting")

View File

@@ -4,7 +4,7 @@
import frappe
from frappe import _
from frappe.utils import cstr, flt
from frappe.utils import flt
import erpnext
from erpnext.accounts.report.financial_statements import (
@@ -31,18 +31,23 @@ def execute(filters=None):
def get_data(filters, dimension_list):
company_currency = erpnext.get_company_currency(filters.company)
acc = frappe.db.sql(
"""
select
name, account_number, parent_account, lft, rgt, root_type,
report_type, account_name, include_in_gross, account_type, is_group
from
`tabAccount`
where
company=%s
order by lft""",
(filters.company),
as_dict=True,
acc = frappe.get_all(
"Account",
filters={"company": filters.company},
fields=[
"name",
"account_number",
"parent_account",
"lft",
"rgt",
"root_type",
"report_type",
"account_name",
"include_in_gross",
"account_type",
"is_group",
],
order_by="lft",
)
if not acc:
@@ -50,16 +55,17 @@ def get_data(filters, dimension_list):
accounts, accounts_by_name, parent_children_map = filter_accounts(acc)
min_lft, max_rgt = frappe.db.sql(
"""select min(lft), max(rgt) from `tabAccount`
where company=%s""",
(filters.company),
lft_rgt = frappe.get_all(
"Account",
filters={"company": filters.company},
fields=[{"MIN": "lft", "as": "min_lft"}, {"MAX": "rgt", "as": "max_rgt"}],
)[0]
min_lft, max_rgt = lft_rgt.min_lft, lft_rgt.max_rgt
account = frappe.db.sql_list(
"""select name from `tabAccount`
where lft >= %s and rgt <= %s and company = %s""",
(min_lft, max_rgt, filters.company),
account = frappe.get_all(
"Account",
filters={"lft": [">=", min_lft], "rgt": ["<=", max_rgt], "company": filters.company},
pluck="name",
)
gl_entries_by_account = {}
@@ -75,42 +81,34 @@ def get_data(filters, dimension_list):
def set_gl_entries_by_account(dimension_list, filters, account, gl_entries_by_account):
condition = get_condition(filters.get("dimension"))
if account:
condition += " and account in ({})".format(", ".join([frappe.db.escape(d) for d in account]))
dimension_field = frappe.scrub(filters.get("dimension"))
gl_filters = {
"company": filters.get("company"),
"from_date": filters.get("from_date"),
"to_date": filters.get("to_date"),
"finance_book": cstr(filters.get("finance_book")),
dimension_field: ["in", list(set(dimension_list))],
"posting_date": ["between", [filters.get("from_date"), filters.get("to_date")]],
"is_cancelled": 0,
}
if account:
gl_filters["account"] = ["in", account]
gl_filters["dimensions"] = tuple(set(dimension_list))
if filters.get("include_default_book_entries"):
gl_filters["company_fb"] = frappe.get_cached_value("Company", filters.company, "default_finance_book")
gl_entries = frappe.db.sql(
"""
select
posting_date, account, {dimension}, debit, credit, is_opening, fiscal_year,
debit_in_account_currency, credit_in_account_currency, account_currency
from
`tabGL Entry`
where
company=%(company)s
{condition}
and posting_date >= %(from_date)s
and posting_date <= %(to_date)s
and is_cancelled = 0
order by account, posting_date""".format(
dimension=frappe.scrub(filters.get("dimension")), condition=condition
),
gl_filters,
as_dict=True,
) # nosec
gl_entries = frappe.get_all(
"GL Entry",
filters=gl_filters,
fields=[
"posting_date",
"account",
dimension_field,
"debit",
"credit",
"is_opening",
"fiscal_year",
"debit_in_account_currency",
"credit_in_account_currency",
"account_currency",
],
order_by="account, posting_date",
)
for entry in gl_entries:
gl_entries_by_account.setdefault(entry.account, []).append(entry)
@@ -178,14 +176,6 @@ def accumulate_values_into_parents(accounts, accounts_by_name, dimension_list):
].get(frappe.scrub(dimension), 0.0) + d.get(frappe.scrub(dimension), 0.0)
def get_condition(dimension):
conditions = []
conditions.append(f"{frappe.scrub(dimension)} in %(dimensions)s")
return " and {}".format(" and ".join(conditions)) if conditions else ""
def get_dimensions(filters):
meta = frappe.get_meta(filters.get("dimension"), cached=False)
query_filters = {}

View File

@@ -179,11 +179,10 @@ def get_data(
company_currency = get_appropriate_currency(company, filters)
gl_entries_by_account = {}
for root in frappe.db.sql(
"""select lft, rgt from tabAccount
where root_type=%s and ifnull(parent_account, '') = ''""",
root_type,
as_dict=1,
for root in frappe.get_all(
"Account",
filters={"root_type": root_type, "parent_account": ["is", "not set"]},
fields=["lft", "rgt"],
):
set_gl_entries_by_account(
company,
@@ -373,13 +372,23 @@ def add_total_row(out, root_type, balance_must_be, period_list, company_currency
def get_accounts(company, root_type):
return frappe.db.sql(
"""
select name, account_number, parent_account, lft, rgt, root_type, report_type, account_name, include_in_gross, account_type, is_group, lft, rgt
from `tabAccount`
where company=%s and root_type=%s order by lft""",
(company, root_type),
as_dict=True,
return frappe.get_all(
"Account",
filters={"company": company, "root_type": root_type},
fields=[
"name",
"account_number",
"parent_account",
"lft",
"rgt",
"root_type",
"report_type",
"account_name",
"include_in_gross",
"account_type",
"is_group",
],
order_by="lft",
)
@@ -529,7 +538,11 @@ def get_accounting_entries(
gl_entry.credit_in_account_currency
if not group_by_account
else Sum(gl_entry.credit_in_account_currency).as_("credit_in_account_currency"),
gl_entry.account_currency,
# when grouping by account the non-aggregated columns must be aggregated for postgres;
# account_currency is constant per account so Max() returns the same value.
gl_entry.account_currency
if not group_by_account
else Max(gl_entry.account_currency).as_("account_currency"),
)
.where(gl_entry.company == filters.company)
)
@@ -547,15 +560,29 @@ def get_accounting_entries(
ignore_is_opening = frappe.get_single_value("Accounts Settings", "ignore_is_opening_check_for_reporting")
if doctype == "GL Entry":
query = query.select(gl_entry.posting_date, gl_entry.is_opening, gl_entry.fiscal_year)
# aggregate the non-grouped columns when grouping by account (postgres requirement)
if group_by_account:
query = query.select(
Max(gl_entry.posting_date).as_("posting_date"),
Max(gl_entry.is_opening).as_("is_opening"),
Max(gl_entry.fiscal_year).as_("fiscal_year"),
)
else:
query = query.select(gl_entry.posting_date, gl_entry.is_opening, gl_entry.fiscal_year)
query = query.where(gl_entry.is_cancelled == 0)
query = query.where(gl_entry.posting_date <= to_date)
query = query.force_index("posting_date_company_index")
# FORCE INDEX is MySQL-only; postgres has no index hints (its planner uses the index anyway)
if frappe.db.db_type != "postgres":
query = query.force_index("posting_date_company_index")
if ignore_opening_entries and not ignore_is_opening:
query = query.where(gl_entry.is_opening == "No")
else:
query = query.select(gl_entry.closing_date.as_("posting_date"))
query = query.select(
Max(gl_entry.closing_date).as_("posting_date")
if group_by_account
else gl_entry.closing_date.as_("posting_date")
)
query = query.where(gl_entry.period_closing_voucher == period_closing_voucher)
query = apply_additional_conditions(doctype, query, from_date, ignore_closing_entries, filters)

View File

@@ -35,7 +35,7 @@ def execute(filters=None):
if filters and filters.get("print_in_account_currency") and not filters.get("account"):
frappe.throw(_("Select an account to print in account currency"))
for acc in frappe.db.sql("""select name, is_group from tabAccount""", as_dict=1):
for acc in frappe.get_all("Account", fields=["name", "is_group"]):
account_details.setdefault(acc.name, acc)
if filters.get("party"):
@@ -650,10 +650,8 @@ def get_result_as_list(data, filters):
def get_supplier_invoice_details():
inv_details = {}
for d in frappe.db.sql(
""" select name, bill_no from `tabPurchase Invoice`
where docstatus = 1 and bill_no is not null and bill_no != '' """,
as_dict=1,
for d in frappe.get_all(
"Purchase Invoice", filters={"docstatus": 1, "bill_no": ["is", "set"]}, fields=["name", "bill_no"]
):
inv_details[d.name] = d.bill_no

View File

@@ -713,20 +713,25 @@ class GrossProfitGenerator:
)
def get_returned_invoice_items(self):
returned_invoices = frappe.db.sql(
"""
select
si.name, si_item.item_code, si_item.stock_qty as qty, si_item.base_net_amount as base_amount, si.return_against
from
`tabSales Invoice` si, `tabSales Invoice Item` si_item
where
si.name = si_item.parent
and si.docstatus = 1
and si.is_return = 1
and si.posting_date between %(from_date)s and %(to_date)s
""",
{"from_date": self.filters.from_date, "to_date": self.filters.to_date},
as_dict=1,
si = frappe.qb.DocType("Sales Invoice")
si_item = frappe.qb.DocType("Sales Invoice Item")
returned_invoices = (
frappe.qb.from_(si)
.inner_join(si_item)
.on(si.name == si_item.parent)
.select(
si.name,
si_item.item_code,
si_item.stock_qty.as_("qty"),
si_item.base_net_amount.as_("base_amount"),
si.return_against,
)
.where(
(si.docstatus == 1)
& (si.is_return == 1)
& si.posting_date.between(self.filters.from_date, self.filters.to_date)
)
.run(as_dict=1)
)
self.returned_invoices = frappe._dict()
@@ -1241,7 +1246,4 @@ class GrossProfitGenerator:
).setdefault(d.parent_item, []).append(d)
def load_non_stock_items(self):
self.non_stock_items = frappe.db.sql_list(
"""select name from tabItem
where is_stock_item=0"""
)
self.non_stock_items = frappe.get_all("Item", filters={"is_stock_item": 0}, pluck="name")

View File

@@ -62,15 +62,10 @@ def get_columns(filters):
def get_all_transfers(date, shareholder):
condition = " "
# if company:
# condition = 'AND company = %(company)s '
return frappe.db.sql(
f"""SELECT * FROM `tabShare Transfer`
WHERE ((DATE(date) <= %(date)s AND from_shareholder = %(shareholder)s {condition})
OR (DATE(date) <= %(date)s AND to_shareholder = %(shareholder)s {condition}))
AND docstatus = 1
ORDER BY date""",
{"date": date, "shareholder": shareholder},
as_dict=1,
return frappe.get_all(
"Share Transfer",
filters={"date": ["<=", date], "docstatus": 1},
or_filters=[["from_shareholder", "=", shareholder], ["to_shareholder", "=", shareholder]],
fields=["*"],
order_by="date",
)

View File

@@ -4,7 +4,7 @@
import frappe
from frappe import _
from frappe.query_builder.functions import Sum
from frappe.query_builder.functions import Max, Sum
from frappe.utils import add_days, cstr, flt, formatdate, getdate
import erpnext
@@ -82,12 +82,21 @@ def validate_filters(filters):
def get_data(filters):
accounts = frappe.db.sql(
"""select name, account_number, parent_account, account_name, root_type, report_type, is_group, lft, rgt
from `tabAccount` where company=%s order by lft""",
filters.company,
as_dict=True,
accounts = frappe.get_all(
"Account",
filters={"company": filters.company},
fields=[
"name",
"account_number",
"parent_account",
"account_name",
"root_type",
"report_type",
"is_group",
"lft",
"rgt",
],
order_by="lft",
)
company_currency = filters.presentation_currency or erpnext.get_company_currency(filters.company)
@@ -240,7 +249,8 @@ def get_opening_balance(
frappe.qb.from_(closing_balance)
.select(
closing_balance.account,
closing_balance.account_currency,
# account_currency is constant per grouped account -> Max() keeps the GROUP BY postgres-valid
Max(closing_balance.account_currency).as_("account_currency"),
Sum(closing_balance.debit).as_("debit"),
Sum(closing_balance.credit).as_("credit"),
Sum(closing_balance.debit_in_account_currency).as_("debit_in_account_currency"),