Merge pull request #56186 from mihir-kandoi/pg-stock-valuation-core

refactor(stock): port valuation-core helpers raw SQL to qb/ORM (Postgres compat)
This commit is contained in:
Mihir Kandoi
2026-06-20 19:10:11 +05:30
committed by GitHub
6 changed files with 310 additions and 185 deletions

View File

@@ -61,19 +61,20 @@ def get_warehouse_account(warehouse, warehouse_account=None):
rebuild_tree("Warehouse")
else:
account = frappe.db.sql(
"""
select
account from `tabWarehouse`
where
lft <= %s and rgt >= %s and company = %s
and account is not null and ifnull(account, '') !=''
order by lft desc limit 1""",
(warehouse.lft, warehouse.rgt, warehouse.company),
as_list=1,
account = frappe.get_all(
"Warehouse",
filters={
"lft": ["<=", warehouse.lft],
"rgt": [">=", warehouse.rgt],
"company": warehouse.company,
"account": ["is", "set"],
},
pluck="account",
order_by="lft desc",
limit=1,
)
account = account[0][0] if account else None
account = account[0] if account else None
if not account and warehouse.company:
account = get_company_default_inventory_account(warehouse.company)

View File

@@ -8,7 +8,7 @@ import frappe
from frappe import _
from frappe.core.doctype.role.role import get_users
from frappe.model.document import Document
from frappe.query_builder.functions import Sum
from frappe.query_builder.functions import Max, Sum
from frappe.utils import add_days, cint, flt, formatdate, get_datetime, getdate
from erpnext.accounts.utils import get_fiscal_year
@@ -311,13 +311,17 @@ class StockLedgerEntry(Document):
if authorized_role:
authorized_users = get_users(authorized_role)
if authorized_users and frappe.session.user not in authorized_users:
last_transaction_time = frappe.db.sql(
"""
select MAX(timestamp(posting_date, posting_time)) as posting_time
from `tabStock Ledger Entry`
where docstatus = 1 and is_cancelled = 0 and item_code = %s
and warehouse = %s""",
(self.item_code, self.warehouse),
sle = frappe.qb.DocType("Stock Ledger Entry")
last_transaction_time = (
frappe.qb.from_(sle)
.select(Max(sle.posting_datetime))
.where(
(sle.docstatus == 1)
& (sle.is_cancelled == 0)
& (sle.item_code == self.item_code)
& (sle.warehouse == self.warehouse)
)
.run()
)[0][0]
cur_doc_posting_datetime = "{} {}".format(

View File

@@ -19,15 +19,11 @@ def repost(only_actual=False, allow_negative_stock=False, allow_zero_rate=False,
existing_allow_negative_stock = frappe.get_single_value("Stock Settings", "allow_negative_stock")
frappe.db.set_single_value("Stock Settings", "allow_negative_stock", 1)
item_warehouses = frappe.db.sql(
"""
select distinct item_code, warehouse
from
(select item_code, warehouse from tabBin
union
select item_code, warehouse from `tabStock Ledger Entry`) a
"""
item_warehouses = frappe.get_all("Bin", fields=["item_code", "warehouse"], as_list=True)
item_warehouses += frappe.get_all(
"Stock Ledger Entry", fields=["item_code", "warehouse"], distinct=True, as_list=True
)
item_warehouses = list({tuple(d) for d in item_warehouses})
for d in item_warehouses:
try:
repost_stock(d[0], d[1], allow_zero_rate, only_actual, only_bin, allow_negative_stock)
@@ -79,102 +75,112 @@ def repost_actual_qty(item_code, warehouse, allow_zero_rate=False, allow_negativ
def get_balance_qty_from_sle(item_code, warehouse):
balance_qty = frappe.db.sql(
"""select qty_after_transaction from `tabStock Ledger Entry`
where item_code=%s and warehouse=%s and is_cancelled=0
order by posting_datetime desc, creation desc
limit 1""",
(item_code, warehouse),
balance_qty = frappe.get_all(
"Stock Ledger Entry",
filters={"item_code": item_code, "warehouse": warehouse, "is_cancelled": 0},
fields=["qty_after_transaction"],
order_by="posting_datetime desc, creation desc",
limit=1,
)
return flt(balance_qty[0][0]) if balance_qty else 0.0
return flt(balance_qty[0].qty_after_transaction) if balance_qty else 0.0
def get_reserved_qty(item_code, warehouse):
dont_reserve_on_return = frappe.get_cached_value(
"Selling Settings", "Selling Settings", "dont_reserve_sales_order_qty_on_sales_return"
)
reserved_qty = frappe.db.sql(
f"""
select
sum(dnpi_qty * ((so_item_qty - so_item_delivered_qty - (case when dont_reserve_qty_on_return = 1 then so_item_returned_qty else 0 end)) / so_item_qty))
from
(
(select
qty as dnpi_qty,
(
select qty from `tabSales Order Item`
where name = dnpi.parent_detail_docname
and (delivered_by_supplier is null or delivered_by_supplier = 0)
) as so_item_qty,
(
select delivered_qty from `tabSales Order Item`
where name = dnpi.parent_detail_docname
and delivered_by_supplier = 0
) as so_item_delivered_qty,
(
select returned_qty from `tabSales Order Item`
where name = dnpi.parent_detail_docname
and delivered_by_supplier = 0
) as so_item_returned_qty,
{dont_reserve_on_return} as dont_reserve_qty_on_return,
parent, name
from
(
select qty, parent_detail_docname, parent, name
from `tabPacked Item` dnpi_in
where item_code = %s and warehouse = %s
and parenttype='Sales Order'
and item_code != parent_item
and exists (select * from `tabSales Order` so
where name = dnpi_in.parent and docstatus = 1 and status not in ('On Hold', 'Closed'))
) dnpi)
union
(select stock_qty as dnpi_qty, qty as so_item_qty,
delivered_qty as so_item_delivered_qty,
returned_qty as so_item_returned_qty,
{dont_reserve_on_return}, parent, name
from `tabSales Order Item` so_item
where item_code = %s and warehouse = %s
and (so_item.delivered_by_supplier is null or so_item.delivered_by_supplier = 0)
and exists(select * from `tabSales Order` so
where so.name = so_item.parent and so.docstatus = 1
and so.status not in ('On Hold', 'Closed')))
) tab
where
so_item_qty >= so_item_delivered_qty
""",
(item_code, warehouse, item_code, warehouse),
so = frappe.qb.DocType("Sales Order")
so_item = frappe.qb.DocType("Sales Order Item")
packed_item = frappe.qb.DocType("Packed Item")
open_so = (so.docstatus == 1) & so.status.notin(["On Hold", "Closed"])
not_delivered_by_supplier = so_item.delivered_by_supplier.isnull() | (so_item.delivered_by_supplier == 0)
# Keep the reserved-qty rollup in the DB (one aggregate per branch) instead of streaming
# every open packed-item / SO-item row into Python. `qty <> 0` mirrors the original
# `where so_item_qty >= so_item_delivered_qty` *and* guards the divide-by-`qty` below
# (MariaDB returned NULL for x/0, postgres raises), so qty=0 rows -- which contributed
# nothing anyway -- are excluded on both databases.
reservable = (so_item.qty != 0) & (so_item.qty >= so_item.delivered_qty)
if dont_reserve_on_return:
net_reserved = so_item.qty - so_item.delivered_qty - so_item.returned_qty
else:
net_reserved = so_item.qty - so_item.delivered_qty
# Bundled (packed) items reserving stock against an open Sales Order
packed_qty = (
frappe.qb.from_(packed_item)
.inner_join(so)
.on(so.name == packed_item.parent)
.inner_join(so_item)
.on(so_item.name == packed_item.parent_detail_docname)
.select(Sum(packed_item.qty * net_reserved / so_item.qty))
.where(
(packed_item.item_code == item_code)
& (packed_item.warehouse == warehouse)
& (packed_item.parenttype == "Sales Order")
& (packed_item.item_code != packed_item.parent_item)
& not_delivered_by_supplier
& open_so
& reservable
)
.run()
)
return flt(reserved_qty[0][0]) if reserved_qty else 0
# Sales Order items directly reserving stock
so_item_qty = (
frappe.qb.from_(so_item)
.inner_join(so)
.on(so.name == so_item.parent)
.select(Sum(so_item.stock_qty * net_reserved / so_item.qty))
.where(
(so_item.item_code == item_code)
& (so_item.warehouse == warehouse)
& not_delivered_by_supplier
& open_so
& reservable
)
.run()
)
return flt(packed_qty[0][0]) + flt(so_item_qty[0][0])
def get_indented_qty(item_code, warehouse):
# Ordered Qty is always maintained in stock UOM
inward_qty = frappe.db.sql(
"""
select sum(mr_item.stock_qty - mr_item.ordered_qty)
from `tabMaterial Request Item` mr_item, `tabMaterial Request` mr
where mr_item.item_code=%s and mr_item.warehouse=%s
and mr.material_request_type in ('Purchase', 'Manufacture', 'Customer Provided', 'Material Transfer')
and mr_item.stock_qty > mr_item.ordered_qty and mr_item.parent=mr.name
and mr.status!='Stopped' and mr.docstatus=1
""",
(item_code, warehouse),
mr_item = frappe.qb.DocType("Material Request Item")
mr = frappe.qb.DocType("Material Request")
base_conditions = (
(mr_item.item_code == item_code)
& (mr_item.warehouse == warehouse)
& (mr_item.stock_qty > mr_item.ordered_qty)
& (mr.status != "Stopped")
& (mr.docstatus == 1)
)
inward_qty = (
frappe.qb.from_(mr_item)
.inner_join(mr)
.on(mr_item.parent == mr.name)
.select(Sum(mr_item.stock_qty - mr_item.ordered_qty))
.where(
base_conditions
& mr.material_request_type.isin(
["Purchase", "Manufacture", "Customer Provided", "Material Transfer"]
)
)
.run()
)
inward_qty = flt(inward_qty[0][0]) if inward_qty else 0
outward_qty = frappe.db.sql(
"""
select sum(mr_item.stock_qty - mr_item.ordered_qty)
from `tabMaterial Request Item` mr_item, `tabMaterial Request` mr
where mr_item.item_code=%s and mr_item.warehouse=%s
and mr.material_request_type = 'Material Issue'
and mr_item.stock_qty > mr_item.ordered_qty and mr_item.parent=mr.name
and mr.status!='Stopped' and mr.docstatus=1
""",
(item_code, warehouse),
outward_qty = (
frappe.qb.from_(mr_item)
.inner_join(mr)
.on(mr_item.parent == mr.name)
.select(Sum(mr_item.stock_qty - mr_item.ordered_qty))
.where(base_conditions & (mr.material_request_type == "Material Issue"))
.run()
)
outward_qty = flt(outward_qty[0][0]) if outward_qty else 0
@@ -248,12 +254,18 @@ def get_subcontracting_order_qty(item_code, warehouse):
def get_planned_qty(item_code, warehouse):
planned_qty = frappe.db.sql(
"""
select sum(qty - produced_qty) from `tabWork Order`
where production_item = %s and fg_warehouse = %s and status not in ('Stopped', 'Completed', 'Closed')
and docstatus=1 and qty > produced_qty""",
(item_code, warehouse),
wo = frappe.qb.DocType("Work Order")
planned_qty = (
frappe.qb.from_(wo)
.select(Sum(wo.qty - wo.produced_qty))
.where(
(wo.production_item == item_code)
& (wo.fg_warehouse == warehouse)
& wo.status.notin(["Stopped", "Completed", "Closed"])
& (wo.docstatus == 1)
& (wo.qty > wo.produced_qty)
)
.run()
)
return flt(planned_qty[0][0]) if planned_qty else 0
@@ -284,27 +296,32 @@ def set_stock_balance_as_per_serial_no(
if not posting_time:
posting_time = nowtime()
condition = " and item.name=%s" % frappe.db.escape(item_code, percent=False) if item_code else ""
bin = frappe.db.sql(
"""select bin.item_code, bin.warehouse, bin.actual_qty, item.stock_uom
from `tabBin` bin, tabItem item
where bin.item_code = item.name and item.has_serial_no = 1 %s"""
% condition
bin_dt = frappe.qb.DocType("Bin")
item = frappe.qb.DocType("Item")
query = (
frappe.qb.from_(bin_dt)
.inner_join(item)
.on(bin_dt.item_code == item.name)
.select(bin_dt.item_code, bin_dt.warehouse, bin_dt.actual_qty, item.stock_uom)
.where(item.has_serial_no == 1)
)
if item_code:
query = query.where(item.name == item_code)
bin = query.run()
for d in bin:
serial_nos = frappe.db.sql(
"""select count(name) from `tabSerial No`
where item_code=%s and warehouse=%s and docstatus < 2""",
(d[0], d[1]),
serial_nos = frappe.db.count(
"Serial No", {"item_code": d[0], "warehouse": d[1], "docstatus": ["<", 2]}
)
sle = frappe.db.sql(
"""select valuation_rate, company from `tabStock Ledger Entry`
where item_code = %s and warehouse = %s and is_cancelled = 0
order by posting_date desc limit 1""",
(d[0], d[1]),
sle = frappe.get_all(
"Stock Ledger Entry",
filters={"item_code": d[0], "warehouse": d[1], "is_cancelled": 0},
fields=["valuation_rate", "company"],
# total order so the latest SLE is picked identically on both engines (was posting_date only)
order_by="posting_date desc, creation desc, name desc",
limit=1,
as_list=True,
)
sle_dict = {
@@ -317,9 +334,9 @@ def set_stock_balance_as_per_serial_no(
"voucher_type": "Stock Reconciliation (Manual)",
"voucher_no": "",
"voucher_detail_no": "",
"actual_qty": flt(serial_nos[0][0]) - flt(d[2]),
"actual_qty": flt(serial_nos) - flt(d[2]),
"stock_uom": d[3],
"incoming_rate": sle and flt(serial_nos[0][0]) > flt(d[2]) and flt(sle[0][0]) or 0,
"incoming_rate": sle and flt(serial_nos) > flt(d[2]) and flt(sle[0][0]) or 0,
"company": sle and cstr(sle[0][1]) or 0,
"batch_no": "",
"serial_no": "",

View File

@@ -0,0 +1,51 @@
# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt
from erpnext.stock.stock_balance import get_indented_qty, get_reserved_qty
from erpnext.tests.utils import ERPNextTestSuite
class TestStockBalance(ERPNextTestSuite):
def test_get_reserved_qty_for_sales_order_item(self):
"""get_reserved_qty (converted from a UNION of correlated subqueries) must add a submitted
Sales Order's open qty for the direct SO-item branch. No delivery, so it stays clear of the
unrelated #39 SLE-repost path and runs on Postgres."""
from erpnext.selling.doctype.sales_order.test_sales_order import make_sales_order
item_code, warehouse = "_Test Item", "_Test Warehouse - _TC"
before = get_reserved_qty(item_code, warehouse)
make_sales_order(item_code=item_code, qty=10, warehouse=warehouse) # submitted
self.assertEqual(get_reserved_qty(item_code, warehouse), before + 10)
def test_get_reserved_qty_for_packed_bundle_item(self):
"""The packed-item branch of get_reserved_qty (the correlated-subquery -> inner_join rewrite)
must reserve the bundle component qty against an open Sales Order: 2 bundles x 3 per bundle = 6."""
from erpnext.selling.doctype.product_bundle.test_product_bundle import make_product_bundle
from erpnext.selling.doctype.sales_order.test_sales_order import make_sales_order
from erpnext.stock.doctype.item.test_item import make_item
warehouse = "_Test Warehouse - _TC"
bundle = make_item(properties={"is_stock_item": 0}).name
component = make_item(properties={"is_stock_item": 1}).name
make_product_bundle(bundle, [component], qty=3)
before = get_reserved_qty(component, warehouse)
make_sales_order(item_code=bundle, qty=2, warehouse=warehouse) # 2 x 3 = 6 component packed
self.assertEqual(get_reserved_qty(component, warehouse), before + 6)
def test_get_indented_qty_for_material_request(self):
"""get_indented_qty inward branch (comma-join -> qb inner_join) must reflect a submitted
Purchase Material Request's not-yet-ordered qty."""
from erpnext.stock.doctype.material_request.test_material_request import make_material_request
item_code, warehouse = "_Test Item", "_Test Warehouse - _TC"
before = get_indented_qty(item_code, warehouse)
make_material_request(item_code=item_code, qty=7, warehouse=warehouse) # Purchase, submitted
self.assertEqual(get_indented_qty(item_code, warehouse), before + 7)

View File

@@ -138,3 +138,59 @@ class TestStockUtilities(ERPNextTestSuite, StockTestMixin):
item_scan_with_ctx = scan_barcode("w12345", ctx=ctx)
self.assertEqual(item_scan_with_ctx["item_code"], item_with_warehouse.name)
self.assertEqual(item_scan_with_ctx["default_warehouse"], warehouse_2.name)
def test_get_latest_stock_qty(self):
"""get_latest_stock_qty (Sum(actual_qty) over Bin; the warehouse-subtree EXISTS converted to
a qb subquery) must reflect received stock for a non-group warehouse."""
from frappe.utils import flt
from erpnext.stock.doctype.stock_entry.stock_entry_utils import make_stock_entry
from erpnext.stock.utils import get_latest_stock_qty
warehouse = "_Test Warehouse - _TC"
item = self.make_item(properties={"is_stock_item": 1}).name
before = flt(get_latest_stock_qty(item, warehouse))
make_stock_entry(item_code=item, target=warehouse, qty=8, basic_rate=100)
self.assertEqual(flt(get_latest_stock_qty(item, warehouse)), before + 8)
def test_get_stock_value_from_bin(self):
"""get_stock_value_from_bin (comma-join -> inner_join, `ifnull(disabled,0)=0` ->
`disabled==0 | isnull`) must sum the Bin stock_value for an item."""
from frappe.utils import flt
from erpnext.stock.doctype.stock_entry.stock_entry_utils import make_stock_entry
from erpnext.stock.utils import get_stock_value_from_bin
warehouse = "_Test Warehouse - _TC"
item = self.make_item(properties={"is_stock_item": 1}).name
make_stock_entry(item_code=item, target=warehouse, qty=5, basic_rate=50)
# returns a single-row result set: [(stock_value,)]
self.assertEqual(flt(get_stock_value_from_bin(item_code=item)[0][0]), 5 * 50)
def test_get_avg_purchase_rate(self):
"""get_avg_purchase_rate must average Serial No purchase_rate via the dict-`AVG` get_all
field (frappe compiles `[{"AVG": "purchase_rate"}]` to `AVG(purchase_rate)` on both engines)."""
from frappe.utils import flt, random_string
from erpnext.stock.utils import get_avg_purchase_rate
item = self.make_item(properties={"is_stock_item": 1, "has_serial_no": 1}).name
serial_nos = []
for rate in (10, 30):
sn = "_TAVG" + random_string(8)
frappe.get_doc(
{
"doctype": "Serial No",
"serial_no": sn,
"item_code": item,
"company": "_Test Company",
"purchase_rate": rate,
}
).insert()
serial_nos.append(sn)
self.assertEqual(flt(get_avg_purchase_rate("\n".join(serial_nos))), 20.0)

View File

@@ -7,7 +7,8 @@ import json
import frappe
from frappe import _
from frappe.query_builder.functions import IfNull, Sum
from frappe.query_builder import Case
from frappe.query_builder.functions import Abs, IfNull, Sum
from frappe.utils import cstr, flt, get_link_to_form, get_time, getdate, nowdate, nowtime
from frappe.utils.data import DateTimeLikeObject
@@ -30,32 +31,32 @@ class PendingRepostingError(frappe.ValidationError):
def get_stock_value_from_bin(warehouse=None, item_code=None):
values = {}
conditions = ""
if warehouse:
conditions += """ and `tabBin`.warehouse in (
select w2.name from `tabWarehouse` w1
join `tabWarehouse` w2 on
w1.name = %(warehouse)s
and w2.lft between w1.lft and w1.rgt
) """
values["warehouse"] = warehouse
if item_code:
conditions += " and `tabBin`.item_code = %(item_code)s"
values["item_code"] = item_code
bin_dt = frappe.qb.DocType("Bin")
item = frappe.qb.DocType("Item")
query = (
"""select sum(stock_value) from `tabBin`, `tabItem` where 1 = 1
and `tabItem`.name = `tabBin`.item_code and ifnull(`tabItem`.disabled, 0) = 0 %s"""
% conditions
frappe.qb.from_(bin_dt)
.inner_join(item)
.on(item.name == bin_dt.item_code)
.select(Sum(bin_dt.stock_value))
.where((item.disabled == 0) | item.disabled.isnull())
)
stock_value = frappe.db.sql(query, values)
if warehouse:
w1 = frappe.qb.DocType("Warehouse").as_("w1")
w2 = frappe.qb.DocType("Warehouse").as_("w2")
descendants = (
frappe.qb.from_(w1)
.join(w2)
.on((w1.name == warehouse) & (w2.lft >= w1.lft) & (w2.lft <= w1.rgt))
.select(w2.name)
)
query = query.where(bin_dt.warehouse.isin(descendants))
return stock_value
if item_code:
query = query.where(bin_dt.item_code == item_code)
return query.run()
def get_stock_value_on(
@@ -177,36 +178,28 @@ def get_serial_nos_data(serial_nos):
@frappe.whitelist()
def get_latest_stock_qty(item_code: str, warehouse: str | None = None):
values, condition = [item_code], ""
bin_dt = frappe.qb.DocType("Bin")
query = frappe.qb.from_(bin_dt).select(Sum(bin_dt.actual_qty)).where(bin_dt.item_code == item_code)
if warehouse:
lft, rgt, is_group = frappe.db.get_value("Warehouse", warehouse, ["lft", "rgt", "is_group"])
if is_group:
values.extend([lft, rgt])
condition += "and exists (\
select name from `tabWarehouse` wh where wh.name = tabBin.warehouse\
and wh.lft >= %s and wh.rgt <= %s)"
wh = frappe.qb.DocType("Warehouse")
query = query.where(
bin_dt.warehouse.isin(
frappe.qb.from_(wh).select(wh.name).where((wh.lft >= lft) & (wh.rgt <= rgt))
)
)
else:
values.append(warehouse)
condition += " AND warehouse = %s"
query = query.where(bin_dt.warehouse == warehouse)
actual_qty = frappe.db.sql(
f"""select sum(actual_qty) from tabBin
where item_code=%s {condition}""",
values,
)[0][0]
return actual_qty
return query.run()[0][0]
def get_latest_stock_balance():
bin_map = {}
for d in frappe.db.sql(
"""SELECT item_code, warehouse, stock_value as stock_value
FROM tabBin""",
as_dict=1,
):
for d in frappe.get_all("Bin", fields=["item_code", "warehouse", "stock_value"]):
bin_map.setdefault(d.warehouse, {}).setdefault(d.item_code, flt(d.stock_value))
return bin_map
@@ -348,12 +341,9 @@ def get_avg_purchase_rate(serial_nos):
serial_nos = get_valid_serial_nos(serial_nos)
return flt(
frappe.db.sql(
"""select avg(purchase_rate) from `tabSerial No`
where name in (%s)"""
% ", ".join(["%s"] * len(serial_nos)),
tuple(serial_nos),
)[0][0]
frappe.get_all(
"Serial No", filters={"name": ["in", serial_nos]}, fields=[{"AVG": "purchase_rate", "as": "rate"}]
)[0].rate
)
@@ -520,13 +510,19 @@ def add_additional_uom_columns(columns, result, include_uom, conversion_factors)
def get_incoming_outgoing_rate_for_cancel(item_code, voucher_type, voucher_no, voucher_detail_no):
outgoing_rate = frappe.db.sql(
"""SELECT CASE WHEN actual_qty = 0 THEN 0 ELSE abs(stock_value_difference / actual_qty) END
FROM `tabStock Ledger Entry`
WHERE voucher_type = %s and voucher_no = %s
and item_code = %s and voucher_detail_no = %s
ORDER BY CREATION DESC limit 1""",
(voucher_type, voucher_no, item_code, voucher_detail_no),
sle = frappe.qb.DocType("Stock Ledger Entry")
outgoing_rate = (
frappe.qb.from_(sle)
.select(Case().when(sle.actual_qty == 0, 0).else_(Abs(sle.stock_value_difference / sle.actual_qty)))
.where(
(sle.voucher_type == voucher_type)
& (sle.voucher_no == voucher_no)
& (sle.item_code == item_code)
& (sle.voucher_detail_no == voucher_detail_no)
)
.orderby(sle.creation, order=frappe.qb.desc)
.limit(1)
.run()
)
outgoing_rate = outgoing_rate[0][0] if outgoing_rate else 0.0