From 2d24eedab2fcb5691e0c7c3a0e38afa0e3cdb280 Mon Sep 17 00:00:00 2001 From: Mihir Kandoi Date: Tue, 16 Jun 2026 21:50:36 +0530 Subject: [PATCH] refactor(manufacturing, projects): make raw SQL portable to PostgreSQL (parity rollout 2/9) (#56008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refactor(manufacturing, projects): make raw SQL portable to PostgreSQL Convert the MariaDB-only raw `frappe.db.sql` in the Manufacturing and Projects modules to the cross-database query builder / ORM, and fix the non-portable constructs that remain. Every change is a no-op on MariaDB (identical rendered SQL / identical results) and only brings PostgreSQL — standards-strict where MySQL is lax — in line. Areas: BOM (cost/where-used/explosion), Work Order (operations, required items, mapper, stock report), Workstation, Production Plan sub-assembly/explosion queries, BOM Stock Analysis / Process Loss / Work Order Stock reports; Projects (project, task, timesheet, activity cost, project update), Daily Timesheet Summary and Project-wise Stock Tracking reports. Part of the staged MariaDB<->PostgreSQL parity rollout (module 2 of 9). Co-authored-by: Claude Opus 4.8 (1M context) --- erpnext/manufacturing/doctype/bom/bom.py | 103 +++++++----- erpnext/manufacturing/doctype/bom/test_bom.py | 18 +- .../production_plan/services/bom_explosion.py | 67 ++++---- .../services/sub_assembly_queries.py | 35 ++-- .../doctype/work_order/mapper.py | 11 +- .../doctype/work_order/services/operations.py | 19 ++- .../work_order/services/required_items.py | 8 +- .../doctype/work_order/test_work_order.py | 9 +- .../doctype/work_order/work_order.py | 33 ++-- .../doctype/workstation/workstation.py | 57 ++++--- .../bom_stock_analysis/bom_stock_analysis.py | 27 +-- .../process_loss_report.py | 16 +- .../work_order_stock_report.py | 5 +- .../doctype/activity_cost/activity_cost.py | 20 ++- erpnext/projects/doctype/project/project.py | 154 +++++++++--------- .../projects/doctype/project/test_project.py | 8 +- .../doctype/project_update/project_update.py | 49 +++--- erpnext/projects/doctype/task/task.py | 99 +++++------ .../projects/doctype/timesheet/timesheet.py | 141 ++++++++-------- .../daily_timesheet_summary.py | 66 ++++---- .../project_wise_stock_tracking.py | 81 +++++---- erpnext/projects/utils.py | 31 ++-- 22 files changed, 590 insertions(+), 467 deletions(-) diff --git a/erpnext/manufacturing/doctype/bom/bom.py b/erpnext/manufacturing/doctype/bom/bom.py index ebe6e04ac03..8dc28edea57 100644 --- a/erpnext/manufacturing/doctype/bom/bom.py +++ b/erpnext/manufacturing/doctype/bom/bom.py @@ -9,7 +9,7 @@ import frappe from frappe import _, bold from frappe.model.document import Document from frappe.query_builder import Field -from frappe.query_builder.functions import Count, IfNull, Sum +from frappe.query_builder.functions import Count, IfNull, Max, Min, Sum from frappe.utils import cint, cstr, flt, get_link_to_form, parse_json from frappe.website.website_generator import WebsiteGenerator @@ -1194,7 +1194,9 @@ def _query_bom_items(bom, company, opts): t = _get_bom_item_tables(opts) query = _build_base_bom_items_query(bom, company, opts.qty, t) query, group_by = _add_bom_item_columns(query, t, bom, opts, track_semi_finished_goods) - return query.groupby(*group_by).orderby(Field("idx")).run(as_dict=True) + # qualify + aggregate idx: bare "idx" is ambiguous across the joined tables and isn't grouped + # (idx is unique per BOM item, so Min() preserves the original ordering) — needed for postgres + return query.groupby(*group_by).orderby(Min(t.bom_item.idx)).run(as_dict=True) def _get_bom_item_tables(opts): @@ -1228,17 +1230,20 @@ def _build_base_bom_items_query(bom, company, qty, t): .on((t.item_default.parent == t.item_doc.name) & (t.item_default.company == company)) .select( t.bom_item.item_code, - t.bom_item.idx, - t.item_doc.item_name, + # every non-grouped column here is functionally dependent on the grouped item_code + # (item attributes / the single BOM's project / per-item Item Default), so Max()/Min() + # returns the value MySQL picked arbitrarily while making the GROUP BY valid on postgres. + Min(t.bom_item.idx).as_("idx"), + Max(t.item_doc.item_name).as_("item_name"), (Sum(t.qty_field_col / IfNull(t.bom_doc.quantity, 1)) * qty).as_("qty"), - t.item_doc.image, - t.bom_doc.project, - t.item_doc.stock_uom, - t.item_doc.item_group, - t.item_doc.allow_alternative_item, - t.item_default.default_warehouse, - t.item_default.expense_account.as_("expense_account"), - t.item_default.buying_cost_center.as_("cost_center"), + Max(t.item_doc.image).as_("image"), + Max(t.bom_doc.project).as_("project"), + Max(t.item_doc.stock_uom).as_("stock_uom"), + Max(t.item_doc.item_group).as_("item_group"), + Max(t.item_doc.allow_alternative_item).as_("allow_alternative_item"), + Max(t.item_default.default_warehouse).as_("default_warehouse"), + Max(t.item_default.expense_account).as_("expense_account"), + Max(t.item_default.buying_cost_center).as_("cost_center"), ) .where((t.bom_item.docstatus < 2) & (t.bom_doc.name == bom)) ) @@ -1247,9 +1252,11 @@ def _build_base_bom_items_query(bom, company, qty, t): def _add_bom_item_columns(query, t, bom, opts, track_semi_finished_goods): is_stock_item = cint(not opts.include_non_stock_items) stock_item_condition = t.item_doc.is_stock_item.isin([1, is_stock_item]) - amount_col = (Sum(t.bom_item.stock_qty / IfNull(t.bom_doc.quantity, 1)) * t.bom_item.rate * opts.qty).as_( - "amount" - ) + # rate is constant per grouped item -> Max() keeps it out of the Sum (preserving the original + # Sum(...) * rate * qty arithmetic) while making the expression postgres-valid under GROUP BY. + amount_col = ( + Sum(t.bom_item.stock_qty / IfNull(t.bom_doc.quantity, 1)) * Max(t.bom_item.rate) * opts.qty + ).as_("amount") if cint(opts.fetch_exploded): return _add_exploded_item_columns(query, t, bom, amount_col, stock_item_condition) @@ -1267,13 +1274,16 @@ def _add_exploded_item_columns(query, t, bom, amount_col, stock_item_condition): .limit(1) ) + # non-grouped columns are constant per grouped item_code -> Max() preserves the value while + # keeping the GROUP BY postgres-valid; the correlated idx subquery references only item_code + # (a grouped column) so it stays valid and still overrides the explosion idx for display. query = query.select( - t.bom_item.source_warehouse, - t.bom_item.operation, - t.bom_item.include_item_in_manufacturing, - t.bom_item.description, - t.bom_item.rate, - t.bom_item.sourced_by_supplier, + Max(t.bom_item.source_warehouse).as_("source_warehouse"), + Max(t.bom_item.operation).as_("operation"), + Max(t.bom_item.include_item_in_manufacturing).as_("include_item_in_manufacturing"), + Max(t.bom_item.description).as_("description"), + Max(t.bom_item.rate).as_("rate"), + Max(t.bom_item.sourced_by_supplier).as_("sourced_by_supplier"), amount_col, idx_subquery.as_("idx"), ).where(stock_item_condition) @@ -1282,33 +1292,37 @@ def _add_exploded_item_columns(query, t, bom, amount_col, stock_item_condition): def _add_secondary_item_columns(query, t, stock_item_condition): + # non-grouped columns are constant per grouped item_code -> Max() keeps the GROUP BY valid on + # postgres while returning the same value MySQL picked arbitrarily. query = query.select( - t.item_doc.description, - t.bom_item.cost_allocation_per, - t.bom_item.process_loss_per, - t.bom_item.secondary_item_type, - t.bom_item.name, - t.bom_item.is_legacy, + Max(t.item_doc.description).as_("description"), + Max(t.bom_item.cost_allocation_per).as_("cost_allocation_per"), + Max(t.bom_item.process_loss_per).as_("process_loss_per"), + Max(t.bom_item.secondary_item_type).as_("secondary_item_type"), + Max(t.bom_item.name).as_("name"), + Max(t.bom_item.is_legacy).as_("is_legacy"), ).where(stock_item_condition) return query, [t.bom_item.item_code] def _add_normal_item_columns(query, t, amount_col, stock_item_condition, track_semi_finished_goods): + # non-grouped columns are constant per grouped item_code (+operation/operation_row_id) -> Max() + # keeps the GROUP BY valid on postgres while returning the value MySQL picked arbitrarily. query = query.select( - t.bom_item.rate, - t.bom_item.uom, - t.bom_item.conversion_factor, - t.bom_item.source_warehouse, - t.bom_item.operation, - t.bom_item.include_item_in_manufacturing, - t.bom_item.sourced_by_supplier, + Max(t.bom_item.rate).as_("rate"), + Max(t.bom_item.uom).as_("uom"), + Max(t.bom_item.conversion_factor).as_("conversion_factor"), + Max(t.bom_item.source_warehouse).as_("source_warehouse"), + Max(t.bom_item.operation).as_("operation"), + Max(t.bom_item.include_item_in_manufacturing).as_("include_item_in_manufacturing"), + Max(t.bom_item.sourced_by_supplier).as_("sourced_by_supplier"), amount_col, - t.bom_item.description, - t.bom_item.base_rate.as_("rate"), - t.bom_item.operation_row_id, - t.bom_item.is_phantom_item, - t.bom_item.bom_no, + Max(t.bom_item.description).as_("description"), + Max(t.bom_item.base_rate).as_("rate"), + Max(t.bom_item.operation_row_id).as_("operation_row_id"), + Max(t.bom_item.is_phantom_item).as_("is_phantom_item"), + Max(t.bom_item.bom_no).as_("bom_no"), ).where(stock_item_condition | (t.bom_item.is_phantom_item == 1)) if track_semi_finished_goods: @@ -1386,16 +1400,19 @@ def validate_bom_no(item, bom_no): def _bom_contains_item(bom, item): - item = item.lower() + # Lower-case only for the case-insensitive item_code comparisons; keep the original `item` + # for the Item lookup, whose name is case-sensitive on postgres (lower-casing it would miss + # the row and wrongly reject a variant's template BOM). + item_code = item.lower() for d in bom.items: - if d.item_code.lower() == item: + if d.item_code.lower() == item_code: return True for d in bom.secondary_items: - if d.item_code.lower() == item: + if d.item_code.lower() == item_code: return True return ( - bom.item.lower() == item + bom.item.lower() == item_code or bom.item.lower() == cstr(frappe.db.get_value("Item", item, "variant_of")).lower() ) diff --git a/erpnext/manufacturing/doctype/bom/test_bom.py b/erpnext/manufacturing/doctype/bom/test_bom.py index 041a3d3899e..1405e73f12b 100644 --- a/erpnext/manufacturing/doctype/bom/test_bom.py +++ b/erpnext/manufacturing/doctype/bom/test_bom.py @@ -97,10 +97,10 @@ class TestBOM(ERPNextTestSuite): update_cost_in_all_boms_in_test() # check if new valuation rate updated in all BOMs - for d in frappe.db.sql( - """select base_rate from `tabBOM Item` - where item_code='_Test Item 2' and docstatus=1 and parenttype='BOM'""", - as_dict=1, + for d in frappe.get_all( + "BOM Item", + filters={"item_code": "_Test Item 2", "docstatus": 1, "parenttype": "BOM"}, + fields=["base_rate"], ): self.assertEqual(d.base_rate, rm_base_rate + 10) @@ -881,12 +881,10 @@ def reset_item_valuation_rate(item_code, warehouse_list=None, qty=None, rate=Non warehouse_list = [warehouse_list] if not warehouse_list: - warehouse_list = frappe.db.sql_list( - """ - select warehouse from `tabBin` - where item_code=%s and actual_qty > 0 - """, - item_code, + warehouse_list = frappe.get_all( + "Bin", + filters={"item_code": item_code, "actual_qty": [">", 0]}, + pluck="warehouse", ) if not warehouse_list: diff --git a/erpnext/manufacturing/doctype/production_plan/services/bom_explosion.py b/erpnext/manufacturing/doctype/production_plan/services/bom_explosion.py index c40c9e1e05f..d0993980342 100644 --- a/erpnext/manufacturing/doctype/production_plan/services/bom_explosion.py +++ b/erpnext/manufacturing/doctype/production_plan/services/bom_explosion.py @@ -4,7 +4,7 @@ """BOM explosion helpers for Production Plan material planning.""" import frappe -from frappe.query_builder.functions import IfNull, Sum +from frappe.query_builder.functions import IfNull, Max, Min, Sum from erpnext.manufacturing.doctype.production_plan.services.planning_queries import get_uom_conversion_factor @@ -38,22 +38,25 @@ def _exploded_items_query(company, bom_no, include_non_stock_items, planned_qty) def _exploded_item_columns(bei, bom, item, item_default, item_uom, planned_qty): + # only item_code/stock_uom are grouped; the rest are functionally dependent on the grouped item + # or arbitrary per BOM Item on MySQL -> Max() keeps the GROUP BY valid on postgres with the same + # value MySQL picked. return [ (IfNull(Sum(bei.stock_qty / IfNull(bom.quantity, 1)), 0) * planned_qty).as_("qty"), - item.item_name, - item.name.as_("item_code"), - bei.description, + Max(item.item_name).as_("item_name"), + Max(item.name).as_("item_code"), + Max(bei.description).as_("description"), bei.stock_uom, - item.min_order_qty, - bei.source_warehouse, - item.default_material_request_type, - item.min_order_qty, - item_default.default_warehouse, - item.purchase_uom, - item_uom.conversion_factor, - item.safety_stock, - bom.item.as_("main_bom_item"), - bom.name.as_("main_bom"), + Max(item.min_order_qty).as_("min_order_qty"), + Max(bei.source_warehouse).as_("source_warehouse"), + Max(item.default_material_request_type).as_("default_material_request_type"), + Max(item.min_order_qty).as_("min_order_qty"), + Max(item_default.default_warehouse).as_("default_warehouse"), + Max(item.purchase_uom).as_("purchase_uom"), + Max(item_uom.conversion_factor).as_("conversion_factor"), + Max(item.safety_stock).as_("safety_stock"), + Max(bom.item).as_("main_bom_item"), + Max(bom.name).as_("main_bom"), ] @@ -106,30 +109,34 @@ def _subitems_query(company, bom_no, include_non_stock_items, parent_qty, planne .select(*_subitem_columns(bom_item, bom, item, item_default, item_uom, parent_qty, planned_qty)) .where(_subitem_filter(bom_item, bom, item, bom_no, include_non_stock_items)) .groupby(bom_item.item_code) - .orderby(bom_item.idx) + # idx is not grouped; Min() preserves the original ordering and is valid on postgres + .orderby(Min(bom_item.idx)) ).run(as_dict=True) def _subitem_columns(bom_item, bom, item, item_default, item_uom, parent_qty, planned_qty): qty = IfNull(parent_qty * Sum(bom_item.stock_qty / IfNull(bom.quantity, 1)) * planned_qty, 0).as_("qty") + # only item_code is grouped; the rest are functionally dependent on the grouped item (item + # attributes) or arbitrary per BOM Item on MySQL -> Max() keeps the GROUP BY valid on postgres + # while returning the same value MySQL picked. return [ bom_item.item_code, - item.default_material_request_type, - item.item_name, + Max(item.default_material_request_type).as_("default_material_request_type"), + Max(item.item_name).as_("item_name"), qty, - item.is_sub_contracted_item.as_("is_sub_contracted"), - bom_item.source_warehouse, - item.default_bom.as_("default_bom"), - bom_item.description.as_("description"), - bom_item.stock_uom.as_("stock_uom"), - item.min_order_qty.as_("min_order_qty"), - item.safety_stock.as_("safety_stock"), - item_default.default_warehouse, - item.purchase_uom, - item_uom.conversion_factor, - bom.item.as_("main_bom_item"), - bom.name.as_("main_bom"), - bom_item.is_phantom_item, + Max(item.is_sub_contracted_item).as_("is_sub_contracted"), + Max(bom_item.source_warehouse).as_("source_warehouse"), + Max(item.default_bom).as_("default_bom"), + Max(bom_item.description).as_("description"), + Max(bom_item.stock_uom).as_("stock_uom"), + Max(item.min_order_qty).as_("min_order_qty"), + Max(item.safety_stock).as_("safety_stock"), + Max(item_default.default_warehouse).as_("default_warehouse"), + Max(item.purchase_uom).as_("purchase_uom"), + Max(item_uom.conversion_factor).as_("conversion_factor"), + Max(bom.item).as_("main_bom_item"), + Max(bom.name).as_("main_bom"), + Max(bom_item.is_phantom_item).as_("is_phantom_item"), ] diff --git a/erpnext/manufacturing/doctype/production_plan/services/sub_assembly_queries.py b/erpnext/manufacturing/doctype/production_plan/services/sub_assembly_queries.py index c4e25cffae5..a86c24521ab 100644 --- a/erpnext/manufacturing/doctype/production_plan/services/sub_assembly_queries.py +++ b/erpnext/manufacturing/doctype/production_plan/services/sub_assembly_queries.py @@ -4,7 +4,7 @@ """Sub-assembly resolution helpers for Production Plan.""" import frappe -from frappe.query_builder.functions import IfNull, Sum +from frappe.query_builder.functions import IfNull, Max, Sum from frappe.utils import flt from erpnext.manufacturing.doctype.bom.bom import get_children as get_bom_children @@ -184,24 +184,27 @@ def _sub_assembly_rm_query(company, bom_no, include_non_stock_items, planned_qty def _sub_assembly_rm_columns(bei, bom, item, item_default, item_uom, planned_qty): + # only item_code/stock_uom are grouped; every other column is functionally dependent on the + # grouped item (item attributes) or arbitrary per BOM Item on MySQL -> Max() keeps the GROUP BY + # valid on postgres while returning the same value MySQL picked. return [ (IfNull(Sum(bei.stock_qty / IfNull(bom.quantity, 1)), 0) * planned_qty).as_("qty"), - item.item_name, - item.name.as_("item_code"), - bei.description, + Max(item.item_name).as_("item_name"), + Max(item.name).as_("item_code"), + Max(bei.description).as_("description"), bei.stock_uom, - bei.is_phantom_item, - bei.bom_no, - item.min_order_qty, - bei.source_warehouse, - item.default_material_request_type, - item.min_order_qty, - item_default.default_warehouse, - item.purchase_uom, - item_uom.conversion_factor, - item.safety_stock, - bom.item.as_("main_bom_item"), - bom.name.as_("main_bom"), + Max(bei.is_phantom_item).as_("is_phantom_item"), + Max(bei.bom_no).as_("bom_no"), + Max(item.min_order_qty).as_("min_order_qty"), + Max(bei.source_warehouse).as_("source_warehouse"), + Max(item.default_material_request_type).as_("default_material_request_type"), + Max(item.min_order_qty).as_("min_order_qty"), + Max(item_default.default_warehouse).as_("default_warehouse"), + Max(item.purchase_uom).as_("purchase_uom"), + Max(item_uom.conversion_factor).as_("conversion_factor"), + Max(item.safety_stock).as_("safety_stock"), + Max(bom.item).as_("main_bom_item"), + Max(bom.name).as_("main_bom"), ] diff --git a/erpnext/manufacturing/doctype/work_order/mapper.py b/erpnext/manufacturing/doctype/work_order/mapper.py index 844952330b0..05ae74df42f 100644 --- a/erpnext/manufacturing/doctype/work_order/mapper.py +++ b/erpnext/manufacturing/doctype/work_order/mapper.py @@ -56,11 +56,12 @@ def _item_master_details(item): def _item_is_alive(item_table): - return ( - item_table.end_of_life.isnull() - | (item_table.end_of_life == "0000-00-00") - | (item_table.end_of_life > nowdate()) - ) + # "not set" end_of_life is NULL on postgres (the MariaDB zero-date '0000-00-00' is an invalid + # date constant there), so only add the zero-date term on MariaDB. + is_alive = item_table.end_of_life.isnull() | (item_table.end_of_life > nowdate()) + if frappe.db.db_type != "postgres": + is_alive |= item_table.end_of_life == "0000-00-00" + return is_alive def _default_bom_for_item(item, project): diff --git a/erpnext/manufacturing/doctype/work_order/services/operations.py b/erpnext/manufacturing/doctype/work_order/services/operations.py index e3249c31204..26bd7ee73e5 100644 --- a/erpnext/manufacturing/doctype/work_order/services/operations.py +++ b/erpnext/manufacturing/doctype/work_order/services/operations.py @@ -11,6 +11,7 @@ are called from other modules. import frappe from dateutil.relativedelta import relativedelta from frappe import _ +from frappe.query_builder.functions import CombineDatetime from frappe.utils import ( cint, date_diff, @@ -268,13 +269,17 @@ class OperationsService: self.doc.actual_end_date = max(end_dates) def _set_dates_from_stock_entries(self): - data = frappe.get_all( - "Stock Entry", - fields=[{"TIMESTAMP": ["posting_date", "posting_time"], "as": "posting_datetime"}], - filters={ - "work_order": self.doc.name, - "purpose": ("in", ["Material Transfer for Manufacture", "Manufacture"]), - }, + # {"TIMESTAMP": [...]} renders MySQL's TIMESTAMP(date, time), invalid on postgres; use the + # portable CombineDatetime via query builder instead. + se = frappe.qb.DocType("Stock Entry") + data = ( + frappe.qb.from_(se) + .select(CombineDatetime(se.posting_date, se.posting_time).as_("posting_datetime")) + .where( + (se.work_order == self.doc.name) + & (se.purpose.isin(["Material Transfer for Manufacture", "Manufacture"])) + ) + .run(as_dict=True) ) if not data: return diff --git a/erpnext/manufacturing/doctype/work_order/services/required_items.py b/erpnext/manufacturing/doctype/work_order/services/required_items.py index 26c9cf155b1..a0e35767de9 100644 --- a/erpnext/manufacturing/doctype/work_order/services/required_items.py +++ b/erpnext/manufacturing/doctype/work_order/services/required_items.py @@ -158,7 +158,13 @@ class RequiredItemsService: frappe.qb.from_(ste) .inner_join(ste_child) .on(ste_child.parent == ste.name) - .select(ste_child.item_code, ste_child.original_item, fn.Sum(ste_child.transfer_qty).as_("qty")) + # original_item is arbitrary per grouped item_code on MySQL -> Max() keeps the GROUP BY valid + # on postgres while returning the same value (it is only used as a dict key fallback below) + .select( + ste_child.item_code, + fn.Max(ste_child.original_item).as_("original_item"), + fn.Sum(ste_child.transfer_qty).as_("qty"), + ) .where(self._material_transfer_filter(ste, is_return)) .groupby(ste_child.item_code) ) diff --git a/erpnext/manufacturing/doctype/work_order/test_work_order.py b/erpnext/manufacturing/doctype/work_order/test_work_order.py index 6182756ef4e..dab925c8f95 100644 --- a/erpnext/manufacturing/doctype/work_order/test_work_order.py +++ b/erpnext/manufacturing/doctype/work_order/test_work_order.py @@ -5136,11 +5136,10 @@ def update_job_card(job_card, jc_qty=None, days=None): def get_secondary_item_details(bom_no): secondary_items = {} - for item in frappe.db.sql( - """select item_code, stock_qty from `tabBOM Secondary Item` - where parent = %s""", - bom_no, - as_dict=1, + for item in frappe.get_all( + "BOM Secondary Item", + filters={"parent": bom_no}, + fields=["item_code", "stock_qty"], ): secondary_items[item.item_code] = item.stock_qty diff --git a/erpnext/manufacturing/doctype/work_order/work_order.py b/erpnext/manufacturing/doctype/work_order/work_order.py index aaa82c9bf74..8191e047d85 100644 --- a/erpnext/manufacturing/doctype/work_order/work_order.py +++ b/erpnext/manufacturing/doctype/work_order/work_order.py @@ -843,15 +843,16 @@ class WorkOrder(Document): frappe.throw(_("Stopped Work Order cannot be cancelled, Unstop it first to cancel")) # Check whether any stock entry exists against this Work Order - stock_entry = frappe.db.sql( - """select name from `tabStock Entry` - where work_order = %s and docstatus = 1""", - self.name, + stock_entry = frappe.get_all( + "Stock Entry", + filters={"work_order": self.name, "docstatus": 1}, + pluck="name", + limit=1, ) if stock_entry: frappe.throw( _("Cannot cancel because submitted Stock Entry {0} exists").format( - frappe.utils.get_link_to_form("Stock Entry", stock_entry[0][0]) + frappe.utils.get_link_to_form("Stock Entry", stock_entry[0]) ) ) @@ -942,14 +943,20 @@ class WorkOrder(Document): @frappe.whitelist() def make_bom(self): - data = frappe.db.sql( - """ select sed.item_code, sed.qty, sed.s_warehouse - from `tabStock Entry Detail` sed, `tabStock Entry` se - where se.name = sed.parent and se.purpose = 'Manufacture' - and (sed.t_warehouse is null or sed.t_warehouse = '') and se.docstatus = 1 - and se.work_order = %s""", - (self.name), - as_dict=1, + sed = frappe.qb.DocType("Stock Entry Detail") + se = frappe.qb.DocType("Stock Entry") + data = ( + frappe.qb.from_(sed) + .inner_join(se) + .on(se.name == sed.parent) + .select(sed.item_code, sed.qty, sed.s_warehouse) + .where( + (se.purpose == "Manufacture") + & (sed.t_warehouse.isnull() | (sed.t_warehouse == "")) + & (se.docstatus == 1) + & (se.work_order == self.name) + ) + .run(as_dict=1) ) bom = frappe.new_doc("BOM") diff --git a/erpnext/manufacturing/doctype/workstation/workstation.py b/erpnext/manufacturing/doctype/workstation/workstation.py index 3ade994a82f..93776503872 100644 --- a/erpnext/manufacturing/doctype/workstation/workstation.py +++ b/erpnext/manufacturing/doctype/workstation/workstation.py @@ -169,15 +169,20 @@ class Workstation(Document): def validate_overlap_for_operation_timings(self): """Check if there is no overlap in setting Workstation Operating Hours""" for d in self.get("working_hours"): - existing = frappe.db.sql_list( - """select idx from `tabWorkstation Working Hour` - where parent = %s and name != %s - and ( - (start_time between %s and %s) or - (end_time between %s and %s) or - (%s between start_time and end_time)) - """, - (self.name, d.name, d.start_time, d.end_time, d.start_time, d.end_time, d.start_time), + wh = frappe.qb.DocType("Workstation Working Hour") + existing = ( + frappe.qb.from_(wh) + .select(wh.idx) + .where( + (wh.parent == self.name) + & (wh.name != d.name) + & ( + wh.start_time.between(d.start_time, d.end_time) + | wh.end_time.between(d.start_time, d.end_time) + | ((wh.start_time <= d.start_time) & (wh.end_time >= d.start_time)) + ) + ) + .run(pluck=True) ) if existing: @@ -187,17 +192,20 @@ class Workstation(Document): ) def update_bom_operation(self): - bom_list = frappe.db.sql( - """select DISTINCT parent from `tabBOM Operation` - where workstation = %s and parenttype = 'routing' """, - self.name, + bom_list = frappe.get_all( + "BOM Operation", + filters={"workstation": self.name, "parenttype": "routing"}, + pluck="parent", + distinct=True, ) - for bom_no in bom_list: - frappe.db.sql( - """update `tabBOM Operation` set hour_rate = %s - where parent = %s and workstation = %s""", - (self.hour_rate, bom_no[0], self.name), + if bom_list: + bom_op = frappe.qb.DocType("BOM Operation") + ( + frappe.qb.update(bom_op) + .set(bom_op.hour_rate, self.hour_rate) + .where(bom_op.parent.isin(bom_list) & (bom_op.workstation == self.name)) + .run() ) def validate_workstation_holiday(self, schedule_date, skip_holiday_list_check=False): @@ -451,12 +459,15 @@ def check_workstation_for_holiday(workstation, from_datetime, to_datetime): holiday_list = frappe.db.get_value("Workstation", workstation, "holiday_list") if holiday_list and from_datetime and to_datetime: applicable_holidays = [] - for d in frappe.db.sql( - """select holiday_date from `tabHoliday` where parent = %s - and holiday_date between %s and %s """, - (holiday_list, getdate(from_datetime), getdate(to_datetime)), + for holiday_date in frappe.get_all( + "Holiday", + filters={ + "parent": holiday_list, + "holiday_date": ["between", [getdate(from_datetime), getdate(to_datetime)]], + }, + pluck="holiday_date", ): - applicable_holidays.append(formatdate(d[0])) + applicable_holidays.append(formatdate(holiday_date)) if applicable_holidays: frappe.throw( diff --git a/erpnext/manufacturing/report/bom_stock_analysis/bom_stock_analysis.py b/erpnext/manufacturing/report/bom_stock_analysis/bom_stock_analysis.py index 568fdf90054..95522adfdb6 100644 --- a/erpnext/manufacturing/report/bom_stock_analysis/bom_stock_analysis.py +++ b/erpnext/manufacturing/report/bom_stock_analysis/bom_stock_analysis.py @@ -3,7 +3,7 @@ import frappe from frappe import _ -from frappe.query_builder.functions import Floor, IfNull, Sum +from frappe.query_builder.functions import Floor, IfNull, Max, Min, Sum from frappe.utils import flt from frappe.utils.data import comma_and from pypika.terms import ExistsCriterion @@ -202,14 +202,15 @@ def get_bom_data(filters): .on(bom_item.item_code == bin.item_code) .select( bom_item.item_code, - bom_item.description, - bom_item.parent.as_("from_bom_no"), + # non-grouped columns are constant per grouped item_code -> Max() keeps the GROUP BY valid + Max(bom_item.description).as_("description"), + Max(bom_item.parent).as_("from_bom_no"), Sum(bom_item.qty_consumed_per_unit).as_("qty_per_unit"), IfNull(Sum(bin.actual_qty), 0).as_("actual_qty"), ) .where((bom_item.parent == filters.get("bom")) & (bom_item.parenttype == "BOM")) .groupby(bom_item.item_code) - .orderby(bom_item.idx) + .orderby(Min(bom_item.idx)) ) if filters.get("warehouse"): @@ -233,7 +234,9 @@ def get_bom_data(filters): query = query.where(bin.warehouse == filters.get("warehouse")) if bom_item_table == "BOM Item": - query = query.select(bom_item.bom_no, bom_item.is_phantom_item) + query = query.select( + Max(bom_item.bom_no).as_("bom_no"), Max(bom_item.is_phantom_item).as_("is_phantom_item") + ) data = query.run(as_dict=True) return explode_phantom_boms(data, filters) if bom_item_table == "BOM Item" else data @@ -312,15 +315,17 @@ def get_producible_fg_items(filters): .on(BOM_ITEM.item_code == bin_subquery.item_code) .select( BOM_ITEM.item_code, - BOM_ITEM.description, - BOM_ITEM.parent.as_("from_bom_no"), - (BOM_ITEM.stock_qty / BOM.quantity).as_("qty_per_unit"), - IfNull(bin_subquery.actual_qty, 0).as_("available_qty"), - Floor(bin_subquery.actual_qty / ((Sum(BOM_ITEM.stock_qty)) / BOM.quantity)), + # Sum() below makes this an aggregate query; the other columns are constant per grouped + # item_code -> Max() keeps them valid on postgres with the same value MySQL picked. + Max(BOM_ITEM.description).as_("description"), + Max(BOM_ITEM.parent).as_("from_bom_no"), + Max(BOM_ITEM.stock_qty / BOM.quantity).as_("qty_per_unit"), + Max(IfNull(bin_subquery.actual_qty, 0)).as_("available_qty"), + Floor(Max(bin_subquery.actual_qty) / ((Sum(BOM_ITEM.stock_qty)) / Max(BOM.quantity))), ) .where((BOM_ITEM.parent == filters.get("bom")) & (BOM_ITEM.parenttype == "BOM")) .groupby(BOM_ITEM.item_code) - .orderby(BOM_ITEM.idx) + .orderby(Min(BOM_ITEM.idx)) ) return query.run(as_list=True) diff --git a/erpnext/manufacturing/report/process_loss_report/process_loss_report.py b/erpnext/manufacturing/report/process_loss_report/process_loss_report.py index 73560dd939b..2ba9f4742fd 100644 --- a/erpnext/manufacturing/report/process_loss_report/process_loss_report.py +++ b/erpnext/manufacturing/report/process_loss_report/process_loss_report.py @@ -4,7 +4,7 @@ import frappe from frappe import _ -from frappe.query_builder.functions import Sum +from frappe.query_builder.functions import Max, Sum Filters = frappe._dict Row = frappe._dict @@ -29,12 +29,14 @@ def get_data(filters: Filters) -> Data: .inner_join(se) .on(wo.name == se.work_order) .select( - wo.name, - wo.status, - wo.production_item, - wo.produced_qty, - wo.process_loss_qty, - wo.qty.as_("qty_to_manufacture"), + # grouped by se.work_order (== wo.name); the work-order columns are constant per group -> + # Max() keeps the GROUP BY valid on postgres with the same value. + Max(wo.name).as_("name"), + Max(wo.status).as_("status"), + Max(wo.production_item).as_("production_item"), + Max(wo.produced_qty).as_("produced_qty"), + Max(wo.process_loss_qty).as_("process_loss_qty"), + Max(wo.qty).as_("qty_to_manufacture"), Sum(se.total_incoming_value).as_("total_fg_value"), Sum(se.total_outgoing_value).as_("total_rm_value"), ) diff --git a/erpnext/manufacturing/report/work_order_stock_report/work_order_stock_report.py b/erpnext/manufacturing/report/work_order_stock_report/work_order_stock_report.py index 998b0e4bccd..0637b14ccc7 100644 --- a/erpnext/manufacturing/report/work_order_stock_report/work_order_stock_report.py +++ b/erpnext/manufacturing/report/work_order_stock_report/work_order_stock_report.py @@ -47,7 +47,10 @@ def get_item_list(wo_list, filters): & (bom_item.item_code == wo_item_details.item_code) & (bom.name == wo_details.bom_no) ) - .groupby(bom_item.item_code) + # build_qty multiplies columns from bin/bom/bom_item that aren't functionally + # dependent on the grouped item_code, so postgres requires them in the GROUP BY. + # The WHERE pins bom, item and warehouse to single rows, so this stays one row. + .groupby(bom_item.item_code, bom.quantity, bom_item.stock_qty, bin.actual_qty) ).run(as_dict=1) stock_qty = 0 diff --git a/erpnext/projects/doctype/activity_cost/activity_cost.py b/erpnext/projects/doctype/activity_cost/activity_cost.py index 257bcc42513..c90aa149c42 100644 --- a/erpnext/projects/doctype/activity_cost/activity_cost.py +++ b/erpnext/projects/doctype/activity_cost/activity_cost.py @@ -43,9 +43,13 @@ class ActivityCost(Document): def check_unique(self): if self.employee: - if frappe.db.sql( - """select name from `tabActivity Cost` where employee_name= %s and activity_type= %s and name != %s""", - (self.employee_name, self.activity_type, self.name), + if frappe.db.exists( + "Activity Cost", + { + "employee_name": self.employee_name, + "activity_type": self.activity_type, + "name": ["!=", self.name], + }, ): frappe.throw( _("Activity Cost exists for Employee {0} against Activity Type - {1}").format( @@ -54,9 +58,13 @@ class ActivityCost(Document): DuplicationError, ) else: - if frappe.db.sql( - """select name from `tabActivity Cost` where ifnull(employee, '')='' and activity_type= %s and name != %s""", - (self.activity_type, self.name), + if frappe.db.exists( + "Activity Cost", + { + "employee": ["is", "not set"], + "activity_type": self.activity_type, + "name": ["!=", self.name], + }, ): frappe.throw( _("Default Activity Cost exists for Activity Type - {0}").format(self.activity_type), diff --git a/erpnext/projects/doctype/project/project.py b/erpnext/projects/doctype/project/project.py index 081ac5dd96c..c5114d44c7a 100644 --- a/erpnext/projects/doctype/project/project.py +++ b/erpnext/projects/doctype/project/project.py @@ -4,15 +4,14 @@ import frappe from email_reply_parser import EmailReplyParser from frappe import _, qb -from frappe.desk.reportview import get_match_cond from frappe.model.document import Document -from frappe.query_builder import Interval -from frappe.query_builder.functions import Count, CurDate, Date, Sum, UnixTimestamp +from frappe.query_builder import Case, Interval +from frappe.query_builder.functions import Count, CurDate, Date, Locate, Sum, UnixTimestamp from frappe.utils import add_days, flt, get_datetime, get_link_to_form, get_time, nowtime, today from frappe.utils.user import is_website_user +from pypika import Order from erpnext import get_default_company -from erpnext.controllers.queries import get_filters_cond from erpnext.controllers.website_list_for_contact import get_customers_suppliers from erpnext.setup.doctype.holiday_list.holiday_list import is_holiday @@ -74,16 +73,15 @@ class Project(Document): # end: auto-generated types def onload(self): + timesheet_detail = frappe.qb.DocType("Timesheet Detail") self.set_onload( "activity_summary", - frappe.db.sql( - """select activity_type, - sum(hours) as total_hours - from `tabTimesheet Detail` where project=%s and docstatus < 2 group by activity_type - order by total_hours desc""", - self.name, - as_dict=True, - ), + frappe.qb.from_(timesheet_detail) + .select(timesheet_detail.activity_type, Sum(timesheet_detail.hours).as_("total_hours")) + .where((timesheet_detail.project == self.name) & (timesheet_detail.docstatus < 2)) + .groupby(timesheet_detail.activity_type) + .orderby("total_hours", order=frappe.qb.desc) + .run(as_dict=True), ) def before_print(self, settings=None): @@ -102,7 +100,7 @@ class Project(Document): """ Copy tasks from template """ - if self.project_template and not frappe.db.get_all("Task", dict(project=self.name), limit=1): + if self.project_template and not frappe.db.exists("Task", {"project": self.name}): # has a template, and no loaded tasks, so lets create if not self.expected_start_date: # project starts today @@ -267,32 +265,25 @@ class Project(Document): if (self.percent_complete_method == "Task Completion" and total > 0) or ( not self.percent_complete_method and total > 0 ): - completed = frappe.db.sql( - """select count(name) from tabTask where - project=%s and status in ('Cancelled', 'Completed')""", - self.name, - )[0][0] + completed = frappe.db.count( + "Task", {"project": self.name, "status": ["in", ["Cancelled", "Completed"]]} + ) self.percent_complete = flt(flt(completed) / total * 100, 2) if self.percent_complete_method == "Task Progress" and total > 0: - progress = frappe.db.sql( - """select sum(progress) from tabTask where - project=%s""", - self.name, + task = frappe.qb.DocType("Task") + progress = ( + frappe.qb.from_(task).select(Sum(task.progress)).where(task.project == self.name).run() )[0][0] self.percent_complete = flt(flt(progress) / total, 2) if self.percent_complete_method == "Task Weight" and total > 0: - weight_sum = frappe.db.sql( - """select sum(task_weight) from tabTask where - project=%s""", - self.name, + task = frappe.qb.DocType("Task") + weight_sum = ( + frappe.qb.from_(task).select(Sum(task.task_weight)).where(task.project == self.name).run() )[0][0] - weighted_progress = frappe.db.sql( - """select progress, task_weight from tabTask where - project=%s""", - self.name, - as_dict=1, + weighted_progress = frappe.get_all( + "Task", filters={"project": self.name}, fields=["progress", "task_weight"] ) pct_complete = 0 for row in weighted_progress: @@ -353,10 +344,12 @@ class Project(Document): self.total_purchase_cost = total_purchase_cost and total_purchase_cost[0][0] or 0 def update_sales_amount(self): - total_sales_amount = frappe.db.sql( - """select sum(base_net_total) - from `tabSales Order` where project = %s and docstatus=1""", - self.name, + so = frappe.qb.DocType("Sales Order") + total_sales_amount = ( + frappe.qb.from_(so) + .select(Sum(so.base_net_total)) + .where((so.project == self.name) & (so.docstatus == 1)) + .run() ) self.total_sales_amount = total_sales_amount and total_sales_amount[0][0] or 0 @@ -365,25 +358,31 @@ class Project(Document): self.total_billed_amount = self.get_billed_amount_from_parent() + self.get_billed_amount_from_child() def get_billed_amount_from_parent(self): - total_billed_amount = frappe.db.sql( - """select sum(base_net_amount) - from `tabSales Invoice` si join `tabSales Invoice Item` si_item on si_item.parent = si.name - where si_item.project is null - and si.project is not null - and si.project = %s - and si.docstatus = 1""", - self.name, + si = frappe.qb.DocType("Sales Invoice") + si_item = frappe.qb.DocType("Sales Invoice Item") + total_billed_amount = ( + frappe.qb.from_(si) + .join(si_item) + .on(si_item.parent == si.name) + .select(Sum(si_item.base_net_amount)) + .where( + si_item.project.isnull() + & si.project.isnotnull() + & (si.project == self.name) + & (si.docstatus == 1) + ) + .run() ) return total_billed_amount and total_billed_amount[0][0] or 0 def get_billed_amount_from_child(self): - total_billed_amount = frappe.db.sql( - """select sum(base_net_amount) - from `tabSales Invoice Item` - where project = %s - and docstatus = 1""", - self.name, + si_item = frappe.qb.DocType("Sales Invoice Item") + total_billed_amount = ( + frappe.qb.from_(si_item) + .select(Sum(si_item.base_net_amount)) + .where((si_item.project == self.name) & (si_item.docstatus == 1)) + .run() ) return total_billed_amount and total_billed_amount[0][0] or 0 @@ -499,28 +498,35 @@ def get_list_context(context=None): @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def get_users_for_project(doctype: str, txt: str, searchfield: str, start: int, page_len: int, filters: dict): - conditions = [] - return frappe.db.sql( - """select name, concat_ws(' ', first_name, middle_name, last_name) - from `tabUser` - where enabled=1 - and name not in ("Guest", "Administrator") - and ({key} like %(txt)s - or full_name like %(txt)s) - {fcond} {mcond} - order by - (case when locate(%(_txt)s, name) > 0 then locate(%(_txt)s, name) else 99999 end), - (case when locate(%(_txt)s, full_name) > 0 then locate(%(_txt)s, full_name) else 99999 end), - idx desc, - name, full_name - limit %(page_len)s offset %(start)s""".format( - **{ - "key": searchfield, - "fcond": get_filters_cond(doctype, filters, conditions), - "mcond": get_match_cond(doctype), - } - ), - {"txt": "%%%s%%" % txt, "_txt": txt.replace("%", ""), "start": start, "page_len": page_len}, + User = frappe.qb.DocType("User") + search_str = f"%{txt}%" + txt_no_percent = txt.replace("%", "") + + query = frappe.qb.get_query( + "User", + fields=["name", "full_name"], + filters=filters, + ignore_permissions=False, + ) + + return ( + query.where(User.enabled == 1) + .where(User.name.notin(["Guest", "Administrator"])) + .where(User[searchfield].like(search_str) | User.full_name.like(search_str)) + .orderby( + Case().when(Locate(txt_no_percent, User.name) > 0, Locate(txt_no_percent, User.name)).else_(99999) + ) + .orderby( + Case() + .when(Locate(txt_no_percent, User.full_name) > 0, Locate(txt_no_percent, User.full_name)) + .else_(99999) + ) + .orderby(User.idx, order=Order.desc) + .orderby(User.name) + .orderby(User.full_name) + .limit(page_len) + .offset(start) + .run() ) @@ -580,11 +586,7 @@ def weekly_reminder(): def allow_to_make_project_update(project, time, frequency): - data = frappe.db.sql( - """ SELECT name from `tabProject Update` - WHERE project = %s and date = %s """, - (project, today()), - ) + data = frappe.get_all("Project Update", filters={"project": project, "date": today()}, pluck="name") # len(data) > 1 condition is checked for twicely frequency if data and (frequency in ["Daily", "Weekly"] or len(data) > 1): diff --git a/erpnext/projects/doctype/project/test_project.py b/erpnext/projects/doctype/project/test_project.py index b24f0d13fe2..5f39516f759 100644 --- a/erpnext/projects/doctype/project/test_project.py +++ b/erpnext/projects/doctype/project/test_project.py @@ -34,7 +34,7 @@ class TestProject(ERPNextTestSuite): def test_project_with_template_having_no_parent_and_depend_tasks(self): project_name = "Test Project with Template - No Parent and Dependend Tasks" - frappe.db.sql(""" delete from tabTask where project = %s """, project_name) + frappe.db.delete("Task", {"project": project_name}) frappe.delete_doc("Project", project_name) task1 = task_exists("Test Template Task with No Parent and Dependency") @@ -67,7 +67,7 @@ class TestProject(ERPNextTestSuite): if frappe.db.get_value("Project", {"project_name": project_name}, "name"): project_name = frappe.db.get_value("Project", {"project_name": project_name}, "name") - frappe.db.sql(""" delete from tabTask where project = %s """, project_name) + frappe.db.delete("Task", {"project": project_name}) frappe.delete_doc("Project", project_name) task1 = task_exists("Test Template Task Parent") @@ -122,7 +122,7 @@ class TestProject(ERPNextTestSuite): def test_project_template_having_dependent_tasks(self): project_name = "Test Project with Template - Dependent Tasks" - frappe.db.sql(""" delete from tabTask where project = %s """, project_name) + frappe.db.delete("Task", {"project": project_name}) frappe.delete_doc("Project", project_name) task1 = task_exists("Test Template Task for Dependency") @@ -218,7 +218,7 @@ class TestProject(ERPNextTestSuite): def test_project_having_no_tasks_complete(self): project_name = "Test Project - No Tasks Completion" - frappe.db.sql(""" delete from tabTask where project = %s """, project_name) + frappe.db.delete("Task", {"project": project_name}) frappe.delete_doc("Project", project_name) project = frappe.get_doc( diff --git a/erpnext/projects/doctype/project_update/project_update.py b/erpnext/projects/doctype/project_update/project_update.py index 2153f1b39e1..14805c2bd99 100644 --- a/erpnext/projects/doctype/project_update/project_update.py +++ b/erpnext/projects/doctype/project_update/project_update.py @@ -4,6 +4,7 @@ import frappe from frappe.model.document import Document +from frappe.utils import add_days, today class ProjectUpdate(Document): @@ -31,30 +32,34 @@ class ProjectUpdate(Document): @frappe.whitelist() def daily_reminder(): - project = frappe.db.sql( - """SELECT `tabProject`.project_name,`tabProject`.frequency,`tabProject`.expected_start_date,`tabProject`.expected_end_date,`tabProject`.percent_complete FROM `tabProject`;""" + projects = frappe.get_all( + "Project", + fields=[ + "project_name", + "frequency", + "expected_start_date", + "expected_end_date", + "percent_complete", + ], ) - for projects in project: - project_name = projects[0] - frequency = projects[1] - date_start = projects[2] - date_end = projects[3] - progress = projects[4] - draft = frappe.db.sql( - """SELECT count(docstatus) from `tabProject Update` WHERE `tabProject Update`.project = %s AND `tabProject Update`.docstatus = 0;""", - project_name, - ) - for drafts in draft: - number_of_drafts = drafts[0] - update = frappe.db.sql( - """SELECT name,date,time,progress,progress_details FROM `tabProject Update` WHERE `tabProject Update`.project = %s AND date = DATE_ADD(CURRENT_DATE, INTERVAL -1 DAY);""", - project_name, + for project in projects: + project_name = project.project_name + frequency = project.frequency + date_start = project.expected_start_date + date_end = project.expected_end_date + progress = project.percent_complete + number_of_drafts = frappe.db.count("Project Update", {"project": project_name, "docstatus": 0}) + update = frappe.get_all( + "Project Update", + filters={"project": project_name, "date": add_days(today(), -1)}, + fields=["name", "date", "time", "progress", "progress_details"], + as_list=True, ) email_sending(project_name, frequency, date_start, date_end, progress, number_of_drafts, update) def email_sending(project_name, frequency, date_start, date_end, progress, number_of_drafts, update): - holiday = frappe.db.sql("""SELECT holiday_date FROM `tabHoliday` where holiday_date = CURRENT_DATE;""") + holiday_today = frappe.db.exists("Holiday", {"holiday_date": today()}) msg = ( "

Project Name: " + project_name @@ -98,9 +103,9 @@ def email_sending(project_name, frequency, date_start, date_end, progress, numbe ) msg += "" - if len(holiday) == 0: - email = frappe.db.sql("""SELECT user from `tabProject User` WHERE parent = %s;""", project_name) - for emails in email: - frappe.sendmail(recipients=emails, subject=frappe._(project_name + " " + "Summary"), message=msg) + if not holiday_today: + recipients = frappe.get_all("Project User", filters={"parent": project_name}, pluck="user") + for user in recipients: + frappe.sendmail(recipients=[user], subject=frappe._(project_name + " " + "Summary"), message=msg) else: pass diff --git a/erpnext/projects/doctype/task/task.py b/erpnext/projects/doctype/task/task.py index 786ae63643f..d7781df5f86 100755 --- a/erpnext/projects/doctype/task/task.py +++ b/erpnext/projects/doctype/task/task.py @@ -76,10 +76,9 @@ class Task(NestedSet): nsm_parent_field = "parent_task" def get_customer_details(self): - cust = frappe.db.sql("select customer_name from `tabCustomer` where name=%s", self.customer) - if cust: - ret = {"customer_name": cust and cust[0][0] or ""} - return ret + customer_name = frappe.db.get_value("Customer", self.customer, "customer_name") + if customer_name: + return {"customer_name": customer_name or ""} def validate(self): self.validate_dates() @@ -252,9 +251,11 @@ class Task(NestedSet): for d in check_list: task_list, count = [self.name], 0 while len(task_list) > count: - tasks = frappe.db.sql( - " select {} from `tabTask Depends On` where {} = {} ".format(d[0], d[1], "%s"), - cstr(task_list[count]), + tasks = frappe.get_all( + "Task Depends On", + filters={d[1]: cstr(task_list[count])}, + fields=[d[0]], + as_list=True, ) count = count + 1 for b in tasks: @@ -268,30 +269,34 @@ class Task(NestedSet): def reschedule_dependent_tasks(self): end_date = self.exp_end_date or self.act_end_date - if end_date: - for task_name in frappe.db.sql( - """ - select name from `tabTask` as parent - where parent.project = %(project)s - and parent.name in ( - select parent from `tabTask Depends On` as child - where child.task = %(task)s and child.project = %(project)s) - """, - {"project": self.project, "task": self.name}, - as_dict=1, + if not end_date: + return + + dependent_parents = frappe.get_all( + "Task Depends On", + filters={"task": self.name, "project": self.project}, + pluck="parent", + ) + if not dependent_parents: + return + + for task_name in frappe.get_all( + "Task", + filters={"project": self.project, "name": ["in", dependent_parents]}, + pluck="name", + ): + task = frappe.get_doc("Task", task_name) + if ( + task.exp_start_date + and task.exp_end_date + and task.exp_start_date < end_date + and task.status == "Open" ): - task = frappe.get_doc("Task", task_name.name) - if ( - task.exp_start_date - and task.exp_end_date - and task.exp_start_date < end_date - and task.status == "Open" - ): - task_duration = date_diff(task.exp_end_date, task.exp_start_date) - task.exp_start_date = add_days(end_date, 1) - task.exp_end_date = add_days(task.exp_start_date, task_duration) - task.flags.ignore_recursion_check = True - task.save() + task_duration = date_diff(task.exp_end_date, task.exp_start_date) + task.exp_start_date = add_days(end_date, 1) + task.exp_end_date = add_days(task.exp_start_date, task_duration) + task.flags.ignore_recursion_check = True + task.save() def has_webform_permission(self): project_user = frappe.db.get_value( @@ -337,27 +342,23 @@ def check_if_child_exists(name: str): @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def get_project(doctype: str, txt: str, searchfield: str, start: int, page_len: int, filters: dict): - from erpnext.controllers.queries import get_match_cond + from frappe.query_builder import Criterion - meta = frappe.get_meta(doctype) - searchfields = meta.get_search_fields() - search_columns = ", " + ", ".join(searchfields) if searchfields else "" - search_cond = " or " + " or ".join(field + " like %(txt)s" for field in searchfields) + searchfields = frappe.get_meta(doctype).get_search_fields() - return frappe.db.sql( - f""" select name {search_columns} from `tabProject` - where %(key)s like %(txt)s - %(mcond)s - {search_cond} - order by name - limit %(page_len)s offset %(start)s""", - { - "key": searchfield, - "txt": "%" + txt + "%", - "mcond": get_match_cond(doctype), - "start": start, - "page_len": page_len, - }, + Project = frappe.qb.DocType("Project") + search_str = f"%{txt}%" + search_fields = list(dict.fromkeys([searchfield, *searchfields])) + search_conditions = [Project[field].like(search_str) for field in search_fields] + + query = frappe.qb.get_query("Project", fields=["name", *searchfields], ignore_permissions=False) + + return ( + query.where(Criterion.any(search_conditions)) + .orderby(Project.name) + .limit(page_len) + .offset(start) + .run() ) diff --git a/erpnext/projects/doctype/timesheet/timesheet.py b/erpnext/projects/doctype/timesheet/timesheet.py index f1a00086464..8d14f9d5ec6 100644 --- a/erpnext/projects/doctype/timesheet/timesheet.py +++ b/erpnext/projects/doctype/timesheet/timesheet.py @@ -7,10 +7,10 @@ import json import frappe from frappe import _ from frappe.model.document import Document +from frappe.query_builder.functions import Concat, Date, Round from frappe.utils import flt, get_datetime, getdate from frappe.utils.deprecations import deprecated -from erpnext.controllers.queries import get_match_cond from erpnext.setup.utils import get_exchange_rate @@ -308,41 +308,41 @@ def get_projectwise_timesheet_data( from_time: str | None = None, to_time: str | None = None, ): - condition = "" + tsd = frappe.qb.DocType("Timesheet Detail") + ts = frappe.qb.DocType("Timesheet") + + query = ( + frappe.qb.from_(tsd) + .inner_join(ts) + .on(ts.name == tsd.parent) + .select( + tsd.name.as_("name"), + tsd.parent.as_("time_sheet"), + tsd.from_time.as_("from_time"), + tsd.to_time.as_("to_time"), + tsd.billing_hours.as_("billing_hours"), + tsd.billing_amount.as_("billing_amount"), + tsd.activity_type.as_("activity_type"), + tsd.description.as_("description"), + ts.currency.as_("currency"), + tsd.project_name.as_("project_name"), + ) + .where( + (tsd.parenttype == "Timesheet") + & (tsd.docstatus == 1) + & (tsd.is_billable == 1) + & tsd.sales_invoice.isnull() + ) + ) + if project: - condition += "AND tsd.project = %(project)s " + query = query.where(tsd.project == project) if parent: - condition += "AND tsd.parent = %(parent)s " + query = query.where(tsd.parent == parent) if from_time and to_time: - condition += "AND CAST(tsd.from_time as DATE) BETWEEN %(from_time)s AND %(to_time)s" + query = query.where(Date(tsd.from_time).between(from_time, to_time)) - query = f""" - SELECT - tsd.name as name, - tsd.parent as time_sheet, - tsd.from_time as from_time, - tsd.to_time as to_time, - tsd.billing_hours as billing_hours, - tsd.billing_amount as billing_amount, - tsd.activity_type as activity_type, - tsd.description as description, - ts.currency as currency, - tsd.project_name as project_name - FROM `tabTimesheet Detail` tsd - INNER JOIN `tabTimesheet` ts - ON ts.name = tsd.parent - WHERE - tsd.parenttype = 'Timesheet' - AND tsd.docstatus = 1 - AND tsd.is_billable = 1 - AND tsd.sales_invoice is NULL - {condition} - ORDER BY tsd.from_time ASC - """ - - filters = {"project": project, "parent": parent, "from_time": from_time, "to_time": to_time} - - return frappe.db.sql(query, filters, as_dict=1) + return query.orderby(tsd.from_time).run(as_dict=1) @frappe.whitelist() @@ -372,25 +372,28 @@ def get_timesheet(doctype: str, txt: str, searchfield: str, start: int, page_len if not filters: filters = {} - condition = "" - if filters.get("project"): - condition = "and tsd.project = %(project)s" + tsd = frappe.qb.DocType("Timesheet Detail") + ts = frappe.qb.DocType("Timesheet") - return frappe.db.sql( - f"""select distinct tsd.parent from `tabTimesheet Detail` tsd, - `tabTimesheet` ts where - ts.status in ('Submitted', 'Payslip') and tsd.parent = ts.name and - tsd.docstatus = 1 and ts.total_billable_amount > 0 - and tsd.parent LIKE %(txt)s {condition} - order by tsd.parent limit %(page_len)s offset %(start)s""", - { - "txt": "%" + txt + "%", - "start": start, - "page_len": page_len, - "project": filters.get("project"), - }, + query = ( + frappe.qb.from_(tsd) + .inner_join(ts) + .on(tsd.parent == ts.name) + .select(tsd.parent) + .distinct() + .where( + ts.status.isin(["Submitted", "Payslip"]) + & (tsd.docstatus == 1) + & (ts.total_billable_amount > 0) + & tsd.parent.like(f"%{txt}%") + ) ) + if filters.get("project"): + query = query.where(tsd.project == filters.get("project")) + + return query.orderby(tsd.parent).limit(page_len).offset(start).run() + @frappe.whitelist() def get_timesheet_data(name: str, project: str): @@ -500,27 +503,37 @@ def get_events(start: str, end: str, filters: str | None = None): :param end: End date-time. :param filters: Filters (JSON). """ - filters = json.loads(filters) if filters else {} from frappe.desk.calendar import get_event_conditions - conditions = get_event_conditions("Timesheet", filters) + filters = json.loads(filters) if filters else {} - return frappe.db.sql( - """select `tabTimesheet Detail`.name as name, - `tabTimesheet Detail`.docstatus as status, `tabTimesheet Detail`.parent as parent, - from_time as start_date, hours, activity_type, - `tabTimesheet Detail`.project, to_time as end_date, - CONCAT(`tabTimesheet Detail`.parent, ' (', ROUND(hours,2),' hrs)') as title - from `tabTimesheet Detail`, `tabTimesheet` - where `tabTimesheet Detail`.parent = `tabTimesheet`.name - and `tabTimesheet`.docstatus < 2 - and (from_time <= %(end)s and to_time >= %(start)s) {conditions} {match_cond} - """.format(conditions=conditions, match_cond=get_match_cond("Timesheet")), - {"start": start, "end": end}, - as_dict=True, - update={"allDay": 0}, + tsd = frappe.qb.DocType("Timesheet Detail") + ts = frappe.qb.DocType("Timesheet") + + query = ( + frappe.qb.from_(tsd) + .inner_join(ts) + .on(tsd.parent == ts.name) + .select( + tsd.name.as_("name"), + tsd.docstatus.as_("status"), + tsd.parent.as_("parent"), + tsd.from_time.as_("start_date"), + tsd.hours, + tsd.activity_type, + tsd.project, + tsd.to_time.as_("end_date"), + Concat(tsd.parent, " (", Round(tsd.hours, 2), " hrs)").as_("title"), + ) + .where((ts.docstatus < 2) & (tsd.from_time <= end) & (tsd.to_time >= start)) ) + # user-permission match conditions + calendar filters on Timesheet (query-builder form) + for condition in get_event_conditions("Timesheet", filters, as_qb=True): + query = query.where(condition) + + return query.run(as_dict=True, update={"allDay": 0}) + def get_timesheets_list(doctype, txt, filters, limit_start, limit_page_length=20, order_by="creation"): user = frappe.session.user diff --git a/erpnext/projects/report/daily_timesheet_summary/daily_timesheet_summary.py b/erpnext/projects/report/daily_timesheet_summary/daily_timesheet_summary.py index 726dd4bac53..81e377ac119 100644 --- a/erpnext/projects/report/daily_timesheet_summary/daily_timesheet_summary.py +++ b/erpnext/projects/report/daily_timesheet_summary/daily_timesheet_summary.py @@ -4,19 +4,17 @@ import frappe from frappe import _ -from frappe.desk.reportview import build_match_conditions +from frappe.desk.reportview import get_match_conditions_qb +from frappe.utils import add_days, getdate + +from erpnext.stock.utils import get_combine_datetime def execute(filters=None): - if not filters: - filters = {} - elif filters.get("from_date") or filters.get("to_date"): - filters["from_time"] = "00:00:00" - filters["to_time"] = "24:00:00" + filters = filters or {} columns = get_column() - conditions = get_conditions(filters) - data = get_data(conditions, filters) + data = get_data(filters) return columns, data @@ -36,30 +34,40 @@ def get_column(): ] -def get_data(conditions, filters): - time_sheet = frappe.db.sql( - """ select `tabTimesheet`.name, `tabTimesheet`.employee, `tabTimesheet`.employee_name, - `tabTimesheet Detail`.from_time, `tabTimesheet Detail`.to_time, `tabTimesheet Detail`.hours, - `tabTimesheet Detail`.activity_type, `tabTimesheet Detail`.task, `tabTimesheet Detail`.project, - `tabTimesheet`.status from `tabTimesheet Detail`, `tabTimesheet` where - `tabTimesheet Detail`.parent = `tabTimesheet`.name and %s order by `tabTimesheet`.name""" - % (conditions), - filters, - as_list=1, +def get_data(filters): + ts = frappe.qb.DocType("Timesheet") + tsd = frappe.qb.DocType("Timesheet Detail") + + query = ( + frappe.qb.from_(tsd) + .inner_join(ts) + .on(tsd.parent == ts.name) + .select( + ts.name, + ts.employee, + ts.employee_name, + tsd.from_time, + tsd.to_time, + tsd.hours, + tsd.activity_type, + tsd.task, + tsd.project, + ts.status, + ) + .where(ts.docstatus == 1) ) - return time_sheet - - -def get_conditions(filters): - conditions = "`tabTimesheet`.docstatus = 1" if filters.get("from_date"): - conditions += " and `tabTimesheet Detail`.from_time >= timestamp(%(from_date)s, %(from_time)s)" + query = query.where(tsd.from_time >= get_combine_datetime(filters.get("from_date"), "00:00:00")) + if filters.get("to_date"): - conditions += " and `tabTimesheet Detail`.to_time <= timestamp(%(to_date)s, %(to_time)s)" + # upper bound is the end of to_date, i.e. midnight of the next day + # (matches the original `timestamp(to_date, '24:00:00')`) + end_of_to_date = get_combine_datetime(add_days(getdate(filters.get("to_date")), 1), "00:00:00") + query = query.where(tsd.to_time <= end_of_to_date) - match_conditions = build_match_conditions("Timesheet") - if match_conditions: - conditions += " and (%s)" % match_conditions + # apply Timesheet user-permission match conditions (query-builder form of build_match_conditions) + for condition in get_match_conditions_qb("Timesheet"): + query = query.where(condition) - return conditions + return query.orderby(ts.name).run(as_list=True) diff --git a/erpnext/projects/report/project_wise_stock_tracking/project_wise_stock_tracking.py b/erpnext/projects/report/project_wise_stock_tracking/project_wise_stock_tracking.py index 41a7c799d9d..b90fb9d0afe 100644 --- a/erpnext/projects/report/project_wise_stock_tracking/project_wise_stock_tracking.py +++ b/erpnext/projects/report/project_wise_stock_tracking/project_wise_stock_tracking.py @@ -3,6 +3,7 @@ import frappe from frappe import _ +from frappe.query_builder.functions import Sum def execute(filters=None): @@ -50,19 +51,28 @@ def get_columns(): def get_project_details(): - return frappe.db.sql( - """ select name, project_name, status, company, customer, estimated_costing, - expected_start_date, expected_end_date from tabProject where docstatus < 2""", - as_dict=1, + return frappe.get_all( + "Project", + filters={"docstatus": ["<", 2]}, + fields=[ + "name", + "project_name", + "status", + "company", + "customer", + "estimated_costing", + "expected_start_date", + "expected_end_date", + ], ) def get_purchased_items_cost(): - pr_items = frappe.db.sql( - """select project, sum(base_net_amount) as amount - from `tabPurchase Receipt Item` where ifnull(project, '') != '' - and docstatus = 1 group by project""", - as_dict=1, + pr_items = frappe.get_all( + "Purchase Receipt Item", + filters={"project": ["is", "set"], "docstatus": 1}, + fields=["project", {"SUM": "base_net_amount", "as": "amount"}], + group_by="project", ) pr_item_map = {} @@ -73,12 +83,20 @@ def get_purchased_items_cost(): def get_issued_items_cost(): - se_items = frappe.db.sql( - """select se.project, sum(se_item.amount) as amount - from `tabStock Entry` se, `tabStock Entry Detail` se_item - where se.name = se_item.parent and se.docstatus = 1 and ifnull(se_item.t_warehouse, '') = '' - and se.project != '' group by se.project""", - as_dict=1, + se = frappe.qb.DocType("Stock Entry") + se_item = frappe.qb.DocType("Stock Entry Detail") + se_items = ( + frappe.qb.from_(se) + .inner_join(se_item) + .on(se.name == se_item.parent) + .select(se.project, Sum(se_item.amount).as_("amount")) + .where( + (se.docstatus == 1) + & (se_item.t_warehouse.isnull() | (se_item.t_warehouse == "")) + & (se.project != "") + ) + .groupby(se.project) + .run(as_dict=1) ) se_item_map = {} @@ -89,21 +107,28 @@ def get_issued_items_cost(): def get_delivered_items_cost(): - dn_items = frappe.db.sql( - """select dn.project, sum(dn_item.base_net_amount) as amount - from `tabDelivery Note` dn, `tabDelivery Note Item` dn_item - where dn.name = dn_item.parent and dn.docstatus = 1 and ifnull(dn.project, '') != '' - group by dn.project""", - as_dict=1, + dn = frappe.qb.DocType("Delivery Note") + dn_item = frappe.qb.DocType("Delivery Note Item") + dn_items = ( + frappe.qb.from_(dn) + .inner_join(dn_item) + .on(dn.name == dn_item.parent) + .select(dn.project, Sum(dn_item.base_net_amount).as_("amount")) + .where((dn.docstatus == 1) & (dn.project != "")) + .groupby(dn.project) + .run(as_dict=1) ) - si_items = frappe.db.sql( - """select si.project, sum(si_item.base_net_amount) as amount - from `tabSales Invoice` si, `tabSales Invoice Item` si_item - where si.name = si_item.parent and si.docstatus = 1 and si.update_stock = 1 - and si.is_pos = 1 and ifnull(si.project, '') != '' - group by si.project""", - as_dict=1, + si = frappe.qb.DocType("Sales Invoice") + si_item = frappe.qb.DocType("Sales Invoice Item") + si_items = ( + frappe.qb.from_(si) + .inner_join(si_item) + .on(si.name == si_item.parent) + .select(si.project, Sum(si_item.base_net_amount).as_("amount")) + .where((si.docstatus == 1) & (si.update_stock == 1) & (si.is_pos == 1) & (si.project != "")) + .groupby(si.project) + .run(as_dict=1) ) dn_item_map = {} diff --git a/erpnext/projects/utils.py b/erpnext/projects/utils.py index f81cd87c862..dca8df659c3 100644 --- a/erpnext/projects/utils.py +++ b/erpnext/projects/utils.py @@ -5,28 +5,25 @@ import frappe +from frappe.query_builder import Case @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def query_task(doctype: str, txt: str, searchfield: str, start: int, page_len: int, filters: dict): - from frappe.desk.reportview import build_match_conditions + search_str = f"%{txt}%" + prefix_str = f"{txt}%" - search_string = "%%%s%%" % txt - order_by_string = "%s%%" % txt - match_conditions = build_match_conditions("Task") - match_conditions = (f"and ({match_conditions})") if match_conditions else "" + Task = frappe.qb.DocType("Task") + query = frappe.qb.get_query("Task", fields=["name", "subject"], ignore_permissions=False) - return frappe.db.sql( - """select name, subject from `tabTask` - where (`{}` like {} or `subject` like {}) {} - order by - case when `subject` like {} then 0 else 1 end, - case when `{}` like {} then 0 else 1 end, - `{}`, - subject - limit {} offset {}""".format( - searchfield, "%s", "%s", match_conditions, "%s", searchfield, "%s", searchfield, "%s", "%s" - ), - (search_string, search_string, order_by_string, order_by_string, page_len, start), + return ( + query.where(Task[searchfield].like(search_str) | Task.subject.like(search_str)) + .orderby(Case().when(Task.subject.like(prefix_str), 0).else_(1)) + .orderby(Case().when(Task[searchfield].like(prefix_str), 0).else_(1)) + .orderby(Task[searchfield]) + .orderby(Task.subject) + .limit(page_len) + .offset(start) + .run() )