From 767a7757095e369da68e017f666c4896fa3cc6a2 Mon Sep 17 00:00:00 2001 From: marination Date: Fri, 27 May 2022 20:33:14 +0530 Subject: [PATCH] perf: `get_next_higher_level_boms` - Separate getting dependants and checking if they are valid (loop within loop led to redundant processing that slowed down function) - Adding to above, the same dependant(parent) was repeatedly processed as many children shared it. Expensive. - Use a parent-child map similar to child-parent map to check if all children are resolved - `map.get()` reduced time: 10 mins -> 0.9s~1 second (as compared to `get_cached_doc` or query) - Total time: 17 seconds to process 6599 leaf boms and 4.2L parent boms - Previous Total time: >10 mins (I terminated it due to not wanting to waste time XD) --- .../bom_update_log/bom_updation_utils.py | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/erpnext/manufacturing/doctype/bom_update_log/bom_updation_utils.py b/erpnext/manufacturing/doctype/bom_update_log/bom_updation_utils.py index 1ec15f0d3ae..790a79b3333 100644 --- a/erpnext/manufacturing/doctype/bom_update_log/bom_updation_utils.py +++ b/erpnext/manufacturing/doctype/bom_update_log/bom_updation_utils.py @@ -159,21 +159,29 @@ def process_if_level_is_complete( def get_next_higher_level_boms( child_boms: Dict[str, bool], processed_boms: Dict[str, bool] ) -> List[str]: - "Generate immediate higher level dependants with no unresolved dependencies." + "Generate immediate higher level dependants with no unresolved dependencies (children)." - def _all_children_are_processed(parent): - bom_doc = frappe.get_cached_doc("BOM", parent) - return all(processed_boms.get(row.bom_no) for row in bom_doc.items if row.bom_no) + def _all_children_are_processed(parent_bom): + child_boms = dependency_map.get(parent_bom) + return all(processed_boms.get(bom) for bom in child_boms) - dependants_map = _generate_dependants_map() - dependants = set() + dependants_map, dependency_map = _generate_dependence_map() + + dependants = [] for bom in child_boms: + # generate list of immediate dependants parents = dependants_map.get(bom) or [] - for parent in parents: - if _all_children_are_processed(parent): - dependants.add(parent) + dependants.extend(parents) - return list(dependants) + dependants = set(dependants) # remove duplicates + resolved_dependants = set() + + # consider only if children are all resolved + for parent_bom in dependants: + if _all_children_are_processed(parent_bom): + resolved_dependants.add(parent_bom) + + return list(resolved_dependants) def get_leaf_boms() -> List[str]: @@ -187,17 +195,19 @@ def get_leaf_boms() -> List[str]: ) -def _generate_dependants_map() -> defaultdict: +def _generate_dependence_map() -> defaultdict: """ - Generate map such as: { BOM-1: [Dependant-BOM-1, Dependant-BOM-2, ..] }. + Generate maps such as: { BOM-1: [Dependant-BOM-1, Dependant-BOM-2, ..] }. Here BOM-1 is the leaf/lower level node/dependency. The list contains one level higher nodes/dependants that depend on BOM-1. + + Generate and return the reverse as well. """ bom = frappe.qb.DocType("BOM") bom_item = frappe.qb.DocType("BOM Item") - bom_parents = ( + bom_items = ( frappe.qb.from_(bom_item) .join(bom) .on(bom_item.parent == bom.name) @@ -212,10 +222,12 @@ def _generate_dependants_map() -> defaultdict: ).run(as_dict=True) child_parent_map = defaultdict(list) - for bom in bom_parents: - child_parent_map[bom.bom_no].append(bom.parent) + parent_child_map = defaultdict(list) + for row in bom_items: + child_parent_map[row.bom_no].append(row.parent) + parent_child_map[row.parent].append(row.bom_no) - return child_parent_map + return child_parent_map, parent_child_map def set_values_in_log(log_name: str, values: Dict[str, Any], commit: bool = False) -> None: