From ba42c876879d34549aaefdf4f620bd90b2b05c2b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 6 Apr 2022 15:40:41 +0530 Subject: [PATCH] refactor: don't use pandas for basic reports (#30597) --- .../opportunity_summary_by_sales_stage.py | 17 +++--- .../sales_pipeline_analytics.py | 15 ++--- .../selling/page/sales_funnel/sales_funnel.py | 56 ++++++++----------- 3 files changed, 37 insertions(+), 51 deletions(-) diff --git a/erpnext/crm/report/opportunity_summary_by_sales_stage/opportunity_summary_by_sales_stage.py b/erpnext/crm/report/opportunity_summary_by_sales_stage/opportunity_summary_by_sales_stage.py index 77e6ae2e04f..3a46fb0879c 100644 --- a/erpnext/crm/report/opportunity_summary_by_sales_stage/opportunity_summary_by_sales_stage.py +++ b/erpnext/crm/report/opportunity_summary_by_sales_stage/opportunity_summary_by_sales_stage.py @@ -1,9 +1,9 @@ # Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt import json +from itertools import groupby import frappe -import pandas from frappe import _ from frappe.utils import flt @@ -101,18 +101,19 @@ class OpportunitySummaryBySalesStage(object): self.convert_to_base_currency() - dataframe = pandas.DataFrame.from_records(self.query_result) - dataframe.replace(to_replace=[None], value="Not Assigned", inplace=True) - result = dataframe.groupby(["sales_stage", based_on], as_index=False)["amount"].sum() + for row in self.query_result: + if not row.get(based_on): + row[based_on] = "Not Assigned" self.grouped_data = [] - for i in range(len(result["amount"])): + grouping_key = lambda o: (o["sales_stage"], o[based_on]) # noqa + for (sales_stage, _based_on), rows in groupby(self.query_result, grouping_key): self.grouped_data.append( { - "sales_stage": result["sales_stage"][i], - based_on: result[based_on][i], - "amount": result["amount"][i], + "sales_stage": sales_stage, + based_on: _based_on, + "amount": sum(flt(r["amount"]) for r in rows), } ) diff --git a/erpnext/crm/report/sales_pipeline_analytics/sales_pipeline_analytics.py b/erpnext/crm/report/sales_pipeline_analytics/sales_pipeline_analytics.py index b0c174be234..d23a22ac46d 100644 --- a/erpnext/crm/report/sales_pipeline_analytics/sales_pipeline_analytics.py +++ b/erpnext/crm/report/sales_pipeline_analytics/sales_pipeline_analytics.py @@ -3,9 +3,9 @@ import json from datetime import date +from itertools import groupby import frappe -import pandas from dateutil.relativedelta import relativedelta from frappe import _ from frappe.utils import cint, flt @@ -109,18 +109,15 @@ class SalesPipelineAnalytics(object): self.convert_to_base_currency() - dataframe = pandas.DataFrame.from_records(self.query_result) - dataframe.replace(to_replace=[None], value="Not Assigned", inplace=True) - result = dataframe.groupby([self.pipeline_by, self.period_by], as_index=False)["amount"].sum() - self.grouped_data = [] - for i in range(len(result["amount"])): + grouping_key = lambda o: (o.get(self.pipeline_by) or "Not Assigned", o[self.period_by]) # noqa + for (pipeline_by, period_by), rows in groupby(self.query_result, grouping_key): self.grouped_data.append( { - self.pipeline_by: result[self.pipeline_by][i], - self.period_by: result[self.period_by][i], - "amount": result["amount"][i], + self.pipeline_by: pipeline_by, + self.period_by: period_by, + "amount": sum(flt(r["amount"]) for r in rows), } ) diff --git a/erpnext/selling/page/sales_funnel/sales_funnel.py b/erpnext/selling/page/sales_funnel/sales_funnel.py index c626f5b05fc..6b33a717531 100644 --- a/erpnext/selling/page/sales_funnel/sales_funnel.py +++ b/erpnext/selling/page/sales_funnel/sales_funnel.py @@ -1,10 +1,11 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt +from itertools import groupby import frappe -import pandas as pd from frappe import _ +from frappe.utils import flt from erpnext.accounts.report.utils import convert @@ -89,28 +90,21 @@ def get_opp_by_lead_source(from_date, to_date, company): for x in opportunities ] - df = ( - pd.DataFrame(cp_opportunities) - .groupby(["source", "sales_stage"], as_index=False) - .agg({"compound_amount": "sum"}) - ) + summary = {} + sales_stages = set() + group_key = lambda o: (o["source"], o["sales_stage"]) # noqa + for (source, sales_stage), rows in groupby(cp_opportunities, group_key): + summary.setdefault(source, {})[sales_stage] = sum(r["compound_amount"] for r in rows) + sales_stages.add(sales_stage) - result = {} - result["labels"] = list(set(df.source.values)) - result["datasets"] = [] - - for s in set(df.sales_stage.values): - result["datasets"].append( - {"name": s, "values": [0] * len(result["labels"]), "chartType": "bar"} - ) - - for row in df.itertuples(): - source_index = result["labels"].index(row.source) - - for dataset in result["datasets"]: - if dataset["name"] == row.sales_stage: - dataset["values"][source_index] = row.compound_amount + pivot_table = [] + for sales_stage in sales_stages: + row = [] + for source, sales_stage_values in summary.items(): + row.append(flt(sales_stage_values.get(sales_stage))) + pivot_table.append({"chartType": "bar", "name": sales_stage, "values": row}) + result = {"datasets": pivot_table, "labels": list(summary.keys())} return result else: @@ -148,20 +142,14 @@ def get_pipeline_data(from_date, to_date, company): for x in opportunities ] - df = ( - pd.DataFrame(cp_opportunities) - .groupby(["sales_stage"], as_index=True) - .agg({"compound_amount": "sum"}) - .to_dict() - ) - - result = {} - result["labels"] = df["compound_amount"].keys() - result["datasets"] = [] - result["datasets"].append( - {"name": _("Total Amount"), "values": df["compound_amount"].values(), "chartType": "bar"} - ) + summary = {} + for sales_stage, rows in groupby(cp_opportunities, lambda o: o["sales_stage"]): + summary[sales_stage] = sum(flt(r["compound_amount"]) for r in rows) + result = { + "labels": list(summary.keys()), + "datasets": [{"name": _("Total Amount"), "values": list(summary.values()), "chartType": "bar"}], + } return result else: