mirror of
https://github.com/frappe/erpnext.git
synced 2026-09-17 02:26:33 +00:00
Merge branch 'develop' into manual_depr_entries_develop
This commit is contained in:
Anand Baburajan
@@ -2,7 +2,8 @@ import frappe
|
||||
from frappe import qb
|
||||
from frappe.query_builder import Case, CustomFunction
|
||||
from frappe.query_builder.custom import ConstantColumn
|
||||
from frappe.query_builder.functions import IfNull
|
||||
from frappe.query_builder.functions import Count, IfNull
|
||||
from frappe.utils import flt
|
||||
|
||||
from erpnext.accounts.doctype.accounting_dimension.accounting_dimension import (
|
||||
get_dimensions,
|
||||
@@ -17,9 +18,9 @@ def create_accounting_dimension_fields():
|
||||
make_dimension_in_accounting_doctypes(dimension, ["Payment Ledger Entry"])
|
||||
|
||||
|
||||
def generate_name_for_payment_ledger_entries(gl_entries):
|
||||
for index, entry in enumerate(gl_entries, 1):
|
||||
entry.name = index
|
||||
def generate_name_for_payment_ledger_entries(gl_entries, start):
|
||||
for index, entry in enumerate(gl_entries, 0):
|
||||
entry.name = start + index
|
||||
|
||||
|
||||
def get_columns():
|
||||
@@ -81,6 +82,14 @@ def insert_chunk_into_payment_ledger(insert_query, gl_entries):
|
||||
|
||||
|
||||
def execute():
|
||||
"""
|
||||
Description:
|
||||
Migrate records from `tabGL Entry` to `tabPayment Ledger Entry`.
|
||||
Patch is non-resumable. if patch failed or is terminatted abnormally, clear 'tabPayment Ledger Entry' table manually before re-running. Re-running is safe only during V13->V14 update.
|
||||
|
||||
Note: Post successful migration to V14, re-running is NOT-SAFE and SHOULD NOT be attempted.
|
||||
"""
|
||||
|
||||
if frappe.reload_doc("accounts", "doctype", "payment_ledger_entry"):
|
||||
# create accounting dimension fields in Payment Ledger
|
||||
create_accounting_dimension_fields()
|
||||
@@ -89,52 +98,90 @@ def execute():
|
||||
account = qb.DocType("Account")
|
||||
ifelse = CustomFunction("IF", ["condition", "then", "else"])
|
||||
|
||||
gl_entries = (
|
||||
qb.from_(gl)
|
||||
.inner_join(account)
|
||||
.on((gl.account == account.name) & (account.account_type.isin(["Receivable", "Payable"])))
|
||||
.select(
|
||||
gl.star,
|
||||
ConstantColumn(1).as_("docstatus"),
|
||||
account.account_type.as_("account_type"),
|
||||
IfNull(
|
||||
ifelse(gl.against_voucher_type == "", None, gl.against_voucher_type), gl.voucher_type
|
||||
).as_("against_voucher_type"),
|
||||
IfNull(ifelse(gl.against_voucher == "", None, gl.against_voucher), gl.voucher_no).as_(
|
||||
"against_voucher_no"
|
||||
),
|
||||
# convert debit/credit to amount
|
||||
Case()
|
||||
.when(account.account_type == "Receivable", gl.debit - gl.credit)
|
||||
.else_(gl.credit - gl.debit)
|
||||
.as_("amount"),
|
||||
# convert debit/credit in account currency to amount in account currency
|
||||
Case()
|
||||
.when(
|
||||
account.account_type == "Receivable",
|
||||
gl.debit_in_account_currency - gl.credit_in_account_currency,
|
||||
)
|
||||
.else_(gl.credit_in_account_currency - gl.debit_in_account_currency)
|
||||
.as_("amount_in_account_currency"),
|
||||
)
|
||||
.where(gl.is_cancelled == 0)
|
||||
.orderby(gl.creation)
|
||||
.run(as_dict=True)
|
||||
# Get Records Count
|
||||
accounts = (
|
||||
qb.from_(account)
|
||||
.select(account.name)
|
||||
.where((account.account_type == "Receivable") | (account.account_type == "Payable"))
|
||||
.orderby(account.name)
|
||||
)
|
||||
un_processed = (
|
||||
qb.from_(gl)
|
||||
.select(Count(gl.name))
|
||||
.where((gl.is_cancelled == 0) & (gl.account.isin(accounts)))
|
||||
.run()
|
||||
)[0][0]
|
||||
|
||||
# primary key(name) for payment ledger records
|
||||
generate_name_for_payment_ledger_entries(gl_entries)
|
||||
if un_processed:
|
||||
print(f"Migrating {un_processed} GL Entries to Payment Ledger")
|
||||
|
||||
# split data into chunks
|
||||
chunk_size = 1000
|
||||
try:
|
||||
for i in range(0, len(gl_entries), chunk_size):
|
||||
insert_query = build_insert_query()
|
||||
insert_chunk_into_payment_ledger(insert_query, gl_entries[i : i + chunk_size])
|
||||
frappe.db.commit()
|
||||
except Exception as err:
|
||||
frappe.db.rollback()
|
||||
ple = qb.DocType("Payment Ledger Entry")
|
||||
qb.from_(ple).delete().where(ple.docstatus >= 0).run()
|
||||
frappe.db.commit()
|
||||
raise err
|
||||
processed = 0
|
||||
last_update_percent = 0
|
||||
batch_size = 5000
|
||||
last_name = None
|
||||
|
||||
while True:
|
||||
if last_name:
|
||||
where_clause = gl.name.gt(last_name) & (gl.is_cancelled == 0)
|
||||
else:
|
||||
where_clause = gl.is_cancelled == 0
|
||||
|
||||
gl_entries = (
|
||||
qb.from_(gl)
|
||||
.inner_join(account)
|
||||
.on((gl.account == account.name) & (account.account_type.isin(["Receivable", "Payable"])))
|
||||
.select(
|
||||
gl.star,
|
||||
ConstantColumn(1).as_("docstatus"),
|
||||
account.account_type.as_("account_type"),
|
||||
IfNull(
|
||||
ifelse(gl.against_voucher_type == "", None, gl.against_voucher_type), gl.voucher_type
|
||||
).as_("against_voucher_type"),
|
||||
IfNull(ifelse(gl.against_voucher == "", None, gl.against_voucher), gl.voucher_no).as_(
|
||||
"against_voucher_no"
|
||||
),
|
||||
# convert debit/credit to amount
|
||||
Case()
|
||||
.when(account.account_type == "Receivable", gl.debit - gl.credit)
|
||||
.else_(gl.credit - gl.debit)
|
||||
.as_("amount"),
|
||||
# convert debit/credit in account currency to amount in account currency
|
||||
Case()
|
||||
.when(
|
||||
account.account_type == "Receivable",
|
||||
gl.debit_in_account_currency - gl.credit_in_account_currency,
|
||||
)
|
||||
.else_(gl.credit_in_account_currency - gl.debit_in_account_currency)
|
||||
.as_("amount_in_account_currency"),
|
||||
)
|
||||
.where(where_clause)
|
||||
.orderby(gl.name)
|
||||
.limit(batch_size)
|
||||
.run(as_dict=True)
|
||||
)
|
||||
|
||||
if gl_entries:
|
||||
last_name = gl_entries[-1].name
|
||||
|
||||
# primary key(name) for payment ledger records
|
||||
generate_name_for_payment_ledger_entries(gl_entries, processed)
|
||||
|
||||
try:
|
||||
insert_query = build_insert_query()
|
||||
insert_chunk_into_payment_ledger(insert_query, gl_entries)
|
||||
frappe.db.commit()
|
||||
|
||||
processed += len(gl_entries)
|
||||
|
||||
# Progress message
|
||||
percent = flt((processed / un_processed) * 100, 2)
|
||||
if percent - last_update_percent > 1:
|
||||
print(f"{percent}% ({processed}) records processed")
|
||||
last_update_percent = percent
|
||||
|
||||
except Exception as err:
|
||||
print("Migration Failed. Clear `tabPayment Ledger Entry` table before re-running")
|
||||
raise err
|
||||
else:
|
||||
break
|
||||
print(f"{processed} records have been sucessfully migrated")
|
||||
|
||||
@@ -1,81 +1,98 @@
|
||||
import frappe
|
||||
from frappe import qb
|
||||
from frappe.utils import create_batch
|
||||
|
||||
|
||||
def remove_duplicate_entries(pl_entries):
|
||||
unique_vouchers = set()
|
||||
for x in pl_entries:
|
||||
unique_vouchers.add(
|
||||
(x.company, x.account, x.party_type, x.party, x.voucher_type, x.voucher_no, x.gle_remarks)
|
||||
)
|
||||
|
||||
entries = []
|
||||
for x in unique_vouchers:
|
||||
entries.append(
|
||||
frappe._dict(
|
||||
company=x[0],
|
||||
account=x[1],
|
||||
party_type=x[2],
|
||||
party=x[3],
|
||||
voucher_type=x[4],
|
||||
voucher_no=x[5],
|
||||
gle_remarks=x[6],
|
||||
)
|
||||
)
|
||||
return entries
|
||||
from frappe.query_builder import CustomFunction
|
||||
from frappe.query_builder.functions import Count, IfNull
|
||||
from frappe.utils import flt
|
||||
|
||||
|
||||
def execute():
|
||||
"""
|
||||
Migrate 'remarks' field from 'tabGL Entry' to 'tabPayment Ledger Entry'
|
||||
"""
|
||||
|
||||
if frappe.reload_doc("accounts", "doctype", "payment_ledger_entry"):
|
||||
|
||||
gle = qb.DocType("GL Entry")
|
||||
ple = qb.DocType("Payment Ledger Entry")
|
||||
|
||||
# get ple and their remarks from GL Entry
|
||||
pl_entries = (
|
||||
qb.from_(ple)
|
||||
.left_join(gle)
|
||||
.on(
|
||||
(ple.account == gle.account)
|
||||
& (ple.party_type == gle.party_type)
|
||||
& (ple.party == gle.party)
|
||||
& (ple.voucher_type == gle.voucher_type)
|
||||
& (ple.voucher_no == gle.voucher_no)
|
||||
& (ple.company == gle.company)
|
||||
)
|
||||
.select(
|
||||
ple.company,
|
||||
ple.account,
|
||||
ple.party_type,
|
||||
ple.party,
|
||||
ple.voucher_type,
|
||||
ple.voucher_no,
|
||||
gle.remarks.as_("gle_remarks"),
|
||||
)
|
||||
.where((ple.delinked == 0) & (gle.is_cancelled == 0))
|
||||
.run(as_dict=True)
|
||||
)
|
||||
# Get empty PLE records
|
||||
un_processed = (
|
||||
qb.from_(ple).select(Count(ple.name)).where((ple.remarks.isnull()) & (ple.delinked == 0)).run()
|
||||
)[0][0]
|
||||
|
||||
pl_entries = remove_duplicate_entries(pl_entries)
|
||||
if un_processed:
|
||||
print(f"Remarks for {un_processed} Payment Ledger records will be updated from GL Entry")
|
||||
|
||||
if pl_entries:
|
||||
# split into multiple batches, update and commit for each batch
|
||||
ifelse = CustomFunction("IF", ["condition", "then", "else"])
|
||||
|
||||
processed = 0
|
||||
last_percent_update = 0
|
||||
batch_size = 1000
|
||||
for batch in create_batch(pl_entries, batch_size):
|
||||
for entry in batch:
|
||||
query = (
|
||||
qb.update(ple)
|
||||
.set(ple.remarks, entry.gle_remarks)
|
||||
.where(
|
||||
(ple.company == entry.company)
|
||||
& (ple.account == entry.account)
|
||||
& (ple.party_type == entry.party_type)
|
||||
& (ple.party == entry.party)
|
||||
& (ple.voucher_type == entry.voucher_type)
|
||||
& (ple.voucher_no == entry.voucher_no)
|
||||
)
|
||||
)
|
||||
query.run()
|
||||
last_name = None
|
||||
|
||||
frappe.db.commit()
|
||||
while True:
|
||||
if last_name:
|
||||
where_clause = (ple.name.gt(last_name)) & (ple.remarks.isnull()) & (ple.delinked == 0)
|
||||
else:
|
||||
where_clause = (ple.remarks.isnull()) & (ple.delinked == 0)
|
||||
|
||||
# results are deterministic
|
||||
names = (
|
||||
qb.from_(ple).select(ple.name).where(where_clause).orderby(ple.name).limit(batch_size).run()
|
||||
)
|
||||
|
||||
if names:
|
||||
last_name = names[-1][0]
|
||||
|
||||
pl_entries = (
|
||||
qb.from_(ple)
|
||||
.left_join(gle)
|
||||
.on(
|
||||
(ple.account == gle.account)
|
||||
& (ple.party_type == gle.party_type)
|
||||
& (ple.party == gle.party)
|
||||
& (ple.voucher_type == gle.voucher_type)
|
||||
& (ple.voucher_no == gle.voucher_no)
|
||||
& (
|
||||
ple.against_voucher_type
|
||||
== IfNull(
|
||||
ifelse(gle.against_voucher_type == "", None, gle.against_voucher_type), gle.voucher_type
|
||||
)
|
||||
)
|
||||
& (
|
||||
ple.against_voucher_no
|
||||
== IfNull(ifelse(gle.against_voucher == "", None, gle.against_voucher), gle.voucher_no)
|
||||
)
|
||||
& (ple.company == gle.company)
|
||||
& (
|
||||
((ple.account_type == "Receivable") & (ple.amount == (gle.debit - gle.credit)))
|
||||
| (ple.account_type == "Payable") & (ple.amount == (gle.credit - gle.debit))
|
||||
)
|
||||
& (gle.remarks.notnull())
|
||||
& (gle.is_cancelled == 0)
|
||||
)
|
||||
.select(ple.name)
|
||||
.distinct()
|
||||
.select(
|
||||
gle.remarks.as_("gle_remarks"),
|
||||
)
|
||||
.where(ple.name.isin(names))
|
||||
.run(as_dict=True)
|
||||
)
|
||||
|
||||
if pl_entries:
|
||||
for entry in pl_entries:
|
||||
query = qb.update(ple).set(ple.remarks, entry.gle_remarks).where((ple.name == entry.name))
|
||||
query.run()
|
||||
|
||||
frappe.db.commit()
|
||||
|
||||
processed += len(pl_entries)
|
||||
percentage = flt((processed / un_processed) * 100, 2)
|
||||
if percentage - last_percent_update > 1:
|
||||
print(f"{percentage}% ({processed}) PLE records updated")
|
||||
last_percent_update = percentage
|
||||
|
||||
else:
|
||||
break
|
||||
print("Remarks succesfully migrated")
|
||||
|
||||
Reference in New Issue
Block a user