diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js index 4652d60bff7..104ac570c69 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js @@ -2,36 +2,49 @@ // For license information, please see license.txt frappe.ui.form.on('Tally Migration', { - refresh: function(frm) { - if (frm.doc.master_data && frm.doc.day_book) { - frm.disable_save(); - if(frm.doc.status != "In Progress") { - frm.page.set_primary_action("Preprocess", () => frm.trigger("preprocess")); + onload: function(frm) { + frappe.realtime.on("tally_migration_progress_update", function (data) { + frm.dashboard.show_progress(data.title, (data.count / data.total) * 100, data.message); + if (data.count == data.total) { + window.setTimeout(title => frm.dashboard.hide_progress(title), 1500, data.title); } - } else { - frm.set_value("status", "Attach File"); - } - if (frm.doc.tally_company && frm.doc.erpnext_company) { - frm.set_df_property("company_section", "hidden", 0); - frm.page.set_primary_action("Start Import", () => frm.trigger("start_import")); - } - }, - preprocess: function(frm) { - frm.call({ - doc: frm.doc, - method: "preprocess", - freeze: true - }).then((r) => { - frm.set_value("status", "Preprocessing In Progress"); }); }, - start_import: function(frm) { - frm.call({ - doc: frm.doc, - method: "start_import", - freeze: true - }).then((r) => { - frm.set_value("status", "Import In Progress"); - }); + refresh: function(frm) { + if (frm.doc.master_data && !frm.doc.is_master_data_imported) { + if (frm.doc.is_master_data_processed) { + if (frm.doc.status != "Importing Master Data") { + frm.events.add_button(frm, __("Import Master Data"), "import_master_data"); + } + } else { + if (frm.doc.status != "Processing Master Data") { + frm.events.add_button(frm, __("Process Master Data"), "process_master_data"); + } + } + } + if (frm.doc.day_book_data && !frm.doc.is_day_book_data_imported) { + if (frm.doc.is_day_book_data_processed) { + if (frm.doc.status != "Importing Day Book Data") { + frm.events.add_button(frm, __("Import Day Book Data"), "import_day_book_data"); + } + } else { + if (frm.doc.status != "Processing Day Book Data") { + frm.events.add_button(frm, __("Process Day Book Data"), "process_day_book_data"); + } + } + } }, + add_button: function(frm, label, method) { + frm.add_custom_button( + label, + () => frm.call({ + doc: frm.doc, + method: method, + freeze: true, + callback: () => { + frm.remove_custom_button(label); + } + }) + ); + } }); diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json index 54e162b4af4..26415caf8d3 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json @@ -1,610 +1,219 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "", - "beta": 0, + "beta": 1, "creation": "2019-02-01 14:27:09.485238", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "status", + "master_data", + "is_master_data_processed", + "is_master_data_imported", + "column_break_2", + "tally_creditors_account", + "tally_debtors_account", + "company_section", + "tally_company", + "column_break_8", + "erpnext_company", + "processed_files_section", + "chart_of_accounts", + "parties", + "addresses", + "column_break_17", + "uoms", + "items", + "vouchers", + "accounts_section", + "default_warehouse", + "round_off_account", + "column_break_21", + "default_cost_center", + "day_book_section", + "day_book_data", + "column_break_27", + "is_day_book_data_processed", + "is_day_book_data_imported" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "status", "fieldtype": "Data", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Status", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Status" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "master_data", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, - "label": "Master Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Master Data" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "default": "Sundry Creditors", "fieldname": "tally_creditors_account", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Tally Creditors Account", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "column_break_2", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Column Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "day_book", - "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Day Book", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "default": "Sundry Debtors", "fieldname": "tally_debtors_account", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Tally Debtors Account", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, + "depends_on": "is_master_data_processed", "fieldname": "company_section", - "fieldtype": "Section Break", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Company Section", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Section Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "tally_company", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Tally Company", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "column_break_8", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Column Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "erpnext_company", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "ERPNext Company", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "ERPNext Company" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "processed_files", + "fieldname": "processed_files_section", "fieldtype": "Section Break", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Processed Files", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Processed Files" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "chart_of_accounts", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Chart of Accounts", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Chart of Accounts" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "parties", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Parties", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Parties" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "addresses", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Addresses", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Addresses" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "column_break_17", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Column Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "uoms", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "UOMs", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "UOMs" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "items", "fieldtype": "Attach", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Items", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Items" + }, + { + "fieldname": "vouchers", + "fieldtype": "Attach", + "label": "Vouchers" + }, + { + "depends_on": "is_master_data_imported", + "fieldname": "accounts_section", + "fieldtype": "Section Break", + "label": "Accounts" + }, + { + "fieldname": "default_warehouse", + "fieldtype": "Link", + "label": "Default Warehouse", + "options": "Warehouse" + }, + { + "fieldname": "round_off_account", + "fieldtype": "Link", + "label": "Round Off Account", + "options": "Account" + }, + { + "fieldname": "column_break_21", + "fieldtype": "Column Break" + }, + { + "fieldname": "default_cost_center", + "fieldtype": "Link", + "label": "Default Cost Center", + "options": "Cost Center" + }, + { + "fieldname": "is_master_data_processed", + "fieldtype": "Check", + "label": "Is Master Data Processed", + "read_only": 1 + }, + { + "fieldname": "is_day_book_data_processed", + "fieldtype": "Check", + "label": "Is Day Book Data Processed", + "read_only": 1 + }, + { + "fieldname": "is_day_book_data_imported", + "fieldtype": "Check", + "label": "Is Day Book Data Imported", + "read_only": 1 + }, + { + "fieldname": "is_master_data_imported", + "fieldtype": "Check", + "label": "Is Master Data Imported", + "read_only": 1 + }, + { + "depends_on": "is_master_data_imported", + "fieldname": "day_book_section", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_27", + "fieldtype": "Column Break" + }, + { + "fieldname": "day_book_data", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Day Book Data" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-03-02 21:51:48.869051", + "modified": "2019-04-29 05:46:54.394967", "modified_by": "Administrator", "module": "ERPNext Integrations", "name": "Tally Migration", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "track_changes": 1 } \ No newline at end of file diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py index cedf0167ab4..5b96ad36f14 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -4,86 +4,65 @@ from __future__ import unicode_literals +from decimal import Decimal import json import re import traceback import zipfile import frappe +from frappe import _ +from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.model.document import Document +from frappe.model.naming import getseries, revert_series_if_last +from frappe.utils.data import format_datetime from bs4 import BeautifulSoup as bs -from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts +from erpnext import encode_company_abbr +from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts PRIMARY_ACCOUNT = "Primary" +VOUCHER_CHUNK_SIZE = 500 + class TallyMigration(Document): - def _preprocess(self): - company, chart_of_accounts_tree, customers, suppliers = self._process_master_data() - parties, addresses = self._process_parties(customers, suppliers) - items, uoms = self._process_stock_items() - self.tally_company = company - self.erpnext_company = company - self.status = "Preprocessed" + def autoname(self): + if not self.name: + self.name = "Tally Migration on " + format_datetime(self.creation) - coa_file = frappe.get_doc({ - "doctype": "File", - "file_name": "COA.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(chart_of_accounts_tree) - }).insert() - self.chart_of_accounts = coa_file.file_url + def get_collection(self, data_file): + def sanitize(string): + return re.sub("", "", string) - parties_file = frappe.get_doc({ - "doctype": "File", - "file_name": "Parties.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(parties) - }).insert() - self.parties = parties_file.file_url + def emptify(string): + string = re.sub(r"<\w+/>", "", string) + string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string) + string = re.sub(r"\r\n", "", string) + return string - addresses_file = frappe.get_doc({ - "doctype": "File", - "file_name": "Addresses.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(addresses) - }).insert() - self.addresses = addresses_file.file_url - - uoms_file = frappe.get_doc({ - "doctype": "File", - "file_name": "UOMs.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(uoms) - }).insert() - self.uoms = uoms_file.file_url - - items_file = frappe.get_doc({ - "doctype": "File", - "file_name": "Items.json", - "attached_to_doctype": self.doctype, - "attached_to_name": self.name, - "content": json.dumps(items) - }).insert() - self.items = items_file.file_url - - - self.save() - - def get_master_collection(self): - master_file = frappe.get_doc("File", {"file_url": self.master_data}) + master_file = frappe.get_doc("File", {"file_url": data_file}) with zipfile.ZipFile(master_file.get_full_path()) as zf: - content = zf.read(zf.namelist()[0]).decode("utf-16") + encoded_content = zf.read(zf.namelist()[0]) + try: + content = encoded_content.decode("utf-8-sig") + except UnicodeDecodeError: + content = encoded_content.decode("utf-16") master = bs(sanitize(emptify(content)), "xml") collection = master.BODY.IMPORTDATA.REQUESTDATA return collection - def _process_master_data(self): + def dump_processed_data(self, data): + for key, value in data.items(): + f = frappe.get_doc({ + "doctype": "File", + "file_name": key + ".json", + "attached_to_doctype": self.doctype, + "attached_to_name": self.name, + "content": json.dumps(value) + }).insert() + setattr(self, key, f.file_url) + def _process_master_data(self): def get_company_name(collection): return collection.find_all("REMOTECMPINFO.LIST")[0].REMOTECMPNAME.string @@ -144,11 +123,11 @@ class TallyMigration(Document): if self.tally_creditors_account in parents[account]: children.pop(account, None) if account not in group_set: - customers.add(account) + suppliers.add(account) elif self.tally_debtors_account in parents[account]: children.pop(account, None) if account not in group_set: - suppliers.add(account) + customers.add(account) return children, customers, suppliers def traverse(tree, children, accounts, roots, group_set): @@ -162,14 +141,6 @@ class TallyMigration(Document): tree[account] = {} return tree - collection = self.get_master_collection() - - company = get_company_name(collection) - chart_of_accounts_tree, customer_names, supplier_names = get_coa_customers_suppliers(collection) - - return company, chart_of_accounts_tree, customer_names, supplier_names - - def _process_parties(self, customers, suppliers): def get_parties_addresses(collection, customers, suppliers): parties, addresses = [], [] for account in collection.find_all("LEDGER"): @@ -210,33 +181,49 @@ class TallyMigration(Document): }) return parties, addresses - collection = self.get_master_collection() + def get_stock_items_uoms(collection): + uoms = [] + for uom in collection.find_all("UNIT"): + uoms.append({"doctype": "UOM", "uom_name": uom.NAME.string}) + + items = [] + for item in collection.find_all("STOCKITEM"): + items.append({ + "doctype": "Item", + "item_code" : item.NAME.string, + "stock_uom": item.BASEUNITS.string, + "is_stock_item": 0, + "item_group": "All Item Groups", + "item_defaults": [{"company": self.erpnext_company}] + }) + return items, uoms + + + self.publish("Process Master Data", _("Reading Uploaded File"), 1, 5) + collection = self.get_collection(self.master_data) + + company = get_company_name(collection) + self.tally_company = company + self.erpnext_company = company + + self.publish("Process Master Data", _("Processing Chart of Accounts and Parties"), 2, 5) + chart_of_accounts, customers, suppliers = get_coa_customers_suppliers(collection) + self.publish("Process Master Data", _("Processing Party Addresses"), 3, 5) parties, addresses = get_parties_addresses(collection, customers, suppliers) - return parties, addresses + self.publish("Process Master Data", _("Processing Items and UOMs"), 4, 5) + items, uoms = get_stock_items_uoms(collection) + data = {"chart_of_accounts": chart_of_accounts, "parties": parties, "addresses": addresses, "items": items, "uoms": uoms} + self.publish("Process Master Data", _("Done"), 5, 5) - def _process_stock_items(self): - collection = self.get_master_collection() - uoms = [] - for uom in collection.find_all("UNIT"): - uoms.append({"doctype": "UOM", "uom_name": uom.NAME.string}) + self.dump_processed_data(data) + self.is_master_data_processed = 1 + self.status = "" + self.save() - items = [] - for item in collection.find_all("STOCKITEM"): - hsn_code = item.find_all("GSTDETAILS.LIST")[0].HSNCODE - items.append({ - "doctype": "Item", - "item_code" : item.NAME.string, - "stock_uom": item.BASEUNITS.string, - "gst_hsn_code": hsn_code.string if hsn_code else None, - "is_stock_item": 0, - "item_group": "All Item Groups", - }) - return items, uoms + def publish(self, title, message, count, total): + frappe.publish_realtime("tally_migration_progress_update", {"title": title, "message": message, "count": count, "total": total}) - def preprocess(self): - frappe.enqueue_doc(self.doctype, self.name, "_preprocess") - - def _start_import(self): + def _import_master_data(self): def create_company_and_coa(coa_file_url): coa_file = frappe.get_doc("File", {"file_url": coa_file_url}) frappe.local.flags.ignore_chart_of_accounts = True @@ -244,56 +231,282 @@ class TallyMigration(Document): "doctype": "Company", "company_name": self.erpnext_company, "default_currency": "INR", + "enable_perpetual_inventory": 0, }).insert() frappe.local.flags.ignore_chart_of_accounts = False - create_charts(company.name, json.loads(coa_file.get_content())) + create_charts(company.name, custom_chart=json.loads(coa_file.get_content())) + company.create_default_warehouses() - def create_parties_addresses(parties_file_url, addresses_file_url): + def create_parties_and_addresses(parties_file_url, addresses_file_url): parties_file = frappe.get_doc("File", {"file_url": parties_file_url}) for party in json.loads(parties_file.get_content()): try: frappe.get_doc(party).insert() except: - log(party) + self.log(party) addresses_file = frappe.get_doc("File", {"file_url": addresses_file_url}) for address in json.loads(addresses_file.get_content()): try: frappe.get_doc(address).insert(ignore_mandatory=True) except: - log(address) + try: + gstin = address.pop("gstin", None) + frappe.get_doc(address).insert(ignore_mandatory=True) + self.log({"address": address, "message": "Invalid GSTIN: {}. Address was created without GSTIN".format(gstin)}) + except: + self.log(address) + def create_items_uoms(items_file_url, uoms_file_url): uoms_file = frappe.get_doc("File", {"file_url": uoms_file_url}) for uom in json.loads(uoms_file.get_content()): - try: - frappe.get_doc(uom).insert() - except: - log(uom) + if not frappe.db.exists(uom): + try: + frappe.get_doc(uom).insert() + except: + self.log(uom) items_file = frappe.get_doc("File", {"file_url": items_file_url}) for item in json.loads(items_file.get_content()): try: frappe.get_doc(item).insert() except: - log(item) + self.log(item) + self.publish("Import Master Data", _("Creating Company and Importing Chart of Accounts"), 1, 4) create_company_and_coa(self.chart_of_accounts) - create_parties_addresses(self.parties, self.addresses) + self.publish("Import Master Data", _("Importing Parties and Addresses"), 2, 4) + create_parties_and_addresses(self.parties, self.addresses) + self.publish("Import Master Data", _("Importing Items and UOMs"), 3, 4) create_items_uoms(self.items, self.uoms) + self.publish("Import Master Data", _("Done"), 4, 4) + self.status = "" + self.is_master_data_imported = 1 + self.save() - def start_import(self): - frappe.enqueue_doc(self.doctype, self.name, "_start_import") + def _process_day_book_data(self): + def get_vouchers(collection): + vouchers = [] + for voucher in collection.find_all("VOUCHER"): + if voucher.ISCANCELLED.string == "Yes": + continue + inventory_entries = voucher.find_all("INVENTORYENTRIES.LIST") + voucher.find_all("ALLINVENTORYENTRIES.LIST") + voucher.find_all("INVENTORYENTRIESIN.LIST") + voucher.find_all("INVENTORYENTRIESOUT.LIST") + if voucher.VOUCHERTYPENAME.string not in ["Journal", "Receipt", "Payment", "Contra"] and inventory_entries: + function = voucher_to_invoice + else: + function = voucher_to_journal_entry + try: + vouchers.append(function(voucher)) + except: + self.log(voucher) + return vouchers + def voucher_to_journal_entry(voucher): + accounts = [] + ledger_entries = voucher.find_all("ALLLEDGERENTRIES.LIST") + voucher.find_all("LEDGERENTRIES.LIST") + for entry in ledger_entries: + account = {"account": encode_company_abbr(entry.LEDGERNAME.string, self.erpnext_company), "cost_center": self.default_cost_center} + if entry.ISPARTYLEDGER.string == "Yes": + party_details = get_party(entry.LEDGERNAME.string) + if party_details: + party_type, party_account = party_details + account["party_type"] = party_type + account["account"] = party_account + account["party"] = entry.LEDGERNAME.string + amount = Decimal(entry.AMOUNT.string) + if amount > 0: + account["credit_in_account_currency"] = str(abs(amount)) + else: + account["debit_in_account_currency"] = str(abs(amount)) + accounts.append(account) -def log(data=None): - message = json.dumps({"data": data, "exception": traceback.format_exc()}, indent=4) - frappe.log_error(title="Tally Migration Error", message=message) + journal_entry = { + "doctype": "Journal Entry", + "tally_guid": voucher.GUID.string, + "posting_date": voucher.DATE.string, + "company": self.erpnext_company, + "accounts": accounts, + } + return journal_entry -def sanitize(string): - return re.sub("", "", string) + def voucher_to_invoice(voucher): + if voucher.VOUCHERTYPENAME.string in ["Sales", "Credit Note"]: + doctype = "Sales Invoice" + party_field = "customer" + account_field = "debit_to" + account_name = encode_company_abbr(self.tally_debtors_account, self.erpnext_company) + price_list_field = "selling_price_list" + elif voucher.VOUCHERTYPENAME.string in ["Purchase", "Debit Note"]: + doctype = "Purchase Invoice" + party_field = "supplier" + account_field = "credit_to" + account_name = encode_company_abbr(self.tally_creditors_account, self.erpnext_company) + price_list_field = "buying_price_list" -def emptify(string): - string = re.sub(r"<\w+/>", "", string) - string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string) - string = re.sub(r"\r\n", "", string) - return string + invoice = { + "doctype": doctype, + party_field: voucher.PARTYNAME.string, + "tally_guid": voucher.GUID.string, + "posting_date": voucher.DATE.string, + "due_date": voucher.DATE.string, + "items": get_voucher_items(voucher, doctype), + "taxes": get_voucher_taxes(voucher), + account_field: account_name, + price_list_field: "Tally Price List", + "set_posting_time": 1, + "disable_rounded_total": 1, + "company": self.erpnext_company, + } + return invoice + + def get_voucher_items(voucher, doctype): + inventory_entries = voucher.find_all("INVENTORYENTRIES.LIST") + voucher.find_all("ALLINVENTORYENTRIES.LIST") + voucher.find_all("INVENTORYENTRIESIN.LIST") + voucher.find_all("INVENTORYENTRIESOUT.LIST") + if doctype == "Sales Invoice": + account_field = "income_account" + elif doctype == "Purchase Invoice": + account_field = "expense_account" + items = [] + for entry in inventory_entries: + qty, uom = entry.ACTUALQTY.string.strip().split() + items.append({ + "item_code": entry.STOCKITEMNAME.string, + "description": entry.STOCKITEMNAME.string, + "qty": qty.strip(), + "uom": uom.strip(), + "conversion_factor": 1, + "price_list_rate": entry.RATE.string.split("/")[0], + "cost_center": self.default_cost_center, + "warehouse": self.default_warehouse, + account_field: encode_company_abbr(entry.find_all("ACCOUNTINGALLOCATIONS.LIST")[0].LEDGERNAME.string, self.erpnext_company), + }) + return items + + def get_voucher_taxes(voucher): + ledger_entries = voucher.find_all("ALLLEDGERENTRIES.LIST") + voucher.find_all("LEDGERENTRIES.LIST") + taxes = [] + for entry in ledger_entries: + if entry.ISPARTYLEDGER.string == "No": + tax_account = encode_company_abbr(entry.LEDGERNAME.string, self.erpnext_company) + taxes.append({ + "charge_type": "Actual", + "account_head": tax_account, + "description": tax_account, + "tax_amount": entry.AMOUNT.string, + "cost_center": self.default_cost_center, + }) + return taxes + + def get_party(party): + if frappe.db.exists({"doctype": "Supplier", "supplier_name": party}): + return "Supplier", encode_company_abbr(self.tally_creditors_account, self.erpnext_company) + elif frappe.db.exists({"doctype": "Customer", "customer_name": party}): + return "Customer", encode_company_abbr(self.tally_debtors_account, self.erpnext_company) + + self.publish("Process Day Book Data", _("Reading Uploaded File"), 1, 3) + collection = self.get_collection(self.day_book_data) + self.publish("Process Day Book Data", _("Processing Vouchers"), 2, 3) + vouchers = get_vouchers(collection) + self.publish("Process Day Book Data", _("Done"), 3, 3) + self.dump_processed_data({"vouchers": vouchers}) + self.status = "" + self.is_day_book_data_processed = 1 + self.save() + + def _import_day_book_data(self): + def create_fiscal_years(vouchers): + from frappe.utils.data import add_years, getdate + earliest_date = getdate(min(voucher["posting_date"] for voucher in vouchers)) + oldest_year = frappe.get_all("Fiscal Year", fields=["year_start_date", "year_end_date"], order_by="year_start_date")[0] + while earliest_date < oldest_year.year_start_date: + new_year = frappe.get_doc({"doctype": "Fiscal Year"}) + new_year.year_start_date = add_years(oldest_year.year_start_date, -1) + new_year.year_end_date = add_years(oldest_year.year_end_date, -1) + if new_year.year_start_date.year == new_year.year_end_date.year: + new_year.year = new_year.year_start_date.year + else: + new_year.year = "{}-{}".format(new_year.year_start_date.year, new_year.year_end_date.year) + new_year.save() + oldest_year = new_year + + def create_custom_fields(doctypes): + for doctype in doctypes: + df = { + "fieldtype": "Data", + "fieldname": "tally_guid", + "read_only": 1, + "label": "Tally GUID" + } + create_custom_field(doctype, df) + + def create_price_list(): + frappe.get_doc({ + "doctype": "Price List", + "price_list_name": "Tally Price List", + "selling": 1, + "buying": 1, + "enabled": 1, + "currency": "INR" + }).insert() + + frappe.db.set_value("Account", encode_company_abbr(self.tally_creditors_account, self.erpnext_company), "account_type", "Payable") + frappe.db.set_value("Account", encode_company_abbr(self.tally_debtors_account, self.erpnext_company), "account_type", "Receivable") + frappe.db.set_value("Company", self.erpnext_company, "round_off_account", self.round_off_account) + + vouchers_file = frappe.get_doc("File", {"file_url": self.vouchers}) + vouchers = json.loads(vouchers_file.get_content()) + + create_fiscal_years(vouchers) + create_price_list() + create_custom_fields(["Journal Entry", "Purchase Invoice", "Sales Invoice"]) + + total = len(vouchers) + is_last = False + for index in range(0, total, VOUCHER_CHUNK_SIZE): + if index + VOUCHER_CHUNK_SIZE >= total: + is_last = True + frappe.enqueue_doc(self.doctype, self.name, "_import_vouchers", queue="long", timeout=3600, start=index+1, total=total, is_last=is_last) + + def _import_vouchers(self, start, total, is_last=False): + frappe.flags.in_migrate = True + vouchers_file = frappe.get_doc("File", {"file_url": self.vouchers}) + vouchers = json.loads(vouchers_file.get_content()) + chunk = vouchers[start: start + VOUCHER_CHUNK_SIZE] + + for index, voucher in enumerate(chunk, start=start): + try: + doc = frappe.get_doc(voucher).insert() + doc.submit() + self.publish("Importing Vouchers", _("{} of {}").format(index, total), index, total) + except: + self.log(voucher) + + if is_last: + self.status = "" + self.is_day_book_data_imported = 1 + self.save() + frappe.db.set_value("Price List", "Tally Price List", "enabled", 0) + frappe.flags.in_migrate = False + + def process_master_data(self): + self.status = "Processing Master Data" + self.save() + frappe.enqueue_doc(self.doctype, self.name, "_process_master_data", queue="long", timeout=3600) + + def import_master_data(self): + self.status = "Importing Master Data" + self.save() + frappe.enqueue_doc(self.doctype, self.name, "_import_master_data", queue="long", timeout=3600) + + def process_day_book_data(self): + self.status = "Processing Day Book Data" + self.save() + frappe.enqueue_doc(self.doctype, self.name, "_process_day_book_data", queue="long", timeout=3600) + + def import_day_book_data(self): + self.status = "Importing Day Book Data" + self.save() + frappe.enqueue_doc(self.doctype, self.name, "_import_day_book_data", queue="long", timeout=3600) + + def log(self, data=None): + message = "\n".join(["Data", json.dumps(data, default=str, indent=4), "Exception", traceback.format_exc()]) + return frappe.log_error(title="Tally Migration Error", message=message)