[email] many fixes, newsletter, unsubscribe

erpnext/crm/doctype/lead/lead.json

@@ -54,6 +54,7 @@
    "label": "Email Id", 
    "oldfieldname": "email_id", 
    "oldfieldtype": "Data", 
+   "options": "Email", 
    "permlevel": 0, 
    "reqd": 0, 
    "search_index": 1
@@ -331,7 +332,7 @@
  ], 
  "icon": "icon-user", 
  "idx": 1, 
- "modified": "2015-02-20 05:07:57.739910", 
+ "modified": "2015-03-18 07:06:34.330943", 
  "modified_by": "Administrator", 
  "module": "CRM", 
  "name": "Lead", 

erpnext/crm/doctype/newsletter/newsletter.js

@@ -0,0 +1,63 @@
+// Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+// License: GNU General Public License v3. See license.txt
+
+cur_frm.cscript.onload = function(doc) {
+	return frappe.call({
+		method: "erpnext.crm.doctype.newsletter.newsletter.get_lead_options",
+		type: "GET",
+		callback: function(r) {
+			set_field_options("lead_source", r.message.sources.join("\n"))
+			set_field_options("lead_status", r.message.statuses.join("\n"))
+		}
+	});
+}
+
+cur_frm.cscript.refresh = function(doc) {
+	erpnext.toggle_naming_series();
+	if(!doc.__islocal && !cint(doc.email_sent) && !doc.__unsaved
+			&& inList(frappe.boot.user.can_write, doc.doctype)) {
+		cur_frm.add_custom_button(__('Send'), function() {
+			return $c_obj(doc, 'send_emails', '', function(r) {
+				cur_frm.refresh();
+			});
+		}, "icon-play", "btn-success");
+	}
+
+	cur_frm.cscript.setup_dashboard();
+
+	if(doc.__islocal && !doc.send_from) {
+		cur_frm.set_value("send_from",
+			repl("%(fullname)s <%(email)s>", frappe.user_info(doc.owner)));
+	}
+}
+
+cur_frm.cscript.setup_dashboard = function() {
+	cur_frm.dashboard.reset();
+	if(!cur_frm.doc.__islocal && cint(cur_frm.doc.email_sent) && cur_frm.doc.__onload && cur_frm.doc.__onload.status_count) {
+		var stat = cur_frm.doc.__onload.status_count;
+		var total = frappe.utils.sum($.map(stat, function(v) { return v; }));
+		if(total) {
+			$.each(stat, function(k, v) {
+				stat[k] = flt(v * 100 / total, 2);
+			});
+
+			cur_frm.dashboard.add_progress("Status", [
+				{
+					title: stat["Sent"] + "% Sent",
+					width: stat["Sent"],
+					progress_class: "progress-bar-success"
+				},
+				{
+					title: stat["Sending"] + "% Sending",
+					width: stat["Sending"],
+					progress_class: "progress-bar-warning"
+				},
+				{
+					title: stat["Error"] + "% Error",
+					width: stat["Error"],
+					progress_class: "progress-bar-danger"
+				}
+			]);
+		}
+	}
+}

erpnext/crm/doctype/newsletter/newsletter.json

@@ -0,0 +1,104 @@
+{
+ "autoname": "field:subject", 
+ "creation": "2013-01-10 16:34:31", 
+ "description": "Create and Send Newsletters", 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "Other", 
+ "fields": [
+  {
+   "fieldname": "newsletter_list", 
+   "fieldtype": "Link", 
+   "label": "Newsletter List", 
+   "options": "Newsletter List", 
+   "permlevel": 0, 
+   "precision": "", 
+   "reqd": 1
+  }, 
+  {
+   "fieldname": "subject", 
+   "fieldtype": "Small Text", 
+   "in_list_view": 0, 
+   "label": "Subject", 
+   "permlevel": 0, 
+   "reqd": 1
+  }, 
+  {
+   "description": "", 
+   "fieldname": "send_from", 
+   "fieldtype": "Data", 
+   "label": "Sender", 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "reqd": 1
+  }, 
+  {
+   "fieldname": "email_sent", 
+   "fieldtype": "Check", 
+   "in_list_view": 0, 
+   "label": "Email Sent?", 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "read_only": 1
+  }, 
+  {
+   "fieldname": "newsletter_content", 
+   "fieldtype": "Section Break", 
+   "label": "", 
+   "permlevel": 0
+  }, 
+  {
+   "fieldname": "message", 
+   "fieldtype": "Text Editor", 
+   "label": "Message", 
+   "permlevel": 0, 
+   "reqd": 0
+  }, 
+  {
+   "description": "", 
+   "fieldname": "test_the_newsletter", 
+   "fieldtype": "Section Break", 
+   "label": "", 
+   "permlevel": 0
+  }, 
+  {
+   "description": "A Lead with this email id should exist", 
+   "fieldname": "test_email_id", 
+   "fieldtype": "Data", 
+   "label": "Test Email Id", 
+   "permlevel": 0
+  }, 
+  {
+   "fieldname": "test_send", 
+   "fieldtype": "Button", 
+   "label": "Test", 
+   "options": "test_send", 
+   "permlevel": 0
+  }
+ ], 
+ "icon": "icon-envelope", 
+ "idx": 1, 
+ "modified": "2015-03-18 08:19:24.393173", 
+ "modified_by": "Administrator", 
+ "module": "CRM", 
+ "name": "Newsletter", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "Newsletter Manager", 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "title_field": "subject"
+}

erpnext/crm/doctype/newsletter/newsletter.py

@@ -0,0 +1,114 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# License: GNU General Public License v3. See license.txt
+
+from __future__ import unicode_literals
+
+import frappe
+import frappe.utils
+from frappe import throw, _
+from frappe.model.document import Document
+import erpnext.tasks
+
+class Newsletter(Document):
+	def onload(self):
+		if self.email_sent:
+			self.get("__onload").status_count = dict(frappe.db.sql("""select status, count(name)
+				from `tabBulk Email` where ref_doctype=%s and ref_docname=%s
+				group by status""", (self.doctype, self.name))) or None
+
+	def test_send(self, doctype="Lead"):
+		self.recipients = self.test_email_id.split(",")
+		self.send_bulk()
+		frappe.msgprint(_("Scheduled to send to {0}").format(self.test_email_id))
+
+	def send_emails(self):
+		"""send emails to leads and customers"""
+		if self.email_sent:
+			throw(_("Newsletter has already been sent"))
+
+		self.recipients = self.get_recipients()
+
+		if getattr(frappe.local, "is_ajax", False):
+			# to avoid request timed out!
+			self.validate_send()
+
+			# hack! event="bulk_long" to queue in longjob queue
+			erpnext.tasks.send_newsletter.delay(frappe.local.site, self.name, event="bulk_long")
+		else:
+			self.send_bulk()
+
+		frappe.msgprint(_("Scheduled to send to {0} recipients").format(len(self.recipients)))
+
+		frappe.db.set(self, "email_sent", 1)
+
+	def get_recipients(self):
+		"""Get recipients from Newsletter List"""
+		return frappe.db.get_all("Newsletter List Subscriber", ["email"], {"unsubscribed": 0})
+
+	def send_bulk(self):
+		if not self.get("recipients"):
+			# in case it is called via worker
+			self.recipients = self.get_recipients()
+
+		self.validate_send()
+
+		sender = self.send_from or frappe.utils.get_formatted_email(self.owner)
+
+		from frappe.email.bulk import send
+
+		if not frappe.flags.in_test:
+			frappe.db.auto_commit_on_many_writes = True
+
+		send(recipients = self.recipients, sender = sender,
+			subject = self.subject, message = self.message,
+			ref_doctype = self.doctype, ref_docname = self.name,
+			unsubscribe_url = "/api/method/erpnext.crm.doctype.newsletter.newsletter.unsubscribe?name=%s&email={email}" % self.name)
+
+		if not frappe.flags.in_test:
+			frappe.db.auto_commit_on_many_writes = False
+
+	def validate_send(self):
+		if self.get("__islocal"):
+			throw(_("Please save the Newsletter before sending"))
+
+@frappe.whitelist()
+def get_lead_options():
+	return {
+		"sources": ["All"] + filter(None,
+			frappe.db.sql_list("""select distinct source from tabLead""")),
+		"statuses": ["All"] + filter(None,
+			frappe.db.sql_list("""select distinct status from tabLead"""))
+	}
+
+
+@frappe.whitelist(allow_guest=True)
+def unsubscribe(email, name):
+	from frappe.email.bulk import return_unsubscribed_page
+
+	name = frappe.db.get_value("Newsletter List Subscriber", {"email": email, "newsletter_list": name})
+	if name:
+		subscriber = frappe.get_doc("Newsletter List Subscriber", name)
+		subscriber.unsubscribe = 1
+		subscriber.save(ignore_permissions=True)
+
+	return_unsubscribed_page(email)
+
+def create_lead(email_id):
+	"""create a lead if it does not exist"""
+	from email.utils import parseaddr
+	from frappe.model.naming import get_default_naming_series
+	real_name, email_id = parseaddr(email_id)
+
+	if frappe.db.get_value("Lead", {"email_id": email_id}):
+		return
+
+	lead = frappe.get_doc({
+		"doctype": "Lead",
+		"email_id": email_id,
+		"lead_name": real_name or email_id,
+		"status": "Lead",
+		"naming_series": get_default_naming_series("Lead"),
+		"company": frappe.db.get_default("company"),
+		"source": "Email"
+	})
+	lead.insert()

erpnext/crm/doctype/newsletter/newsletter_list.js

@@ -0,0 +1,10 @@
+frappe.listview_settings['Newsletter'] = {
+	add_fields: ["subject", "send_to_type", "email_sent"],
+	get_indicator: function(doc) {
+		if(doc.email_sent) {
+			return [__("Sent"), "green", "email_sent,=,Yes"];
+		} else {
+			return [__("Not Sent"), "orange", "email_sent,=,No"];
+		}
+	}
+};

erpnext/crm/doctype/newsletter/test_newsletter.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# License: GNU General Public License v3. See license.txt
+from __future__ import unicode_literals
+
+import frappe, unittest
+
+class TestNewsletter(unittest.TestCase):
+	def test_get_recipients_lead(self):
+		w = frappe.get_doc(test_records[0])
+		w.insert()
+		self.assertTrue("test_lead@example.com" in w.get_recipients())
+		frappe.db.sql("""delete from `tabBulk Email`""")
+		w.send_emails()
+		self.assertTrue(frappe.db.get_value("Bulk Email", {"recipient": "test_lead@example.com"}))
+
+	def test_get_recipients_lead_by_status(self):
+		w = frappe.get_doc(test_records[0])
+		w.lead_status="Converted"
+		w.insert()
+		self.assertTrue("test_lead3@example.com" in w.get_recipients())
+
+	def test_get_recipients_contact_customer(self):
+		w = frappe.get_doc(test_records[1])
+		w.insert()
+		self.assertTrue("test_contact_customer@example.com" in w.get_recipients())
+
+	def test_get_recipients_contact_supplier(self):
+		w = frappe.get_doc(test_records[1])
+		w.contact_type="Supplier"
+		w.insert()
+		self.assertTrue("test_contact_supplier@example.com" in w.get_recipients())
+
+	def test_get_recipients_custom(self):
+		w = frappe.get_doc(test_records[2])
+		w.insert()
+		self.assertTrue("test_custom2@example.com" in w.get_recipients())
+		self.assertTrue(frappe.db.get("Lead",
+			{"email_id": "test_custom2@example.com"}))
+
+
+test_dependencies = ["Lead", "Contact"]
+
+test_records = frappe.get_test_records('Newsletter')

erpnext/crm/doctype/newsletter/test_records.json

@@ -0,0 +1,26 @@
+[
+ {
+  "doctype": "Newsletter", 
+  "lead_source": "All", 
+  "message": "This is a test newsletter", 
+  "send_from": "admin@example.com", 
+  "send_to_type": "Lead", 
+  "subject": "_Test Newsletter to Lead"
+ }, 
+ {
+  "contact_type": "Customer", 
+  "doctype": "Newsletter", 
+  "message": "This is a test newsletter", 
+  "send_from": "admin@example.com", 
+  "send_to_type": "Contact", 
+  "subject": "_Test Newsletter to Contact"
+ }, 
+ {
+  "doctype": "Newsletter", 
+  "email_list": "test_custom@example.com, test_custom1@example.com, test_custom2@example.com", 
+  "message": "This is a test newsletter", 
+  "send_from": "admin@example.com", 
+  "send_to_type": "Custom", 
+  "subject": "_Test Newsletter to Custom"
+ }
+]

erpnext/crm/doctype/newsletter_list/newsletter_list.js

@@ -0,0 +1,30 @@
+frappe.ui.form.on("Newsletter List", "refresh", function(frm) {
+	if(!frm.is_new()) {
+		frm.add_custom_button(__("View Subscribers"), function() {
+			frappe.route_options = {"newsletter_list": frm.doc.name};
+			frappe.set_route("Report", "Newsletter List Subscriber");
+		});
+
+		frm.add_custom_button(__("Import Subscribers"), function() {
+			frappe.prompt({fieldtype:"Select", options: frm.doc.__onload.import_types,
+				label:__("Import Email From"), fieldname:"doctype", reqd:1}, function(data) {
+					frappe.call({
+						method: "erpnext.crm.doctype.newsletter_list.newsletter_list.import_from",
+						args: {
+							"name": frm.doc.name,
+							"doctype": data.doctype
+						},
+						callback: function(r) {
+							frm.set_value("total_subscribers", r.message);
+						}
+					})
+				}, __("Import Subscribers"), __("Import"));
+		});
+
+		frm.add_custom_button(__("New Newsletter"), function() {
+			frappe.route_options = {"newsletter_list": frm.doc.name};
+			new_doc("Newsletter");
+		});
+
+	}
+});

erpnext/crm/doctype/newsletter_list/newsletter_list.json

@@ -0,0 +1,79 @@
+{
+ "allow_copy": 0, 
+ "allow_import": 1, 
+ "allow_rename": 0, 
+ "autoname": "field:title", 
+ "creation": "2015-03-18 06:08:32.729800", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "Master", 
+ "fields": [
+  {
+   "allow_on_submit": 0, 
+   "fieldname": "title", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "in_filter": 0, 
+   "in_list_view": 0, 
+   "label": "Title", 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "read_only": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0
+  }, 
+  {
+   "default": "0", 
+   "fieldname": "total_subscribers", 
+   "fieldtype": "Int", 
+   "label": "Total Subscribers", 
+   "permlevel": 0, 
+   "precision": "", 
+   "read_only": 1
+  }
+ ], 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "in_create": 0, 
+ "in_dialog": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "modified": "2015-03-18 08:08:37.692367", 
+ "modified_by": "Administrator", 
+ "module": "CRM", 
+ "name": "Newsletter List", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "import": 1, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "Newsletter Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC"
+}

erpnext/crm/doctype/newsletter_list/newsletter_list.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+
+class NewsletterList(Document):
+	def onload(self):
+		singles = [d.name for d in frappe.db.get_all("DocType", "name", {"issingle": 1})]
+		self.get("__onload").import_types = [d.parent \
+			for d in frappe.db.get_all("DocField", "parent", {"options": "Email"}) if d.parent not in singles]
+
+	def import_from(self, doctype):
+		"""Extract email ids from given doctype and add them to the current list"""
+		meta = frappe.get_meta(doctype)
+		email_field = [d.fieldname for d in meta.fields if d.fieldtype in ("Data", "Small Text") and d.options=="Email"][0]
+		unsubscribed_field = "unsubscribed" if meta.get_field("unsubscribed") else None
+		added = 0
+
+		for user in frappe.db.get_all(doctype, [email_field, unsubscribed_field or "name"]):
+			try:
+				if user.get(email_field):
+					frappe.get_doc({
+						"doctype": "Newsletter List Subscriber",
+						"newsletter_list": self.name,
+						"email": user.get(email_field),
+						"unsubscribed": user.get(unsubscribed_field) if unsubscribed_field else 0
+					}).insert(ignore_permissions=True)
+
+					added += 1
+			except Exception, e:
+				# already added, ignore
+				if e.args[0]!=1062:
+					raise
+
+		frappe.msgprint(frappe._("{0} subscribers added").format(added))
+
+		return self.update_total_subscribers()
+
+	def update_total_subscribers(self):
+		self.total_subscribers = frappe.db.sql("""select count(*) from `tabNewsletter List Subscriber`
+			where newsletter_list=%s""", self.name)[0][0]
+		self.db_update()
+		return self.total_subscribers
+
+@frappe.whitelist()
+def import_from(name, doctype):
+	nlist = frappe.get_doc("Newsletter List", name)
+	if nlist.has_permission("write"):
+		return nlist.import_from(doctype)

erpnext/crm/doctype/newsletter_list/test_newsletter_list.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import frappe
+import unittest
+
+# test_records = frappe.get_test_records('Newletter List')
+
+class TestNewletterList(unittest.TestCase):
+	pass

erpnext/crm/doctype/newsletter_list_subscriber/newsletter_list_subscriber.json

@@ -0,0 +1,109 @@
+{
+ "allow_copy": 0, 
+ "allow_import": 1, 
+ "allow_rename": 0, 
+ "autoname": "hash", 
+ "creation": "2015-03-18 06:15:59.321619", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "Transaction", 
+ "fields": [
+  {
+   "allow_on_submit": 0, 
+   "fieldname": "newsletter_list", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "in_filter": 0, 
+   "in_list_view": 1, 
+   "label": "Newsletter List", 
+   "no_copy": 0, 
+   "options": "Newsletter List", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "read_only": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0
+  }, 
+  {
+   "allow_on_submit": 0, 
+   "fieldname": "email", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "in_filter": 0, 
+   "in_list_view": 1, 
+   "label": "Email", 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "read_only": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0
+  }, 
+  {
+   "allow_on_submit": 0, 
+   "fieldname": "unsubscribed", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "in_filter": 0, 
+   "in_list_view": 1, 
+   "label": "Unsubscribed", 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "read_only": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0
+  }
+ ], 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "in_create": 0, 
+ "in_dialog": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "modified": "2015-03-18 08:07:24.748431", 
+ "modified_by": "Administrator", 
+ "module": "CRM", 
+ "name": "Newsletter List Subscriber", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "import": 1, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "Newsletter Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC"
+}

erpnext/crm/doctype/newsletter_list_subscriber/newsletter_list_subscriber.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+
+class NewsletterListSubscriber(Document):
+	pass
+
+def after_doctype_insert():
+	frappe.db.add_unique("Newsletter List Subscriber", ("name", "email"))

erpnext/crm/doctype/newsletter_list_subscriber/test_newsletter_list_subscriber.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import frappe
+import unittest
+
+# test_records = frappe.get_test_records('Newsletter List Subscriber')
+
+class TestNewsletterListSubscriber(unittest.TestCase):
+	pass