snapshot of https://github.com/frappe/crm.git (develop, 2026-09-18; blobs > 131072B omitted)

crm/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "2.0.0-dev"
+__title__ = "Frappe CRM"

crm/activation.py

@@ -0,0 +1,36 @@
+import frappe
+
+
+def get_site_info(site_info):
+	# called via hook
+	return {"activation": get_sales_data(site_info)}
+
+
+def get_sales_data(site_info):
+	activation_level = site_info.get("activation", {}).get("activation_level", 0)
+	sales_data = site_info.get("activation", {}).get("sales_data", [])
+	doctypes = [
+		"CRM Lead",
+		"CRM Deal",
+		"CRM Organization",
+		"Contact",
+		"CRM Task",
+		"FCRM Note",
+		"CRM Call Log",
+		"CRM Lead Status",
+		"CRM Deal Status",
+		"CRM Lead Source",
+		"CRM Industry",
+		"CRM Territory",
+		"CRM Lost Reason",
+		"CRM Product",
+		"CRM Form Script",
+		"CRM Fields Layout",
+		"CRM View Settings",
+	]
+
+	for doctype in doctypes:
+		count = frappe.db.count(doctype)
+		sales_data.append({doctype: count})
+
+	return {"activation_level": activation_level, "sales_data": sales_data}

crm/api/__init__.py

@@ -0,0 +1,179 @@
+import frappe
+from bs4 import BeautifulSoup
+from frappe import _
+from frappe.core.api.file import get_max_file_size
+from frappe.rate_limiter import rate_limit
+from frappe.translate import get_all_translations
+from frappe.utils import cstr, split_emails, validate_email_address
+
+from crm.utils import is_frappe_version
+
+
+@frappe.whitelist(allow_guest=True)
+def get_translations():
+	if frappe.session.user != "Guest":
+		language = frappe.db.get_value("User", frappe.session.user, "language")
+	else:
+		language = frappe.db.get_single_value("System Settings", "language")
+
+	return get_all_translations(language)
+
+
+@frappe.whitelist()
+def get_user_signature():
+	user = frappe.session.user
+	user_email_signature = (
+		frappe.db.get_value(
+			"User",
+			user,
+			"email_signature",
+		)
+		if user
+		else None
+	)
+
+	signature = user_email_signature or frappe.db.get_value(
+		"Email Account",
+		{"default_outgoing": 1, "add_signature": 1},
+		"signature",
+	)
+
+	if not signature:
+		return
+
+	soup = BeautifulSoup(signature, "html.parser")
+	html_signature = soup.find("div", {"class": "ql-editor read-mode"})
+	_signature = None
+	if html_signature:
+		_signature = html_signature.renderContents()
+	content = ""
+	if cstr(_signature) or signature:
+		content = f'<br><p class="signature">{signature}</p>'
+	return content
+
+
+def check_app_permission():
+	if frappe.session.user == "Administrator":
+		return True
+
+	allowed_modules = []
+
+	if is_frappe_version("15"):
+		allowed_modules = frappe.config.get_modules_from_all_apps_for_user()
+	elif is_frappe_version("16", above=True):
+		from frappe.utils.modules import get_modules_from_all_apps_for_user
+
+		allowed_modules = get_modules_from_all_apps_for_user()
+
+	allowed_modules = [x["module_name"] for x in allowed_modules]
+	if "FCRM" not in allowed_modules:
+		return False
+
+	roles = frappe.get_roles()
+	if any(role in ["System Manager", "Sales User", "Sales Manager"] for role in roles):
+		return True
+
+	return False
+
+
+@frappe.whitelist(allow_guest=True)
+@rate_limit(limit=10, seconds=60 * 60)
+def accept_invitation(key: str | None = None):
+	if not key:
+		frappe.throw(_("Invalid or expired key"))
+
+	result = frappe.db.get_all("CRM Invitation", filters={"key": key}, pluck="name")
+	if not result:
+		frappe.throw(_("Invalid or expired key"))
+	invitation = frappe.get_doc("CRM Invitation", result[0])
+	is_new_user = invitation.accept()
+	invitation.reload()
+
+	# this is a GET request, which is rolled back unless a commit is requested
+	frappe.local.flags.commit = True
+
+	if invitation.status == "Accepted":
+		frappe.local.response["type"] = "redirect"
+		if is_new_user:
+			# a new user has no password yet, send them to the set password page
+			# which logs them in and redirects to /crm once the password is set
+			user = frappe.get_doc("User", invitation.email)
+			frappe.local.response["location"] = user._reset_password()
+		else:
+			frappe.local.login_manager.login_as(invitation.email)
+			frappe.local.response["location"] = "/crm"
+
+
+@frappe.whitelist()
+def invite_by_email(emails: str, role: str):
+	frappe.only_for(["Sales Manager", "System Manager"], True)
+
+	user_roles = frappe.get_roles(frappe.session.user)
+
+	if role == "System Manager" and "System Manager" not in user_roles:
+		frappe.throw(_("You are not allowed to invite System Managers"), frappe.PermissionError)
+
+	if role == "Sales Manager" and "System Manager" not in user_roles:
+		frappe.throw(_("You are not allowed to invite Sales Managers"), frappe.PermissionError)
+
+	if role not in ["System Manager", "Sales Manager", "Sales User"]:
+		frappe.throw(_("Cannot invite for this role"), frappe.PermissionError)
+
+	if not emails:
+		return
+
+	email_string = validate_email_address(emails, throw=False)
+	email_list = split_emails(email_string)
+	if not email_list:
+		return
+	existing_members = frappe.db.get_all("User", filters={"email": ["in", email_list]}, pluck="email")
+	existing_invites = frappe.db.get_all(
+		"CRM Invitation",
+		filters={
+			"email": ["in", email_list],
+			"role": ["in", ["System Manager", "Sales Manager", "Sales User"]],
+		},
+		pluck="email",
+	)
+
+	to_invite = list(set(email_list) - set(existing_members) - set(existing_invites))
+
+	for email in to_invite:
+		frappe.get_doc(doctype="CRM Invitation", email=email, role=role).insert(ignore_permissions=True)
+
+	return {
+		"existing_members": existing_members,
+		"existing_invites": existing_invites,
+		"to_invite": to_invite,
+	}
+
+
+@frappe.whitelist(methods=["DELETE", "POST"])
+def delete_attachment(doctype: str, docname: str, file_url: str):
+	if not frappe.has_permission(doctype, doc=docname, ptype="write"):
+		frappe.throw(_("You don't have permission to delete this attachment"), frappe.PermissionError)
+
+	file_name = frappe.db.get_value(
+		"File",
+		{"file_url": file_url, "attached_to_doctype": doctype, "attached_to_name": docname},
+		"name",
+	)
+	if file_name:
+		frappe.delete_doc("File", file_name)
+
+
+@frappe.whitelist()
+def get_file_uploader_defaults(doctype: str):
+	max_number_of_files = None
+	make_attachments_public = False
+	if doctype:
+		meta = frappe.get_meta(doctype)
+		max_number_of_files = meta.get("max_attachments")
+		make_attachments_public = meta.get("make_attachments_public")
+
+	return {
+		"allowed_file_types": frappe.get_system_settings("allowed_file_extensions"),
+		"max_file_size": get_max_file_size(),
+		"max_number_of_files": max_number_of_files,
+		"make_attachments_public": bool(make_attachments_public),
+	}

crm/api/activities.py

@@ -0,0 +1,536 @@
+import json
+
+import frappe
+from bs4 import BeautifulSoup
+from frappe import _
+from frappe.desk.form.load import get_docinfo
+from frappe.query_builder import JoinType
+from frappe.translate import get_translated_doctypes
+
+from crm.fcrm.doctype.crm_call_log.crm_call_log import parse_call_log
+from crm.fcrm.doctype.crm_fields_layout.crm_fields_layout import get_permlevel_access
+
+
+@frappe.whitelist()
+def get_activities(name: str):
+	if frappe.db.exists("CRM Deal", name):
+		return get_deal_activities(name)
+	elif frappe.db.exists("CRM Lead", name):
+		return get_lead_activities(name)
+	else:
+		frappe.throw(_("Document not found"), frappe.DoesNotExistError)
+
+
+def get_deal_activities(name: str):
+	if not frappe.has_permission("CRM Deal", "read", name):
+		frappe.throw(_("Not permitted"), frappe.PermissionError)
+
+	get_docinfo("", "CRM Deal", name)
+	docinfo = frappe.response["docinfo"]
+	deal_fields = get_readable_fields("CRM Deal")
+	avoid_fields = [
+		"lead",
+		"response_by",
+		"sla_creation",
+		"sla",
+		"first_response_time",
+		"first_responded_on",
+	]
+
+	doc = frappe.db.get_values("CRM Deal", name, ["creation", "owner", "lead"])[0]
+	lead = doc[2]
+
+	activities = []
+	calls = []
+	notes = []
+	tasks = []
+	attachments = []
+	creation_text = _("created this deal")
+
+	if lead:
+		creation_text = _("converted the lead to this deal")
+		# a user can have access to the deal but not the lead it came from, so
+		# skip the lead's history instead of failing the whole timeline
+		if frappe.has_permission("CRM Lead", "read", lead):
+			activities, calls, notes, tasks, attachments = get_lead_activities(lead)
+
+	activities.append(
+		{
+			"activity_type": "creation",
+			"creation": doc[0],
+			"owner": doc[1],
+			"data": creation_text,
+			"is_lead": False,
+		}
+	)
+
+	docinfo.versions.reverse()
+
+	for version in docinfo.versions:
+		data = json.loads(version.data)
+		if not data.get("changed"):
+			continue
+
+		if change := data.get("changed")[0]:
+			field = deal_fields.get(change[0], None)
+
+			if not field or change[0] in avoid_fields or (not change[1] and not change[2]):
+				continue
+
+			field_label = field.get("label") or change[0]
+			field_option = field.get("options") or None
+
+			activity_type = "changed"
+			data = {
+				"field": change[0],
+				"field_label": field_label,
+				"old_value": change[1],
+				"value": change[2],
+			}
+
+			if not change[1] and change[2]:
+				activity_type = "added"
+				data = {
+					"field": change[0],
+					"field_label": field_label,
+					"value": change[2],
+				}
+			elif change[1] and not change[2]:
+				activity_type = "removed"
+				data = {
+					"field": change[0],
+					"field_label": field_label,
+					"value": change[1],
+				}
+
+			if data.get("value") and field_option and is_translatable(field_option):
+				data["value"] = _(data["value"])
+
+				if data.get("old_value"):
+					data["old_value"] = _(data["old_value"])
+
+		activity = {
+			"activity_type": activity_type,
+			"creation": version.creation,
+			"owner": version.owner,
+			"data": data,
+			"is_lead": False,
+			"options": field_option,
+		}
+		activities.append(activity)
+
+	for comment in docinfo.comments:
+		activity = {
+			"name": comment.name,
+			"activity_type": "comment",
+			"creation": comment.creation,
+			"owner": comment.owner,
+			"content": comment.content,
+			"attachments": get_attachments("Comment", comment.name),
+			"is_lead": False,
+		}
+		activities.append(activity)
+
+	for communication in docinfo.communications + docinfo.automated_messages:
+		activity = {
+			"activity_type": "communication",
+			"communication_type": communication.communication_type,
+			"communication_date": communication.communication_date or communication.creation,
+			"creation": communication.creation,
+			"data": {
+				"subject": communication.subject,
+				"content": communication.content,
+				"sender_full_name": communication.sender_full_name,
+				"sender": communication.sender,
+				"recipients": communication.recipients,
+				"cc": communication.cc,
+				"bcc": communication.bcc,
+				"attachments": get_attachments("Communication", communication.name),
+				"read_by_recipient": communication.read_by_recipient,
+				"delivery_status": communication.delivery_status,
+			},
+			"is_lead": False,
+		}
+		activities.append(activity)
+
+	for attachment_log in docinfo.attachment_logs:
+		activity = {
+			"name": attachment_log.name,
+			"activity_type": "attachment_log",
+			"creation": attachment_log.creation,
+			"owner": attachment_log.owner,
+			"data": parse_attachment_log(attachment_log.content, attachment_log.comment_type),
+			"is_lead": False,
+		}
+		activities.append(activity)
+
+	calls = calls + get_linked_calls(name).get("calls", [])
+	notes = notes + get_linked_notes(name) + get_linked_calls(name).get("notes", [])
+	tasks = tasks + get_linked_tasks(name) + get_linked_calls(name).get("tasks", [])
+	attachments = attachments + get_attachments("CRM Deal", name)
+
+	activities.sort(key=lambda x: x["creation"], reverse=True)
+	activities = handle_multiple_versions(activities)
+
+	return activities, calls, notes, tasks, attachments
+
+
+def get_lead_activities(name: str):
+	if not frappe.has_permission("CRM Lead", "read", name):
+		frappe.throw(_("Not permitted"), frappe.PermissionError)
+
+	get_docinfo("", "CRM Lead", name)
+	docinfo = frappe.response["docinfo"]
+	lead_fields = get_readable_fields("CRM Lead")
+	avoid_fields = [
+		"converted",
+		"response_by",
+		"sla_creation",
+		"sla",
+		"first_response_time",
+		"first_responded_on",
+	]
+
+	doc = frappe.db.get_values("CRM Lead", name, ["creation", "owner"])[0]
+	activities = [
+		{
+			"activity_type": "creation",
+			"creation": doc[0],
+			"owner": doc[1],
+			"data": _("created this lead"),
+			"is_lead": True,
+		}
+	]
+
+	docinfo.versions.reverse()
+
+	for version in docinfo.versions:
+		data = json.loads(version.data)
+		if not data.get("changed"):
+			continue
+
+		if change := data.get("changed")[0]:
+			field = lead_fields.get(change[0], None)
+
+			if not field or change[0] in avoid_fields or (not change[1] and not change[2]):
+				continue
+
+			field_label = field.get("label") or change[0]
+			field_option = field.get("options") or None
+
+			activity_type = "changed"
+			data = {
+				"field": change[0],
+				"field_label": field_label,
+				"old_value": change[1],
+				"value": change[2],
+			}
+
+			if not change[1] and change[2]:
+				activity_type = "added"
+				data = {
+					"field": change[0],
+					"field_label": field_label,
+					"value": change[2],
+				}
+			elif change[1] and not change[2]:
+				activity_type = "removed"
+				data = {
+					"field": change[0],
+					"field_label": field_label,
+					"value": change[1],
+				}
+
+			if data.get("value") and field_option and is_translatable(field_option):
+				data["value"] = _(data["value"])
+
+				if data.get("old_value"):
+					data["old_value"] = _(data["old_value"])
+
+		activity = {
+			"activity_type": activity_type,
+			"creation": version.creation,
+			"owner": version.owner,
+			"data": data,
+			"is_lead": True,
+			"options": field_option,
+		}
+		activities.append(activity)
+
+	for comment in docinfo.comments:
+		activity = {
+			"name": comment.name,
+			"activity_type": "comment",
+			"creation": comment.creation,
+			"owner": comment.owner,
+			"content": comment.content,
+			"attachments": get_attachments("Comment", comment.name),
+			"is_lead": True,
+		}
+		activities.append(activity)
+
+	for communication in docinfo.communications + docinfo.automated_messages:
+		activity = {
+			"activity_type": "communication",
+			"communication_type": communication.communication_type,
+			"communication_date": communication.communication_date or communication.creation,
+			"creation": communication.creation,
+			"data": {
+				"subject": communication.subject,
+				"content": communication.content,
+				"sender_full_name": communication.sender_full_name,
+				"sender": communication.sender,
+				"recipients": communication.recipients,
+				"cc": communication.cc,
+				"bcc": communication.bcc,
+				"attachments": get_attachments("Communication", communication.name),
+				"read_by_recipient": communication.read_by_recipient,
+				"delivery_status": communication.delivery_status,
+			},
+			"is_lead": True,
+		}
+		activities.append(activity)
+
+	for attachment_log in docinfo.attachment_logs:
+		activity = {
+			"name": attachment_log.name,
+			"activity_type": "attachment_log",
+			"creation": attachment_log.creation,
+			"owner": attachment_log.owner,
+			"data": parse_attachment_log(attachment_log.content, attachment_log.comment_type),
+			"is_lead": True,
+		}
+		activities.append(activity)
+
+	calls = get_linked_calls(name).get("calls", [])
+	notes = get_linked_notes(name) + get_linked_calls(name).get("notes", [])
+	tasks = get_linked_tasks(name) + get_linked_calls(name).get("tasks", [])
+	attachments = get_attachments("CRM Lead", name)
+
+	activities.sort(key=lambda x: x["creation"], reverse=True)
+	activities = handle_multiple_versions(activities)
+
+	return activities, calls, notes, tasks, attachments
+
+
+def get_readable_fields(doctype: str):
+	"""Map of fieldname to label & options, skipping fields the user cannot read.
+
+	Permlevel restrictions already hide these fields on the form layout, so the
+	activity timeline has to hide them too instead of leaking their values.
+	"""
+	allowed_permlevels = get_permlevel_access("read", doctype)
+
+	return {
+		field.fieldname: {"label": field.label, "options": field.options}
+		for field in frappe.get_meta(doctype).fields
+		if field.permlevel == 0 or field.permlevel in allowed_permlevels
+	}
+
+
+def get_attachments(doctype: str, name: str):
+	return (
+		frappe.db.get_all(
+			"File",
+			filters={"attached_to_doctype": doctype, "attached_to_name": name},
+			fields=[
+				"name",
+				"file_name",
+				"file_type",
+				"file_url",
+				"file_size",
+				"is_private",
+				"modified",
+				"creation",
+				"owner",
+			],
+		)
+		or []
+	)
+
+
+def handle_multiple_versions(versions: list):
+	activities = []
+	grouped_versions = []
+	old_version = None
+	for version in versions:
+		is_version = version["activity_type"] in ["changed", "added", "removed"]
+		if not is_version:
+			activities.append(version)
+		if not old_version:
+			old_version = version
+			if is_version:
+				grouped_versions.append(version)
+			continue
+		if is_version and old_version.get("owner") and version["owner"] == old_version["owner"]:
+			grouped_versions.append(version)
+		else:
+			if grouped_versions:
+				activities.append(parse_grouped_versions(grouped_versions))
+			grouped_versions = []
+			if is_version:
+				grouped_versions.append(version)
+		old_version = version
+		if version == versions[-1] and grouped_versions:
+			activities.append(parse_grouped_versions(grouped_versions))
+
+	return activities
+
+
+def parse_grouped_versions(versions: list):
+	version = versions[0]
+	if len(versions) == 1:
+		return version
+	other_versions = versions[1:]
+	version["other_versions"] = other_versions
+	return version
+
+
+def get_linked_calls(name: str):
+	calls = frappe.db.get_all(
+		"CRM Call Log",
+		filters={"reference_docname": name},
+		fields=[
+			"name",
+			"caller",
+			"receiver",
+			"from",
+			"to",
+			"duration",
+			"start_time",
+			"end_time",
+			"status",
+			"type",
+			"recording_url",
+			"creation",
+			"note",
+		],
+	)
+
+	linked_calls = frappe.db.get_all(
+		"Dynamic Link", filters={"link_name": name, "parenttype": "CRM Call Log"}, pluck="parent"
+	)
+
+	notes = []
+	tasks = []
+
+	if linked_calls:
+		CallLog = frappe.qb.DocType("CRM Call Log")
+		Link = frappe.qb.DocType("Dynamic Link")
+		query = (
+			frappe.qb.from_(CallLog)
+			.select(
+				CallLog.name,
+				CallLog.caller,
+				CallLog.receiver,
+				CallLog["from"],
+				CallLog.to,
+				CallLog.duration,
+				CallLog.start_time,
+				CallLog.end_time,
+				CallLog.status,
+				CallLog.type,
+				CallLog.recording_url,
+				CallLog.creation,
+				CallLog.note,
+				Link.link_doctype,
+				Link.link_name,
+			)
+			.join(Link, JoinType.inner)
+			.on(Link.parent == CallLog.name)
+			.where(CallLog.name.isin(linked_calls))
+		)
+		_calls = query.run(as_dict=True)
+
+		for call in _calls:
+			if call.get("link_doctype") == "FCRM Note":
+				notes.append(call.link_name)
+			elif call.get("link_doctype") == "CRM Task":
+				tasks.append(call.link_name)
+
+		_calls = [call for call in _calls if call.get("link_doctype") not in ["FCRM Note", "CRM Task"]]
+		if _calls:
+			calls = calls + _calls
+
+	if notes:
+		notes = frappe.db.get_all(
+			"FCRM Note",
+			filters={"name": ("in", notes)},
+			fields=["name", "title", "content", "owner", "modified"],
+		)
+
+	if tasks:
+		tasks = frappe.db.get_all(
+			"CRM Task",
+			filters={"name": ("in", tasks)},
+			fields=[
+				"name",
+				"title",
+				"description",
+				"assigned_to",
+				"due_date",
+				"priority",
+				"status",
+				"modified",
+			],
+		)
+
+	calls = [parse_call_log(call) for call in calls] if calls else []
+
+	return {"calls": calls, "notes": notes, "tasks": tasks}
+
+
+def get_linked_notes(name: str):
+	notes = frappe.db.get_all(
+		"FCRM Note",
+		filters={"reference_docname": name},
+		fields=["name", "title", "content", "owner", "modified", "creation"],
+	)
+	return notes or []
+
+
+def get_linked_tasks(name: str):
+	tasks = frappe.db.get_all(
+		"CRM Task",
+		filters={"reference_docname": name},
+		fields=[
+			"name",
+			"title",
+			"description",
+			"assigned_to",
+			"due_date",
+			"priority",
+			"status",
+			"modified",
+			"creation",
+		],
+	)
+	return tasks or []
+
+
+def parse_attachment_log(html: str, type: str):
+	soup = BeautifulSoup(html, "html.parser")
+	a_tag = soup.find("a")
+	type = "added" if type == "Attachment" else "removed"
+	if not a_tag:
+		return {
+			"type": type,
+			"file_name": html.replace("Removed ", ""),
+			"file_url": "",
+			"is_private": False,
+		}
+
+	is_private = False
+	if "private/files" in a_tag["href"]:
+		is_private = True
+
+	return {
+		"type": type,
+		"file_name": a_tag.text,
+		"file_url": a_tag["href"],
+		"is_private": is_private,
+	}
+
+
+def is_translatable(doctype: str) -> bool:
+	return doctype in get_translated_doctypes()

crm/api/assignment_rule.py

@@ -0,0 +1,32 @@
+import frappe
+
+
+@frappe.whitelist()
+def get_assignment_rules_list():
+	assignment_rules = []
+	for docname in frappe.get_all(
+		"Assignment Rule", filters={"document_type": ["in", ["CRM Lead", "CRM Deal"]]}
+	):
+		doc = frappe.get_value(
+			"Assignment Rule",
+			docname,
+			fieldname=[
+				"name",
+				"description",
+				"disabled",
+				"priority",
+			],
+			as_dict=True,
+		)
+		users_exists = bool(frappe.db.exists("Assignment Rule User", {"parent": docname.name}))
+		assignment_rules.append({**doc, "users_exists": users_exists})
+	return assignment_rules
+
+
+@frappe.whitelist()
+def duplicate_assignment_rule(docname: str, new_name: str):
+	doc = frappe.get_doc("Assignment Rule", docname)
+	doc.name = new_name
+	doc.assignment_rule_name = new_name
+	doc.insert()
+	return doc

crm/api/auth.py

@@ -0,0 +1,39 @@
+import frappe
+
+
+@frappe.whitelist(allow_guest=True)
+def oauth_providers():
+	from frappe.utils.html_utils import get_icon_html
+	from frappe.utils.oauth import get_oauth2_authorize_url, get_oauth_keys
+	from frappe.utils.password import get_decrypted_password
+
+	out = []
+	providers = frappe.get_all(
+		"Social Login Key",
+		filters={"enable_social_login": 1},
+		fields=["name", "client_id", "base_url", "provider_name", "icon"],
+		order_by="name",
+	)
+
+	for provider in providers:
+		client_secret = get_decrypted_password("Social Login Key", provider.name, "client_secret")
+		if not client_secret:
+			continue
+
+		icon = None
+		if provider.icon:
+			if provider.provider_name == "Custom":
+				icon = get_icon_html(provider.icon, small=True)
+			else:
+				icon = f"<img src='{provider.icon}' alt={provider.provider_name}>"
+
+		if provider.client_id and provider.base_url and get_oauth_keys(provider.name):
+			out.append(
+				{
+					"name": provider.name,
+					"provider_name": provider.provider_name,
+					"auth_url": get_oauth2_authorize_url(provider.name, "/crm"),
+					"icon": icon,
+				}
+			)
+	return out

crm/api/comment.py

@@ -0,0 +1,128 @@
+from collections.abc import Iterable
+
+import frappe
+from bs4 import BeautifulSoup
+from frappe import _
+from frappe.desk.form.utils import add_comment as frappe_add_comment
+from frappe.utils import get_fullname
+
+from crm.fcrm.doctype.crm_notification.crm_notification import notify_user
+
+
+def on_update(self, method):
+	notify_mentions(self)
+
+
+def notify_mentions(doc):
+	"""
+	Extract mentions from `content`, and notify.
+	`content` must have `HTML` content.
+	"""
+	if doc.reference_doctype not in ["CRM Lead", "CRM Deal"]:
+		return
+	content = getattr(doc, "content", None)
+	if not content:
+		return
+	mentions = extract_mentions(content)
+	reference_doc = frappe.get_doc(doc.reference_doctype, doc.reference_name)
+	for mention in mentions:
+		owner = frappe.get_cached_value("User", doc.owner, "full_name")
+		doctype = doc.reference_doctype
+		if doctype.startswith("CRM "):
+			doctype = doctype[4:].lower()
+		name = (
+			reference_doc.lead_name
+			if doctype == "lead"
+			else reference_doc.organization or reference_doc.lead_name
+		)
+		notification_text = f"""
+            <div class="mb-2 leading-5 text-ink-gray-5">
+                <span class="font-medium text-ink-gray-9">{owner}</span>
+                <span>{_("mentioned you in {0}").format(doctype)}</span>
+                <span class="font-medium text-ink-gray-9">{name}</span>
+            </div>
+        """
+		notify_user(
+			{
+				"owner": doc.owner,
+				"assigned_to": mention.email,
+				"notification_type": "Mention",
+				"message": doc.content,
+				"notification_text": notification_text,
+				"reference_doctype": "Comment",
+				"reference_docname": doc.name,
+				"redirect_to_doctype": doc.reference_doctype,
+				"redirect_to_docname": doc.reference_name,
+			}
+		)
+
+
+def extract_mentions(html):
+	if not html:
+		return []
+	soup = BeautifulSoup(html, "html.parser")
+	mentions = []
+	for d in soup.find_all("span", attrs={"data-type": "mention"}):
+		mentions.append(frappe._dict(full_name=d.get("data-label"), email=d.get("data-id")))
+	return mentions
+
+
+@frappe.whitelist()
+def add_comment(reference_doctype: str, reference_name: str, content: str, attachments: list | None = None):
+	"""Add a comment to the given document
+
+	:param reference_doctype: Reference Doctype
+	:param reference_name: Reference Document Name
+	:param content: Comment Content (HTML)
+	:param attachments: List of File names or dicts with keys "fname" and "fcontent"
+	:return: Comment Document
+	"""
+	comment = frappe_add_comment(
+		reference_doctype,
+		reference_name,
+		content,
+		comment_email=frappe.session.user,
+		comment_by=get_fullname(frappe.session.user),
+	)
+
+	if attachments and comment.name:
+		add_attachments(comment.name, attachments)
+
+	return comment
+
+
+def add_attachments(name: str, attachments: Iterable[str | dict]) -> None:
+	"""Add attachments to the given Comment
+
+	:param name: Comment name
+	:param attachments: File names or dicts with keys "fname" and "fcontent"
+	"""
+	# loop through attachments
+	for a in attachments:
+		if isinstance(a, str):
+			attach = frappe.db.get_value("File", {"name": a}, ["file_url", "is_private"], as_dict=1)
+			file_args = {
+				"file_url": attach.file_url,
+				"is_private": attach.is_private,
+			}
+		elif isinstance(a, dict) and "fcontent" in a and "fname" in a:
+			# dict returned by frappe.attach_print()
+			file_args = {
+				"file_name": a["fname"],
+				"content": a["fcontent"],
+				"is_private": 1,
+			}
+		else:
+			continue
+
+		file_args.update(
+			{
+				"attached_to_doctype": "Comment",
+				"attached_to_name": name,
+				"folder": "Home/Attachments",
+			}
+		)
+
+		_file = frappe.new_doc("File")
+		_file.update(file_args)
+		_file.save(ignore_permissions=True)

crm/api/contact.py

@@ -0,0 +1,145 @@
+import frappe
+from frappe import _
+
+
+def validate(doc, method):
+	update_deals_email_mobile_no(doc)
+
+
+def update_deals_email_mobile_no(doc):
+	linked_deals = frappe.get_all(
+		"CRM Contacts",
+		filters={"contact": doc.name, "is_primary": 1},
+		fields=["parent"],
+	)
+
+	for linked_deal in linked_deals:
+		deal = frappe.db.get_values("CRM Deal", linked_deal.parent, ["email", "mobile_no"], as_dict=True)[0]
+		if deal.email != doc.email_id or deal.mobile_no != doc.mobile_no:
+			frappe.db.set_value(
+				"CRM Deal",
+				linked_deal.parent,
+				{
+					"email": doc.email_id,
+					"mobile_no": doc.mobile_no,
+				},
+			)
+
+
+@frappe.whitelist()
+def get_linked_deals(contact: str):
+	"""Get linked deals for a contact"""
+
+	if not frappe.has_permission("Contact", "read", contact):
+		frappe.throw(_("Not permitted"), frappe.PermissionError)
+
+	deal_names = frappe.get_all(
+		"CRM Contacts",
+		filters={"contact": contact, "parenttype": "CRM Deal"},
+		fields=["parent"],
+		distinct=True,
+	)
+
+	# get deals data
+	deals = []
+	for d in deal_names:
+		deal = frappe.get_cached_doc(
+			"CRM Deal",
+			d.parent,
+			fields=[
+				"name",
+				"organization",
+				"currency",
+				"deal_value",
+				"status",
+				"email",
+				"mobile_no",
+				"deal_owner",
+				"modified",
+			],
+		)
+		deals.append(deal.as_dict())
+
+	return deals
+
+
+@frappe.whitelist()
+def create_new(contact: str, field: str, value: str):
+	"""Create new email or phone for a contact"""
+	if not frappe.has_permission("Contact", "write", contact):
+		frappe.throw(_("Not permitted"), frappe.PermissionError)
+
+	contact = frappe.get_cached_doc("Contact", contact)
+
+	if field == "email":
+		email = {"email_id": value, "is_primary": 1 if len(contact.email_ids) == 0 else 0}
+		contact.append("email_ids", email)
+	elif field in ("mobile_no", "phone"):
+		mobile_no = {"phone": value, "is_primary_mobile_no": 1 if len(contact.phone_nos) == 0 else 0}
+		contact.append("phone_nos", mobile_no)
+	else:
+		frappe.throw(_("Invalid field"))
+
+	contact.save()
+	return True
+
+
+@frappe.whitelist()
+def set_as_primary(contact: str, field: str, value: str):
+	"""Set email or phone as primary for a contact"""
+	if not frappe.has_permission("Contact", "write", contact):
+		frappe.throw(_("Not permitted"), frappe.PermissionError)
+
+	contact = frappe.get_doc("Contact", contact)
+
+	if field == "email":
+		for email in contact.email_ids:
+			if email.email_id == value:
+				email.is_primary = 1
+			else:
+				email.is_primary = 0
+	elif field in ("mobile_no", "phone"):
+		name = "is_primary_mobile_no" if field == "mobile_no" else "is_primary_phone"
+		for phone in contact.phone_nos:
+			if phone.phone == value:
+				phone.set(name, 1)
+			else:
+				phone.set(name, 0)
+	else:
+		frappe.throw(_("Invalid field"))
+
+	contact.save()
+	return True
+
+
+@frappe.whitelist()
+def search_emails(txt: str):
+	doctype = "Contact"
+	meta = frappe.get_meta(doctype)
+	filters = [["Contact", "email_id", "is", "set"]]
+
+	if meta.get("fields", {"fieldname": "enabled", "fieldtype": "Check"}):
+		filters.append([doctype, "enabled", "=", 1])
+	if meta.get("fields", {"fieldname": "disabled", "fieldtype": "Check"}):
+		filters.append([doctype, "disabled", "!=", 1])
+
+	or_filters = []
+	search_fields = ["full_name", "email_id", "name"]
+	if txt:
+		for f in search_fields:
+			or_filters.append([doctype, f.strip(), "like", f"%{txt}%"])
+
+	results = frappe.get_list(
+		doctype,
+		filters=filters,
+		fields=search_fields,
+		or_filters=or_filters,
+		limit_start=0,
+		limit_page_length=20,
+		order_by="email_id, full_name, name",
+		ignore_permissions=False,
+		as_list=True,
+		strict=False,
+	)
+
+	return results

crm/api/doc.py

@@ -0,0 +1,857 @@
+import json
+
+import frappe
+from frappe import _
+from frappe.custom.doctype.property_setter.property_setter import make_property_setter
+from frappe.desk.form.assign_to import set_status
+from frappe.model import no_value_fields
+from frappe.model.delete_doc import get_dynamic_linked_docs, get_linked_docs
+from frappe.model.document import get_controller
+from frappe.utils import make_filter_tuple
+from pypika import Criterion
+
+from crm.api.views import get_views
+from crm.fcrm.doctype.crm_form_script.crm_form_script import get_form_script
+from crm.utils import is_frappe_version
+
+COUNT_NAME = (
+	{"COUNT": "name", "as": "total_count"}
+	if is_frappe_version("16", above=True)
+	else "count(name) as total_count"
+)
+
+
+@frappe.whitelist()
+def sort_options(doctype: str):
+	fields = frappe.get_meta(doctype).fields
+	fields = [field for field in fields if field.fieldtype not in no_value_fields]
+	fields = [
+		{
+			"label": _(field.label),
+			"value": field.fieldname,
+			"fieldname": field.fieldname,
+		}
+		for field in fields
+		if field.label and field.fieldname
+	]
+
+	standard_fields = [
+		{"label": "Name", "fieldname": "name"},
+		{"label": "Created On", "fieldname": "creation"},
+		{"label": "Last Modified", "fieldname": "modified"},
+		{"label": "Modified By", "fieldname": "modified_by"},
+		{"label": "Owner", "fieldname": "owner"},
+	]
+
+	for field in standard_fields:
+		field["label"] = _(field["label"])
+		field["value"] = field["fieldname"]
+		fields.append(field)
+
+	return fields
+
+
+@frappe.whitelist()
+def get_filterable_fields(doctype: str):
+	allowed_fieldtypes = [
+		"Check",
+		"Data",
+		"Float",
+		"Int",
+		"Currency",
+		"Dynamic Link",
+		"Link",
+		"Long Text",
+		"Select",
+		"Small Text",
+		"Text Editor",
+		"Text",
+		"Duration",
+		"Rating",
+		"Date",
+		"Datetime",
+	]
+
+	c = get_controller(doctype)
+	restricted_fields = []
+	if hasattr(c, "get_non_filterable_fields"):
+		restricted_fields = c.get_non_filterable_fields()
+
+	fields = []
+
+	meta = frappe.get_meta(doctype).as_dict()
+
+	# append standard fields (getting error when using frappe.model.std_fields)
+	standard_fields = [
+		{"fieldname": "name", "fieldtype": "Link", "label": "Name", "options": doctype},
+		{"fieldname": "owner", "fieldtype": "Link", "label": "Created By", "options": "User"},
+		{
+			"fieldname": "modified_by",
+			"fieldtype": "Link",
+			"label": "Last Updated By",
+			"options": "User",
+		},
+		{"fieldname": "_user_tags", "fieldtype": "Data", "label": "Tags"},
+		{"fieldname": "_liked_by", "fieldtype": "Data", "label": "Like"},
+		{"fieldname": "_comments", "fieldtype": "Text", "label": "Comments"},
+		{"fieldname": "_assign", "fieldtype": "Text", "label": "Assigned To"},
+		{"fieldname": "creation", "fieldtype": "Datetime", "label": "Created On"},
+		{"fieldname": "modified", "fieldtype": "Datetime", "label": "Last Updated On"},
+	]
+
+	for field in standard_fields + meta.get("fields", []):
+		if field.get("fieldname") not in restricted_fields and field.get("fieldtype") in allowed_fieldtypes:
+			field["name"] = field.get("fieldname")
+			field["label"] = _(field.get("label"))
+			field["value"] = field.get("fieldname")
+			fields.append(field)
+
+	return fields
+
+
+@frappe.whitelist()
+def get_group_by_fields(doctype: str):
+	allowed_fieldtypes = [
+		"Check",
+		"Data",
+		"Float",
+		"Int",
+		"Currency",
+		"Dynamic Link",
+		"Link",
+		"Select",
+		"Duration",
+		"Date",
+		"Datetime",
+	]
+
+	fields = frappe.get_meta(doctype).fields
+	fields = [
+		field
+		for field in fields
+		if field.fieldtype not in no_value_fields and field.fieldtype in allowed_fieldtypes
+	]
+	fields = [
+		{
+			"label": _(field.label),
+			"fieldname": field.fieldname,
+		}
+		for field in fields
+		if field.label and field.fieldname
+	]
+
+	standard_fields = [
+		{"label": "Name", "fieldname": "name"},
+		{"label": "Created On", "fieldname": "creation"},
+		{"label": "Last Modified", "fieldname": "modified"},
+		{"label": "Modified By", "fieldname": "modified_by"},
+		{"label": "Owner", "fieldname": "owner"},
+		{"label": "Like", "fieldname": "_liked_by"},
+		{"label": "Assigned To", "fieldname": "_assign"},
+		{"label": "Comments", "fieldname": "_comments"},
+		{"label": "Created On", "fieldname": "creation"},
+		{"label": "Modified On", "fieldname": "modified"},
+	]
+
+	for field in standard_fields:
+		field["label"] = _(field["label"])
+		fields.append(field)
+
+	return fields
+
+
+@frappe.whitelist()
+def get_quick_filters(doctype: str, cached: bool = True):
+	meta = frappe.get_meta(doctype, cached)
+	quick_filters = []
+
+	if global_settings := frappe.db.exists("CRM Global Settings", {"dt": doctype, "type": "Quick Filters"}):
+		_quick_filters = frappe.db.get_value("CRM Global Settings", global_settings, "json")
+		_quick_filters = json.loads(_quick_filters) or []
+
+		fields = []
+
+		for filter in _quick_filters:
+			if filter == "name":
+				fields.append({"label": "Name", "fieldname": "name", "fieldtype": "Data"})
+			else:
+				field = next((f for f in meta.fields if f.fieldname == filter), None)
+				if field:
+					fields.append(field)
+
+	else:
+		fields = [field for field in meta.fields if field.in_standard_filter]
+
+	for field in fields:
+		options = field.get("options")
+		if field.get("fieldtype") == "Select" and options and isinstance(options, str):
+			options = options.split("\n")
+			options = [{"label": option, "value": option} for option in options]
+			if not any([not option.get("value") for option in options]):
+				options.insert(0, {"label": "", "value": ""})
+		quick_filters.append(
+			{
+				"label": _(field.get("label")),
+				"fieldname": field.get("fieldname"),
+				"fieldtype": field.get("fieldtype"),
+				"options": options,
+			}
+		)
+
+	if doctype == "CRM Lead":
+		quick_filters = [filter for filter in quick_filters if filter.get("fieldname") != "converted"]
+
+	return quick_filters
+
+
+@frappe.whitelist()
+def update_quick_filters(quick_filters: str, old_filters: str, doctype: str):
+	quick_filters = json.loads(quick_filters)
+	old_filters = json.loads(old_filters)
+
+	new_filters = [filter for filter in quick_filters if filter not in old_filters]
+	removed_filters = [filter for filter in old_filters if filter not in quick_filters]
+
+	# update or create global quick filter settings
+	create_update_global_settings(doctype, quick_filters)
+
+	# remove old filters
+	for filter in removed_filters:
+		update_in_standard_filter(filter, doctype, 0)
+
+	# add new filters
+	for filter in new_filters:
+		update_in_standard_filter(filter, doctype, 1)
+
+
+def create_update_global_settings(doctype, quick_filters):
+	if global_settings := frappe.db.exists("CRM Global Settings", {"dt": doctype, "type": "Quick Filters"}):
+		frappe.db.set_value("CRM Global Settings", global_settings, "json", json.dumps(quick_filters))
+	else:
+		# create CRM Global Settings doc
+		doc = frappe.new_doc("CRM Global Settings")
+		doc.dt = doctype
+		doc.type = "Quick Filters"
+		doc.json = json.dumps(quick_filters)
+		doc.insert()
+
+
+def update_in_standard_filter(fieldname, doctype, value):
+	if property_name := frappe.db.exists(
+		"Property Setter",
+		{"doc_type": doctype, "field_name": fieldname, "property": "in_standard_filter"},
+	):
+		frappe.db.set_value("Property Setter", property_name, "value", value)
+	else:
+		make_property_setter(
+			doctype,
+			fieldname,
+			"in_standard_filter",
+			value,
+			"Check",
+			validate_fields_for_doctype=False,
+		)
+
+
+@frappe.whitelist()
+def get_data(
+	doctype: str,
+	filters: dict,
+	order_by: str,
+	page_length: int = 20,
+	page_length_count: int = 20,
+	column_field: str | None = None,
+	title_field: str | None = None,
+	columns: str | list | None = None,
+	rows: str | list | None = None,
+	kanban_columns: str | list | None = None,
+	kanban_fields: str | list | None = None,
+	view: str | dict | None = None,
+	default_filters: dict | None = None,
+):
+	custom_view = False
+	filters = frappe._dict(filters)
+	rows = frappe.parse_json(rows or "[]")
+	columns = frappe.parse_json(columns or "[]")
+	kanban_fields = frappe.parse_json(kanban_fields or "[]")
+	kanban_columns = frappe.parse_json(kanban_columns or "[]")
+
+	custom_view_name = view.get("custom_view_name") if view else None
+	view_type = view.get("view_type") if view else None
+	group_by_field = view.get("group_by_field") if view else None
+
+	for key in filters:
+		value = filters[key]
+		if isinstance(value, list):
+			if "@me" in value:
+				value[value.index("@me")] = frappe.session.user
+			elif "%@me%" in value:
+				index = [i for i, v in enumerate(value) if v == "%@me%"]
+				for i in index:
+					value[i] = "%" + frappe.session.user + "%"
+		elif value == "@me":
+			filters[key] = frappe.session.user
+
+	if default_filters:
+		default_filters = frappe.parse_json(default_filters)
+		filters.update(default_filters)
+
+	is_default = True
+	data = []
+	_list = get_controller(doctype)
+	default_rows = []
+	default_column_keys = []
+	if hasattr(_list, "default_list_data"):
+		default_list_data = _list.default_list_data()
+		default_rows = default_list_data.get("rows")
+		default_column_keys = [column.get("key") for column in default_list_data.get("columns", [])]
+
+	meta = frappe.get_meta(doctype)
+
+	if view_type != "kanban":
+		if columns or rows:
+			custom_view = True
+			is_default = False
+			columns = frappe.parse_json(columns)
+			rows = frappe.parse_json(rows)
+
+		if not columns:
+			columns = [
+				{"label": "Name", "type": "Data", "key": "name", "width": "16rem"},
+				{"label": "Last Modified", "type": "Datetime", "key": "modified", "width": "8rem"},
+			]
+
+		if not rows:
+			rows = ["name"]
+
+		default_view_filters = {
+			"dt": doctype,
+			"type": view_type or "list",
+			"is_standard": 1,
+			"user": frappe.session.user,
+		}
+
+		if not custom_view and frappe.db.exists("CRM View Settings", default_view_filters):
+			list_view_settings = frappe.get_doc("CRM View Settings", default_view_filters)
+			columns = frappe.parse_json(list_view_settings.columns)
+			rows = frappe.parse_json(list_view_settings.rows)
+			is_default = False
+		elif not custom_view or (is_default and hasattr(_list, "default_list_data")):
+			rows = default_rows
+			columns = _list.default_list_data().get("columns")
+
+		visible_columns = []
+		for column in columns:
+			key = column.get("key")
+			# hidden fields are dropped unless the doctype lists them in its own defaults
+			column_meta = meta.get_field(key)
+			if column_meta and column_meta.get("hidden") and key not in default_column_keys:
+				continue
+
+			if key not in rows:
+				rows.append(key)
+			column["label"] = _(column.get("label"))
+
+			if key == "_liked_by" and column.get("width") == "10rem":
+				column["width"] = "50px"
+
+			visible_columns.append(column)
+		columns = visible_columns
+
+		# check if rows has group_by_field if not add it
+		if group_by_field and group_by_field not in rows:
+			rows.append(group_by_field)
+
+		if meta.track_seen and "_seen" not in rows:
+			rows.append("_seen")
+
+		data = (
+			frappe.get_list(
+				doctype,
+				fields=rows,
+				filters=filters,
+				order_by=order_by,
+				page_length=page_length,
+			)
+			or []
+		)
+		data = parse_list_data(data, doctype)
+
+	if view_type == "kanban":
+		if not rows:
+			rows = default_rows
+
+		if not kanban_columns and column_field:
+			field_meta = frappe.get_meta(doctype).get_field(column_field)
+			if field_meta.fieldtype == "Link":
+				kanban_columns = frappe.get_all(
+					field_meta.options,
+					fields=["name"],
+					order_by="modified asc",
+				)
+			elif field_meta.fieldtype == "Select":
+				kanban_columns = [{"name": option} for option in field_meta.options.split("\n")]
+
+		if not title_field:
+			title_field = "name"
+			if hasattr(_list, "default_kanban_settings"):
+				title_field = _list.default_kanban_settings().get("title_field")
+
+		if title_field not in rows:
+			rows.append(title_field)
+
+		if not kanban_fields:
+			kanban_fields = ["name"]
+			if hasattr(_list, "default_kanban_settings"):
+				kanban_fields = json.loads(_list.default_kanban_settings().get("kanban_fields"))
+
+		for field in kanban_fields:
+			if field not in rows:
+				rows.append(field)
+
+		for kc in kanban_columns:
+			# Start with base filters
+			column_filters = []
+
+			# Convert and add the main filters first
+			if filters:
+				base_filters = convert_filter_to_tuple(doctype, filters)
+				column_filters.extend(base_filters)
+
+			# Add the column-specific filter
+			if column_field and kc.get("name"):
+				column_filters.append([doctype, column_field, "=", kc.get("name")])
+
+			order = kc.get("order")
+			if kc.get("delete"):
+				column_data = []
+			else:
+				page_length = kc.get("page_length", 20)
+
+				if order:
+					column_data = get_records_based_on_order(
+						doctype, rows, column_filters, page_length, order
+					)
+				else:
+					column_data = frappe.get_list(
+						doctype,
+						fields=rows,
+						filters=column_filters,
+						order_by=order_by,
+						page_length=page_length,
+					)
+
+				all_count = frappe.get_list(
+					doctype,
+					filters=column_filters,
+					fields=[COUNT_NAME],
+				)[0].total_count
+
+				kc["all_count"] = all_count
+				kc["count"] = len(column_data)
+
+			if order:
+				column_data = sorted(
+					column_data,
+					key=lambda x: order.index(x.get("name")) if x.get("name") in order else len(order),
+				)
+
+			data.append({"column": kc, "fields": kanban_fields, "data": column_data})
+
+	fields = frappe.get_meta(doctype).fields
+	fields = [field for field in fields if field.fieldtype not in no_value_fields]
+	fields = [
+		{
+			"label": _(field.label),
+			"fieldtype": field.fieldtype,
+			"fieldname": field.fieldname,
+			"options": field.options,
+		}
+		for field in fields
+		if field.label and field.fieldname
+	]
+
+	std_fields = [
+		{"label": "Name", "fieldtype": "Data", "fieldname": "name"},
+		{"label": "Created On", "fieldtype": "Datetime", "fieldname": "creation"},
+		{"label": "Last Modified", "fieldtype": "Datetime", "fieldname": "modified"},
+		{
+			"label": "Modified By",
+			"fieldtype": "Link",
+			"fieldname": "modified_by",
+			"options": "User",
+		},
+		{"label": "Assigned To", "fieldtype": "Text", "fieldname": "_assign"},
+		{"label": "Owner", "fieldtype": "Link", "fieldname": "owner", "options": "User"},
+		{"label": "Like", "fieldtype": "Data", "fieldname": "_liked_by"},
+	]
+
+	for field in std_fields:
+		if field.get("fieldname") not in rows:
+			rows.append(field.get("fieldname"))
+		if field not in fields:
+			field["label"] = _(field["label"])
+			fields.append(field)
+
+	if not is_default and custom_view_name:
+		is_default = frappe.db.get_value("CRM View Settings", custom_view_name, "load_default_columns")
+
+	if group_by_field and view_type == "group_by":
+
+		def get_options(type, options):
+			if type == "Select":
+				return [option for option in options.split("\n")]
+			else:
+				has_empty_values = any([not d.get(group_by_field) for d in data])
+				options = list(set([d.get(group_by_field) for d in data]))
+				options = [u for u in options if u]
+				if has_empty_values:
+					options.append("")
+
+				if order_by and group_by_field in order_by:
+					order_by_fields = order_by.split(",")
+					order_by_fields = [
+						(field.split(" ")[0], field.split(" ")[1]) for field in order_by_fields
+					]
+					if (group_by_field, "asc") in order_by_fields:
+						options.sort()
+					elif (group_by_field, "desc") in order_by_fields:
+						options.sort(reverse=True)
+				else:
+					options.sort()
+				return options
+
+		for field in fields:
+			if field.get("fieldname") == group_by_field:
+				group_by_field = {
+					"label": field.get("label"),
+					"fieldname": field.get("fieldname"),
+					"fieldtype": field.get("fieldtype"),
+					"options": get_options(field.get("fieldtype"), field.get("options")),
+				}
+
+	return {
+		"data": data,
+		"columns": columns,
+		"rows": rows,
+		"fields": fields,
+		"column_field": column_field,
+		"title_field": title_field,
+		"kanban_columns": kanban_columns,
+		"kanban_fields": kanban_fields,
+		"group_by_field": group_by_field,
+		"page_length": page_length,
+		"page_length_count": page_length_count,
+		"is_default": is_default,
+		"views": get_views(doctype),
+		"total_count": frappe.get_list(doctype, filters=filters, fields=[COUNT_NAME])[0].total_count,
+		"row_count": len(data),
+		"form_script": get_form_script(doctype),
+		"list_script": get_form_script(doctype, "List"),
+		"view_type": view_type,
+	}
+
+
+def parse_list_data(data, doctype):
+	_list = get_controller(doctype)
+	if hasattr(_list, "parse_list_data"):
+		data = _list.parse_list_data(data)
+	return data
+
+
+def convert_filter_to_tuple(doctype, filters):
+	if isinstance(filters, dict):
+		filters_items = filters.items()
+		filters = []
+		for key, value in filters_items:
+			filters.append(make_filter_tuple(doctype, key, value))
+	return filters
+
+
+def get_records_based_on_order(doctype, rows, filters, page_length, order):
+	records = []
+	filters = convert_filter_to_tuple(doctype, filters)
+	in_filters = filters.copy()
+	in_filters.append([doctype, "name", "in", order[:page_length]])
+	records = frappe.get_list(
+		doctype,
+		fields=rows,
+		filters=in_filters,
+		order_by="creation desc",
+		page_length=page_length,
+	)
+
+	if len(records) < page_length:
+		not_in_filters = filters.copy()
+		not_in_filters.append([doctype, "name", "not in", order])
+		remaining_records = frappe.get_list(
+			doctype,
+			fields=rows,
+			filters=not_in_filters,
+			order_by="creation desc",
+			page_length=page_length - len(records),
+		)
+		for record in remaining_records:
+			records.append(record)
+
+	return records
+
+
+@frappe.whitelist()
+def remove_assignments(doctype: str, name: str, assignees: str | list, ignore_permissions: bool = False):
+	assignees = frappe.parse_json(assignees)
+
+	if not assignees:
+		return
+
+	for assign_to in assignees:
+		set_status(
+			doctype,
+			name,
+			todo=None,
+			assign_to=assign_to,
+			status="Cancelled",
+			ignore_permissions=ignore_permissions,
+		)
+
+
+@frappe.whitelist()
+def add_seen(doctype: str, name: str):
+	doc = frappe.get_doc(doctype, name)
+	doc.check_permission("read")
+	doc.add_seen()
+
+
+@frappe.whitelist()
+def get_assigned_users(doctype: str, name: str | int, default_assigned_to: str | None = None):
+	assigned_users = frappe.get_all(
+		"ToDo",
+		fields=["allocated_to"],
+		filters={
+			"reference_type": doctype,
+			"reference_name": name,
+			"status": ("not in", ("Closed", "Cancelled")),
+		},
+		pluck="allocated_to",
+	)
+
+	users = list(set(assigned_users))
+
+	# if users is empty, add default_assigned_to
+	if not users and default_assigned_to:
+		users = [default_assigned_to]
+	return users
+
+
+@frappe.whitelist()
+def get_fields(doctype: str, allow_all_fieldtypes: bool = False):
+	not_allowed_fieldtypes = [*list(frappe.model.no_value_fields), "Read Only"]
+	if allow_all_fieldtypes:
+		not_allowed_fieldtypes = []
+	fields = frappe.get_meta(doctype).fields
+
+	_fields = []
+
+	for field in fields:
+		if field.fieldtype not in not_allowed_fieldtypes and field.fieldname:
+			_fields.append(field)
+
+	return _fields
+
+
+def getCounts(d, doctype):
+	d["_email_count"] = (
+		frappe.db.count(
+			"Communication",
+			filters={
+				"reference_doctype": doctype,
+				"reference_name": d.get("name"),
+				"communication_type": "Communication",
+			},
+		)
+		or 0
+	)
+	d["_email_count"] = d["_email_count"] + frappe.db.count(
+		"Communication",
+		filters={
+			"reference_doctype": doctype,
+			"reference_name": d.get("name"),
+			"communication_type": "Automated Message",
+		},
+	)
+	d["_comment_count"] = frappe.db.count(
+		"Comment",
+		filters={"reference_doctype": doctype, "reference_name": d.get("name"), "comment_type": "Comment"},
+	)
+	d["_task_count"] = frappe.db.count(
+		"CRM Task", filters={"reference_doctype": doctype, "reference_docname": d.get("name")}
+	)
+	d["_note_count"] = frappe.db.count(
+		"FCRM Note", filters={"reference_doctype": doctype, "reference_docname": d.get("name")}
+	)
+	return d
+
+
+@frappe.whitelist()
+def get_linked_docs_of_document(doctype: str, docname: str):
+	try:
+		doc = frappe.get_doc(doctype, docname)
+	except frappe.DoesNotExistError:
+		return []
+
+	linked_docs = get_linked_docs(doc)
+	dynamic_linked_docs = get_dynamic_linked_docs(doc)
+
+	linked_docs.extend(dynamic_linked_docs)
+	linked_docs = list({doc["reference_docname"]: doc for doc in linked_docs}.values())
+
+	docs_data = []
+	for doc in linked_docs:
+		if not doc.get("reference_doctype") or not doc.get("reference_docname"):
+			continue
+
+		try:
+			data = frappe.get_doc(doc["reference_doctype"], doc["reference_docname"])
+		except (frappe.DoesNotExistError, frappe.ValidationError):
+			continue
+
+		title = data.get("title")
+		if data.doctype == "CRM Call Log":
+			title = f"Call from {data.get('from')} to {data.get('to')}"
+
+		if data.doctype == "CRM Deal":
+			title = data.get("organization")
+
+		if data.doctype == "CRM Notification":
+			title = data.get("message")
+
+		docs_data.append(
+			{
+				"doc": data.doctype,
+				"title": title or data.get("name"),
+				"reference_docname": doc["reference_docname"],
+				"reference_doctype": doc["reference_doctype"],
+			}
+		)
+	return docs_data
+
+
+def remove_doc_link(doctype, docname):
+	if not doctype or not docname:
+		return
+
+	try:
+		linked_doc_data = frappe.get_doc(doctype, docname)
+		if doctype == "CRM Notification":
+			delete_notification_type = {
+				"notification_type_doctype": "",
+				"notification_type_doc": "",
+			}
+			delete_references = {
+				"reference_doctype": "",
+				"reference_name": "",
+			}
+
+			if linked_doc_data.get("notification_type_doctype") == linked_doc_data.get("reference_doctype"):
+				delete_references.update(delete_notification_type)
+
+			linked_doc_data.update(delete_references)
+		else:
+			linked_doc_data.update(
+				{
+					"reference_doctype": "",
+					"reference_docname": "",
+				}
+			)
+		linked_doc_data.save(ignore_permissions=True)
+	except (frappe.DoesNotExistError, frappe.ValidationError):
+		pass
+
+
+def remove_contact_link(doctype, docname):
+	if not doctype or not docname:
+		return
+
+	try:
+		linked_doc_data = frappe.get_doc(doctype, docname)
+		linked_doc_data.update(
+			{
+				"contact": None,
+				"contacts": [],
+			}
+		)
+		linked_doc_data.save(ignore_permissions=True)
+	except (frappe.DoesNotExistError, frappe.ValidationError):
+		pass
+
+
+@frappe.whitelist()
+def remove_linked_doc_reference(items: str | list, remove_contact: bool = False, delete: bool = False):
+	if isinstance(items, str):
+		items = frappe.parse_json(items)
+
+	for item in items:
+		if not item.get("doctype") or not item.get("docname"):
+			continue
+
+		if not frappe.has_permission(item["doctype"], "write", item["docname"]):
+			continue
+
+		try:
+			if remove_contact:
+				remove_contact_link(item["doctype"], item["docname"])
+			else:
+				remove_doc_link(item["doctype"], item["docname"])
+
+			if delete:
+				frappe.delete_doc(item["doctype"], item["docname"])
+		except (frappe.DoesNotExistError, frappe.ValidationError):
+			# Skip if document doesn't exist or has validation errors
+			continue
+
+	return "success"
+
+
+@frappe.whitelist()
+def delete_bulk_docs(doctype: str, items: str | list, delete_linked: bool = False):
+	from frappe.desk.reportview import delete_bulk
+
+	if not doctype:
+		frappe.throw(_("Doctype is required"))
+
+	if not items:
+		frappe.throw(_("Items are required"))
+
+	items = frappe.parse_json(items)
+	if not isinstance(items, list):
+		frappe.throw(_("Items must be a list"))
+
+	for doc in items:
+		try:
+			if not frappe.db.exists(doctype, doc):
+				frappe.log_error(f"Document {doctype} {doc} does not exist", "Bulk Delete Error")
+				continue
+
+			linked_docs = get_linked_docs_of_document(doctype, doc)
+			for linked_doc in linked_docs:
+				if not linked_doc.get("reference_doctype") or not linked_doc.get("reference_docname"):
+					continue
+
+				remove_linked_doc_reference(
+					[
+						{
+							"doctype": linked_doc["reference_doctype"],
+							"docname": linked_doc["reference_docname"],
+						}
+					],
+					remove_contact=doctype == "Contact",
+					delete=delete_linked,
+				)
+		except Exception as e:
+			frappe.log_error(f"Error processing linked docs for {doctype} {doc}: {e!s}", "Bulk Delete Error")
+
+	if len(items) > 10:
+		frappe.enqueue("frappe.desk.reportview.delete_bulk", doctype=doctype, items=items)
+	else:
+		delete_bulk(doctype, items)
+	return "success"

crm/api/event.py

@@ -0,0 +1,372 @@
+"""
+Event notification handling for CRM.
+
+This module handles event notifications for the CRM system, supporting both:
+1. Custom event notifications set on individual events
+2. Global default notifications from CRM Settings for events without custom notifications
+
+The notification system processes events that are either:
+- Starting from now or in the future (normal case)
+- Currently in progress (between starts_on and ends_on) to handle delayed notifications
+
+Global notifications are configured in CRM Settings under the Calendar tab and are applied
+to events that don't have custom notifications set. This ensures all events can receive
+reminders even if users don't configure them individually.
+"""
+
+from datetime import datetime, timedelta
+
+import frappe
+from frappe.utils import add_to_date, now_datetime
+
+
+def trigger_offset_event_notifications():
+	"""Trigger event notifications for offset-based intervals (minutes)."""
+	_process_event_notifications_by_interval("minutes")
+	_process_event_notifications_by_interval("hours")
+
+
+def trigger_hourly_event_notifications():
+	"""Trigger event notifications for hourly intervals."""
+	_process_event_notifications_by_interval("hours")
+
+
+def trigger_daily_event_notifications():
+	"""Trigger event notifications for daily intervals."""
+	_process_event_notifications_by_interval("days")
+
+
+def trigger_weekly_event_notifications():
+	"""Trigger event notifications for weekly intervals."""
+	_process_event_notifications_by_interval("weeks")
+
+
+def _process_event_notifications_by_interval(interval):
+	"""
+	Process event notifications for a specific interval.
+
+	Args:
+		interval (str): The interval type ('minutes', 'hours', 'days', 'weeks')
+	"""
+
+	if frappe.flags.in_import or frappe.flags.in_patch:
+		return
+
+	current_time = now_datetime()
+	current_user = frappe.session.user
+	all_events_data = frappe.db.sql(
+		"""
+		SELECT
+			e.name as event_name,
+			e.subject,
+			e.starts_on,
+			e.ends_on,
+			e.owner,
+			e.description,
+			e.all_day as all_day_event,
+			en.type as notification_type,
+			en.before as before_value,
+			en.time as time_of_day,
+			en.interval as notification_interval,
+			ep.email as participant_email,
+			ep_all.participant_emails_csv,
+			CASE WHEN en.parent IS NULL THEN 0 ELSE 1 END as has_custom_notifications
+		FROM `tabEvent` e
+		LEFT JOIN `tabEvent Notifications` en ON e.name = en.parent AND en.interval = %s
+		LEFT JOIN `tabEvent Participants` ep ON e.name = ep.parent AND ep.email = %s
+		LEFT JOIN (
+			SELECT parent, GROUP_CONCAT(email) AS participant_emails_csv
+			FROM `tabEvent Participants`
+			GROUP BY parent
+		) AS ep_all ON ep_all.parent = e.name
+		WHERE (e.starts_on >= %s OR (%s >= e.starts_on AND %s < e.ends_on))
+		AND (e.owner = %s OR ep.email = %s)
+		AND e.status != 'Cancelled'
+		ORDER BY e.starts_on, e.name
+	""",
+		(interval, current_user, current_time, current_time, current_time, current_user, current_user),
+		as_dict=True,
+	)
+
+	for event_data in all_events_data:
+		participant_emails_csv = event_data.pop("participant_emails_csv", None)
+		event_data["event_participants"] = _split_participant_emails(participant_emails_csv)
+
+	notifications = _process_unified_event_data(all_events_data, interval)
+
+	for notification in notifications:
+		try:
+			event_start = notification.get("starts_on")
+			event_end = notification.get("ends_on")
+			before_value = notification.get("before_value", 1)
+
+			trigger_datetime = _calculate_trigger_datetime(
+				event_start,
+				before_value,
+				interval,
+				notification.get("all_day_event"),
+				notification.get("time_of_day"),
+			)
+
+			event_is_in_progress = event_start <= current_time < event_end
+			trigger_time_passed = current_time > trigger_datetime
+
+			if event_is_in_progress and trigger_time_passed:
+				trigger_window_start = trigger_datetime
+				trigger_window_end = event_end
+			else:
+				window_duration = _get_trigger_window_duration(interval)
+				trigger_window_start = add_to_date(
+					trigger_datetime, **{k: -v for k, v in window_duration.items()}
+				)
+				trigger_window_end = add_to_date(trigger_datetime, **window_duration)
+
+			if not (trigger_window_start <= current_time <= trigger_window_end):
+				continue
+
+			if notification.get("notification_type") == "Email":
+				_send_email_notification(notification, event_start, before_value, interval)
+			elif notification.get("notification_type") == "Notification":
+				_send_system_notification(notification)
+
+		except Exception as e:
+			frappe.log_error(
+				f"Error processing {interval} notification for event {notification.get('event_name', 'Unknown')}: {e!s}"
+			)
+			continue
+
+
+def _process_unified_event_data(all_events_data, interval):
+	"""
+	Process unified event data that includes both events with and without custom notifications.
+
+	Args:
+		all_events_data (list): List of event data from the unified query
+		interval (str): The interval type ('minutes', 'hours', 'days', 'weeks')
+
+	Returns:
+		list: List of processed notifications ready for sending
+	"""
+	notifications = []
+	events_without_notifications = []
+
+	for event_data in all_events_data:
+		if event_data.get("has_custom_notifications") == 1:
+			notifications.append(event_data)
+		else:
+			event_key = event_data.get("event_name")
+			if not any(e.get("event_name") == event_key for e in events_without_notifications):
+				events_without_notifications.append(event_data)
+	if events_without_notifications:
+		global_notifications = _apply_global_notifications_to_events(events_without_notifications, interval)
+		notifications.extend(global_notifications)
+
+	return notifications
+
+
+def _apply_global_notifications_to_events(events_without_notifications, interval):
+	"""
+	Apply global CRM Settings notifications to events that don't have custom notifications.
+
+	Args:
+		events_without_notifications (list): List of events without custom notifications
+		interval (str): The interval type ('minutes', 'hours', 'days', 'weeks')
+
+	Returns:
+		list: List of notification dictionaries using global settings
+	"""
+
+	fcrm_settings = frappe.get_single("FCRM Settings")
+	global_notifications = []
+	if hasattr(fcrm_settings, "event_notifications"):
+		for notification in fcrm_settings.event_notifications:
+			if notification.interval == interval:
+				notification._table_type = "regular"
+				global_notifications.append(notification)
+
+	if hasattr(fcrm_settings, "all_day_event_notifications"):
+		for notification in fcrm_settings.all_day_event_notifications:
+			if notification.interval == interval:
+				notification._table_type = "all_day"
+				global_notifications.append(notification)
+
+	if not global_notifications:
+		return []
+	notifications = []
+	for event in events_without_notifications:
+		for global_notification in global_notifications:
+			if (global_notification._table_type == "all_day" and not event.all_day_event) or (
+				global_notification._table_type == "regular" and event.all_day_event
+			):
+				continue
+
+			notification = {
+				"event_name": event.event_name,
+				"notification_type": global_notification.type,
+				"before_value": global_notification.before,
+				"time_of_day": global_notification.time,
+				"subject": event.subject,
+				"starts_on": event.starts_on,
+				"ends_on": event.ends_on,
+				"owner": event.owner,
+				"description": event.description,
+				"all_day_event": event.all_day_event,
+				"event_participants": event.get("event_participants", []),
+			}
+			notifications.append(notification)
+
+	return notifications
+
+
+def _calculate_trigger_datetime(event_start, before_value, interval, all_day_event, time_of_day):
+	"""
+	Calculate when the notification should be triggered based on event start time and interval.
+
+	Args:
+		event_start (datetime): Event start datetime
+		before_value (int): How many units before the event
+		interval (str): The interval type ('minutes', 'hours', 'days', 'weeks')
+		all_day_event (bool): Whether this is an all-day event
+		time_of_day (time): Specific time to send notification for all-day events
+
+	Returns:
+		datetime: When the notification should be triggered
+	"""
+
+	if all_day_event and time_of_day and interval in ["days", "weeks"]:
+		if interval == "days":
+			trigger_date = event_start.date() - timedelta(days=before_value)
+		elif interval == "weeks":
+			trigger_date = event_start.date() - timedelta(weeks=before_value)
+
+		trigger_datetime = datetime.combine(trigger_date, time_of_day)
+	else:
+		interval_kwargs = _get_interval_kwargs(interval, before_value)
+		trigger_datetime = add_to_date(event_start, **interval_kwargs)
+
+	return trigger_datetime
+
+
+def _get_interval_kwargs(interval, before_value):
+	"""
+	Get the appropriate keyword arguments for add_to_date based on interval type.
+
+	Args:
+		interval (str): The interval type ('minutes', 'hours', 'days', 'weeks')
+		before_value (int): How many units before the event
+
+	Returns:
+		dict: Keyword arguments for add_to_date with negative values
+	"""
+	interval_mapping = {
+		"minutes": {"minutes": -before_value},
+		"hours": {"hours": -before_value},
+		"days": {"days": -before_value},
+		"weeks": {"weeks": -before_value},
+	}
+
+	return interval_mapping.get(interval, {"hours": -before_value})
+
+
+def _get_trigger_window_duration(interval):
+	"""
+	Get the trigger window duration based on interval type.
+
+	Args:
+		interval (str): The interval type ('minutes', 'hours', 'days', 'weeks')
+
+	Returns:
+		dict: Window duration to be used symmetrically around the trigger time
+	"""
+	window_mapping = {
+		"minutes": {"minutes": 5},
+		"hours": {"hours": 1},
+		"days": {"hours": 8},
+		"weeks": {"days": 4},
+	}
+
+	return window_mapping.get(interval, {"hours": 1})
+
+
+def _split_participant_emails(participant_emails_csv):
+	"""Return a clean list of participant emails from a comma-separated string."""
+
+	if not participant_emails_csv:
+		return []
+
+	return [email.strip() for email in participant_emails_csv.split(",") if email and email.strip()]
+
+
+def _send_email_notification(notification, event_start, before_value, interval):
+	"""Send email notification for an event"""
+
+	try:
+		recipients = set()
+		subject = f"Event Reminder: {notification.subject}"
+
+		if notification.owner and notification.owner != "Administrator":
+			recipients.add(notification.owner)
+
+		participant_emails = notification.get("event_participants") or []
+		if participant_emails:
+			recipients.update(participant_emails)
+		else:
+			event_doc = frappe.get_doc("Event", notification.event_name)
+			for participant in event_doc.get("event_participants", []):
+				email = getattr(participant, "email", None)
+				if email:
+					recipients.add(email)
+
+		recipients = [email for email in recipients if email]
+
+		if not recipients:
+			return
+		time_remaining_text = _format_time_remaining(before_value, interval)
+
+		message = f"""
+		<div style="font-family: Arial, sans-serif; max-width: 600px;">
+			<h2 style="color: #333;">Event Reminder</h2>
+			<p>This is a reminder for your upcoming event:</p>
+			<div style="background-color: #f8f9fa; padding: 15px; border-left: 4px solid #007bff; margin: 20px 0;">
+				<h3 style="margin: 0; color: #007bff;">{notification.subject}</h3>
+				{f'<p style="margin: 10px 0; color: #666;">{notification.description}</p>' if notification.description else ""}
+				<p style="margin: 5px 0;"><strong>Start Time:</strong> {event_start.strftime("%Y-%m-%d %H:%M:%S")}</p>
+				<p style="margin: 5px 0;"><strong>Time Remaining:</strong> {time_remaining_text}</p>
+			</div>
+			<p style="color: #666; font-size: 12px;">This is an automated reminder from your calendar system.</p>
+		</div>
+		"""
+
+		frappe.sendmail(
+			recipients=recipients,
+			subject=subject,
+			message=message,
+			reference_doctype="Event",
+			reference_name=notification.event_name,
+			now=True,
+		)
+
+	except Exception as e:
+		frappe.log_error(f"Failed to send email for event {notification.event_name}: {e!s}")
+
+
+def _format_time_remaining(before_value, interval):
+	"""
+	Format the time remaining message based on the interval and before value.
+
+	Args:
+		before_value (int): The number of units before the event
+		interval (str): The interval type ('minutes', 'hours', 'days', 'weeks')
+
+	Returns:
+		str: Formatted time remaining message
+	"""
+	interval_labels = {"minutes": "minute(s)", "hours": "hour(s)", "days": "day(s)", "weeks": "week(s)"}
+
+	interval_label = interval_labels.get(interval, "unit(s)")
+	return f"{before_value} {interval_label}"
+
+
+def _send_system_notification(notification):
+	"""Send system notification for an event"""
+	frappe.publish_realtime("event_notification", notification)

crm/api/exchange_rate.py

@@ -0,0 +1,158 @@
+import frappe
+import requests
+from frappe import _
+
+from crm.fcrm.doctype.fcrm_settings.fcrm_settings import FCRMSettings
+
+
+@frappe.whitelist()
+def get_exchange_rate(from_currency: str, to_currency: str, date: str | None = None):
+	if not date:
+		date = "latest"
+
+	# "latest" is keyed by today's date so tomorrow's call automatically misses the cache
+	cache_date = frappe.utils.today() if date == "latest" else date
+	cache_key = f"exchange_rate_{from_currency}_{to_currency}_{cache_date}"
+
+	cached_rate = frappe.cache().get_value(cache_key)
+	if cached_rate is not None:
+		return cached_rate
+
+	rate, api_used = _fetch_exchange_rate(from_currency, to_currency, date)
+
+	if rate is not None:
+		frappe.cache().set_value(cache_key, rate)
+		return rate
+
+	_raise_exchange_rate_error(from_currency, to_currency, date, api_used)
+
+
+def _fetch_exchange_rate(from_currency: str, to_currency: str, date: str):
+	settings = frappe.get_single("FCRM Settings")
+	provider = settings.service_provider
+
+	# Paid providers — no fallback, fail explicitly
+	if provider == "exchangerate.host":
+		return _fetch_from_exchangerate_host(settings, from_currency, to_currency, date), provider
+
+	if provider == "exchangerate-api":
+		return _fetch_from_exchangerate_api(settings, from_currency, to_currency), provider
+
+	# Free providers — try both as fallbacks, regardless of which is "primary"
+	if provider == "frankfurter.app":
+		rate = _fetch_from_frankfurter(from_currency, to_currency, date)
+		if rate is not None:
+			return rate, provider
+		# Frankfurter is down — silently fall back to fawaz
+		rate = _fetch_from_fawaz_api(from_currency, to_currency, date)
+		return rate, "fawazahmed-exchange-api"
+
+	if provider == "fawazahmed-exchange-api":
+		rate = _fetch_from_fawaz_api(from_currency, to_currency, date)
+		if rate is not None:
+			return rate, provider
+		# fawaz is down — silently fall back to frankfurter
+		rate = _fetch_from_frankfurter(from_currency, to_currency, date)
+		return rate, "frankfurter"
+
+	# Unknown provider — try both free ones
+	rate = _fetch_from_frankfurter(from_currency, to_currency, date)
+	if rate is not None:
+		return rate, "frankfurter"
+	return _fetch_from_fawaz_api(from_currency, to_currency, date), "fawazahmed-exchange-api"
+
+
+def _fetch_from_frankfurter(from_currency: str, to_currency: str, date: str):
+	try:
+		res = requests.get(
+			f"https://api.frankfurter.app/{date}?from={from_currency}&to={to_currency}", timeout=5
+		)
+		if res.ok:
+			return res.json().get("rates", {}).get(to_currency)
+	except Exception:
+		pass
+	return None
+
+
+def _fetch_from_fawaz_api(from_currency: str, to_currency: str, date: str):
+	from_lower = from_currency.lower()
+	to_lower = to_currency.lower()
+	date_str = "latest" if date == "latest" else date
+	urls = [
+		f"https://cdn.jsdelivr.net/npm/@fawazahmed0/currency-api@{date_str}/v1/currencies/{from_lower}.json",
+		f"https://{date_str}.currency-api.pages.dev/v1/currencies/{from_lower}.json",
+	]
+	for url in urls:
+		try:
+			res = requests.get(url, timeout=5)
+			if res.ok:
+				return res.json()[from_lower][to_lower]
+		except Exception:
+			continue
+	return None
+
+
+def _fetch_from_exchangerate_host(settings: FCRMSettings, from_currency: str, to_currency: str, date: str):
+	if not settings.access_key:
+		frappe.throw(
+			_("Access Key is required for Service Provider: {0}").format(
+				frappe.bold(settings.service_provider)
+			)
+		)
+	params = {"access_key": settings.access_key, "from": from_currency, "to": to_currency, "amount": 1}
+	if date != "latest":
+		params["date"] = date
+	res = requests.get("https://api.exchangerate.host/convert", params=params, timeout=5)
+	if res.ok:
+		return res.json()["result"]
+	return None
+
+
+def _fetch_from_exchangerate_api(settings: FCRMSettings, from_currency: str, to_currency: str):
+	if not settings.access_key:
+		frappe.throw(
+			_("Access Key is required for Service Provider: {0}").format(
+				frappe.bold(settings.service_provider)
+			)
+		)
+	res = requests.get(
+		f"https://v6.exchangerate-api.com/v6/{settings.access_key}/pair/{from_currency}/{to_currency}",
+		timeout=5,
+	)
+	if res.ok:
+		data = res.json()
+		if data["result"] == "success":
+			return data["conversion_rate"]
+	return None
+
+
+def _raise_exchange_rate_error(from_currency: str, to_currency: str, date: str, api_used: str):
+	frappe.log_error(
+		title="Exchange Rate Fetch Error",
+		message=f"Failed to fetch exchange rate from {from_currency} to {to_currency} using {api_used} API.",
+	)
+
+	if api_used == "frankfurter":
+		user = frappe.session.user
+		is_manager = (
+			"System Manager" in frappe.get_roles(user)
+			or "Sales Manager" in frappe.get_roles(user)
+			or user == "Administrator"
+		)
+		if not is_manager:
+			frappe.throw(
+				_(
+					"Ask your manager to set up the Exchange Rate Provider, as default provider does not support currency conversion for {0} to {1}."
+				).format(from_currency, to_currency)
+			)
+		frappe.throw(
+			_(
+				"Setup the Exchange Rate Provider other than 'Frankfurter' in settings, as default provider does not support currency conversion for {0} to {1}."
+			).format(from_currency, to_currency)
+		)
+
+	frappe.throw(
+		_(
+			"Failed to fetch exchange rate from {0} to {1} on {2}. Please check your internet connection or try again later."
+		).format(from_currency, to_currency, date)
+	)

crm/api/form.py

@@ -0,0 +1,638 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Hybrid forms API.
+
+Storage/submission/validation is delegated to Frappe's built-in `Web Form`
+doctype; this module only:
+  1. curates which target fields a CRM form may collect,
+  2. gives the CRM builder a small, scoped CRUD surface over Web Form records,
+  3. re-applies CRM-specific enrichment (source, organization, primary contact)
+     on submission, which the framework's generic insert does not do.
+"""
+
+import json
+
+import frappe
+from frappe import _
+from frappe.utils.telemetry import capture
+
+ALLOWED_DOCTYPES = ("CRM Lead", "CRM Deal")
+FORM_SOURCE = "Web Form"
+FORM_MODULE = "FCRM"
+
+
+def ensure_form_source() -> str:
+	"""Return the 'Web Form' CRM Lead Source, creating it once if needed."""
+	if not frappe.db.exists("CRM Lead Source", FORM_SOURCE):
+		frappe.get_doc({"doctype": "CRM Lead Source", "source_name": FORM_SOURCE}).insert(
+			ignore_permissions=True
+		)
+	return FORM_SOURCE
+
+
+# Fieldtypes a form can render/collect. A `Link` renders as a server-populated,
+# pick-only dropdown (see crm_form.py); excludes types needing a live widget or file
+# upload (Dynamic Link, Table, Attach, …).
+SUPPORTED_FIELDTYPES = (
+	"Data",
+	"Small Text",
+	"Text",
+	"Long Text",
+	"Text Editor",
+	"HTML Editor",
+	"Markdown Editor",
+	"Select",
+	"Link",
+	"Int",
+	"Float",
+	"Currency",
+	"Percent",
+	"Check",
+	"Date",
+	"Datetime",
+	"Time",
+	"Phone",
+	"Color",
+)
+
+# Never expose these as mappable fields even if their type is supported.
+DENIED_FIELDNAMES = (
+	"naming_series",
+	# a mandatory Link the system sets — seeded as a hidden field with a default, not
+	# offered in the picker.
+	"status",
+	"lead_name",
+	"converted",
+	"sla_status",
+	"response_by",
+	"first_response_time",
+	"first_responded_on",
+	"facebook_form_id",
+	"facebook_lead_id",
+)
+
+LAYOUT_BREAKS = ("Section Break", "Column Break")
+
+# Starting layout of a brand-new form, per target doctype: labelled sections,
+# each a list of columns, each column a list of fieldnames. A sensible
+# contact-capture starting point the author can then edit.
+SEED_LAYOUT = {
+	"CRM Lead": [
+		{
+			"label": "Personal Details",
+			"columns": [["first_name", "email"], ["last_name", "phone"]],
+		},
+	],
+	"CRM Deal": [
+		{
+			"label": "Personal Details",
+			"columns": [["first_name", "email"], ["last_name", "phone"]],
+		},
+		{
+			"label": "Organization Details",
+			"columns": [["organization_name"]],
+		},
+	],
+}
+
+
+def _seeded_visible_fieldnames(document_type: str) -> set:
+	names = set()
+	for section in SEED_LAYOUT.get(document_type, []):
+		for col in section["columns"]:
+			names.update(col)
+	return names
+
+
+def guest_can_select(doctype: str) -> bool:
+	"""Whether Guest may reference `doctype` in a Link field. Gates on `select` (never
+	`read`) — Frappe's minimal tier that lists record names without exposing full
+	documents — so a doctype is linkable only when Guest select is deliberately granted."""
+	return bool(doctype) and frappe.has_permission(doctype, ptype="select", user="Guest")
+
+
+def _link_target_doctypes() -> set:
+	"""Doctypes reachable via a Link field on any form-mappable target — the only
+	doctypes `grant_guest_link_access` is allowed to touch, so the endpoint can't be
+	used to open Guest select on an arbitrary doctype."""
+	targets = set()
+	for document_type in ALLOWED_DOCTYPES:
+		for df in frappe.get_meta(document_type).fields:
+			if df.fieldtype == "Link" and df.options:
+				targets.add(df.options)
+	return targets
+
+
+def _mappable_fields(document_type: str) -> list[dict]:
+	"""Fields of a target DocType a form may collect (shared by the picker and by
+	the brand-new-form seeding)."""
+	meta = frappe.get_meta(document_type)
+	fields = []
+	for df in meta.fields:
+		if df.fieldtype not in SUPPORTED_FIELDTYPES:
+			continue
+		if not df.fieldname or df.fieldname in DENIED_FIELDNAMES:
+			continue
+		if df.hidden or df.read_only:
+			continue
+		# a Link is offered even when guests can't select the target yet; the builder
+		# warns and offers a one-click grant (see grant_guest_link_access).
+		fields.append(
+			{
+				"fieldname": df.fieldname,
+				"label": df.label or df.fieldname,
+				"fieldtype": df.fieldtype,
+				"options": df.options,
+				"reqd": df.reqd,
+				"default": df.default,
+			}
+		)
+	return fields
+
+
+def _default_status(document_type: str) -> str | None:
+	"""The status a new Lead/Deal defaults to — mirrors the doctype controllers so
+	the hidden 'Status' field is pre-filled with the value the CRM would use."""
+	status_dt = "CRM Lead Status" if document_type == "CRM Lead" else "CRM Deal Status"
+	preferred = "New" if document_type == "CRM Lead" else "Qualification"
+	if frappe.db.exists(status_dt, preferred):
+		return preferred
+	rows = frappe.get_all(status_dt, {"type": "Open"}, pluck="name")
+	return rows[0] if rows else None
+
+
+def _seed_visible_fields(document_type: str) -> list[dict]:
+	"""Web-form rows for a new form's visible layout: the curated contact set,
+	arranged into labelled sections and columns (with Section/Column breaks)."""
+	catalog = {f["fieldname"]: f for f in _mappable_fields(document_type)}
+
+	def _break(fieldtype, i, label=""):
+		prefix = "section_break" if fieldtype == "Section Break" else "column_break"
+		return {
+			"fieldname": f"{prefix}_seed{i}",
+			"label": label,
+			"fieldtype": fieldtype,
+			"options": "",
+			"reqd": 0,
+			"placeholder": "",
+			"field_description": "",
+		}
+
+	rows = []
+	n = 0
+	for section in SEED_LAYOUT.get(document_type, []):
+		n += 1
+		rows.append(_break("Section Break", n, section.get("label") or ""))
+		for ci, col in enumerate(section["columns"]):
+			if ci > 0:
+				n += 1
+				rows.append(_break("Column Break", n))
+			for fn in col:
+				f = catalog.get(fn)
+				if not f:
+					continue
+				rows.append(
+					{
+						"fieldname": f["fieldname"],
+						"label": f["label"],
+						"fieldtype": f["fieldtype"],
+						"options": f["options"],
+						"reqd": 1 if f["reqd"] else 0,
+						"placeholder": "",
+						"field_description": "",
+					}
+				)
+	return rows
+
+
+def _seed_hidden_fields(document_type: str) -> list[dict]:
+	"""Mandatory fields a public visitor should not fill (e.g. Status, a Link the
+	system sets). Seeded into the hidden section with a sensible default so the
+	author sees them and can override the value applied on submission."""
+	meta = frappe.get_meta(document_type)
+	visible = _seeded_visible_fieldnames(document_type)
+	hidden = []
+	for df in meta.fields:
+		if not df.reqd or df.fieldname in visible:
+			continue
+		if df.fieldtype in SUPPORTED_FIELDTYPES and df.fieldname not in DENIED_FIELDNAMES:
+			continue  # a fillable mandatory field belongs in the visible layout
+		default = _default_status(document_type) if df.fieldname == "status" else (df.default or "")
+		hidden.append(
+			{
+				"fieldname": df.fieldname,
+				"label": df.label or df.fieldname,
+				"fieldtype": df.fieldtype,
+				"options": df.options or "",
+				"default": default,
+			}
+		)
+	return hidden
+
+
+def _check_manager():
+	"""CRM forms are managed by CRM managers, not Website Managers. Gate here and
+	then write the Web Form with ignore_permissions (the role mismatch, in code)."""
+	roles = set(frappe.get_roles())
+	if not roles & {"System Manager", "Sales Manager"}:
+		frappe.throw(_("Not permitted"), frappe.PermissionError)
+
+
+@frappe.whitelist()
+def get_form_fields(document_type: str) -> list[dict]:
+	"""Mappable fields of a target DocType, for the builder's field picker."""
+	if document_type not in ALLOWED_DOCTYPES:
+		frappe.throw(_("Forms can only map to: {0}").format(", ".join(ALLOWED_DOCTYPES)))
+	return _mappable_fields(document_type)
+
+
+@frappe.whitelist()
+def get_hidden_seed(document_type: str) -> list[dict]:
+	"""The system-managed hidden required fields for a doctype (e.g. Status) with
+	their default value. Used by the builder to reconcile on a doctype switch."""
+	if document_type not in ALLOWED_DOCTYPES:
+		frappe.throw(_("Forms can only map to: {0}").format(", ".join(ALLOWED_DOCTYPES)))
+	return _seed_hidden_fields(document_type)
+
+
+@frappe.whitelist()
+def link_field_guest_access(doctype: str) -> dict:
+	"""Whether guests can already select `doctype`. Drives the builder's Link-field
+	notice: if False, the author is warned that the public dropdown will be empty and
+	is offered a one-click grant (grant_guest_link_access)."""
+	_check_manager()
+	return {"doctype": doctype, "guest_can_select": guest_can_select(doctype)}
+
+
+@frappe.whitelist()
+def grant_guest_link_access(doctype: str) -> dict:
+	"""Grant Guest `select` on `doctype` so a public Link dropdown can list its records — a
+	deliberate choice by a form manager (the builder warns that anyone with the form link
+	will then see them). Limited to doctypes that are Link targets on a CRM form; stored as
+	a site-level Custom DocPerm."""
+	_check_manager()
+	if doctype not in _link_target_doctypes():
+		frappe.throw(_("{0} isn't a linkable field on a CRM form.").format(doctype))
+
+	if not guest_can_select(doctype):
+		from frappe.permissions import add_permission
+
+		# a Sales Manager may run the builder but can't normally write Custom DocPerm; do
+		# this narrow, doctype-scoped grant with permission checks off (as this module
+		# already does to write the Web Form).
+		had_flag = frappe.flags.ignore_permissions
+		frappe.flags.ignore_permissions = True
+		try:
+			perm_name = frappe.db.get_value(
+				"Custom DocPerm",
+				{"parent": doctype, "role": "Guest", "permlevel": 0, "if_owner": 0},
+			)
+			if not perm_name:
+				add_permission(doctype, "Guest", 0, ptype="select")
+				perm_name = frappe.db.get_value(
+					"Custom DocPerm",
+					{"parent": doctype, "role": "Guest", "permlevel": 0, "if_owner": 0},
+				)
+			# add_permission defaults the row to read/export; a public Link only lists
+			# names, so trim the Guest row to `select` alone.
+			perm = frappe.get_doc("Custom DocPerm", perm_name)
+			perm.update(
+				{
+					"select": 1,
+					"read": 0,
+					"write": 0,
+					"create": 0,
+					"delete": 0,
+					"submit": 0,
+					"cancel": 0,
+					"amend": 0,
+					"report": 0,
+					"export": 0,
+					"import": 0,
+					"print": 0,
+					"email": 0,
+					"share": 0,
+				}
+			)
+			perm.save(ignore_permissions=True)
+		finally:
+			frappe.flags.ignore_permissions = had_flag
+		frappe.clear_cache(doctype=doctype)
+
+	return {"doctype": doctype, "guest_can_select": guest_can_select(doctype)}
+
+
+@frappe.whitelist()
+def list_forms() -> list[dict]:
+	"""CRM forms only (native Web Form records mapped to Lead/Deal)."""
+	_check_manager()
+	return frappe.get_all(
+		"Web Form",
+		filters={"module": FORM_MODULE, "doc_type": ["in", ALLOWED_DOCTYPES]},
+		fields=[
+			"name",
+			"title",
+			"route",
+			"doc_type as document_type",
+			"crm_published as published",
+			"modified",
+		],
+		order_by="modified desc",
+	)
+
+
+@frappe.whitelist()
+def get_form_config(name: str) -> dict:
+	"""Full config for the builder, read from the native Web Form record."""
+	_check_manager()
+	doc = _get_crm_form(name)
+	return {
+		"name": doc.name,
+		"title": doc.title,
+		"route": doc.route,
+		"document_type": doc.doc_type,
+		"published": doc.crm_published,
+		"submit_button_label": doc.button_label or _("Submit"),
+		"description": doc.introduction_text or "",
+		"success_message": doc.success_message or "",
+		"redirect_url": doc.success_url or "",
+		"allowed_embedding_domains": doc.allowed_embedding_domains or "",
+		"fields": [
+			{
+				"fieldname": f.fieldname,
+				"label": f.label,
+				"fieldtype": f.fieldtype,
+				"options": f.options,
+				"reqd": f.reqd,
+				"placeholder": f.placeholder,
+				"field_description": f.description,
+				"depends_on": f.depends_on or "",
+				"mandatory_depends_on": f.mandatory_depends_on or "",
+				"read_only_depends_on": f.read_only_depends_on or "",
+			}
+			for f in doc.web_form_fields
+		],
+		"hidden_fields": _load_hidden_fields(doc),
+	}
+
+
+def _load_hidden_fields(doc) -> list[dict]:
+	try:
+		return json.loads(doc.get("crm_hidden_defaults") or "[]")
+	except Exception:
+		return []
+
+
+def _assert_hidden_defaults_set(hidden: list[dict]):
+	"""Publishing requires every hidden required field to carry a default — a blank
+	one would break record creation on submission. Drafts may save without them."""
+	missing = [
+		h.get("label") or h.get("fieldname") for h in hidden if not str(h.get("default") or "").strip()
+	]
+	if missing:
+		frappe.throw(_("Set a default value before publishing for: {0}").format(", ".join(missing)))
+
+
+def _validated_visible_fields(document_type: str, fields: list[dict]) -> list[dict]:
+	"""Re-apply the catalog the picker filters by: `accept()` lets a public visitor
+	write anything listed in `web_form_fields`."""
+	meta = frappe.get_meta(document_type)
+	catalog = {f["fieldname"]: f for f in _mappable_fields(document_type)}
+	rows = []
+	for f in fields:
+		fieldname = f.get("fieldname")
+		if f.get("fieldtype") in LAYOUT_BREAKS:
+			# a break carries no data, but `accept()` still writes any row naming a field
+			if meta.has_field(fieldname):
+				frappe.throw(_("{0} can't be used as a layout break").format(fieldname))
+			rows.append(f)
+			continue
+		allowed = catalog.get(fieldname)
+		if not allowed:
+			frappe.throw(_("{0} can't be collected by a form").format(fieldname or _("Unnamed field")))
+		rows.append({**f, "fieldtype": allowed["fieldtype"], "options": allowed["options"]})
+	return rows
+
+
+def _validated_hidden_fields(document_type: str, hidden: list[dict]) -> list[dict]:
+	"""Same gate for the hidden section: `_apply_hidden_defaults` writes these onto
+	every submission."""
+	catalog = {h["fieldname"]: h for h in _seed_hidden_fields(document_type)}
+	rows = []
+	for h in hidden:
+		fieldname = h.get("fieldname")
+		allowed = catalog.get(fieldname)
+		if not allowed:
+			frappe.throw(_("{0} can't be set as a hidden field").format(fieldname or _("Unnamed field")))
+		rows.append({**h, "fieldtype": allowed["fieldtype"], "options": allowed["options"]})
+	return rows
+
+
+@frappe.whitelist()
+def save_form(name: str | None, form: dict | str) -> dict:
+	"""Create/update a native Web Form scoped to CRM doctypes."""
+	_check_manager()
+	if isinstance(form, str):
+		form = json.loads(form or "{}")
+
+	if form.get("document_type") not in ALLOWED_DOCTYPES:
+		frappe.throw(_("Forms can only map to: {0}").format(", ".join(ALLOWED_DOCTYPES)))
+
+	doc = _get_crm_form(name) if name else frappe.new_doc("Web Form")
+
+	doc.title = form.get("title")
+	doc.route = form.get("route")
+	doc.doc_type = form.get("document_type")
+	doc.introduction_text = form.get("description")
+	doc.button_label = form.get("submit_button_label") or "Submit"
+	doc.success_message = form.get("success_message")
+	# redirect visitors here after a successful submission (native Web Form field);
+	# blank falls back to showing the success message
+	doc.success_url = form.get("redirect_url") or ""
+	doc.allowed_embedding_domains = form.get("allowed_embedding_domains")
+	# `crm_published` drives the CRM's own branded page at /crm-form/<route>; mirror
+	# it onto the native `published` so the Desk Web Form view shows the right state.
+	# (This also makes Frappe serve its generic form at /<route> — an additional,
+	# unbranded public URL alongside /crm-form/<route>.)
+	doc.crm_published = 1 if form.get("published") else 0
+	doc.published = doc.crm_published
+	# CRM forms are public, single-purpose lead/deal capture
+	doc.login_required = 0
+	doc.allow_multiple = 1
+	doc.is_standard = 0
+	doc.module = FORM_MODULE
+
+	# brand-new form → seed a starting contact-capture layout (only on create)
+	fields = form.get("fields")
+	if not name and not fields:
+		fields = _seed_visible_fields(form["document_type"])
+
+	# only rewrite the layout when fields were actually sent — an update that omits
+	# `fields` (e.g. a settings-only save) must not wipe the existing layout
+	if fields is not None:
+		fields = _validated_visible_fields(form["document_type"], fields)
+		doc.set("web_form_fields", [])
+		for i, f in enumerate(fields):
+			doc.append(
+				"web_form_fields",
+				{
+					"fieldname": f.get("fieldname"),
+					"label": f.get("label"),
+					"fieldtype": f.get("fieldtype"),
+					"options": f.get("options"),
+					"reqd": 1 if f.get("reqd") else 0,
+					"placeholder": f.get("placeholder"),
+					"description": f.get("field_description"),
+					# conditional-logic rules authored per field in the builder (same as
+					# the framework's Web Form fields; evaluated client-side)
+					"depends_on": f.get("depends_on") or "",
+					"mandatory_depends_on": f.get("mandatory_depends_on") or "",
+					"read_only_depends_on": f.get("read_only_depends_on") or "",
+					"idx": i + 1,
+				},
+			)
+
+	# hidden required fields: mandatory fields kept out of the visible form, with
+	# the default value applied on submission (see enrich_form_submission). Seeded
+	# once on create (e.g. Status); thereafter whatever the builder sends wins.
+	hidden = form.get("hidden_fields")
+	if hidden is None and not name:
+		hidden = _seed_hidden_fields(form["document_type"])
+	if isinstance(hidden, str):
+		hidden = json.loads(hidden or "[]")
+	hidden = _validated_hidden_fields(form["document_type"], hidden or [])
+	if doc.crm_published:
+		_assert_hidden_defaults_set(hidden)
+	doc.crm_hidden_defaults = json.dumps(hidden) if hidden else ""
+
+	doc.save(ignore_permissions=True)
+	return {"name": doc.name, "route": doc.route}
+
+
+def _get_crm_form(name: str):
+	doc = frappe.get_doc("Web Form", name)
+	# scope to CRM's own forms — a Web Form from another app that happens to target
+	# CRM Lead/Deal must not be readable/mutable/deletable through this API
+	if doc.module != FORM_MODULE or doc.doc_type not in ALLOWED_DOCTYPES:
+		frappe.throw(_("Not a CRM form"))
+	return doc
+
+
+@frappe.whitelist()
+def set_published(name: str, published: int) -> None:
+	"""Publish/unpublish from the list, bypassing native Web Form role perms."""
+	_check_manager()
+	doc = _get_crm_form(name)
+	if int(published):
+		_assert_hidden_defaults_set(_load_hidden_fields(doc))
+	doc.crm_published = 1 if int(published) else 0
+	doc.published = doc.crm_published  # mirror onto native flag (see save_form)
+	doc.save(ignore_permissions=True)
+
+
+@frappe.whitelist()
+def delete_form(name: str) -> None:
+	_check_manager()
+	_get_crm_form(name)
+	frappe.delete_doc("Web Form", name, ignore_permissions=True)
+
+
+@frappe.whitelist()
+def test_submit_form(name: str, values: dict | str) -> dict:
+	"""Dry-run a submission for an author previewing a draft: validate required
+	fields the same way the live form does, but create no record."""
+	_check_manager()
+	if isinstance(values, str):
+		values = json.loads(values or "{}")
+
+	doc = _get_crm_form(name)
+	for f in doc.web_form_fields:
+		if f.fieldtype in ("Section Break", "Column Break"):
+			continue
+		value = values.get(f.fieldname)
+		if f.reqd and (value is None or value == ""):
+			frappe.throw(_("{0} is required").format(f.label or f.fieldname))
+	return {"test": True}
+
+
+# Public form serving + submission run through the framework's own Web Form engine:
+# the CRM page (`www/crm_form.py`) renders the published form and posts to the
+# built-in `accept()`, which triggers `enrich_form_submission` below via the
+# `in_web_form` flag. No CRM-owned guest endpoint is needed for that path.
+
+
+def enrich_form_submission(doc):
+	"""Called from the CRM Lead/Deal `before_insert`: when the record is created via a
+	web form, apply the same enrichment the CRM applies on manual creation.
+
+	The framework's Web Form `accept()` just inserts the target doc, so without this
+	web submissions would miss Source / Organization / primary Contact. `accept()`
+	sets `frappe.flags.in_web_form`, which is how we scope this to web submissions.
+	"""
+	if not frappe.flags.get("in_web_form"):
+		return
+	if doc.doctype not in ALLOWED_DOCTYPES:
+		return
+
+	_apply_hidden_defaults(doc)
+
+	# stamp the source so form records are identifiable/filterable
+	if doc.meta.has_field("source") and not doc.get("source"):
+		doc.source = ensure_form_source()
+
+	capture(
+		"web_form_submitted",
+		"crm",
+		properties={"doctype": doc.doctype, "form": frappe.form_dict.get("web_form")},
+	)
+
+	if doc.doctype != "CRM Deal":
+		return
+
+	from crm.fcrm.doctype.crm_deal.crm_deal import create_contact, create_organization
+
+	# auto-create the linked Organization from the org name (as the CRM does for deals)
+	if doc.get("organization_name") and not doc.get("organization"):
+		created = create_organization(doc)
+		if created:
+			doc.organization = created
+
+	# auto-create & link a primary Contact from the person's details
+	if not doc.get("contacts") and (
+		doc.get("first_name") or doc.get("last_name") or doc.get("email") or doc.get("mobile_no")
+	):
+		contact = create_contact(doc)
+		if contact:
+			doc.append("contacts", {"contact": contact, "is_primary": 1})
+
+
+def _apply_hidden_defaults(doc):
+	"""Apply the submitting form's hidden-field defaults (e.g. Status) to fields the
+	visitor didn't fill, so mandatory values are present before the record is saved."""
+	web_form = frappe.form_dict.get("web_form")
+	if not web_form:
+		return
+	# `web_form` comes from the (client-controllable) POST body — only trust a CRM
+	# form that targets this exact doctype, else a submission could pull in another
+	# form's defaults (or a different doctype's).
+	form = frappe.db.get_value("Web Form", web_form, ["doc_type", "crm_hidden_defaults"], as_dict=True)
+	if not form or form.doc_type not in ALLOWED_DOCTYPES or form.doc_type != doc.doctype:
+		return
+	raw = form.crm_hidden_defaults
+	if not raw:
+		return
+	try:
+		hidden = json.loads(raw)
+	except Exception:
+		return
+	for h in hidden:
+		fieldname = h.get("fieldname")
+		default = h.get("default")
+		if (
+			fieldname
+			and default not in (None, "")
+			and doc.meta.has_field(fieldname)
+			and not doc.get(fieldname)
+		):
+			doc.set(fieldname, default)

crm/api/live_demo.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2024, Frappe Technologies and contributors
+# This file is used to handle live demo site (https://frappecrm-demo.frappe.cloud) related API calls and hooks
+
+import frappe
+from frappe import _
+from frappe.auth import LoginManager
+
+
+@frappe.whitelist(allow_guest=True)
+def login():
+	if not frappe.conf.demo_username or not frappe.conf.demo_password:
+		return
+	frappe.local.response["redirect_to"] = "/crm"
+	login_manager = LoginManager()
+	login_manager.authenticate(frappe.conf.demo_username, frappe.conf.demo_password)
+	login_manager.post_login()
+	frappe.local.response["type"] = "redirect"
+	frappe.local.response["location"] = frappe.local.response["redirect_to"]
+
+
+def validate_reset_password(doc, event):
+	if frappe.conf.demo_username and frappe.session.user == frappe.conf.demo_username:
+		frappe.throw(
+			_("Password cannot be reset by Demo User {}").format(frappe.bold(frappe.conf.demo_username)),
+			frappe.PermissionError,
+		)
+
+
+def validate_user(doc, event):
+	if frappe.conf.demo_username and frappe.session.user == frappe.conf.demo_username and doc.new_password:
+		frappe.throw(
+			_("Password cannot be reset by Demo User {}").format(frappe.bold(frappe.conf.demo_username)),
+			frappe.PermissionError,
+		)

crm/api/notifications.py

@@ -0,0 +1,69 @@
+import frappe
+from frappe.query_builder import Order
+
+
+@frappe.whitelist()
+def get_notifications():
+	Notification = frappe.qb.DocType("CRM Notification")
+	query = (
+		frappe.qb.from_(Notification)
+		.select("*")
+		.where(Notification.to_user == frappe.session.user)
+		.orderby("creation", order=Order.desc)
+	)
+	notifications = query.run(as_dict=True)
+
+	_notifications = []
+	for notification in notifications:
+		_notifications.append(
+			{
+				"creation": notification.creation,
+				"from_user": {
+					"name": notification.from_user,
+					"full_name": frappe.get_value("User", notification.from_user, "full_name"),
+				},
+				"type": notification.type,
+				"to_user": notification.to_user,
+				"read": notification.read,
+				"hash": get_hash(notification),
+				"notification_text": notification.notification_text,
+				"notification_type_doctype": notification.notification_type_doctype,
+				"notification_type_doc": notification.notification_type_doc,
+				"reference_doctype": ("deal" if notification.reference_doctype == "CRM Deal" else "lead"),
+				"reference_name": notification.reference_name,
+				"route_name": ("Deal" if notification.reference_doctype == "CRM Deal" else "Lead"),
+			}
+		)
+
+	return _notifications
+
+
+@frappe.whitelist()
+def mark_as_read(doc: str | None = None):
+	user = frappe.session.user
+	filters = {"to_user": user, "read": False}
+	or_filters = []
+	if doc:
+		or_filters = [
+			{"comment": doc},
+			{"notification_type_doc": doc},
+		]
+	for n in frappe.get_all("CRM Notification", filters=filters, or_filters=or_filters):
+		d = frappe.get_doc("CRM Notification", n.name)
+		d.read = True
+		d.save()
+
+
+def get_hash(notification):
+	_hash = ""
+	if notification.type == "Mention" and notification.notification_type_doc:
+		_hash = "#" + notification.notification_type_doc
+
+	if notification.type == "WhatsApp":
+		_hash = "#whatsapp"
+
+	if notification.type == "Assignment" and notification.notification_type_doctype == "CRM Task":
+		_hash = "#tasks"
+		if "has been removed by" in notification.message:
+			_hash = ""
+	return _hash

crm/api/onboarding.py

@@ -0,0 +1,24 @@
+import frappe
+
+
+@frappe.whitelist()
+def get_first_lead():
+	lead = frappe.get_all(
+		"CRM Lead",
+		filters={"converted": 0},
+		fields=["name"],
+		order_by="creation",
+		limit=1,
+	)
+	return lead[0].name if lead else None
+
+
+@frappe.whitelist()
+def get_first_deal():
+	deal = frappe.get_all(
+		"CRM Deal",
+		fields=["name"],
+		order_by="creation",
+		limit=1,
+	)
+	return deal[0].name if deal else None

crm/api/session.py

@@ -0,0 +1,171 @@
+import frappe
+from frappe import _
+
+CRM_ALLOWED_ROLES = ["System Manager", "Sales Manager", "Sales User"]
+
+
+def get_session_role_flags():
+	roles = set(frappe.get_roles())
+
+	if not roles.intersection(set(CRM_ALLOWED_ROLES)):
+		frappe.throw(_("You are not permitted to access CRM resources."), frappe.PermissionError)
+
+	return {
+		"is_system_manager": "System Manager" in roles,
+		"is_sales_manager": "Sales Manager" in roles and "System Manager" not in roles,
+		"is_sales_user": "Sales User" in roles
+		and "Sales Manager" not in roles
+		and "System Manager" not in roles,
+	}
+
+
+USER_FIELDS = [
+	"name",
+	"email",
+	"enabled",
+	"user_image",
+	"first_name",
+	"last_name",
+	"full_name",
+	"user_type",
+	"language",
+]
+
+
+@frappe.whitelist()
+def get_users(include_all: bool = False):
+	"""Return (users, crm_users) for the CRM frontend.
+
+	By default (`include_all=False`) the User query is filtered at SQL level
+	to just users with CRM roles — typically a handful of rows. This unblocks
+	the UI on initial load even on sites with hundreds of thousands of users.
+
+	When `include_all=True` and the session has System Manager, the full
+	enabled user list is returned. The frontend uses this path for a
+	non-blocking background fetch that populates name/avatar info for
+	non-CRM users referenced in CRM activity. Non-System-Manager sessions
+	cannot escalate by passing this param.
+	"""
+	session_roles = get_session_role_flags()
+
+	# Param arrives as a string from the HTTP layer; coerce.
+	if isinstance(include_all, str):
+		include_all = include_all.lower() in ("1", "true", "yes")
+	if not session_roles["is_system_manager"]:
+		include_all = False
+
+	# Always need the CRM user name set — used both as the filter for the
+	# fast path and as the membership check on the full path.
+	crm_user_names = set(
+		frappe.get_all(
+			"Has Role",
+			filters={"parenttype": "User", "role": ["in", CRM_ALLOWED_ROLES]},
+			pluck="parent",
+			distinct=True,
+		)
+	)
+	crm_user_names.add("Administrator")
+
+	user_filters = {"enabled": 1}
+	if not include_all:
+		user_filters["name"] = ["in", list(crm_user_names)]
+
+	users = frappe.qb.get_query(
+		"User",
+		fields=USER_FIELDS,
+		order_by="full_name asc",
+		filters=user_filters,
+	).run(as_dict=1)
+
+	if not users:
+		return [], []
+
+	system_language = frappe.db.get_single_value("System Settings", "language")
+	session_user = frappe.session.user
+
+	# Has Role lookup — restrict by parent on the fast path (tiny IN list);
+	# unfiltered scan on the full path because the IN list would otherwise
+	# carry every enabled user name (see commit dropping IN-list filters).
+	if include_all:
+		role_filters = {"parenttype": "User"}
+	else:
+		role_filters = {"parenttype": "User", "parent": ["in", list(crm_user_names)]}
+	role_rows = frappe.get_all("Has Role", filters=role_filters, fields=["parent", "role"])
+	roles_by_user = {}
+	for row in role_rows:
+		roles_by_user.setdefault(row.parent, []).append(row.role)
+
+	# Telephony agent table is tiny on any real site; full pluck is cheaper
+	# than serializing an IN list and gives identical results.
+	telephony_agents = set(frappe.get_all("CRM Telephony Agent", pluck="user"))
+
+	role_priority = ("System Manager", "Sales Manager", "Sales User", "Guest")
+	crm_users = []
+
+	for user in users:
+		if session_user == user.name:
+			user.session_user = True
+
+		# Administrator has every role implicitly via frappe.get_roles() but
+		# its Has Role child table is not guaranteed to contain System Manager
+		# on every install — special-case it to avoid locking the admin out.
+		if user.name == "Administrator":
+			user.roles = ["System Manager", "All"]
+			user.role = "System Manager"
+		else:
+			# Mirror frappe.get_roles() which appends implicit "All" and "Guest"
+			user.roles = [*roles_by_user.get(user.name, []), "All", "Guest"]
+			user.role = ""
+			for role in role_priority:
+				if role in user.roles:
+					user.role = role
+					break
+
+		user.is_telephony_agent = user.name in telephony_agents
+		user.language = user.language or system_language
+
+		if user.role in CRM_ALLOWED_ROLES:
+			crm_users.append(user)
+
+	if not include_all:
+		# Fast path — both positions are the CRM users.
+		return crm_users, crm_users
+
+	return users, crm_users
+
+
+@frappe.whitelist()
+def get_user_info(users: str | list):
+	"""Resolve display info for a batch of User names.
+
+	Used by the frontend to fill in name/avatar info for non-CRM users
+	referenced in CRM activity (comment authors, doc owners, etc.) when
+	the background full-list fetch has not yet landed. Gated behind a
+	CRM role; capped at 200 names per call to limit enumeration cost.
+	"""
+	get_session_role_flags()
+
+	if isinstance(users, str):
+		users = frappe.parse_json(users)
+	if not users:
+		return []
+
+	return frappe.get_all(
+		"User",
+		filters={"name": ["in", list(users)[:200]]},
+		fields=["name", "email", "full_name", "user_image", "user_type"],
+	)
+
+
+@frappe.whitelist()
+def get_organizations():
+	get_session_role_flags()
+
+	organizations = frappe.qb.get_query(
+		"CRM Organization",
+		fields=["*"],
+		order_by="name asc",
+		distinct=True,
+	).run(as_dict=1)
+
+	return organizations

crm/api/settings.py

@@ -0,0 +1,99 @@
+import frappe
+
+
+@frappe.whitelist()
+def create_email_account(data: dict):
+	service = data.get("service")
+	service_config = email_service_config.get(service)
+	if not service_config:
+		return "Service not supported"
+
+	try:
+		email_doc = frappe.get_doc(
+			{
+				"doctype": "Email Account",
+				"email_id": data.get("email_id"),
+				"email_account_name": data.get("email_account_name"),
+				"service": service,
+				"enable_incoming": data.get("enable_incoming"),
+				"enable_outgoing": data.get("enable_outgoing"),
+				"default_incoming": data.get("default_incoming"),
+				"default_outgoing": data.get("default_outgoing"),
+				"email_sync_option": "ALL",
+				"initial_sync_count": 100,
+				"create_contact": 1,
+				"track_email_status": 1,
+				"use_tls": 1,
+				"use_imap": 1,
+				"smtp_port": 587,
+				**service_config,
+			}
+		)
+		if service == "Frappe Mail":
+			email_doc.api_key = data.get("api_key")
+			email_doc.api_secret = data.get("api_secret")
+			email_doc.frappe_mail_site = data.get("frappe_mail_site")
+			email_doc.append_to = "CRM Lead"
+		else:
+			email_doc.append("imap_folder", {"append_to": "CRM Lead", "folder_name": "INBOX"})
+			email_doc.password = data.get("password")
+			# validate whether the credentials are correct
+			email_doc.get_incoming_server()
+
+		# if correct credentials, save the email account
+		email_doc.save()
+	except Exception as e:
+		frappe.throw(str(e))
+
+
+email_service_config = {
+	"Frappe Mail": {
+		"domain": None,
+		"password": None,
+		"awaiting_password": 0,
+		"ascii_encode_password": 0,
+		"login_id_is_different": 0,
+		"login_id": None,
+		"use_imap": 0,
+		"use_ssl": 0,
+		"validate_ssl_certificate": 0,
+		"use_starttls": 0,
+		"email_server": None,
+		"incoming_port": 0,
+		"always_use_account_email_id_as_sender": 1,
+		"use_tls": 0,
+		"use_ssl_for_outgoing": 0,
+		"smtp_server": None,
+		"smtp_port": None,
+		"no_smtp_authentication": 0,
+	},
+	"GMail": {
+		"email_server": "imap.gmail.com",
+		"use_ssl": 1,
+		"smtp_server": "smtp.gmail.com",
+	},
+	"Outlook": {
+		"email_server": "imap-mail.outlook.com",
+		"use_ssl": 1,
+		"smtp_server": "smtp-mail.outlook.com",
+	},
+	"Sendgrid": {
+		"smtp_server": "smtp.sendgrid.net",
+		"smtp_port": 587,
+	},
+	"SparkPost": {
+		"smtp_server": "smtp.sparkpostmail.com",
+	},
+	"Yahoo": {
+		"email_server": "imap.mail.yahoo.com",
+		"use_ssl": 1,
+		"smtp_server": "smtp.mail.yahoo.com",
+		"smtp_port": 587,
+	},
+	"Yandex": {
+		"email_server": "imap.yandex.com",
+		"use_ssl": 1,
+		"smtp_server": "smtp.yandex.com",
+		"smtp_port": 587,
+	},
+}

crm/api/todo.py

@@ -0,0 +1,144 @@
+import frappe
+from frappe import _
+
+from crm.fcrm.doctype.crm_notification.crm_notification import notify_user
+
+
+def validate(doc, method):
+	"""Require write access when directly assigning a CRM lead or deal."""
+	# assign_to and assignment rules insert with ignore_permissions after their
+	# own permission check.
+	if (
+		doc.is_new()
+		and doc.reference_type in ["CRM Lead", "CRM Deal"]
+		and doc.reference_name
+		and not doc.flags.ignore_permissions
+	):
+		frappe.get_doc(doc.reference_type, doc.reference_name).check_permission("write")
+
+
+def after_insert(doc, method):
+	if doc.reference_type in ["CRM Lead", "CRM Deal"] and doc.reference_name and doc.allocated_to:
+		fieldname = "lead_owner" if doc.reference_type == "CRM Lead" else "deal_owner"
+		# Mirror assign_to: the latest assignment owns the record, overriding any prior owner.
+		frappe.db.set_value(
+			doc.reference_type, doc.reference_name, fieldname, doc.allocated_to, update_modified=False
+		)
+
+	if doc.reference_type in ["CRM Lead", "CRM Deal", "CRM Task"] and doc.reference_name and doc.allocated_to:
+		notify_assigned_user(doc)
+
+
+def on_update(doc, method):
+	if (
+		doc.has_value_changed("status")
+		and doc.status == "Cancelled"
+		and doc.reference_type in ["CRM Lead", "CRM Deal", "CRM Task"]
+		and doc.reference_name
+		and doc.allocated_to
+	):
+		notify_assigned_user(doc, is_cancelled=True)
+		clear_owner_on_unassign(doc)
+
+
+def clear_owner_on_unassign(doc):
+	# Owner concept only exists for Lead/Deal, not Task.
+	if doc.reference_type not in ["CRM Lead", "CRM Deal"]:
+		return
+	fieldname = "lead_owner" if doc.reference_type == "CRM Lead" else "deal_owner"
+	# Mirror assign_to: cancelling an assignment clears the owner. Wrinkle (accepted):
+	# removing one of several manual co-assignees also clears, since owner is single-valued.
+	frappe.db.set_value(doc.reference_type, doc.reference_name, fieldname, None, update_modified=False)
+
+
+def notify_assigned_user(doc, is_cancelled=False):
+	_doc = frappe.get_doc(doc.reference_type, doc.reference_name)
+	owner = frappe.get_cached_value("User", frappe.session.user, "full_name")
+	notification_text = get_notification_text(owner, doc, _doc, is_cancelled)
+
+	message = (
+		_("Your assignment on {0} {1} has been removed by {2}").format(
+			doc.reference_type, doc.reference_name, owner
+		)
+		if is_cancelled
+		else _("{0} assigned a {1} {2} to you").format(owner, doc.reference_type, doc.reference_name)
+	)
+
+	redirect_to_doctype, redirect_to_name = get_redirect_to_doc(doc)
+
+	notify_user(
+		{
+			"owner": frappe.session.user,
+			"assigned_to": doc.allocated_to,
+			"notification_type": "Assignment",
+			"message": message,
+			"notification_text": notification_text,
+			"reference_doctype": doc.reference_type,
+			"reference_docname": doc.reference_name,
+			"redirect_to_doctype": redirect_to_doctype,
+			"redirect_to_docname": redirect_to_name,
+		}
+	)
+
+
+def get_notification_text(owner, doc, reference_doc, is_cancelled=False):
+	name = doc.reference_name
+	doctype = doc.reference_type
+
+	if doctype.startswith("CRM "):
+		doctype = doctype[4:].lower()
+
+	if doctype in ["lead", "deal"]:
+		name = (
+			reference_doc.lead_name or name
+			if doctype == "lead"
+			else reference_doc.organization or reference_doc.lead_name or name
+		)
+
+		if is_cancelled:
+			return f"""
+                <div class="mb-2 leading-5 text-ink-gray-5">
+                    <span>{ _('Your assignment on {0} {1} has been removed by {2}').format(
+                        doctype,
+                        f'<span class="font-medium text-ink-gray-9">{ name }</span>',
+                        f'<span class="font-medium text-ink-gray-9">{ owner }</span>'
+                    ) }</span>
+                </div>
+            """
+
+		return f"""
+            <div class="mb-2 leading-5 text-ink-gray-5">
+                <span class="font-medium text-ink-gray-9">{ owner }</span>
+                <span>{ _('assigned a {0} {1} to you').format(
+                    doctype,
+                    f'<span class="font-medium text-ink-gray-9">{ name }</span>'
+                ) }</span>
+            </div>
+        """
+
+	if doctype == "task":
+		if is_cancelled:
+			return f"""
+                <div class="mb-2 leading-5 text-ink-gray-5">
+                    <span>{ _('Your assignment on task {0} has been removed by {1}').format(
+                        f'<span class="font-medium text-ink-gray-9">{ reference_doc.title }</span>',
+                        f'<span class="font-medium text-ink-gray-9">{ owner }</span>'
+                    ) }</span>
+                </div>
+            """
+		return f"""
+            <div class="mb-2 leading-5 text-ink-gray-5">
+                <span class="font-medium text-ink-gray-9">{ owner }</span>
+                <span>{ _('assigned a new task {0} to you').format(
+                    f'<span class="font-medium text-ink-gray-9">{ reference_doc.title }</span>'
+                ) }</span>
+            </div>
+        """
+
+
+def get_redirect_to_doc(doc):
+	if doc.reference_type == "CRM Task":
+		reference_doc = frappe.get_doc(doc.reference_type, doc.reference_name)
+		return reference_doc.reference_doctype, reference_doc.reference_docname
+
+	return doc.reference_type, doc.reference_name

crm/api/user.py

@@ -0,0 +1,238 @@
+import frappe
+from frappe import _
+from frappe.auth import LoginAttemptTracker
+from frappe.rate_limiter import rate_limit
+from frappe.utils.password import check_password, update_password
+
+
+@frappe.whitelist()
+@rate_limit(limit=5, seconds=300)  # 5 attempts per 5 minutes per user/IP
+def change_password(old_password: str, new_password: str):
+	"""
+	Change password for the current logged-in user.
+	Uses Frappe's LoginAttemptTracker for attempt counting/lockout, and rate_limit for API abuse protection.
+	"""
+	user = frappe.session.user
+	if user == "Guest":
+		frappe.throw(_("You must be logged in to change your password"), frappe.AuthenticationError)
+
+	tracker = LoginAttemptTracker(user)
+	if not tracker.is_user_allowed():
+		frappe.throw(_("Too many failed attempts. Please try again after some time."))
+
+	if old_password == new_password:
+		frappe.throw(
+			_("New password cannot be the same as current password. Please choose a different password.")
+		)
+
+	try:
+		check_password(user, old_password)
+	except frappe.AuthenticationError:
+		tracker.add_failure_attempt()
+		frappe.throw(_("Incorrect current password. Please try again."))
+	else:
+		tracker.add_success_attempt()
+
+	# Validate new password strength (server-side enforcement)
+	from frappe.core.doctype.user.user import test_password_strength
+
+	result = test_password_strength(new_password)
+	feedback = result.get("feedback", {})
+	if not feedback.get("password_policy_validation_passed", False):
+		suggestions = feedback.get("suggestions", [])
+		frappe.throw(_("Password is too weak. {0}").format(" ".join(suggestions) if suggestions else ""))
+
+	update_password(user=user, pwd=new_password, logout_all_sessions=False)
+	return _("Password Updated Successfully")
+
+
+def needs_password_setup() -> bool:
+	"""Whether the session user has no password of their own and must set one.
+
+	True for users provisioned without a password — the Frappe Cloud site owner,
+	users created by a script — who are logged into a session they never
+	authenticated for and so have no way back in once it expires.
+
+	Users who sign in through SSO are excluded: having no password is the
+	correct state for them, not something to fix.
+	"""
+	user = frappe.session.user
+
+	# Checked first: it is the discriminating one, and it short-circuits the
+	# other queries for everyone who already has a password — which is almost
+	# everyone, on every CRM page load.
+	if has_password(user):
+		return False
+
+	if frappe.get_system_settings("disable_user_pass_login"):
+		return False
+
+	# Every user carries a `frappe` provider row (Frappe issues one on insert
+	# so the site can act as an identity provider), so only other providers
+	# mean the user actually signs in through SSO.
+	if frappe.db.exists(
+		"User Social Login",
+		{
+			"parenttype": "User",
+			"parent": user,
+			"provider": ["!=", "frappe"],
+			"userid": ["is", "set"],
+		},
+	):
+		return False
+
+	return True
+
+
+def has_password(user: str) -> bool:
+	"""Whether a password is stored for the user.
+
+	User passwords are hashed rows in `__Auth` with `encrypted = 0`, so
+	`get_decrypted_password` (which only looks at encrypted rows) cannot answer
+	this — hence the direct query.
+	"""
+	Auth = frappe.qb.Table("__Auth")
+
+	return bool(
+		(
+			frappe.qb.from_(Auth)
+			.select(Auth.name)
+			.where(
+				(Auth.doctype == "User")
+				& (Auth.name == user)
+				& (Auth.fieldname == "password")
+				& (Auth.encrypted == 0)
+			)
+			.limit(1)
+		).run()
+	)
+
+
+@frappe.whitelist()
+def add_existing_users(users: str | list, role: str = "Sales User"):
+	"""
+	Add existing users to the CRM by assigning them a role (Sales User or Sales Manager).
+	:param users: List of user names to be added
+	"""
+	frappe.only_for(["System Manager", "Sales Manager"], True)
+	is_system_manager = "System Manager" in frappe.get_roles()
+
+	if role == "System Manager" and not is_system_manager:
+		frappe.throw(_("Only System Managers can assign the System Manager role"), frappe.PermissionError)
+
+	if role == "Sales Manager" and not is_system_manager:
+		frappe.throw(_("Only System Managers can assign the Sales Manager role"), frappe.PermissionError)
+
+	users = frappe.parse_json(users)
+
+	for user in users:
+		update_user_role(user, role)
+
+
+@frappe.whitelist()
+def update_user_role(user: str, new_role: str):
+	"""
+	Update the role of the user to Sales Manager, Sales User, or System Manager.
+	:param user: The name of the user
+	:param new_role: The new role to assign (Sales Manager or Sales User)
+	"""
+
+	frappe.only_for(["System Manager", "Sales Manager"], True)
+	is_system_manager = "System Manager" in frappe.get_roles()
+
+	if new_role not in ["System Manager", "Sales Manager", "Sales User"]:
+		frappe.throw(_("Cannot assign this role"))
+
+	user_doc = frappe.get_doc("User", user)
+	target_roles = [d.role for d in user_doc.roles]
+	target_is_system_manager = "System Manager" in target_roles
+
+	if new_role == "System Manager" and not is_system_manager:
+		frappe.throw(_("Only System Managers can assign the System Manager role"), frappe.PermissionError)
+
+	if target_is_system_manager and not is_system_manager:
+		frappe.throw(_("Only System Managers can modify other System Managers"), frappe.PermissionError)
+
+	if new_role == "Sales Manager" and not is_system_manager:
+		frappe.throw(_("Only System Managers can assign the Sales Manager role"), frappe.PermissionError)
+
+	if new_role == "System Manager":
+		user_doc.append_roles("System Manager", "Sales Manager", "Sales User")
+		user_doc.set("block_modules", [])
+	if new_role == "Sales Manager":
+		user_doc.append_roles("Sales Manager", "Sales User")
+		remove_roles(user_doc, "System Manager")
+	if new_role == "Sales User":
+		node = frappe.db.get_value(
+			"CRM Sales Hierarchy", {"user": user}, ["name", "reports_to"], as_dict=True
+		)
+		if node:
+			has_reports = frappe.db.exists("CRM Sales Hierarchy", {"reports_to": node.name})
+			if has_reports or not node.reports_to:
+				frappe.throw(
+					_("Remove this user from the sales hierarchy before changing their role to Sales User")
+				)
+		user_doc.append_roles("Sales User")
+		remove_roles(user_doc, "Sales Manager", "System Manager")
+		update_module_in_user(user_doc, "FCRM")
+
+	user_doc.save(ignore_permissions=True)
+
+
+@frappe.whitelist()
+def remove_crm_roles_from_user(user: str):
+	"""
+	Remove a user means removing Sales User & Sales Manager roles from the user.
+	:param user: The name of the user to be removed
+	"""
+	frappe.only_for(["System Manager", "Sales Manager"], True)
+
+	if user == frappe.session.user:
+		frappe.throw(_("You cannot remove yourself."), frappe.PermissionError)
+
+	user_doc = frappe.get_doc("User", user)
+	roles = [d.role for d in user_doc.roles]
+
+	current_user_is_system_manager = "System Manager" in frappe.get_roles()
+
+	if "System Manager" in roles and not current_user_is_system_manager:
+		frappe.throw(_("Only System Managers can modify other System Managers"), frappe.PermissionError)
+
+	if user_doc.get("role_profiles") or user_doc.get("role_profile_name"):
+		return frappe.throw(
+			_("User {0} cannot be removed as it has a Role Profile assigned to it.").format(user)
+		)
+
+	if "Sales User" in roles:
+		remove_roles(user_doc, "Sales User")
+	if "Sales Manager" in roles:
+		remove_roles(user_doc, "Sales Manager")
+	if "System Manager" in roles and current_user_is_system_manager:
+		remove_roles(user_doc, "System Manager")
+		update_module_in_user(user_doc, "FCRM")
+
+	user_doc.save(ignore_permissions=True)
+
+	node_name = frappe.db.get_value("CRM Sales Hierarchy", {"user": user}, "name")
+	if node_name:
+		frappe.delete_doc("CRM Sales Hierarchy", node_name, ignore_permissions=True)
+
+	frappe.msgprint(_("User {0} has been removed from CRM roles.").format(user))
+
+
+def remove_roles(self, *roles):
+	existing_roles = {d.role: d for d in self.get("roles")}
+	for role in roles:
+		if role in existing_roles:
+			self.get("roles").remove(existing_roles[role])
+
+
+def update_module_in_user(user, module):
+	block_modules = frappe.get_all(
+		"Module Def",
+		fields=["name as module"],
+		filters={"name": ["!=", module]},
+	)
+
+	if block_modules:
+		user.set("block_modules", block_modules)

crm/api/views.py

@@ -0,0 +1,16 @@
+import frappe
+from pypika import Criterion
+
+
+@frappe.whitelist()
+def get_views(doctype: str):
+	View = frappe.qb.DocType("CRM View Settings")
+	query = (
+		frappe.qb.from_(View)
+		.select("*")
+		.where(Criterion.any([View.user == "", View.user == frappe.session.user]))
+	)
+	if doctype:
+		query = query.where(View.dt == doctype)
+	views = query.run(as_dict=True)
+	return views

crm/api/whatsapp.py

@@ -0,0 +1,387 @@
+import json
+
+import frappe
+from frappe import _
+from frappe.permissions import add_permission, update_permission_property
+
+from crm.api.doc import get_assigned_users
+from crm.fcrm.doctype.crm_notification.crm_notification import notify_user
+from crm.integrations.api import get_contact_lead_or_deal_from_number
+
+ALLOWED_WHATSAPP_ROLES = ["System Manager", "Sales Manager", "Sales User"]
+
+
+def validate_access(reference_doctype=None, reference_name=None, permtype="read"):
+	if not any(role in ALLOWED_WHATSAPP_ROLES for role in frappe.get_roles()):
+		frappe.throw(_("Only sales users can access WhatsApp features."), frappe.PermissionError)
+
+	if reference_doctype and reference_name:
+		if not frappe.db.exists(reference_doctype, reference_name):
+			frappe.throw(
+				_("Reference document {0} {1} does not exist.").format(reference_doctype, reference_name),
+				frappe.DoesNotExistError,
+			)
+		reference_doc = frappe.get_doc(reference_doctype, reference_name)
+		if not reference_doc.has_permission(permtype):
+			frappe.throw(
+				_("Not permitted to access reference document {0} {1}.").format(
+					reference_doctype, reference_name
+				),
+				frappe.PermissionError,
+			)
+		return reference_doc
+
+	return None
+
+
+def validate(doc, method):
+	phone_number = doc.get("from") if doc.type == "Incoming" else doc.get("to")
+	if phone_number:
+		try:
+			name, doctype = get_contact_lead_or_deal_from_number(phone_number)
+			if doctype and name is not None:
+				doc.reference_doctype = doctype
+				doc.reference_name = name
+		except Exception:
+			frappe.log_error(frappe.get_traceback(), "CRM WhatsApp: failed to resolve contact from number")
+
+
+def on_update(doc, method):
+	frappe.publish_realtime(
+		"whatsapp_message",
+		{
+			"reference_doctype": doc.reference_doctype,
+			"reference_name": doc.reference_name,
+		},
+	)
+
+	notify_agent(doc)
+
+
+def notify_agent(doc):
+	if doc.type == "Incoming":
+		if not doc.reference_doctype or not doc.reference_name:
+			return
+		doctype = doc.reference_doctype
+		if doctype and doctype.startswith("CRM "):
+			doctype = doctype[4:].lower()
+		safe_reference_name = frappe.utils.escape_html(doc.reference_name)
+		notification_text = f"""
+            <div class="mb-2 leading-5 text-ink-gray-5">
+                <span class="font-medium text-ink-gray-9">{_("You")}</span>
+                <span>{_("received a whatsapp message in {0}").format(doctype)}</span>
+                <span class="font-medium text-ink-gray-9">{safe_reference_name}</span>
+            </div>
+        """
+		assigned_users = get_assigned_users(doc.reference_doctype, doc.reference_name)
+		for user in assigned_users:
+			notify_user(
+				{
+					"owner": doc.owner,
+					"assigned_to": user,
+					"notification_type": "WhatsApp",
+					"message": doc.message,
+					"notification_text": notification_text,
+					"reference_doctype": "WhatsApp Message",
+					"reference_docname": doc.name,
+					"redirect_to_doctype": doc.reference_doctype,
+					"redirect_to_docname": doc.reference_name,
+				}
+			)
+
+
+@frappe.whitelist()
+def is_whatsapp_enabled():
+	if not frappe.db.exists("DocType", "WhatsApp Settings"):
+		return False
+	default_outgoing = frappe.get_cached_value(
+		"WhatsApp Settings", "WhatsApp Settings", "default_outgoing_account"
+	)
+	if not default_outgoing:
+		return False
+	status = frappe.get_cached_value("WhatsApp Account", default_outgoing, "status")
+	return status == "Active"
+
+
+@frappe.whitelist()
+def is_whatsapp_installed():
+	if not frappe.db.exists("DocType", "WhatsApp Settings"):
+		return False
+	return True
+
+
+@frappe.whitelist()
+def get_whatsapp_messages(reference_doctype: str, reference_name: str):
+	reference_doc = validate_access(reference_doctype, reference_name)
+	# twilio integration app is not compatible with crm app
+	# crm has its own twilio integration in built
+	if "twilio_integration" in frappe.get_installed_apps():
+		return []
+	if not frappe.db.exists("DocType", "WhatsApp Message"):
+		return []
+	messages = []
+
+	if reference_doctype == "CRM Deal":
+		lead = reference_doc.get("lead")
+		if lead:
+			validate_access("CRM Lead", lead)
+			messages = frappe.get_all(
+				"WhatsApp Message",
+				filters={
+					"reference_doctype": "CRM Lead",
+					"reference_name": lead,
+				},
+				fields=[
+					"name",
+					"type",
+					"to",
+					"from",
+					"content_type",
+					"message_type",
+					"attach",
+					"template",
+					"use_template",
+					"message_id",
+					"is_reply",
+					"reply_to_message_id",
+					"creation",
+					"message",
+					"status",
+					"reference_doctype",
+					"reference_name",
+					"template_parameters",
+					"template_header_parameters",
+				],
+			)
+
+	messages += frappe.get_all(
+		"WhatsApp Message",
+		filters={
+			"reference_doctype": reference_doctype,
+			"reference_name": reference_name,
+		},
+		fields=[
+			"name",
+			"type",
+			"to",
+			"from",
+			"content_type",
+			"message_type",
+			"attach",
+			"template",
+			"use_template",
+			"message_id",
+			"is_reply",
+			"reply_to_message_id",
+			"creation",
+			"message",
+			"status",
+			"reference_doctype",
+			"reference_name",
+			"template_parameters",
+			"template_header_parameters",
+		],
+	)
+
+	# Filter messages to get only Template messages
+	template_messages = [message for message in messages if message["message_type"] == "Template"]
+
+	# Iterate through template messages
+	for template_message in template_messages:
+		# Find the template that this message is using
+		if not frappe.db.exists("WhatsApp Templates", template_message["template"]):
+			continue
+		template = frappe.get_doc("WhatsApp Templates", template_message["template"])
+
+		if template:
+			template_message["template_name"] = template.template_name
+			if template_message["template_parameters"]:
+				parameters = json.loads(template_message["template_parameters"])
+				template.template = parse_template_parameters(template.template, parameters)
+
+			template_message["template"] = template.template
+			if template_message["template_header_parameters"]:
+				header_parameters = json.loads(template_message["template_header_parameters"])
+				template.header = parse_template_parameters(template.header, header_parameters)
+			template_message["header"] = template.header
+			template_message["footer"] = template.footer
+
+	# Filter messages to get only reaction messages
+	reaction_messages = [message for message in messages if message["content_type"] == "reaction"]
+	reaction_messages.reverse()
+
+	# Iterate through reaction messages
+	for reaction_message in reaction_messages:
+		# Find the message that this reaction is reacting to
+		reacted_message = next(
+			(m for m in messages if m["message_id"] == reaction_message["reply_to_message_id"]),
+			None,
+		)
+
+		# If the reacted message is found, add the reaction to it
+		if reacted_message:
+			reacted_message["reaction"] = reaction_message["message"]
+
+	for message in messages:
+		from_name = get_from_name(message) if message["from"] else _("You")
+		message["from_name"] = from_name
+	# Filter messages to get only replies
+	reply_messages = [message for message in messages if message["is_reply"]]
+
+	# Iterate through reply messages
+	for reply_message in reply_messages:
+		# Find the message that this message is replying to
+		replied_message = next(
+			(m for m in messages if m["message_id"] == reply_message["reply_to_message_id"]),
+			None,
+		)
+
+		# If the replied message is found, add the reply details to the reply message
+		if replied_message:
+			from_name = get_from_name(reply_message) if replied_message["from"] else _("You")
+			message = replied_message["message"]
+			if replied_message["message_type"] == "Template":
+				message = replied_message["template"]
+			reply_message["reply_message"] = message
+			reply_message["header"] = replied_message.get("header") or ""
+			reply_message["footer"] = replied_message.get("footer") or ""
+			reply_message["reply_to"] = replied_message["name"]
+			reply_message["reply_to_type"] = replied_message["type"]
+			reply_message["reply_to_from"] = from_name
+
+	return [message for message in messages if message["content_type"] != "reaction"]
+
+
+@frappe.whitelist()
+def create_whatsapp_message(
+	reference_doctype: str,
+	reference_name: str,
+	message: str,
+	to: str,
+	attach: str,
+	reply_to: str,
+	content_type: str = "text",
+):
+	validate_access(reference_doctype, reference_name)
+	doc = frappe.new_doc("WhatsApp Message")
+
+	if reply_to:
+		if not frappe.db.exists("WhatsApp Message", reply_to):
+			frappe.throw(_("Referenced WhatsApp message does not exist."), frappe.DoesNotExistError)
+		reply_doc = frappe.get_doc("WhatsApp Message", reply_to)
+		if not reply_doc.has_permission("read"):
+			frappe.throw(
+				_("Not permitted to access the referenced WhatsApp message."), frappe.PermissionError
+			)
+		validate_access(reply_doc.reference_doctype, reply_doc.reference_name)
+		doc.update(
+			{
+				"is_reply": True,
+				"reply_to_message_id": reply_doc.message_id,
+			}
+		)
+
+	doc.update(
+		{
+			"reference_doctype": reference_doctype,
+			"reference_name": reference_name,
+			"message": message or attach,
+			"to": to,
+			"attach": attach,
+			"content_type": content_type,
+		}
+	)
+	doc.insert(ignore_permissions=True)
+	return doc.name
+
+
+@frappe.whitelist()
+def send_whatsapp_template(reference_doctype: str, reference_name: str, template: str, to: str):
+	validate_access(reference_doctype, reference_name)
+	doc = frappe.new_doc("WhatsApp Message")
+	doc.update(
+		{
+			"reference_doctype": reference_doctype,
+			"reference_name": reference_name,
+			"message_type": "Template",
+			"message": "Template message",
+			"content_type": "text",
+			"use_template": True,
+			"template": template,
+			"to": to,
+		}
+	)
+	doc.insert(ignore_permissions=True)
+	return doc.name
+
+
+@frappe.whitelist()
+def react_on_whatsapp_message(emoji: str, reply_to_name: str):
+	validate_access()
+	if not frappe.db.exists("WhatsApp Message", reply_to_name):
+		frappe.throw(_("Referenced WhatsApp message does not exist."), frappe.DoesNotExistError)
+	reply_to_doc = frappe.get_doc("WhatsApp Message", reply_to_name)
+
+	if not reply_to_doc.has_permission("read"):
+		frappe.throw(_("Not permitted to access the referenced WhatsApp message."), frappe.PermissionError)
+
+	validate_access(reply_to_doc.reference_doctype, reply_to_doc.reference_name)
+
+	to = (reply_to_doc.type == "Incoming" and reply_to_doc.get("from")) or reply_to_doc.to
+	doc = frappe.new_doc("WhatsApp Message")
+	doc.update(
+		{
+			"reference_doctype": reply_to_doc.reference_doctype,
+			"reference_name": reply_to_doc.reference_name,
+			"message": emoji,
+			"to": to,
+			"reply_to_message_id": reply_to_doc.message_id,
+			"content_type": "reaction",
+		}
+	)
+	doc.insert(ignore_permissions=True)
+	return doc.name
+
+
+def parse_template_parameters(string, parameters):
+	for i, parameter in enumerate(parameters, start=1):
+		placeholder = "{{" + str(i) + "}}"
+		string = string.replace(placeholder, str(parameter))
+
+	return string
+
+
+def get_from_name(message):
+	doc = frappe.get_doc(message["reference_doctype"], message["reference_name"])
+	from_name = ""
+	if message["reference_doctype"] == "CRM Deal":
+		if doc.get("contacts"):
+			for c in doc.get("contacts"):
+				if c.is_primary:
+					from_name = c.full_name or c.mobile_no
+					break
+		else:
+			from_name = doc.get("lead_name")
+	else:
+		from_name = " ".join(name for name in [doc.get("first_name"), doc.get("last_name")] if name)
+	return from_name
+
+
+def add_roles():
+	if "frappe_whatsapp" not in frappe.get_installed_apps():
+		return
+
+	role_list = ["Sales Manager", "Sales User"]
+	doctypes = ["WhatsApp Message", "WhatsApp Templates", "WhatsApp Settings"]
+	for doctype in doctypes:
+		for role in role_list:
+			if frappe.db.exists("Custom DocPerm", {"parent": doctype, "role": role}):
+				continue
+			add_permission(doctype, role, 0, "write")
+			update_permission_property(doctype, role, 0, "create", 1)
+			update_permission_property(doctype, role, 0, "delete", 1)
+			update_permission_property(doctype, role, 0, "share", 1)
+			update_permission_property(doctype, role, 0, "email", 1)
+			update_permission_property(doctype, role, 0, "print", 1)
+			update_permission_property(doctype, role, 0, "report", 1)
+			update_permission_property(doctype, role, 0, "export", 1)

crm/crowdin.yml

@@ -0,0 +1,8 @@
+files:
+  - source: /crm/locale/main.pot
+    translation: /crm/locale/%two_letters_code%.po
+pull_request_title: "chore: sync translations from crowdin"
+pull_request_labels:
+  - translation
+commit_message: "chore: %language% translations"
+append_commit_message: false

crm/demo/activities.py

@@ -0,0 +1,191 @@
+from datetime import datetime, timedelta
+
+import frappe
+
+from crm.demo.utils import (
+	build_full_names,
+	insert_comment,
+	insert_communication,
+	insert_version,
+	resolve_owners,
+)
+
+
+def create_demo_activities(lead_names, demo_users):
+	"""Create comments, email communications, and field-change versions for demo leads."""
+	session_user, owner_1, owner_2, _ = resolve_owners(demo_users)
+	_full_names = build_full_names(session_user)
+
+	now = datetime.now()
+
+	comment_names = _create_comments(lead_names, session_user, owner_1, owner_2, _full_names, now)
+	communication_names = _create_communications(lead_names, session_user, owner_1, _full_names, now)
+	_create_versions(lead_names, session_user, owner_1, owner_2, now)
+
+	return {"comments": comment_names, "communications": communication_names}
+
+
+def _create_comments(lead_names, session_user, owner_1, owner_2, full_names, now):
+	comments_data = [
+		{
+			"lead": lead_names[0],
+			"owner": session_user,
+			"content": (
+				"<p>Initial discovery call was very productive. Alice's team of ~300 engineers is actively "
+				"evaluating CRM solutions. She mentioned budget has already been approved — just need "
+				"to get sign-off from the VP of Sales.</p>"
+			),
+			"days_ago": 56,  # lead[0] deal created at 50 days ago — keep before that
+		},
+		{
+			"lead": lead_names[1],
+			"owner": owner_1,
+			"content": (
+				"<p>Bob mentioned they had a poor experience with their last CRM. He emphasized that "
+				"onboarding support and data migration are top priorities. Need to loop in the "
+				"customer success team before the next call.</p>"
+			),
+			"days_ago": 9,
+		},
+		{
+			"lead": lead_names[2],
+			"owner": owner_2,
+			"content": (
+				"<p>Carol is temporarily unavailable — on leave until end of month. Her colleague "
+				"Mark is the interim contact. Revisit in 3 weeks with updated pricing deck.</p>"
+			),
+			"days_ago": 11,
+		},
+		{
+			"lead": lead_names[3],
+			"owner": owner_1,
+			"content": (
+				"<p>Technical demo completed successfully. David's engineering team was impressed with "
+				"the API and custom field options. He's requesting pricing for 15 seats — send "
+				"the proposal by Friday.</p>"
+			),
+			"days_ago": 41,  # lead[3] deal created at 37 days ago — keep before that
+		},
+	]
+
+	return [
+		insert_comment(
+			"CRM Lead",
+			data["lead"],
+			data["owner"],
+			data["content"],
+			full_names,
+			now - timedelta(days=data["days_ago"]),
+		)
+		for data in comments_data
+	]
+
+
+def _create_communications(lead_names, session_user, owner_1, full_names, now):
+	# Lead emails used for recipient/sender fields
+	lead_emails = {
+		lead_names[0]: "alice.johnson@example.com",
+		lead_names[1]: "bob.martinez@example.com",
+		lead_names[3]: "david.lee@example.com",
+	}
+
+	comms_data = [
+		{
+			"lead": lead_names[0],
+			"owner": session_user,
+			"sent_or_received": "Sent",
+			"subject": "Following up on our discovery call — Acme Corp",
+			"content": (
+				"<p>Hi Alice,</p>"
+				"<p>It was great speaking with you earlier today. As discussed, I'm sharing our "
+				"product overview and a link to schedule a full demo with your team.</p>"
+				"<p>Please let me know if you have any questions before we meet.</p>"
+				"<p>Best regards</p>"
+			),
+			"sender": session_user,
+			"recipients": lead_emails[lead_names[0]],
+			"days_ago": 54,  # lead[0] deal at 50 days ago — keep before that
+		},
+		{
+			"lead": lead_names[1],
+			"owner": owner_1,
+			"sent_or_received": "Received",
+			"subject": "Re: CRM Demo Request — Globex Industries",
+			"content": (
+				"<p>Hi,</p>"
+				"<p>Thanks for reaching out. We'd definitely like to see a live demo, specifically "
+				"around pipeline management and email integration. Can we schedule something for "
+				"next Tuesday at 2 PM?</p>"
+				"<p>Best,<br>Bob Martinez<br>CEO, Globex Industries</p>"
+			),
+			"sender": lead_emails[lead_names[1]],
+			"recipients": owner_1,
+			"days_ago": 8,
+		},
+		{
+			"lead": lead_names[3],
+			"owner": owner_1,
+			"sent_or_received": "Sent",
+			"subject": "CRM Pricing Proposal — TechStart Inc (15 seats)",
+			"content": (
+				"<p>Hi David,</p>"
+				"<p>Great connecting with your team today — really glad the API walkthrough was "
+				"useful. Attached is our pricing proposal for 15 seats including onboarding and "
+				"priority support.</p>"
+				"<p>Happy to jump on a call if you have questions.</p>"
+				"<p>Cheers</p>"
+			),
+			"sender": owner_1,
+			"recipients": lead_emails[lead_names[3]],
+			"days_ago": 39,  # lead[3] deal at 37 days ago — keep before that
+		},
+	]
+
+	return [
+		insert_communication(
+			"CRM Lead", data["lead"], data, full_names, now - timedelta(days=data["days_ago"])
+		)
+		for data in comms_data
+	]
+
+
+def _create_versions(lead_names, session_user, owner_1, owner_2, now):
+	versions_data = [
+		{
+			"lead": lead_names[0],
+			"owner": session_user,
+			"changed": [["no_of_employees", "201-500", "501-1000"]],
+			"days_ago": 52,  # lead[0] deal at 50 days ago — keep before that
+		},
+		{
+			"lead": lead_names[1],
+			"owner": owner_1,
+			"changed": [["annual_revenue", None, "10000000"]],
+			"days_ago": 10,
+		},
+		{
+			"lead": lead_names[2],
+			"owner": owner_2,
+			"changed": [["source", "Email", "Referral"]],
+			"days_ago": 12,
+		},
+		{
+			"lead": lead_names[3],
+			"owner": owner_1,
+			"changed": [["status", "Contacted", "Qualified"]],
+			"days_ago": 38,  # lead[3] deal at 37 days ago — keep before that
+		},
+	]
+
+	for data in versions_data:
+		insert_version(
+			"CRM Lead", data["lead"], data["owner"], data["changed"], now - timedelta(days=data["days_ago"])
+		)
+
+
+def delete_demo_activities(activity_data):
+	"""Delete communications (comments and versions are cascade-deleted with their lead)."""
+	communication_names = activity_data.get("communications", [])
+	for name in communication_names:
+		if frappe.db.exists("Communication", name):
+			frappe.delete_doc("Communication", name, ignore_permissions=True, force=True)

crm/demo/api.py

@@ -0,0 +1,91 @@
+import json
+
+import frappe
+from frappe.utils.telemetry import capture
+
+DEMO_STATE_KEY = "crm_demo_data_created"
+DEMO_LEADS_KEY = "crm_demo_leads"
+DEMO_NOTES_KEY = "crm_demo_notes"
+DEMO_TASKS_KEY = "crm_demo_tasks"
+DEMO_CALL_LOGS_KEY = "crm_demo_call_logs"
+DEMO_ACTIVITIES_KEY = "crm_demo_activities"
+DEMO_DEALS_KEY = "crm_demo_deals"
+
+
+def create_demo_data(_args: dict | None = None):
+	if frappe.db.get_default(DEMO_STATE_KEY):
+		return
+
+	from crm.demo.activities import create_demo_activities
+	from crm.demo.call_logs import create_demo_call_logs
+	from crm.demo.deals import create_demo_deals
+	from crm.demo.leads import create_demo_leads
+	from crm.demo.notes import create_demo_notes
+	from crm.demo.tasks import create_demo_tasks
+	from crm.demo.users import create_demo_users
+
+	demo_users = create_demo_users()
+	lead_names = create_demo_leads(demo_users)
+	note_names = create_demo_notes(lead_names, demo_users)
+	task_names = create_demo_tasks(lead_names, demo_users)
+	call_log_names = create_demo_call_logs(lead_names, demo_users)
+	activity_data = create_demo_activities(lead_names, demo_users)
+
+	from crm.demo.leads import rebackdate_demo_leads
+
+	rebackdate_demo_leads(lead_names, demo_users)
+
+	deal_data = create_demo_deals(lead_names, demo_users)
+	frappe.db.set_default(DEMO_LEADS_KEY, json.dumps(lead_names))
+	frappe.db.set_default(DEMO_NOTES_KEY, json.dumps(note_names))
+	frappe.db.set_default(DEMO_TASKS_KEY, json.dumps(task_names))
+	frappe.db.set_default(DEMO_CALL_LOGS_KEY, json.dumps(call_log_names))
+	frappe.db.set_default(DEMO_ACTIVITIES_KEY, json.dumps(activity_data))
+	frappe.db.set_default(DEMO_DEALS_KEY, json.dumps(deal_data))
+	frappe.db.set_default(DEMO_STATE_KEY, "1")
+
+	capture("demo_data_created", "crm")
+
+
+@frappe.whitelist()
+def clear_demo_data():
+	frappe.only_for(["Sales Manager", "System Manager"], True)
+
+	if not frappe.db.get_default(DEMO_STATE_KEY):
+		return
+
+	from crm.demo.activities import delete_demo_activities
+	from crm.demo.call_logs import delete_demo_call_logs
+	from crm.demo.deals import delete_demo_deals
+	from crm.demo.leads import delete_demo_leads
+	from crm.demo.notes import delete_demo_notes
+	from crm.demo.tasks import delete_demo_tasks
+	from crm.demo.users import DEMO_USER_EMAILS, delete_demo_users
+
+	lead_names = json.loads(frappe.db.get_default(DEMO_LEADS_KEY) or "[]")
+	note_names = json.loads(frappe.db.get_default(DEMO_NOTES_KEY) or "[]")
+	task_names = json.loads(frappe.db.get_default(DEMO_TASKS_KEY) or "[]")
+	call_log_names = json.loads(frappe.db.get_default(DEMO_CALL_LOGS_KEY) or "[]")
+	activity_data = json.loads(frappe.db.get_default(DEMO_ACTIVITIES_KEY) or "{}")
+	deal_data = json.loads(frappe.db.get_default(DEMO_DEALS_KEY) or "{}")
+	delete_demo_deals(deal_data, lead_names)
+	delete_demo_activities(activity_data)
+	delete_demo_notes(note_names)
+	delete_demo_tasks(task_names)
+	delete_demo_call_logs(call_log_names)
+	delete_demo_leads(lead_names)
+	delete_demo_users(DEMO_USER_EMAILS)
+	frappe.db.set_default(DEMO_LEADS_KEY, None)
+	frappe.db.set_default(DEMO_NOTES_KEY, None)
+	frappe.db.set_default(DEMO_TASKS_KEY, None)
+	frappe.db.set_default(DEMO_CALL_LOGS_KEY, None)
+	frappe.db.set_default(DEMO_ACTIVITIES_KEY, None)
+	frappe.db.set_default(DEMO_DEALS_KEY, None)
+	frappe.db.set_default(DEMO_STATE_KEY, None)
+
+	capture("demo_data_cleared", "crm")
+
+
+@frappe.whitelist()
+def get_demo_state():
+	return {"demo_data_created": bool(frappe.db.get_default(DEMO_STATE_KEY))}

crm/demo/call_logs.py

@@ -0,0 +1,122 @@
+import frappe
+
+from crm.demo.users import DEMO_USERS
+from crm.demo.utils import resolve_owners
+
+
+def create_demo_call_logs(lead_names, demo_users):
+	from datetime import datetime, timedelta
+
+	now = datetime.now()
+	session_user, owner_1, owner_2, _ = resolve_owners(demo_users)
+
+	# Build rep phone number map from DEMO_USERS mobile numbers
+	_user_mobile = {u["email"]: u["mobile_no"] for u in DEMO_USERS}
+	rep_numbers = {
+		session_user: _user_mobile.get(session_user, "+1 555 100 0001"),
+		owner_1: _user_mobile.get(owner_1, "+1 555 100 0002"),
+		owner_2: _user_mobile.get(owner_2, "+1 555 100 0003"),
+	}
+
+	call_logs_data = [
+		{
+			"type": "Outgoing",
+			"status": "Completed",
+			"telephony_medium": "Manual",
+			"duration": 720,
+			"from": rep_numbers[session_user],
+			"to": "+1 555 000 1234",
+			"caller": session_user,
+			"start_time": now - timedelta(days=5, hours=2),
+			"end_time": now - timedelta(days=5, hours=2) + timedelta(seconds=720),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[0],
+		},
+		{
+			"type": "Incoming",
+			"status": "Completed",
+			"telephony_medium": "Manual",
+			"duration": 300,
+			"from": "+1 555 000 1234",
+			"to": rep_numbers[session_user],
+			"receiver": session_user,
+			"start_time": now - timedelta(days=2, hours=1),
+			"end_time": now - timedelta(days=2, hours=1) + timedelta(seconds=300),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[0],
+		},
+		{
+			"type": "Outgoing",
+			"status": "Completed",
+			"telephony_medium": "Manual",
+			"duration": 540,
+			"from": rep_numbers[owner_1],
+			"to": "+1 555 000 5678",
+			"caller": owner_1,
+			"start_time": now - timedelta(days=4),
+			"end_time": now - timedelta(days=4) + timedelta(seconds=540),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[1],
+		},
+		{
+			"type": "Outgoing",
+			"status": "No Answer",
+			"telephony_medium": "Manual",
+			"duration": 0,
+			"from": rep_numbers[owner_1],
+			"to": "+1 555 000 5678",
+			"caller": owner_1,
+			"start_time": now - timedelta(days=1),
+			"end_time": now - timedelta(days=1),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[1],
+		},
+		{
+			"type": "Outgoing",
+			"status": "Completed",
+			"telephony_medium": "Manual",
+			"duration": 420,
+			"from": rep_numbers[owner_2],
+			"to": "+1 555 000 9012",
+			"caller": owner_2,
+			"start_time": now - timedelta(days=3),
+			"end_time": now - timedelta(days=3) + timedelta(seconds=420),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[2],
+		},
+		{
+			"type": "Incoming",
+			"status": "Completed",
+			"telephony_medium": "Manual",
+			"duration": 600,
+			"from": "+1 555 000 3456",
+			"to": rep_numbers[owner_1],
+			"receiver": owner_1,
+			"start_time": now - timedelta(days=6),
+			"end_time": now - timedelta(days=6) + timedelta(seconds=600),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[3],
+		},
+	]
+
+	created = []
+	for data in call_logs_data:
+		log_owner = data.get("caller") or data.get("receiver") or frappe.session.user
+		log_ts = data.get("start_time")
+		log = frappe.get_doc(
+			{"doctype": "CRM Call Log", "id": frappe.generate_hash(length=10), **data}
+		).insert(ignore_permissions=True)
+		patch = {"owner": log_owner, "modified_by": log_owner}
+		if log_ts:
+			patch["creation"] = log_ts
+			patch["modified"] = log_ts
+		frappe.db.set_value("CRM Call Log", log.name, patch, update_modified=False)
+		created.append(log.name)
+
+	return created
+
+
+def delete_demo_call_logs(call_log_names):
+	for name in call_log_names:
+		if frappe.db.exists("CRM Call Log", name):
+			frappe.delete_doc("CRM Call Log", name, ignore_permissions=True, force=True)

crm/demo/deals.py

@@ -0,0 +1,348 @@
+from datetime import datetime, timedelta
+
+import frappe
+from frappe.query_builder import DocType
+
+from crm.demo.utils import (
+	backdate,
+	build_full_names,
+	fix_auto_records,
+	insert_comment,
+	insert_communication,
+	insert_version,
+	resolve_owners,
+)
+
+
+def create_demo_deals(lead_names, demo_users):
+	"""Convert seven leads into deals and add deal-specific activity."""
+	from crm.fcrm.doctype.crm_lead.crm_lead import convert_to_deal
+
+	session_user, owner_1, owner_2, _ = resolve_owners(demo_users)
+	_full_names = build_full_names(session_user)
+
+	# leads[0] Alice, [3] David, [7] Henry, [8] Iris, [9] Jack → 5 active/won deals
+	# leads[10] Karen, [11] Leo → 2 lost deals
+	d_alice = convert_to_deal(
+		lead=lead_names[0],
+		deal={"status": "Demo/Making", "deal_value": 120000, "probability": 50, "deal_owner": session_user},
+	)
+	d_david = convert_to_deal(
+		lead=lead_names[3],
+		deal={"status": "Proposal/Quotation", "deal_value": 45000, "probability": 70, "deal_owner": owner_1},
+	)
+	d_henry = convert_to_deal(
+		lead=lead_names[7],
+		deal={"status": "Negotiation", "deal_value": 85000, "probability": 60, "deal_owner": owner_2},
+	)
+	d_iris = convert_to_deal(
+		lead=lead_names[8],
+		deal={"status": "Qualification", "deal_value": 60000, "probability": 35, "deal_owner": session_user},
+	)
+	d_jack = convert_to_deal(
+		lead=lead_names[9],
+		deal={"status": "Won", "deal_value": 175000, "probability": 100, "deal_owner": owner_1},
+	)
+	d_karen = convert_to_deal(
+		lead=lead_names[10],
+		deal={
+			"status": "Lost",
+			"deal_value": 95000,
+			"probability": 0,
+			"deal_owner": owner_2,
+			"lost_reason": "Competition",
+			"lost_notes": "Prospect chose a competitor offering deeper BI integrations out of the box. Price was not the issue — feature parity was.",
+		},
+	)
+	d_leo = convert_to_deal(
+		lead=lead_names[11],
+		deal={
+			"status": "Lost",
+			"deal_value": 55000,
+			"probability": 0,
+			"deal_owner": session_user,
+			"lost_reason": "Budget constraints",
+			"lost_notes": "Q2 budget was cut. Leo said they would revisit in Q4 once headcount hiring is complete. Added to nurture sequence.",
+		},
+	)
+
+	deal_names_list = [d_alice, d_david, d_henry, d_iris, d_jack, d_karen, d_leo]
+	# Converted lead indices — must match deal_names_list order
+	_converted_lead_indices = [0, 3, 7, 8, 9, 10, 11]
+	# Days ago each deal was created (always after its lead)
+	_deal_days = [50, 37, 18, 11, 24, 9, 5]
+	_deal_owners = [session_user, owner_1, owner_2, session_user, owner_1, owner_2, session_user]
+	# Organization logos keyed by org name (index order: alice, david, henry, iris, jack, karen, leo)
+	_org_logos = {
+		"Acme Corp": "/assets/crm/images/demo/acme-corp.png",
+		"TechStart Inc": "/assets/crm/images/demo/techstart-inc.png",
+		"PivotTech Solutions": "/assets/crm/images/demo/pivottech-solutions.png",
+		"ScaleUp Labs": "/assets/crm/images/demo/scaleup-labs.png",
+		"Meridian Systems": "/assets/crm/images/demo/meridian-systems.png",
+		"Vertex Analytics": "/assets/crm/images/demo/vertex-analytics.png",
+		"Forge Digital": "/assets/crm/images/demo/forge-digital.png",
+	}
+	now = datetime.now()
+
+	for d_name, days, d_owner, li in zip(
+		deal_names_list, _deal_days, _deal_owners, _converted_lead_indices, strict=False
+	):
+		ts = now - timedelta(days=days)
+		backdate("CRM Deal", d_name, d_owner, ts)
+		org = frappe.db.get_value("CRM Deal", d_name, "organization")
+		if org:
+			logo = _org_logos.get(org)
+			if logo:
+				frappe.db.set_value("CRM Organization", org, "organization_logo", logo, update_modified=False)
+			backdate("CRM Organization", org, d_owner, ts)
+		contacts = frappe.get_all(
+			"CRM Contacts", filters={"parent": d_name, "parenttype": "CRM Deal"}, pluck="contact"
+		)
+		for contact in contacts:
+			if contact:
+				backdate("Contact", contact, d_owner, ts)
+		backdate("CRM Lead", lead_names[li], d_owner, ts, set_creation=False)
+		fix_auto_records("CRM Deal", d_name, d_owner, ts)
+		fix_auto_records("CRM Lead", lead_names[li], d_owner, ts)
+
+	comment_names = _create_deal_comments(deal_names_list, session_user, owner_1, owner_2, _full_names, now)
+	communication_names = _create_deal_communications(
+		deal_names_list, session_user, owner_1, _full_names, now
+	)
+	_create_deal_versions(deal_names_list, session_user, owner_1, owner_2, now)
+
+	# Re-backdate modified to last-activity date so active deals sort to the top of the list.
+	# Index order: alice, david, henry, iris, jack, karen, leo
+	_deal_last_touched_days = [2, 4, 7, 10, 6, 8, 12]
+	for d_name, days, d_owner in zip(deal_names_list, _deal_last_touched_days, _deal_owners, strict=False):
+		backdate("CRM Deal", d_name, d_owner, now - timedelta(days=days), set_creation=False)
+
+	return {
+		"deals": deal_names_list,
+		"comments": comment_names,
+		"communications": communication_names,
+	}
+
+
+def _create_deal_comments(deal_names, session_user, owner_1, owner_2, full_names, now):
+	comments_data = [
+		{
+			"deal": deal_names[0],  # Alice / Acme Corp
+			"owner": session_user,
+			"content": (
+				"<p>Live demo went well — Alice's product team joined and had great questions "
+				"about workflow automation and bulk import. They want a custom sandbox environment "
+				"to evaluate with their own data before signing. Following up to arrange access.</p>"
+			),
+			"days_ago": 3,
+		},
+		{
+			"deal": deal_names[1],  # David / TechStart Inc
+			"owner": owner_1,
+			"content": (
+				"<p>David reviewed the proposal with his co-founder. They're happy with the pricing "
+				"but want a 3-month pilot before committing to annual. Checking with management on "
+				"whether we can offer a pilot discount.</p>"
+			),
+			"days_ago": 5,
+		},
+		{
+			"deal": deal_names[2],  # Henry / PivotTech Solutions
+			"owner": owner_2,
+			"content": (
+				"<p>Negotiation call with Henry went long but productive. He’s pushing for a 15% "
+				"discount on the annual plan citing budget constraints. Legal team is now reviewing "
+				"the contract — decision expected by end of week.</p>"
+			),
+			"days_ago": 7,
+		},
+		{
+			"deal": deal_names[3],  # Iris / ScaleUp Labs
+			"owner": session_user,
+			"content": (
+				"<p>Qualification call completed with Iris and her co-founder. Small but fast-moving "
+				"team of 8 engineers. Main concerns are API depth and self-serve onboarding. "
+				"Sending over the developer docs and scheduling a technical deep-dive.</p>"
+			),
+			"days_ago": 10,
+		},
+		{
+			"deal": deal_names[4],  # Jack / Meridian Systems
+			"owner": owner_1,
+			"content": (
+				"<p>Deal closed — Jack signed the contract this morning. Final value $175k annual. "
+				"Onboarding is scheduled for next Monday. Handoff notes sent to the customer "
+				"success team. Great result for the quarter!</p>"
+			),
+			"days_ago": 2,
+		},
+		{
+			"deal": deal_names[5],  # Karen / Vertex Analytics
+			"owner": owner_2,
+			"content": (
+				"<p>Karen's team went with a competitor — they had deeper BI integrations that we "
+				"currently don't support. Not a pricing issue. Flagged to product team for roadmap "
+				"consideration. Karen asked us to follow up in 6 months.</p>"
+			),
+			"days_ago": 8,
+		},
+		{
+			"deal": deal_names[6],  # Leo / Forge Digital
+			"owner": session_user,
+			"content": (
+				"<p>Budget was cut for the rest of the year — Leo was very apologetic and said the "
+				"product was exactly what they needed. Re-added to nurture sequence for Q4. "
+				"Good candidate for a comeback deal.</p>"
+			),
+			"days_ago": 12,
+		},
+	]
+
+	return [
+		insert_comment(
+			"CRM Deal",
+			data["deal"],
+			data["owner"],
+			data["content"],
+			full_names,
+			now - timedelta(days=data["days_ago"]),
+		)
+		for data in comments_data
+	]
+
+
+def _create_deal_communications(deal_names, session_user, owner_1, full_names, now):
+	# Use a placeholder email for recipients where we don't have a real one handy
+	comms_data = [
+		{
+			"deal": deal_names[0],
+			"owner": session_user,
+			"sent_or_received": "Sent",
+			"subject": "Sandbox access details — Acme Corp CRM Evaluation",
+			"content": (
+				"<p>Hi Alice,</p>"
+				"<p>Thanks for joining yesterday's demo — really great session. I've created a "
+				"dedicated sandbox environment pre-loaded with sample data. Login details and "
+				"setup instructions are attached.</p>"
+				"<p>Our onboarding specialist Sarah will also be available for a walkthrough "
+				"session this week — just let us know a convenient time.</p>"
+				"<p>Best regards</p>"
+			),
+			"sender": session_user,
+			"recipients": "alice.johnson@example.com",
+			"days_ago": 2,
+		},
+		{
+			"deal": deal_names[1],
+			"owner": owner_1,
+			"sent_or_received": "Received",
+			"subject": "Re: CRM Proposal — Pilot terms query",
+			"content": (
+				"<p>Hi,</p>"
+				"<p>We've reviewed the proposal and the pricing looks good. Before we proceed, "
+				"could you clarify what's included in the onboarding package? Specifically, we "
+				"need help migrating ~4,000 contacts from our existing system.</p>"
+				"<p>Also, is there flexibility on the contract start date?</p>"
+				"<p>Thanks,<br>David Lee<br>CTO, TechStart Inc</p>"
+			),
+			"sender": "david.lee@example.com",
+			"recipients": owner_1,
+			"days_ago": 4,
+		},
+	]
+
+	return [
+		insert_communication(
+			"CRM Deal", data["deal"], data, full_names, now - timedelta(days=data["days_ago"])
+		)
+		for data in comms_data
+	]
+
+
+def _create_deal_versions(deal_names, session_user, owner_1, owner_2, now):
+	versions_data = [
+		{
+			"deal": deal_names[0],  # Alice
+			"owner": session_user,
+			"changed": [["probability", "30", "50"]],
+			"days_ago": 4,
+		},
+		{
+			"deal": deal_names[1],  # David
+			"owner": owner_1,
+			"changed": [["deal_value", "40000", "45000"]],
+			"days_ago": 6,
+		},
+		{
+			"deal": deal_names[2],  # Henry
+			"owner": owner_2,
+			"changed": [["probability", "50", "60"]],
+			"days_ago": 8,
+		},
+		{
+			"deal": deal_names[3],  # Iris
+			"owner": session_user,
+			"changed": [["probability", "25", "35"]],
+			"days_ago": 11,
+		},
+		{
+			"deal": deal_names[4],  # Jack
+			"owner": owner_1,
+			"changed": [["status", "Ready to Close", "Won"]],
+			"days_ago": 3,
+		},
+		{
+			"deal": deal_names[5],  # Karen
+			"owner": owner_2,
+			"changed": [["status", "Proposal/Quotation", "Lost"]],
+			"days_ago": 9,
+		},
+		{
+			"deal": deal_names[6],  # Leo
+			"owner": session_user,
+			"changed": [["status", "Demo/Making", "Lost"]],
+			"days_ago": 13,
+		},
+	]
+
+	for data in versions_data:
+		insert_version(
+			"CRM Deal", data["deal"], data["owner"], data["changed"], now - timedelta(days=data["days_ago"])
+		)
+
+
+def delete_demo_deals(deal_data, lead_names):
+	"""
+	Delete deals, their linked contacts, organizations, and deal-specific communications.
+	Comments and Versions are cascade-deleted when the deal is deleted.
+	"""
+	communication_names = deal_data.get("communications", [])
+	deal_names = set(deal_data.get("deals", []))
+
+	# Find all deals converted from the provided demo leads
+	if lead_names:
+		demo_lead_deals = frappe.get_all("CRM Deal", filters={"lead": ["in", lead_names]}, pluck="name")
+		deal_names.update(demo_lead_deals)
+
+	for name in communication_names:
+		if frappe.db.exists("Communication", name):
+			frappe.delete_doc("Communication", name, ignore_permissions=True, force=True)
+
+	for name in deal_names:
+		if not frappe.db.exists("CRM Deal", name):
+			continue
+		# Collect linked contacts and organization before deleting deal
+		contact_names = frappe.get_all(
+			"CRM Contacts", filters={"parent": name, "parenttype": "CRM Deal"}, pluck="contact"
+		)
+		org = frappe.db.get_value("CRM Deal", name, "organization")
+		frappe.delete_doc("CRM Deal", name, ignore_permissions=True, force=True)
+		for contact in contact_names:
+			if contact and frappe.db.exists("Contact", contact):
+				for child_doctype in ("Contact Email", "Contact Phone", "Dynamic Link"):
+					Child = DocType(child_doctype)
+					frappe.qb.from_(Child).delete().where(Child.parent == contact).run()
+				Contact = DocType("Contact")
+				frappe.qb.from_(Contact).delete().where(Contact.name == contact).run()
+		if org and frappe.db.exists("CRM Organization", org):
+			frappe.delete_doc("CRM Organization", org, ignore_permissions=True, force=True)

crm/demo/leads.py

@@ -0,0 +1,253 @@
+from datetime import datetime, timedelta
+
+import frappe
+
+from crm.demo.utils import backdate, fix_auto_records, resolve_owners
+
+# Days ago each lead was created — index matches leads_data order
+_LEAD_DAYS = [58, 52, 47, 43, 38, 34, 28, 22, 17, 30, 13, 8]
+
+# Last-activity date per lead (used for modified so active leads sort to top).
+# Converted leads will be overwritten by create_demo_deals anyway.
+# Index: [0]alice [1]bob [2]carol [3]david [4]eve [5]frank [6]grace [7]henry [8]iris [9]jack [10]karen [11]leo
+_LEAD_LAST_TOUCHED_DAYS = [50, 8, 11, 37, 38, 34, 28, 18, 11, 24, 9, 5]
+
+
+def create_demo_leads(demo_users):
+	session_user, owner_1, owner_2, owner_3 = resolve_owners(demo_users)
+
+	leads_data = [
+		# [0] Alice — will be converted to a deal
+		{
+			"first_name": "Alice",
+			"last_name": "Johnson",
+			"email": "alice.johnson@example.com",
+			"mobile_no": "+1 555 000 1234",
+			"organization": "Acme Corp",
+			"website": "https://acmecorp.example.com",
+			"industry": "Technology",
+			"job_title": "VP of Engineering",
+			"status": "Qualified",
+			"source": "Cold Calling",
+			"no_of_employees": "201-500",
+			"annual_revenue": 5000000,
+			"lead_owner": session_user,
+		},
+		# [1] Bob — stays as a lead
+		{
+			"first_name": "Bob",
+			"last_name": "Martinez",
+			"email": "bob.martinez@example.com",
+			"mobile_no": "+1 555 000 5678",
+			"organization": "Globex Industries",
+			"website": "https://globexindustries.example.com",
+			"industry": "Manufacturing",
+			"job_title": "CEO",
+			"status": "Contacted",
+			"source": "Advertisement",
+			"no_of_employees": "51-200",
+			"annual_revenue": 8000000,
+			"lead_owner": owner_1,
+		},
+		# [2] Carol — stays as a lead
+		{
+			"first_name": "Carol",
+			"last_name": "Smith",
+			"email": "carol.smith@example.com",
+			"mobile_no": "+1 555 000 9012",
+			"organization": "BrightPath Ltd",
+			"website": "https://brightpath.example.com",
+			"industry": "Consulting",
+			"job_title": "Head of Operations",
+			"status": "Nurture",
+			"source": "Email",
+			"no_of_employees": "11-50",
+			"annual_revenue": 1500000,
+			"lead_owner": owner_2,
+		},
+		# [3] David — will be converted to a deal
+		{
+			"first_name": "David",
+			"last_name": "Lee",
+			"email": "david.lee@example.com",
+			"mobile_no": "+1 555 000 3456",
+			"organization": "TechStart Inc",
+			"website": "https://techstart.example.com",
+			"industry": "Technology",
+			"job_title": "CTO",
+			"status": "Qualified",
+			"source": "Reference",
+			"no_of_employees": "1-10",
+			"annual_revenue": 500000,
+			"lead_owner": owner_1,
+		},
+		# [4] Emma — stays as a lead
+		{
+			"first_name": "Emma",
+			"last_name": "Williams",
+			"email": "emma.williams@example.com",
+			"mobile_no": "+1 555 000 2345",
+			"organization": "CloudBase LLC",
+			"website": "https://cloudbase.example.com",
+			"industry": "Technology",
+			"job_title": "Procurement Manager",
+			"status": "New",
+			"source": "Website",
+			"no_of_employees": "51-200",
+			"annual_revenue": 3000000,
+			"lead_owner": owner_1,
+		},
+		# [5] Frank — stays as a lead
+		{
+			"first_name": "Frank",
+			"last_name": "Turner",
+			"email": "frank.turner@example.com",
+			"mobile_no": "+1 555 000 6789",
+			"organization": "DataSync Co",
+			"website": "https://datasync.example.com",
+			"industry": "Technology",
+			"job_title": "Head of Sales",
+			"status": "Nurture",
+			"source": "Email",
+			"no_of_employees": "11-50",
+			"annual_revenue": 2000000,
+			"lead_owner": owner_2,
+		},
+		# [6] Grace — stays as a lead
+		{
+			"first_name": "Grace",
+			"last_name": "Park",
+			"email": "grace.park@example.com",
+			"mobile_no": "+1 555 000 0123",
+			"organization": "NextWave Corp",
+			"website": "https://nextwavecorp.example.com",
+			"industry": "Telecommunications",
+			"job_title": "COO",
+			"status": "Contacted",
+			"source": "Reference",
+			"no_of_employees": "201-500",
+			"annual_revenue": 12000000,
+			"lead_owner": session_user,
+		},
+		# [7] Henry — will be converted to a deal
+		{
+			"first_name": "Henry",
+			"last_name": "Adams",
+			"email": "henry.adams@example.com",
+			"mobile_no": "+1 555 000 4567",
+			"organization": "PivotTech Solutions",
+			"website": "https://pivottech.example.com",
+			"industry": "Technology",
+			"job_title": "CFO",
+			"status": "Qualified",
+			"source": "Advertisement",
+			"no_of_employees": "51-200",
+			"annual_revenue": 6000000,
+			"lead_owner": owner_3,
+		},
+		# [8] Iris — will be converted to a deal
+		{
+			"first_name": "Iris",
+			"last_name": "Chen",
+			"email": "iris.chen@example.com",
+			"mobile_no": "+1 555 000 7890",
+			"organization": "ScaleUp Labs",
+			"website": "https://scaleuplabs.example.com",
+			"industry": "Technology",
+			"job_title": "VP Engineering",
+			"status": "Qualified",
+			"source": "Cold Calling",
+			"no_of_employees": "1-10",
+			"annual_revenue": 750000,
+			"lead_owner": session_user,
+		},
+		# [9] Jack — will be converted to a deal
+		{
+			"first_name": "Jack",
+			"last_name": "Morrison",
+			"email": "jack.morrison@example.com",
+			"mobile_no": "+1 555 000 8901",
+			"organization": "Meridian Systems",
+			"website": "https://meridiansystems.example.com",
+			"industry": "Defense",
+			"job_title": "CEO",
+			"status": "Qualified",
+			"source": "Reference",
+			"no_of_employees": "501-1000",
+			"annual_revenue": 25000000,
+			"lead_owner": owner_1,
+		},
+		# [10] Karen — will be converted to a Lost deal
+		{
+			"first_name": "Karen",
+			"last_name": "White",
+			"email": "karen.white@example.com",
+			"mobile_no": "+1 555 000 2222",
+			"organization": "Vertex Analytics",
+			"website": "https://vertexanalytics.example.com",
+			"industry": "Financial Services",
+			"job_title": "Director of Strategy",
+			"status": "Qualified",
+			"source": "Advertisement",
+			"no_of_employees": "201-500",
+			"annual_revenue": 9000000,
+			"lead_owner": owner_2,
+		},
+		# [11] Leo — will be converted to a Lost deal
+		{
+			"first_name": "Leo",
+			"last_name": "Brown",
+			"email": "leo.brown@example.com",
+			"mobile_no": "+1 555 000 3333",
+			"organization": "Forge Digital",
+			"website": "https://forgedigital.example.com",
+			"industry": "Advertising",
+			"job_title": "Head of Product",
+			"status": "Qualified",
+			"source": "Cold Calling",
+			"no_of_employees": "51-200",
+			"annual_revenue": 4000000,
+			"lead_owner": session_user,
+		},
+	]
+
+	now = datetime.now()
+	created = []
+	for data, days in zip(leads_data, _LEAD_DAYS, strict=False):
+		ts = now - timedelta(days=days)
+		owner = data["lead_owner"]
+		lead = frappe.get_doc({"doctype": "CRM Lead", **data}).insert(ignore_permissions=True)
+		backdate("CRM Lead", lead.name, owner, ts)
+		fix_auto_records("CRM Lead", lead.name, owner, ts)
+		created.append(lead.name)
+
+	return created
+
+
+def rebackdate_demo_leads(lead_names, demo_users):
+	"""Re-apply modified timestamps after activity creation resets them."""
+	session_user, owner_1, owner_2, owner_3 = resolve_owners(demo_users)
+	# Must match the lead_owner assignments in create_demo_leads (same index order)
+	_owners = [
+		session_user,
+		owner_1,
+		owner_2,
+		owner_1,
+		owner_1,
+		owner_2,
+		session_user,
+		owner_3,
+		session_user,
+		owner_1,
+		owner_2,
+		session_user,
+	]
+	now = datetime.now()
+	for name, days, owner in zip(lead_names, _LEAD_LAST_TOUCHED_DAYS, _owners, strict=False):
+		backdate("CRM Lead", name, owner, now - timedelta(days=days), set_creation=False)
+
+
+def delete_demo_leads(lead_names):
+	for name in lead_names:
+		if frappe.db.exists("CRM Lead", name):
+			frappe.delete_doc("CRM Lead", name, ignore_permissions=True, force=True)

crm/demo/notes.py

@@ -0,0 +1,86 @@
+from datetime import datetime, timedelta
+
+import frappe
+
+from crm.demo.utils import backdate, resolve_owners
+
+
+def create_demo_notes(lead_names, demo_users):
+	session_user, owner_1, owner_2, _ = resolve_owners(demo_users)
+	now = datetime.now()
+
+	# Two notes per lead — a call summary and a follow-up observation
+	notes_data = [
+		{
+			"title": "Initial Discovery Call",
+			"content": "<p>Had a productive intro call. They're evaluating solutions for Q3. Key stakeholders include the CTO and VP of Sales. Budget is pre-approved. Follow up with a tailored proposal.</p>",
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[0],
+			"owner": session_user,
+			"days_ago": 56,
+		},
+		{
+			"title": "Pain Points Identified",
+			"content": "<p>Current system lacks reporting and integrations. Team size is growing fast — 40% headcount increase this year. Decision timeline is 6–8 weeks.</p>",
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[0],
+			"owner": session_user,
+			"days_ago": 51,
+		},
+		{
+			"title": "Demo Feedback",
+			"content": "<p>Very positive response to the pipeline view. Asked about custom fields and API access. Needs sign-off from CFO before moving forward.</p>",
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[1],
+			"owner": owner_1,
+			"days_ago": 46,
+		},
+		{
+			"title": "Competitor Mentioned",
+			"content": "<p>Currently trialing a competitor. Our differentiators: better UX, open source, and lower TCO. Send a comparison doc before next call.</p>",
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[1],
+			"owner": owner_1,
+			"days_ago": 39,
+		},
+		{
+			"title": "Requirements Gathered",
+			"content": "<p>Needs CRM + email integration + team collaboration. Currently using spreadsheets. Strong motivation to move to a structured system this quarter.</p>",
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[2],
+			"owner": owner_2,
+			"days_ago": 44,
+		},
+		{
+			"title": "Pricing Discussion",
+			"content": "<p>Walked through pricing tiers. They prefer an annual plan. Requested a 2-week trial extension. Champion is very engaged — internal buy-in looks strong.</p>",
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[3],
+			"owner": owner_1,
+			"days_ago": 40,
+		},
+	]
+
+	created = []
+	for data in notes_data:
+		ts = now - timedelta(days=data["days_ago"])
+		owner = data["owner"]
+		note = frappe.get_doc(
+			{
+				"doctype": "FCRM Note",
+				"title": data["title"],
+				"content": data["content"],
+				"reference_doctype": data["reference_doctype"],
+				"reference_docname": data["reference_docname"],
+			}
+		).insert(ignore_permissions=True)
+		backdate("FCRM Note", note.name, owner, ts)
+		created.append(note.name)
+
+	return created
+
+
+def delete_demo_notes(note_names):
+	for name in note_names:
+		if frappe.db.exists("FCRM Note", name):
+			frappe.delete_doc("FCRM Note", name, ignore_permissions=True, force=True)

crm/demo/tasks.py

@@ -0,0 +1,96 @@
+import frappe
+
+from crm.demo.utils import backdate, resolve_owners
+
+
+def create_demo_tasks(lead_names, demo_users):
+	from datetime import date, datetime, timedelta
+
+	today = date.today()
+	now = datetime.now()
+	session_user, owner_1, owner_2, _ = resolve_owners(demo_users)
+
+	tasks_data = [
+		{
+			"title": "Send proposal document",
+			"priority": "High",
+			"status": "Todo",
+			"assigned_to": session_user,
+			"due_date": today + timedelta(days=2),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[0],
+			"description": "Prepare and send a tailored proposal covering pricing, integrations, and onboarding timeline.",
+			"days_ago": 1,
+		},
+		{
+			"title": "Schedule technical demo",
+			"priority": "Medium",
+			"status": "In Progress",
+			"assigned_to": owner_1,
+			"due_date": today + timedelta(days=5),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[0],
+			"description": "Coordinate with the CTO to set up a 1-hour technical deep-dive session.",
+			"days_ago": 3,
+		},
+		{
+			"title": "Follow up on trial access",
+			"priority": "High",
+			"status": "Todo",
+			"assigned_to": owner_1,
+			"due_date": today + timedelta(days=1),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[1],
+			"description": "Check in on trial progress and address any blockers before the next call.",
+			"days_ago": 2,
+		},
+		{
+			"title": "Send competitor comparison doc",
+			"priority": "Medium",
+			"status": "Done",
+			"assigned_to": owner_2,
+			"due_date": today - timedelta(days=2),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[1],
+			"description": "Share the feature comparison sheet highlighting our advantages over the competitor they're trialing.",
+			"days_ago": 6,
+		},
+		{
+			"title": "Set up trial environment",
+			"priority": "High",
+			"status": "In Progress",
+			"assigned_to": owner_2,
+			"due_date": today + timedelta(days=3),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[2],
+			"description": "Provision a trial instance with sample data pre-loaded for their team to evaluate.",
+			"days_ago": 3,
+		},
+		{
+			"title": "Confirm annual plan details",
+			"priority": "Low",
+			"status": "Backlog",
+			"assigned_to": session_user,
+			"due_date": today + timedelta(days=10),
+			"reference_doctype": "CRM Lead",
+			"reference_docname": lead_names[3],
+			"description": "Finalize seat count and send updated quote for annual subscription.",
+			"days_ago": 1,
+		},
+	]
+
+	created = []
+	for data in tasks_data:
+		owner = data["assigned_to"]
+		ts = now - timedelta(days=data.pop("days_ago"))
+		task = frappe.get_doc({"doctype": "CRM Task", **data}).insert(ignore_permissions=True)
+		backdate("CRM Task", task.name, owner, ts)
+		created.append(task.name)
+
+	return created
+
+
+def delete_demo_tasks(task_names):
+	for name in task_names:
+		if frappe.db.exists("CRM Task", name):
+			frappe.delete_doc("CRM Task", name, ignore_permissions=True, force=True)

crm/demo/users.py

@@ -0,0 +1,89 @@
+from datetime import datetime, timedelta
+
+import frappe
+from frappe.query_builder import DocType
+
+DEMO_USERS = [
+	{
+		"email": "sarah.demo@example.com",
+		"first_name": "Sarah",
+		"last_name": "Connor",
+		"mobile_no": "+1 555 100 0002",
+		"roles": ["Sales Manager", "Sales User"],
+		"avatar": "/assets/crm/images/demo/sarah-connor.png",
+	},
+	{
+		"email": "john.demo@example.com",
+		"first_name": "John",
+		"last_name": "Parker",
+		"mobile_no": "+1 555 100 0003",
+		"roles": ["Sales User"],
+		"avatar": "/assets/crm/images/demo/john-parker.png",
+	},
+	{
+		"email": "emily.demo@example.com",
+		"first_name": "Emily",
+		"last_name": "Chen",
+		"mobile_no": "+1 555 100 0004",
+		"roles": ["Sales User"],
+		"avatar": "/assets/crm/images/demo/emily-chen.png",
+	},
+]
+
+DEMO_USER_EMAILS = [u["email"] for u in DEMO_USERS]
+
+
+def create_demo_users():
+	for user_data in DEMO_USERS:
+		if not frappe.db.exists("User", user_data["email"]):
+			frappe.get_doc(
+				{
+					"doctype": "User",
+					"email": user_data["email"],
+					"first_name": user_data["first_name"],
+					"last_name": user_data["last_name"],
+					"send_welcome_email": 0,
+					"user_type": "System User",
+					"mobile_no": user_data["mobile_no"],
+					"user_image": user_data["avatar"],
+					"roles": [{"role": r} for r in user_data["roles"]],
+				}
+			).insert(ignore_permissions=True)
+
+	# Backdate auto-created contacts and set their image
+	_ts = datetime.now() - timedelta(days=70)
+	_avatar_by_email = {u["email"]: u["avatar"] for u in DEMO_USERS}
+	contact_names = frappe.get_all(
+		"Contact", filters={"user": ["in", DEMO_USER_EMAILS]}, fields=["name", "user"]
+	)
+	for row in contact_names:
+		frappe.db.set_value(
+			"Contact",
+			row.name,
+			{"image": _avatar_by_email.get(row.user, ""), "creation": _ts, "modified": _ts},
+			update_modified=False,
+		)
+
+	return DEMO_USER_EMAILS
+
+
+def delete_demo_users(demo_user_emails):
+	# delete notifications first — they link to users and block deletion
+	Notification = DocType("CRM Notification")
+	frappe.qb.from_(Notification).delete().where(
+		(Notification.from_user.isin(demo_user_emails)) | (Notification.to_user.isin(demo_user_emails))
+	).run()
+
+	# collect contact names before deleting users — User.on_trash NULLs contact.user
+	contact_names = frappe.get_all("Contact", filters={"user": ["in", demo_user_emails]}, pluck="name")
+
+	for email in demo_user_emails:
+		if frappe.db.exists("User", email):
+			frappe.delete_doc("User", email, ignore_permissions=True, force=True)
+
+	if contact_names:
+		for child_doctype in ("Contact Email", "Contact Phone", "Dynamic Link"):
+			Child = DocType(child_doctype)
+			frappe.qb.from_(Child).delete().where(Child.parent.isin(contact_names)).run()
+		Contact = DocType("Contact")
+		frappe.qb.from_(Contact).delete().where(Contact.name.isin(contact_names)).run()

crm/demo/utils.py

@@ -0,0 +1,117 @@
+import json
+
+import frappe
+from frappe.query_builder import DocType
+
+
+def resolve_owners(demo_users):
+	"""Return (session_user, owner_1, owner_2, owner_3) each falling back to session_user."""
+	s = frappe.session.user
+	if s in ("Administrator", "Guest"):
+		s = demo_users[0] if demo_users else s
+	owners = [demo_users[i] if i < len(demo_users) else s for i in range(3)]
+	return (s, *owners)
+
+
+def build_full_names(session_user):
+	"""Return {email: full_name} mapping for all demo users plus session_user."""
+	from crm.demo.users import DEMO_USERS
+
+	names = {u["email"]: f"{u['first_name']} {u['last_name']}" for u in DEMO_USERS}
+	names[session_user] = frappe.utils.get_fullname(session_user)
+	return names
+
+
+def backdate(doctype, name, owner, ts, *, set_creation=True):
+	"""Set owner, modified_by, modified (and optionally creation) without triggering hooks."""
+	fields = {"owner": owner, "modified_by": owner, "modified": ts}
+	if set_creation:
+		fields["creation"] = ts
+	frappe.db.set_value(doctype, name, fields, update_modified=False)
+
+
+def fix_auto_records(doctype, docname, owner, ts):
+	"""Backdate the Version and assignment Comment auto-created with owner='Administrator' on insert."""
+	Version = DocType("Version")
+	(
+		frappe.qb.update(Version)
+		.set(Version.owner, owner)
+		.set(Version.modified_by, owner)
+		.set(Version.creation, ts)
+		.set(Version.modified, ts)
+		.where(
+			(Version.ref_doctype == doctype)
+			& (Version.docname == docname)
+			& (Version.owner == "Administrator")
+		)
+		.run()
+	)
+	Comment = DocType("Comment")
+	(
+		frappe.qb.update(Comment)
+		.set(Comment.owner, owner)
+		.set(Comment.modified_by, owner)
+		.set(Comment.comment_email, owner)
+		.set(Comment.creation, ts)
+		.set(Comment.modified, ts)
+		.where(
+			(Comment.reference_doctype == doctype)
+			& (Comment.reference_name == docname)
+			& (Comment.owner == "Administrator")
+		)
+		.run()
+	)
+
+
+def insert_comment(reference_doctype, reference_name, owner, content, full_names, ts):
+	"""Insert a Comment, backdate it, and return its name."""
+	comment = frappe.get_doc(
+		{
+			"doctype": "Comment",
+			"comment_type": "Comment",
+			"reference_doctype": reference_doctype,
+			"reference_name": reference_name,
+			"content": content,
+			"comment_email": owner,
+			"comment_by": full_names.get(owner, owner),
+		}
+	).insert(ignore_permissions=True)
+	backdate("Comment", comment.name, owner, ts)
+	return comment.name
+
+
+def insert_communication(reference_doctype, reference_name, data, full_names, ts):
+	"""Insert an email Communication, backdate it, and return its name."""
+	comm = frappe.get_doc(
+		{
+			"doctype": "Communication",
+			"communication_type": "Communication",
+			"communication_medium": "Email",
+			"status": "Linked",
+			"sent_or_received": data["sent_or_received"],
+			"reference_doctype": reference_doctype,
+			"reference_name": reference_name,
+			"subject": data["subject"],
+			"content": data["content"],
+			"sender": data["sender"],
+			"sender_full_name": full_names.get(data["sender"], data["sender"]),
+			"recipients": data["recipients"],
+			"communication_date": ts,
+		}
+	).insert(ignore_permissions=True)
+	backdate("Communication", comm.name, data["owner"], ts)
+	return comm.name
+
+
+def insert_version(ref_doctype, docname, owner, changed, ts):
+	"""Insert a Version record, backdate it, and return its name."""
+	version = frappe.get_doc(
+		{
+			"doctype": "Version",
+			"ref_doctype": ref_doctype,
+			"docname": docname,
+			"data": json.dumps({"changed": changed}),
+		}
+	).insert(ignore_permissions=True)
+	backdate("Version", version.name, owner, ts)
+	return version.name

crm/domain_enrichment/README.md

@@ -0,0 +1,316 @@
+# Domain Enrichment
+
+Auto-enrich a CRM Lead, Deal or Organization from its **website** — no paid data
+providers. Given a domain, the engine crawls the public site, runs **admin-tunable,
+rule-based** extractors (company name, description, logo, industry,
+emails, phones, social links), records everything with full provenance, and fills
+the mapped CRM fields according to configurable write policies.
+
+The guiding principle: **config is data, the engine is code.** Everything a
+sales-ops admin would tune — keywords, social patterns,
+field mappings, crawl limits, enabled sources, the SSRF allow/block lists — lives
+in **doctypes you edit in the desk**, never in Python. The engine knows no keywords
+itself; it loads rules and executes them.
+
+---
+
+## What & why
+
+- **Domain-based.** The only input is a website URL. No API keys, no per-record
+  cost, no third-party enrichment vendor.
+- **Rule-based / public-data.** Industry and social profiles
+  are all matched by **rules an admin owns** (`CRM Enrichment Rule`).
+  The mechanics (crawling, HTML parsing, the favicon scorer, JSON-LD parsing,
+  readability diagnosis, email/phone regexes) are pure code because they are
+  algorithms, not tunable knowledge.
+- **Provenance preserved.** Every extracted value records `{value, source, method}`
+  — which page it came from and how it was found — so a reviewer can judge how much
+  to trust it. The full provenance JSON is stored on each run.
+- **Self-contained.** The only outbound traffic is fetching the target site itself —
+  no third-party APIs.
+- **Framework-native.** Outbound HTTP uses `frappe.utils.get_request_session`;
+  background work uses `frappe.enqueue`; progress streams over
+  `frappe.publish_realtime`.
+
+---
+
+## Architecture
+
+```
+DESK ADMIN (data)                         ENGINE (code, rule-agnostic)        RESULT (data)
+ CRM Enrichment Settings (Single) ─┐       crm/domain_enrichment/
+ CRM Enrichment Rule (+Pattern) ───┼─►get_config()──► config.py              CRM Enrichment Run
+ CRM Enrichment Field Mapping ─────┘      http.py (session+SSRF+cap)           (Dynamic Link to
+                                          crawler.py (BFS; caps from cfg)       Lead/Deal/Org)
+        │                                 extractors.py (rule executors)              ▲
+        ▼                                 pipeline.py → EnrichmentResult              │
+   bench migrate syncs                    api.py (@whitelist) / tasks.py        mapper.py writes
+   native fields onto                     (enqueue + realtime)                  mapped fields onto
+   Lead/Deal/Organization                                                      the record (policy)
+```
+
+### Module map (`crm/domain_enrichment/`)
+
+| File | Responsibility |
+|---|---|
+| `config.py` | `get_config()` → `EnrichmentConfig` (settings + `Rule`/`Mapping` objects), built on demand (not cached). `get_settings()` / `auto_enrich_enabled_for()` are the cheap Settings-only reads for hot paths. |
+| `http.py` | `fetch(url, cfg)` on the framework session; the **SSRF guard** (`validate_url`); byte cap; HTML-only filter; never-raise `(status, html, error, final_url)` contract. |
+| `crawler.py` | Same-domain, depth-limited BFS. Caps / link-priority order / skip patterns from config. |
+| `extractors.py` | **Generic rule executor** (`apply_keyword_rules`) + the pure mechanics. No literal keyword tables. |
+| `result.py` | `EnrichmentResult` and its `{value, source, method}` provenance schema. Pure dataclasses, no framework import. |
+| `pipeline.py` | `run(website, cfg, progress)` orchestrates crawl → extract → assemble `EnrichmentResult`. Never writes to the DB. |
+| `mapper.py` | `apply_to_document(doc, result, cfg)` — the **single** result→CRM-field authority, driven by Field Mapping records + write policies. |
+| `tasks.py` | `run_enrichment` (enqueued worker) + `write_run` (the **single** run-history writer). Streams realtime progress; never raises. |
+| `api.py` | Whitelisted `enrich` (enqueue) + `retry` (re-run from a Run). Type-annotated; permission- and rate-limited. |
+| `cross_record.py` | `copy_enrichment_from_organization` — the one link-time Org→Lead/Deal copy. |
+| `install.py` | Idempotent seeder: translates the original constant tables into Rule + Field Mapping records. |
+
+---
+
+## Configuration guide
+
+Three doctypes hold all the tunable knowledge. `get_config()` reads them on demand
+(no cache), so an edit takes effect on the very next enrichment without a restart.
+
+### 1. CRM Enrichment Settings (Single)
+
+| Field | Meaning |
+|---|---|
+| `enabled` | Master on/off for the whole feature, across Lead, Deal and Organization. |
+| `auto_enrich` | When `1`, a newly-created CRM **Lead, Deal or Organization** that has a website is enriched automatically in a background job (`after_insert`). A new Deal with a new Organization enriches both (each crawls and writes its own fields). Default `0` — enrichment is otherwise manual via the Enrich button. |
+| `max_pages` / `max_depth` | Crawl breadth/depth caps (BFS). |
+| `request_timeout` | Per-request timeout (seconds). |
+| `max_download_bytes` | Hard cap on bytes read per page (streamed). |
+| `retry_count` | Transient-error retries on the session. |
+| `user_agent` | Crawler User-Agent header. |
+| `allowed_domains` / `blocked_domains` (child) | SSRF allow/block lists (subdomain-aware). A blocked host is always rejected; if an allow list exists, only listed hosts pass. |
+| `link_priority_order` (child) | `keyword` + `weight`: a crawl-ordering hint — links whose URL/anchor contain these are fetched first (higher weight = sooner). Defaults are seeded on install. |
+| `skip_patterns` (child) | URL substrings/regexes to never crawl. |
+
+If the Single has never been saved, the engine falls back to sane defaults
+(`config.DEFAULT_SETTINGS`).
+
+### 2. CRM Enrichment Rule (+ CRM Enrichment Rule Pattern)
+
+The heart of the knowledge base. One record = one rule.
+
+| Field | Meaning |
+|---|---|
+| `rule_name` | Unique name (the record's id). |
+| `rule_type` | `Industry` / `Social`. |
+| `enabled` | Disabled rules are skipped by the config builder. |
+| `target_value` | The label a match emits (tech name, network). Ignored for `Industry`. |
+| `industry` | (Industry rules only) the linked `CRM Industry` a match classifies to. |
+| `weight` | Multiplier on the hit count (industry scoring). |
+| `match_scope` | `Headline` / `Full Text` / `HTML` / `Headers` / `URL` — which slice of the page the rule matches. |
+| `patterns` (child) | One or more `pattern` strings; `is_regex` toggles substring vs. regex. Substrings are matched case-insensitively. |
+
+### 3. CRM Enrichment Field Mapping
+
+The single source of result→field truth. One record = one field written.
+
+| Field | Meaning |
+|---|---|
+| `enabled` | Disabled mappings are skipped. |
+| `source_key` | One of the 13 frozen result keys (below). |
+| `target_doctype` | `CRM Lead` / `CRM Deal` / `CRM Organization`. |
+| `target_fieldname` | The field on that doctype to fill (validated to exist). |
+| `write_policy` | `Fill if empty` / `Always refresh` / `Override defaults` (below). |
+| `default_values` | (Override defaults) newline-separated values treated as "unset". |
+| `create_missing_link` | (Link fields) auto-create the linked master if missing (e.g. a new `CRM Industry`). |
+
+**The 13 `source_key`s:** `company_name`, `description`, `logo`, `industry`,
+`primary_email`, `primary_phone`, `secondary_phone`, `linkedin`,
+`twitter`, `github`, `facebook`, `instagram`, `youtube`.
+
+**Write policies:**
+- **Fill if empty** — write only when the target field is currently empty.
+- **Always refresh** — overwrite (or clear) with the fresh value, even if the field
+  had a previous value. Used for fully engine-derived fields that should always
+  reflect the latest crawl.
+- **Override defaults** — treat the listed `default_values` (plus empty) as "unset"
+  and fill over them, but respect any real user value. Useful for a field that ships
+  with a meaningless placeholder default that enrichment should replace.
+
+### Worked examples
+
+**Add a new industry.** Create a `CRM Industry` (or let `create_missing_link` do it),
+then a `CRM Enrichment Rule`: `rule_type = Industry`, `industry = <your industry>`,
+`match_scope = Headline`, and add patterns (e.g. `biotech`, `genomics`). Save — the
+classifier picks it up on the next run.
+
+**Add a social network.** New `CRM Enrichment Rule`: `rule_type = Social`,
+`target_value = "mastodon"`, `match_scope = HTML`, and a regex pattern
+(`is_regex = 1`) such as `@[A-Za-z0-9_]+@mastodon\.social`.
+
+**Remap a result field / change a write policy.** Edit the relevant
+`CRM Enrichment Field Mapping` — change `target_fieldname`, or flip `write_policy`
+from `Fill if empty` to `Always refresh`. To stop a field being written, untick
+`enabled` on its mapping.
+
+**Tune crawl limits / SSRF.** Raise `max_pages`/`max_depth` for deeper crawls; add a
+`skip_patterns` row (e.g. `/blog/`) to skip noisy sections; add a `blocked_domains`
+row to hard-block a host, or `allowed_domains` rows to restrict crawling to an
+allow-list only.
+
+---
+
+## How a run works (end to end)
+
+1. **Trigger** — the SPA `EnrichFromWebsite` button (or the desk form button) calls
+   the whitelisted `api.enrich(reference_doctype, reference_name)`.
+2. **Validate + enqueue** — `enrich` checks the doctype is enabled, the user has
+   `write` permission, and the record has a `website`, then `frappe.enqueue`s
+   `tasks.run_enrichment` on the `long` queue with a per-doc `job_id` and
+   `deduplicate=True` (a second click while one is in-flight is a no-op).
+3. **Crawl** — `pipeline.run` builds the session, runs the BFS crawler (SSRF-guarded
+   `http.fetch` per page), and emits progress callbacks.
+4. **Rule execution** — the extractors run the config rules: industry classification,
+   social profiles, plus the mechanics (company info, emails, phones).
+5. **Map** — `mapper.apply_to_document` writes the mapped fields onto the origin doc
+   per each mapping's write policy; the doc is saved (normal, permission-respecting).
+6. **Run record** — `tasks.write_run` persists exactly one `CRM Enrichment Run` with
+   the summary fields + full `raw_json` provenance.
+7. **Realtime** — progress and the terminal result stream to the triggering user over
+   the `domain_enrichment_progress` event.
+
+Progress event payload (mirrors `crm/api/whatsapp.py`):
+
+```json
+{"reference_doctype": "...", "reference_name": "...", "step": 3, "total": 9,
+ "message": "...", "status": "running|completed|error", "payload": {}}
+```
+
+The terminal `completed` event's `payload` carries `{filled_fields, notes, **result.flat()}`.
+
+### Run lifecycle (state machine)
+
+```
+        enqueue (dedupe)      worker start          all steps ok
+ [Queued] ───────────► [Running] ─────────────────────────────► [Completed]
+                           │
+                           │  exception / blocked / unreachable
+                           ▼
+                       [Failed]   (reason in notes; frappe.log_error)
+```
+
+`tasks.run_enrichment` never raises to the worker: on any exception the Run is
+written as `Failed` (with the traceback truncated into `notes`), the error is logged
+via `frappe.log_error`, and an `error` event is published.
+
+---
+
+## Cross-record behavior
+
+There is exactly **one** cross-record copy:
+`cross_record.copy_enrichment_from_organization(target_doc)`. When a **Lead** or
+**Deal** links an already-enriched **Organization**, the Organization's stored
+enriched fields are copied onto the **empty** fields of the Lead/Deal
+(fill-empty semantics, even for mappings normally configured "Always refresh"), so
+user-entered data is never touched.
+
+- **Lead seam:** `CRMLead.create_organization` (existing-org branch).
+- **Deal seam:** `CRMDeal.validate` when `organization` is set/changed.
+- An Organization is "enriched" if it has a **completed** `CRM Enrichment Run`, or
+  (fallback) any populated native enriched field.
+
+**No background fan-out.** The copy is synchronous, in-request, Organization →
+Lead/Deal only. There is no job, no propagation to other linked records, and no
+write back to the Organization. The field-writing reuses `mapper.apply_to_document`
+(the single authority) — no second copy implementation exists.
+
+---
+
+## SSRF Guard
+
+- **SSRF guard (`http.validate_url`).** Mandatory and not provided by the framework.
+  It rejects non-`http(s)` schemes, resolves the hostname, and rejects any URL that
+  resolves to a loopback / private / link-local / reserved / multicast / unspecified
+  address — including the cloud-metadata endpoint `169.254.169.254`. It honors the
+  `blocked_domains` / `allowed_domains` lists and **re-validates after every
+  redirect** (redirects are followed manually). The private-network rejection is
+  **unconditional** — there is deliberately no setting to disable it (a crawler that
+  fetches user-supplied URLs must never be turnable into an internal-network probe).
+- **DNS-rebinding TOCTOU — closed by IP pinning.** The guard validates the resolved
+  addresses and the connection is then made to one of those exact addresses
+  (`http._pinned_get`): the URL netloc is rewritten to the validated IP while the
+  Host header, TLS SNI and certificate verification stay on the original hostname
+  (urllib3 `server_hostname`). A host that alternates DNS answers (short TTL) can no
+  longer pass validation with a public IP and serve the request from a private one.
+  Each redirect hop is re-validated and re-pinned.
+- **Permissions.** `api.enrich` enforces the Settings allow-list and
+  `doc.check_permission("write")` — a user who cannot edit the record cannot enrich
+  it. The worker save is a normal permission-respecting save. `create_missing_link`
+  auto-creation is permission-respecting too: `_ensure_link_target` only creates a
+  master (`CRM Industry`) when the enriching user has create rights on it, so a
+  scraped value can never escalate into an insert the user couldn't do by hand. The
+  only `ignore_permissions` write is the engine-owned `CRM Enrichment Run` audit log
+  — never a cross-record write to data the user can't edit.
+- **Rate limiting.** Every entry point that triggers a fetch — `enrich` and
+  `retry` — is decorated with `rate_limit(limit=ENRICH_RATE_LIMIT, seconds=60)`
+  (per user, per route). Both only enqueue a per-doc-deduplicated job.
+
+---
+
+## Extending in code
+
+- **Add an extractor mechanic** (e.g. a new heuristic): add a pure function to
+  `extractors.py`, call it from `pipeline.run`, and surface its output on
+  `EnrichmentResult` (`result.py`). Keep tunable knowledge in rules, not constants.
+- **Add a new `rule_type`:** add the option to the `CRM Enrichment Rule` Select, add
+  a generic executor (or reuse `apply_keyword_rules`) in
+  `extractors.py`, and wire it into `pipeline.run` via `cfg.rules("<Type>")`. Keep the
+  rule vocabulary small (substring/regex + scope + weight) — do **not** grow a DSL.
+- **Add a new `source_key`:** extend `mapper.get_value_for_source_key`, add the option
+  to the Field Mapping Select, and (if it needs new data) populate it on
+  `EnrichmentResult`.
+
+---
+
+## Limitations & future
+
+- **JS-rendered sites.** Enrichment reads server-delivered HTML only (`requests` +
+  BeautifulSoup), so a fully client-rendered site yields only `<head>` metadata; the
+  run records the "main content is JavaScript-rendered" note (`READABILITY_MESSAGES["empty"]`)
+  and fields may be blank. A headless-Chromium render fallback was prototyped but
+  **intentionally removed** — the per-worker Chromium memory footprint isn't worth it
+  for a fallback. (The implementation is preserved locally in the gitignored
+  `CHROMIUM_FALLBACK.local.md` if it ever needs to be reintroduced.)
+- **Logo vs image.** The `logo` is the company's **link icon** (`extract_logo`): the
+  best declared `<link rel=icon>` — scalable SVG > largest raster / apple-touch — with
+  `/favicon.ico` as the fallback. It's the crisp, square brand mark suited to an avatar.
+  The larger social/brand image (`extract_image`: JSON-LD `Organization.logo` →
+  `og:image`/`twitter:image`) is captured separately as `image` in the run JSON — it is
+  **not** used as the logo (`og:image` is usually a wide banner) and isn't mapped to a
+  record field by default.
+- **Scheduled re-enrichment — _future feature, not implemented._** Triggering is
+  either manual (Enrich button) or automatic on Lead/Deal/Organization creation (`auto_enrich`);
+  there is no *scheduled* re-enrichment of existing records. A future version could enqueue periodic re-enrichment of
+  stale records (e.g. via a scheduled job that re-runs `tasks.run_enrichment` for
+  records whose last `CRM Enrichment Run` is older than N days).
+- **TLS fingerprinting.** The fetcher uses `requests`, so its TLS ClientHello
+  fingerprints (JA3/JA4) as a non-browser client. Bot walls that fingerprint TLS
+  (Cloudflare, Akamai, DataDome) can still block it even though `http.build_session`
+  sends browser-like headers. Real evasion needs a TLS-impersonating client
+  (`curl_cffi`) or a headless browser — deliberately out of scope here.
+
+---
+
+## Tests
+
+Framework-native tests live in `crm/domain_enrichment/tests/` and are discoverable by
+`bench run-tests --app crm`:
+
+- **Pure unit (`frappe.tests.UnitTestCase`, no DB, offline fixtures):**
+  `test_crawler.py`, `test_extractors.py`, `test_pipeline.py`, `test_ssrf.py`. The
+  rule executors are exercised with in-memory `Rule` objects (`tests/fixtures.py`);
+  the SSRF guard mocks `socket.getaddrinfo` so DNS is deterministic and offline.
+- **Integration (`frappe.tests.IntegrationTestCase`, real test DB):**
+  `test_integration.py` — seeder idempotency, mapper write-policies, the Run writer,
+  API permission/allow-list enforcement, and the link-time Org→Lead/Deal copy. The
+  crawl is monkeypatched to a canned `EnrichmentResult`, so the suite is network-free.
+
+```bash
+bench --site <site> run-tests --app crm --module crm.domain_enrichment.tests.test_extractors
+bench --site <site> run-tests --app crm --module crm.domain_enrichment.tests.test_integration
+```

crm/domain_enrichment/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Domain Enrichment engine.
+
+Config-driven, Frappe-idiomatic feature that enriches CRM Lead/Deal/Organization
+records from their website domain. Configuration (rules, field mappings, crawl
+limits, sources) lives in desk doctypes; this package is the rule-agnostic engine.
+"""

crm/domain_enrichment/api.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Whitelisted entry points: ``enrich`` (enqueue a full run for a record) and
+``retry`` (re-run one from its Run history).
+
+Both are type-annotated -- hooks.py sets ``require_type_annotated_api_methods``.
+Security: each enforces the doctype allow-list (from Settings) + a ``write``
+permission check, and is rate-limited per user (see ``ENRICH_RATE_LIMIT``).
+"""
+
+from __future__ import annotations
+
+import frappe
+from frappe import _
+from frappe.rate_limiter import rate_limit
+
+from .config import ENRICHABLE_DOCTYPES, EnrichmentConfig, get_config
+from .tasks import enqueue_enrichment
+
+# Per-user, per-minute cap on each enrich entry point (each gets its own bucket).
+# Both only enqueue a per-doc-deduplicated job -- 10/min is far above any real human
+# burst while capping scripted queue-flooding.
+ENRICH_RATE_LIMIT = 10
+
+
+def _enabled_doctypes(cfg: EnrichmentConfig) -> list[str]:
+	"""The doctypes enrichment is enabled for, per Settings (allow-list)."""
+	if not cfg.setting("enabled"):
+		return []
+	return list(ENRICHABLE_DOCTYPES)
+
+
+def _enqueue_run(cfg, reference_doctype: str, reference_name: str, website: str) -> dict:
+	"""Validate + enqueue one enrichment run for a target record. Shared by
+	``enrich`` (initial trigger from the record) and ``retry`` (re-run from a Run).
+
+	Caller must have already resolved the website. Enforces the Settings allow-list
+	and ``write`` permission on the target. Uses the per-doc ``job_id`` +
+	``deduplicate`` so a second trigger while one is in-flight is a no-op.
+	"""
+	if reference_doctype not in _enabled_doctypes(cfg):
+		frappe.throw(
+			_("Enrichment is not enabled for {0}.").format(reference_doctype),
+			frappe.ValidationError,
+		)
+
+	doc = frappe.get_doc(reference_doctype, reference_name)
+	doc.check_permission("write")
+
+	website = (website or "").strip()
+	if not website:
+		frappe.throw(_("Set a website on this record before enriching."), frappe.ValidationError)
+
+	return enqueue_enrichment(reference_doctype, reference_name, website, frappe.session.user)
+
+
+@frappe.whitelist()
+@rate_limit(limit=ENRICH_RATE_LIMIT, seconds=60)
+def enrich(reference_doctype: str, reference_name: str) -> dict:
+	"""Enqueue a full enrichment run for one CRM record, using the record's own
+	``website`` field. The initial trigger (the "Enrich from Website" button).
+
+	Returns ``{queued: bool, job_id: str, website: str}``.
+	"""
+	cfg = get_config()
+	doc = frappe.get_doc(reference_doctype, reference_name)
+	website = (doc.get("website") or "").strip()
+	return _enqueue_run(cfg, reference_doctype, reference_name, website)
+
+
+@frappe.whitelist()
+@rate_limit(limit=ENRICH_RATE_LIMIT, seconds=60)
+def retry(run: str) -> dict:
+	"""Re-run the enrichment recorded by a ``CRM Enrichment Run`` (the desk "Retry"
+	button on each run). Re-enriches the run's linked record, preferring the record's
+	current ``website`` and falling back to the website this run originally scraped.
+
+	Returns ``{queued: bool, job_id: str, website: str}``.
+	"""
+	run_doc = frappe.get_doc("CRM Enrichment Run", run)
+	if not run_doc.reference_doctype or not run_doc.reference_name:
+		frappe.throw(_("This run has no linked record to re-enrich."), frappe.ValidationError)
+
+	cfg = get_config()
+	target = frappe.get_doc(run_doc.reference_doctype, run_doc.reference_name)
+	website = (target.get("website") or "").strip() or (run_doc.source_website or "").strip()
+	return _enqueue_run(cfg, run_doc.reference_doctype, run_doc.reference_name, website)

crm/domain_enrichment/config.py

@@ -0,0 +1,261 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Loads enrichment configuration (Settings + Rules + Field Mappings) from the desk.
+
+The engine is rule-agnostic: it loads admin-edited config here and executes it.
+``get_config()`` assembles an ``EnrichmentConfig`` from the config doctypes on demand
+(not cached -- see the note on ``get_config``). Hot paths that only need the Settings
+toggles use ``get_settings`` / ``auto_enrich_enabled_for`` instead of the full build.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+import frappe
+
+# The doctypes enrichment runs on. Single source of truth shared by the manual (api)
+# and auto-enrich (tasks) paths, and the allow-list the whitelisted API checks a
+# caller-supplied doctype against.
+ENRICHABLE_DOCTYPES = ("CRM Lead", "CRM Deal", "CRM Organization")
+
+
+# Sensible fallbacks applied when the Single doctype has not been saved yet (the
+# JSON field defaults only populate a freshly-created row, which may not exist).
+DEFAULT_SETTINGS = {
+	"enabled": 1,
+	"auto_enrich": 0,
+	"max_pages": 10,
+	"max_depth": 2,
+	"use_sitemap": 1,
+	"request_timeout": 10,
+	"max_download_bytes": 3_000_000,
+	"retry_count": 2,
+	"user_agent": (
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+		"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
+	),
+}
+
+
+def _setting(doc, key):
+	"""One Settings field, falling back to DEFAULT_SETTINGS when unset."""
+	val = doc.get(key)
+	return DEFAULT_SETTINGS.get(key) if val in (None, "") else val
+
+
+def get_settings():
+	"""The CRM Enrichment Settings Single, framework-cached via ``get_cached_doc``.
+
+	Cheap: reads only the Single, never assembling Rules/Mappings. For hot paths --
+	the per-insert auto-enrich flag check and the enqueue timeout -- that must not
+	build the full config on every record creation.
+	"""
+	return frappe.get_cached_doc("CRM Enrichment Settings")
+
+
+def auto_enrich_enabled_for(doctype: str) -> bool:
+	"""True if auto-enrich-on-create should fire for ``doctype``. A cheap,
+	Settings-only check (feature enabled + auto_enrich on + an enrichable doctype) --
+	no Rules/Mappings assembled."""
+	s = get_settings()
+	return bool(_setting(s, "enabled") and _setting(s, "auto_enrich") and doctype in ENRICHABLE_DOCTYPES)
+
+
+# Industry classifier thresholds. These are mechanics (how confident the winner
+# must be), not tunable knowledge, so they stay constants. If an admin ever needs
+# to tune them they can be promoted to Settings fields without touching callers.
+INDUSTRY_MIN_SCORE = 2  # at least one headline keyword hit (weighted)
+INDUSTRY_MIN_CONFIDENCE = 0.34  # the winner must clearly lead
+
+# Default link-priority ordering (BFS) -- seeded into Settings.link_priority_order
+# on install, and used as a fallback when that table is empty.
+DEFAULT_LINK_PRIORITY = [
+	("about", 1.0),
+	("contact", 1.0),
+	("team", 1.0),
+	("leadership", 1.0),
+	("careers", 1.0),
+	("company", 1.0),
+	("people", 1.0),
+	("founders", 1.0),
+	("management", 1.0),
+	("jobs", 1.0),
+]
+
+
+@dataclass
+class Rule:
+	"""A single config-driven rule with its compiled patterns.
+
+	``patterns`` is a list of (compiled_regex, raw_pattern) tuples. Substring
+	patterns (is_regex=0) are compiled as escaped, case-insensitive regexes so the
+	executors can use one uniform match path.
+	"""
+
+	rule_type: str
+	target_value: str = ""
+	industry: str = ""
+	weight: float = 1.0
+	match_scope: str = "Full Text"
+	patterns: list = field(default_factory=list)
+
+	@property
+	def label(self) -> str:
+		"""The value a match yields: industry name for Industry rules, else
+		target_value (tech name / network)."""
+		return self.industry if self.rule_type == "Industry" else self.target_value
+
+	def matches(self, text: str) -> int:
+		"""Total number of pattern hits across this rule's patterns in ``text``."""
+		if not text:
+			return 0
+		return sum(len(rx.findall(text)) for rx, _raw in self.patterns)
+
+
+@dataclass
+class Mapping:
+	source_key: str
+	target_doctype: str
+	target_fieldname: str
+	write_policy: str = "Fill if empty"
+	create_missing_link: int = 0
+	default_values: str = ""
+
+
+@dataclass
+class EnrichmentConfig:
+	settings: dict = field(default_factory=dict)
+	rules_by_type: dict = field(default_factory=dict)  # rule_type -> list[Rule]
+	mappings_by_doctype: dict = field(default_factory=dict)  # doctype -> list[Mapping]
+	link_priority: list = field(default_factory=list)  # list[(keyword, weight)]
+	skip_patterns: list = field(default_factory=list)  # list[str]
+	allowed_domains: list = field(default_factory=list)
+	blocked_domains: list = field(default_factory=list)
+
+	# Convenience accessors with fallbacks --------------------------------- #
+	def setting(self, key, default=None):
+		val = self.settings.get(key)
+		if val in (None, ""):
+			return DEFAULT_SETTINGS.get(key, default)
+		return val
+
+	def rules(self, rule_type: str) -> list:
+		return self.rules_by_type.get(rule_type, [])
+
+
+def _compile_pattern(pattern: str, is_regex: int):
+	try:
+		if is_regex:
+			return re.compile(pattern, re.IGNORECASE)
+		# Word-boundary-anchor substring patterns: an un-anchored short keyword like
+		# "erp" or "api" matches inside completely unrelated words ("waterproofing",
+		# "capital"), causing false industry-classification hits. An admin who wants
+		# unanchored substring matching can still opt in via is_regex=1.
+		return re.compile(rf"\b{re.escape(pattern)}\b", re.IGNORECASE)
+	except re.error:
+		# A malformed admin-entered regex must not break the whole config build.
+		frappe.log_error(
+			title="Domain Enrichment: invalid rule pattern",
+			message=f"Could not compile pattern: {pattern!r}",
+		)
+		return None
+
+
+def _build_rules() -> dict:
+	rules_by_type: dict = {}
+	names = frappe.get_all("CRM Enrichment Rule", filters={"enabled": 1}, pluck="name")
+	for name in names:
+		doc = frappe.get_doc("CRM Enrichment Rule", name)
+		compiled = []
+		for pat in doc.patterns or []:
+			rx = _compile_pattern(pat.pattern, pat.is_regex)
+			if rx is not None:
+				compiled.append((rx, pat.pattern))
+		if not compiled:
+			continue
+		rule = Rule(
+			rule_type=doc.rule_type,
+			target_value=doc.target_value or "",
+			industry=doc.industry or "",
+			weight=doc.weight or 1.0,
+			match_scope=doc.match_scope or "Full Text",
+			patterns=compiled,
+		)
+		rules_by_type.setdefault(doc.rule_type, []).append(rule)
+	return rules_by_type
+
+
+def _build_mappings() -> dict:
+	mappings_by_doctype: dict = {}
+	rows = frappe.get_all(
+		"CRM Enrichment Field Mapping",
+		filters={"enabled": 1},
+		fields=[
+			"source_key",
+			"target_doctype",
+			"target_fieldname",
+			"write_policy",
+			"create_missing_link",
+			"default_values",
+		],
+	)
+	for row in rows:
+		mapping = Mapping(
+			source_key=row.source_key,
+			target_doctype=row.target_doctype,
+			target_fieldname=row.target_fieldname,
+			write_policy=row.write_policy or "Fill if empty",
+			create_missing_link=row.create_missing_link or 0,
+			default_values=row.default_values or "",
+		)
+		mappings_by_doctype.setdefault(row.target_doctype, []).append(mapping)
+	return mappings_by_doctype
+
+
+def _build_settings(settings_doc) -> dict:
+	out = {}
+	for key in DEFAULT_SETTINGS:
+		out[key] = settings_doc.get(key)
+	return out
+
+
+def _build_config() -> EnrichmentConfig:
+	settings_doc = frappe.get_cached_doc("CRM Enrichment Settings")
+
+	link_priority = [
+		(kw.keyword.lower(), kw.weight or 1.0)
+		for kw in (settings_doc.link_priority_order or [])
+		if kw.keyword
+	] or list(DEFAULT_LINK_PRIORITY)
+
+	skip_patterns = [sp.pattern for sp in (settings_doc.skip_patterns or []) if sp.pattern]
+
+	allowed_domains = [d.domain.lower().strip() for d in (settings_doc.allowed_domains or []) if d.domain]
+	blocked_domains = [d.domain.lower().strip() for d in (settings_doc.blocked_domains or []) if d.domain]
+
+	return EnrichmentConfig(
+		settings=_build_settings(settings_doc),
+		rules_by_type=_build_rules(),
+		mappings_by_doctype=_build_mappings(),
+		link_priority=link_priority,
+		skip_patterns=skip_patterns,
+		allowed_domains=allowed_domains,
+		blocked_domains=blocked_domains,
+	)
+
+
+def get_config() -> EnrichmentConfig:
+	"""Assemble the full enrichment config (Settings + Rules + Field Mappings) fresh.
+
+	Deliberately NOT cached. It is built at most a couple of times per enrichment run,
+	and a run is a multi-second, network-bound crawl -- so the handful of small queries
+	here is noise. A cache, by contrast, buys nothing measurable and adds a real
+	stale-config surface (any write that bypasses ``on_update`` -- a bulk ``db_set``,
+	direct SQL -- would serve stale rules until a restart). The per-insert hot path does
+	NOT call this; it uses ``auto_enrich_enabled_for`` (Settings-only). Do not re-add a
+	cache here without a measured hot-path caller that justifies it.
+	"""
+	return _build_config()

crm/domain_enrichment/crawler.py

@@ -0,0 +1,449 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Same-domain, depth-limited BFS crawler. Caps, priorities and skip-patterns
+come from config (Settings child tables), not module constants.
+
+Exposes:
+    crawl()           -> ranked list of (CrawledPage, soup) tuples, homepage first
+    crawl_page()      -> fetch + parse a single page into a CrawledPage
+    extract_content() -> pull title/headings/visible text from soup
+"""
+
+from __future__ import annotations
+
+import re
+from collections import deque
+from functools import lru_cache
+from urllib.parse import urldefrag, urljoin, urlparse
+from urllib.robotparser import RobotFileParser
+from xml.etree import ElementTree as ET
+
+import tldextract
+from bs4 import BeautifulSoup
+
+from .http import build_session, fetch
+from .result import CrawledPage
+
+SKIP_EXTENSIONS = (
+	".pdf",
+	".jpg",
+	".jpeg",
+	".png",
+	".gif",
+	".svg",
+	".webp",
+	".ico",
+	".css",
+	".js",
+	".zip",
+	".gz",
+	".mp4",
+	".mp3",
+	".avi",
+	".woff",
+	".woff2",
+	".ttf",
+	".eot",
+	".xml",
+	".json",
+	".rss",
+)
+SKIP_SCHEMES = ("mailto:", "tel:", "javascript:", "data:", "#")
+
+
+def normalize_url(url: str) -> str:
+	"""Query strings are dropped, not canonicalized -- almost always tracking
+	noise, not a distinct page identity, for the pages this crawler cares about."""
+	url, _frag = urldefrag(url)
+	parsed = urlparse(url)
+	if parsed.query:
+		url = parsed._replace(query="").geturl()
+	if url.endswith("/") and len(urlparse(url).path) > 1:
+		url = url.rstrip("/")
+	return url
+
+
+@lru_cache(maxsize=1)
+def _tld_extractor() -> tldextract.TLDExtract:
+	"""Pinned to the bundled PSL snapshot (no network fetch); private suffixes like
+	``github.io``/``vercel.app`` stay registrable so distinct tenants count as distinct sites."""
+	return tldextract.TLDExtract(
+		suffix_list_urls=(),
+		cache_dir=None,
+		include_psl_private_domains=True,
+	)
+
+
+def registrable_domain(netloc: str) -> str:
+	"""``www.acme.co.uk`` -> ``acme.co.uk`` -- avoids a naive last-two-labels rule
+	collapsing all of ``*.co.uk`` into one same-site bucket."""
+	netloc = netloc.lower().split(":")[0]
+	extracted = _tld_extractor()(netloc)
+	if extracted.domain and extracted.suffix:
+		return f"{extracted.domain}.{extracted.suffix}"
+	return extracted.domain or netloc
+
+
+def same_site(url: str, base_netloc: str) -> bool:
+	try:
+		return registrable_domain(urlparse(url).netloc) == registrable_domain(base_netloc)
+	except Exception:
+		return False
+
+
+def _link_priority(url: str, anchor_text: str, link_priority: list) -> float:
+	"""Lower number = crawl sooner."""
+	path = urlparse(url).path.rstrip("/")
+	leaf = path.rsplit("/", 1)[-1] if path else ""
+	haystack = (leaf + " " + (anchor_text or "")).lower()
+	best = None
+	for idx, (kw, weight) in enumerate(link_priority):
+		if kw and kw in haystack:
+			# Higher weight -> earlier; ties broken by config order.
+			score = idx - (weight or 1.0)
+			if best is None or score < best:
+				best = score
+	return best if best is not None else len(link_priority) + 1
+
+
+def _is_crawlable(url: str, skip_patterns: list) -> bool:
+	low = url.lower()
+	if any(low.startswith(s) for s in SKIP_SCHEMES):
+		return False
+	path = urlparse(url).path.lower()
+	if path.endswith(SKIP_EXTENSIONS):
+		return False
+	if urlparse(url).scheme not in ("http", "https"):
+		return False
+	for pat in skip_patterns:
+		try:
+			if re.search(pat, url, re.IGNORECASE):
+				return False
+		except re.error:
+			if pat.lower() in low:
+				return False
+	return True
+
+
+def extract_content(soup: BeautifulSoup) -> dict:
+	"""Mutates ``soup`` (decomposes script/style) -- pass a throwaway copy if the
+	caller needs the original DOM afterwards."""
+	title = ""
+	if soup.title and soup.title.string:
+		title = soup.title.string.strip()
+
+	headings = []
+	for tag in soup.find_all(["h1", "h2", "h3"]):
+		txt = tag.get_text(" ", strip=True)
+		if txt:
+			headings.append(txt)
+
+	for node in soup(["script", "style", "noscript", "template", "svg"]):
+		node.decompose()
+	text = soup.get_text(" ", strip=True)
+	text = " ".join(text.split())
+
+	return {"title": title, "headings": headings, "text": text}
+
+
+def _links_on_page(soup, page_url, base_netloc, skip_patterns):
+	for a in soup.find_all("a", href=True):
+		href = a["href"].strip()
+		if not href or any(href.lower().startswith(s) for s in SKIP_SCHEMES):
+			continue
+		absolute = normalize_url(urljoin(page_url, href))
+		if not _is_crawlable(absolute, skip_patterns):
+			continue
+		if not same_site(absolute, base_netloc):
+			continue
+		yield absolute, a.get_text(" ", strip=True)
+
+
+def _parse_and_fill(page, html):
+	"""``extract_content`` runs on a throwaway parse so the returned soup stays unmutated."""
+	soup = BeautifulSoup(html, "html.parser")
+	content = extract_content(BeautifulSoup(html, "html.parser"))
+	page.title = content["title"]
+	page.headings = content["headings"]
+	page.text = content["text"]
+	return soup
+
+
+def crawl_page(url, cfg, session=None):
+	status, html, error, final_url = fetch(url, cfg, session=session)
+	# Resolved URL: links and provenance match the host that actually served the body.
+	page = CrawledPage(url=final_url or url, status_code=status, html=html or "", error=error)
+	soup = _parse_and_fill(page, html) if (html and not error) else None
+	return page, soup
+
+
+ABOUT_PROBE_PATHS = ("/about", "/about-us", "/company", "/our-story")
+ABOUT_MIN_TEXT = 200
+
+
+def load_robots(start_url, cfg, session=None):
+	"""Returns ``None`` when absent/unreadable (convention: allow all)."""
+	try:
+		parsed = urlparse(normalize_url(start_url))
+		if not parsed.netloc:
+			return None
+		robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+		own = session is None
+		session = session or build_session(cfg)
+		try:
+			status, body, error, _final = fetch(robots_url, cfg, session=session, html_only=False)
+		finally:
+			if own:
+				session.close()
+		if error or status != 200 or not body:
+			return None
+		rp = RobotFileParser()
+		rp.parse(body.splitlines())
+		return rp
+	except Exception:
+		return None
+
+
+def _robots_allows(robots, user_agent, url):
+	if robots is None:
+		return True
+	try:
+		return robots.can_fetch(user_agent or "*", url)
+	except Exception:
+		return True
+
+
+def probe_about_pages(home_url, cfg, session=None, skip_urls=()):
+	skip = {normalize_url(u) for u in skip_urls}
+	user_agent = cfg.setting("user_agent")
+	own_session = session is None
+	session = session or build_session(cfg)
+	robots = load_robots(home_url, cfg, session)
+	try:
+		for path in ABOUT_PROBE_PATHS:
+			url = normalize_url(urljoin(home_url, path))
+			if url in skip:
+				continue
+			skip.add(url)
+			if not _robots_allows(robots, user_agent, url):
+				continue
+			page, soup = crawl_page(url, cfg, session=session)
+			if (
+				soup is not None
+				and page.status_code == 200
+				and not page.error
+				and len(page.text) >= ABOUT_MIN_TEXT
+			):
+				return [(page, soup)]
+	finally:
+		if own_session:
+			session.close()
+	return []
+
+
+# --------------------------------------------------------------------------- #
+# Sitemap discovery
+# --------------------------------------------------------------------------- #
+# Additive on top of link-following BFS: seeds pages the sitemap author flagged.
+SITEMAP_MAX_SITEMAPS = 5  # sitemap files fetched; guards pathological sitemap indexes
+SITEMAP_MAX_URLS = 200  # total <loc> entries collected across all fetched sitemaps
+
+
+def _local_tag(tag: str) -> str:
+	"""Strip the XML namespace off a tag: '{http://.../0.9}urlset' -> 'urlset'."""
+	return tag.rsplit("}", 1)[-1] if tag else tag
+
+
+def _parse_sitemap(xml_text: str):
+	"""Returns ('urlset' | 'sitemapindex' | None, [loc, ...]); never raises.
+
+	Rejects any DOCTYPE before parsing as an XML entity-expansion guard -- a
+	legitimate sitemap never has one.
+	"""
+	if "<!doctype" in xml_text.lower():
+		return None, []
+	try:
+		root = ET.fromstring(xml_text)
+	except ET.ParseError:
+		return None, []
+	kind = _local_tag(root.tag)
+	if kind not in ("urlset", "sitemapindex"):
+		return None, []
+	child_tag = "sitemap" if kind == "sitemapindex" else "url"
+	locs = []
+	for el in root:
+		if _local_tag(el.tag) != child_tag:
+			continue
+		for child in el:
+			if _local_tag(child.tag) == "loc" and child.text:
+				locs.append(child.text.strip())
+				break
+	return kind, locs
+
+
+def discover_sitemap_urls(start_url, cfg, session, robots):
+	"""Falls back from robots.txt ``Sitemap:`` to ``/sitemap.xml``, bounded by
+	``SITEMAP_MAX_SITEMAPS``/``SITEMAP_MAX_URLS``."""
+	try:
+		parsed = urlparse(normalize_url(start_url))
+		if not parsed.netloc:
+			return []
+
+		candidates = []
+		if robots is not None:
+			try:
+				candidates.extend(robots.site_maps() or [])
+			except Exception:
+				pass
+		if not candidates:
+			candidates.append(f"{parsed.scheme}://{parsed.netloc}/sitemap.xml")
+
+		to_fetch = deque(candidates[:SITEMAP_MAX_SITEMAPS])
+		sitemaps_queued = len(to_fetch)  # running total: seed + discovered children
+		fetched_sitemaps = set()
+		urls = []
+		seen_urls = set()
+
+		while to_fetch and len(fetched_sitemaps) < SITEMAP_MAX_SITEMAPS and len(urls) < SITEMAP_MAX_URLS:
+			sitemap_url = to_fetch.popleft()
+			if sitemap_url in fetched_sitemaps:
+				continue
+			fetched_sitemaps.add(sitemap_url)
+			if not same_site(sitemap_url, parsed.netloc):
+				continue
+
+			status, body, error, _final = fetch(sitemap_url, cfg, session=session, html_only=False)
+			if error or status != 200 or not body:
+				continue
+
+			kind, locs = _parse_sitemap(body)
+
+			if kind == "sitemapindex":
+				for loc in locs:
+					if sitemaps_queued >= SITEMAP_MAX_SITEMAPS:
+						break
+					to_fetch.append(loc)
+					sitemaps_queued += 1
+
+			elif kind == "urlset":
+				for loc in locs:
+					norm = normalize_url(loc)
+					if norm in seen_urls:
+						continue
+					seen_urls.add(norm)
+					urls.append(norm)
+					if len(urls) >= SITEMAP_MAX_URLS:
+						break
+
+		return urls
+	except Exception:
+		return []
+
+
+def crawl(start_url, cfg, session=None, progress=None):
+	max_pages = int(cfg.setting("max_pages"))
+	max_depth = int(cfg.setting("max_depth"))
+	link_priority = cfg.link_priority
+	skip_patterns = cfg.skip_patterns
+
+	user_agent = cfg.setting("user_agent")
+	own_session = session is None
+	session = session or build_session(cfg)
+	start_url = normalize_url(start_url)
+	base_netloc = urlparse(start_url).netloc
+
+	# robots.txt only gates discovered links; skip the fetch for homepage-only runs.
+	crawl_beyond_homepage = max_depth >= 1 and max_pages > 1
+	robots = load_robots(start_url, cfg, session) if crawl_beyond_homepage else None
+
+	visited = set()
+	# Resolved (post-redirect) URLs already crawled -- catches a duplicate the
+	# pre-fetch `visited` check can't (e.g. /contact.html -> /contact).
+	visited_resolved = set()
+	results = []
+	# Queue holds (url, depth, anchor_text).
+	queue = deque([(start_url, 0, "")])
+
+	# Sitemap-seeded links join at depth 1, never 0, so the homepage still wins
+	# the first pop. Additive only -- a site with no sitemap is unaffected.
+	if crawl_beyond_homepage and cfg.setting("use_sitemap", True):
+		queued_from_sitemap = set()
+		for loc in discover_sitemap_urls(start_url, cfg, session, robots):
+			link = normalize_url(loc)
+			if link == start_url or link in queued_from_sitemap:
+				continue
+			if not same_site(link, base_netloc) or not _is_crawlable(link, skip_patterns):
+				continue
+			if not _robots_allows(robots, user_agent, link):
+				continue
+			queued_from_sitemap.add(link)
+			queue.append((link, 1, ""))
+
+	try:
+		while queue and len(results) < max_pages:
+			# Pick the highest-priority queued URL (homepage first, then by keyword).
+			queue = deque(
+				sorted(
+					queue,
+					key=lambda item: (
+						0 if item[1] == 0 else 1,
+						_link_priority(item[0], item[2], link_priority),
+					),
+				)
+			)
+			url, depth, _anchor = queue.popleft()
+			url = normalize_url(url)
+			if url in visited:
+				continue
+			visited.add(url)
+
+			if progress:
+				progress(f"Crawling {url}")
+			page, soup = crawl_page(url, cfg, session=session)
+			resolved = normalize_url(page.url) if page.url else ""
+
+			if not results:
+				# Homepage: can't be a duplicate yet. Adopt its resolved host as the
+				# same-site base if it redirected elsewhere.
+				if resolved:
+					base_netloc = urlparse(page.url).netloc or base_netloc
+			else:
+				# A redirect can land here under a different requested URL (e.g.
+				# /contact.html -> /contact) -- drop the duplicate, no budget spent.
+				if resolved and resolved in visited_resolved:
+					continue
+				if soup is not None and page.url and not same_site(page.url, base_netloc):
+					# Same-site link redirected off-domain; fetch() only checks SSRF
+					# per hop, not same-site scope. Clear every parsed field (not just
+					# html/text) so nothing -- including title/headings, already
+					# filled in by crawl_page() -- leaks into extraction.
+					page.error = page.error or f"redirected off-site to {urlparse(page.url).netloc}"
+					page.html = ""
+					page.text = ""
+					page.title = ""
+					page.headings = []
+					soup = None
+
+			if resolved:
+				visited_resolved.add(resolved)
+
+			results.append((page, soup))
+
+			if soup is None or depth >= max_depth:
+				continue
+
+			# Resolve links against the page's resolved URL, not the requested one.
+			seen_here = set()
+			for link, anchor in _links_on_page(soup, page.url, base_netloc, skip_patterns):
+				if link in visited or link in seen_here:
+					continue
+				if not _robots_allows(robots, user_agent, link):
+					continue
+				seen_here.add(link)
+				queue.append((link, depth + 1, anchor))
+	finally:
+		if own_session:
+			session.close()
+
+	return results

crm/domain_enrichment/cross_record.py

@@ -0,0 +1,112 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Link-time Organization -> Lead/Deal enrichment copy.
+
+The ONLY cross-record behaviour in Domain Enrichment: when a Lead or Deal links an
+already-enriched ``CRM Organization``, synchronously copy that Organization's stored
+enriched fields onto the *empty* fields of the Lead/Deal (fill-empty). The direction
+is strictly Organization -> Lead/Deal, in-request, on the origin save. There is no
+background job, no fan-out, and no write back to the Organization.
+
+The actual field writing reuses :func:`crm.domain_enrichment.mapper.apply_to_document`
+-- the single result->field authority -- by wrapping the Organization's stored values
+in an :class:`~crm.domain_enrichment.result.EnrichmentResult`. No crawling happens
+here; this is a pure record-to-record copy.
+"""
+
+from __future__ import annotations
+
+import frappe
+
+from crm.domain_enrichment import mapper
+from crm.domain_enrichment.config import get_config
+from crm.domain_enrichment.result import (
+	EnrichmentResult,
+	Field,
+	SocialProfile,
+)
+
+# Native enriched fields stored on a CRM Organization. If at least one of these is
+# populated (or the org has a completed enrichment run) we treat the org as enriched
+# and worth copying from. Empty source values never overwrite anything (mapper guards).
+_ORG_ENRICHED_FIELDS = (
+	"company_description",
+	"organization_logo",
+	"industry",
+	"linkedin",
+	"twitter",
+	"facebook",
+)
+
+
+def _is_org_enriched(org) -> bool:
+	"""Has this Organization actually been enriched?
+
+	Cleanest signal: a completed ``CRM Enrichment Run`` for the org. We also accept
+	any populated native enriched field as a fallback (an org may carry enriched data
+	without a run row, e.g. seeded/imported data).
+	"""
+	if frappe.db.exists(
+		"CRM Enrichment Run",
+		{
+			"reference_doctype": "CRM Organization",
+			"reference_name": org.name,
+			"status": "Completed",
+		},
+	):
+		return True
+	return any(org.get(fieldname) for fieldname in _ORG_ENRICHED_FIELDS)
+
+
+def _result_from_organization(org) -> EnrichmentResult:
+	"""Wrap an Organization's stored enriched fields in an EnrichmentResult so the
+	shared mapper can copy them onto the target via the same Field Mapping records."""
+	result = EnrichmentResult(website=org.get("website") or "")
+	result.company_name = Field(value=org.get("organization_name") or "")
+	result.description = Field(value=org.get("company_description") or "")
+	result.logo = Field(value=org.get("organization_logo") or "")
+	result.industry = Field(value=org.get("industry") or "")
+
+	for network in ("linkedin", "twitter", "facebook"):
+		value = org.get(network)
+		if value:
+			result.social_profiles[network] = SocialProfile(value=value)
+
+	return result
+
+
+def copy_enrichment_from_organization(target_doc) -> list[str]:
+	"""Copy enriched fields from the linked CRM Organization onto ``target_doc``.
+
+	``target_doc`` is a CRM Lead or CRM Deal that has just linked an Organization
+	(Lead: ``organization`` is free-text resolved to an org name; Deal: ``organization``
+	is a Link to CRM Organization). If that Organization is actually enriched, its
+	stored enriched fields are copied onto the *empty* fields of ``target_doc`` using
+	the shared mapper with fill-empty semantics, so user-entered data is never touched.
+
+	Best-effort: never raises, never blocks the Lead/Deal save. Returns the list of
+	filled field labels (empty when nothing was copied).
+	"""
+	try:
+		org_name = target_doc.get("organization")
+		if not org_name or not frappe.db.exists("CRM Organization", org_name):
+			return []
+
+		org = frappe.get_cached_doc("CRM Organization", org_name)
+		if not _is_org_enriched(org):
+			return []
+
+		cfg = get_config()
+		result = _result_from_organization(org)
+		# Fill-empty only: force fill-empty semantics for every mapping (even ones
+		# configured "Always refresh") so the link-time copy never
+		# overwrites or clears user-entered data. apply_to_document mutates target_doc
+		# in place and does NOT save (the caller's save persists it).
+		return mapper.apply_to_document(target_doc, result, cfg, fill_empty_only=True)
+	except Exception:
+		frappe.log_error(
+			title="Domain Enrichment: Org -> Lead/Deal copy failed",
+			message=f"{target_doc.doctype} {target_doc.name}: {frappe.get_traceback()}",
+		)
+		return []

crm/domain_enrichment/doctype/crm_enrichment_domain/crm_enrichment_domain.json

@@ -0,0 +1,33 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2026-06-17 00:00:00.000000",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "domain"
+ ],
+ "fields": [
+  {
+   "fieldname": "domain",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Domain",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2026-06-17 00:00:00.000000",
+ "modified_by": "Administrator",
+ "module": "Domain Enrichment",
+ "name": "CRM Enrichment Domain",
+ "owner": "Administrator",
+ "permissions": [],
+ "row_format": "Dynamic",
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

crm/domain_enrichment/doctype/crm_enrichment_domain/crm_enrichment_domain.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class CRMEnrichmentDomain(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		domain: DF.Data
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+	# end: auto-generated types
+
+	pass

crm/domain_enrichment/doctype/crm_enrichment_field_mapping/crm_enrichment_field_mapping.json

@@ -0,0 +1,125 @@
+{
+ "actions": [],
+ "allow_import": 1,
+ "allow_rename": 1,
+ "creation": "2026-06-17 00:00:00.000000",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "enabled",
+  "source_key",
+  "column_break_main",
+  "target_doctype",
+  "target_fieldname",
+  "policy_section",
+  "write_policy",
+  "create_missing_link",
+  "default_values"
+ ],
+ "fields": [
+  {
+   "default": "1",
+   "fieldname": "enabled",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Enabled"
+  },
+  {
+   "description": "The enrichment result key to read the value from.",
+   "fieldname": "source_key",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Source Key",
+   "options": "company_name\ndescription\nlogo\nindustry\nprimary_email\nprimary_phone\nsecondary_phone\nlinkedin\ntwitter\ngithub\nfacebook\ninstagram\nyoutube",
+   "reqd": 1
+  },
+  {
+   "fieldname": "column_break_main",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "target_doctype",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Target Doctype",
+   "options": "DocType",
+   "reqd": 1
+  },
+  {
+   "description": "The fieldname on the target doctype to write the value to.",
+   "fieldname": "target_fieldname",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Target Fieldname",
+   "reqd": 1
+  },
+  {
+   "fieldname": "policy_section",
+   "fieldtype": "Section Break",
+   "label": "Write Policy"
+  },
+  {
+   "default": "Fill if empty",
+   "description": "Fill if empty: only write when the field is blank. Always refresh: overwrite on every run (for fully enrichment-derived fields). Override defaults: write even when the current value is a known default (see Default Values).",
+   "fieldname": "write_policy",
+   "fieldtype": "Select",
+   "label": "Write Policy",
+   "options": "Fill if empty\nAlways refresh\nOverride defaults",
+   "reqd": 1
+  },
+  {
+   "default": "0",
+   "description": "If the value names a missing linked record (e.g. a CRM Industry), create it. Opt-in only.",
+   "fieldname": "create_missing_link",
+   "fieldtype": "Check",
+   "label": "Create Missing Link"
+  },
+  {
+   "depends_on": "eval:doc.write_policy=='Override defaults'",
+   "description": "Newline-separated values treated as \"unset\" defaults that may be overwritten, in addition to a truly empty field.",
+   "fieldname": "default_values",
+   "fieldtype": "Small Text",
+   "label": "Default Values"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2026-06-17 00:00:00.000000",
+ "modified_by": "Administrator",
+ "module": "Domain Enrichment",
+ "name": "CRM Enrichment Field Mapping",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Sales Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "row_format": "Dynamic",
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

crm/domain_enrichment/doctype/crm_enrichment_field_mapping/crm_enrichment_field_mapping.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+import frappe
+from frappe import _
+from frappe.model.document import Document
+
+# Field Mapping only targets the three app-owned CRM doctypes that can be enriched.
+ALLOWED_TARGET_DOCTYPES = ("CRM Lead", "CRM Deal", "CRM Organization")
+
+
+class CRMEnrichmentFieldMapping(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		create_missing_link: DF.Check
+		default_values: DF.SmallText | None
+		enabled: DF.Check
+		source_key: DF.Literal[
+			"company_name",
+			"description",
+			"logo",
+			"industry",
+			"primary_email",
+			"primary_phone",
+			"secondary_phone",
+			"linkedin",
+			"twitter",
+			"github",
+			"facebook",
+			"instagram",
+			"youtube",
+		]
+		target_doctype: DF.Link
+		target_fieldname: DF.Data
+		write_policy: DF.Literal["Fill if empty", "Always refresh", "Override defaults"]
+	# end: auto-generated types
+
+	def validate(self):
+		if self.target_doctype not in ALLOWED_TARGET_DOCTYPES:
+			frappe.throw(_("Target Doctype must be one of: {0}").format(", ".join(ALLOWED_TARGET_DOCTYPES)))
+		if not frappe.get_meta(self.target_doctype).has_field(self.target_fieldname):
+			frappe.throw(_("{0} has no field {1}").format(self.target_doctype, self.target_fieldname))

crm/domain_enrichment/doctype/crm_enrichment_link_priority/crm_enrichment_link_priority.json

@@ -0,0 +1,42 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2026-06-17 00:00:00.000000",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "keyword",
+  "weight"
+ ],
+ "fields": [
+  {
+   "description": "Substring matched against a candidate URL/anchor to prioritise crawling it (e.g. \"about\", \"team\", \"contact\").",
+   "fieldname": "keyword",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Keyword",
+   "reqd": 1
+  },
+  {
+   "default": "1",
+   "fieldname": "weight",
+   "fieldtype": "Float",
+   "in_list_view": 1,
+   "label": "Weight"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2026-06-17 00:00:00.000000",
+ "modified_by": "Administrator",
+ "module": "Domain Enrichment",
+ "name": "CRM Enrichment Link Priority",
+ "owner": "Administrator",
+ "permissions": [],
+ "row_format": "Dynamic",
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

crm/domain_enrichment/doctype/crm_enrichment_link_priority/crm_enrichment_link_priority.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class CRMEnrichmentLinkPriority(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		keyword: DF.Data
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+		weight: DF.Float
+	# end: auto-generated types
+
+	pass

crm/domain_enrichment/doctype/crm_enrichment_rule/crm_enrichment_rule.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+// frappe.ui.form.on("CRM Enrichment Rule", {
+// 	refresh(frm) {
+
+// 	},
+// });

crm/domain_enrichment/doctype/crm_enrichment_rule/crm_enrichment_rule.json

@@ -0,0 +1,131 @@
+{
+ "actions": [],
+ "allow_import": 1,
+ "allow_rename": 1,
+ "autoname": "{rule_type}.-.####",
+ "creation": "2026-06-17 00:00:00",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "rule_name",
+  "rule_type",
+  "enabled",
+  "column_break_main",
+  "target_value",
+  "industry",
+  "weight",
+  "match_scope",
+  "patterns_section",
+  "patterns"
+ ],
+ "fields": [
+  {
+   "fieldname": "rule_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Rule Name",
+   "reqd": 1,
+   "unique": 1
+  },
+  {
+   "fieldname": "rule_type",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Rule Type",
+   "options": "Industry\nSocial",
+   "reqd": 1
+  },
+  {
+   "default": "1",
+   "fieldname": "enabled",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Enabled"
+  },
+  {
+   "fieldname": "column_break_main",
+   "fieldtype": "Column Break"
+  },
+  {
+   "description": "The value this rule resolves to when it matches (e.g. the social network \"linkedin\"). For Industry rules, use the Industry link instead.",
+   "fieldname": "target_value",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Target Value"
+  },
+  {
+   "depends_on": "eval:doc.rule_type=='Industry'",
+   "fieldname": "industry",
+   "fieldtype": "Link",
+   "label": "Industry",
+   "options": "CRM Industry"
+  },
+  {
+   "default": "1",
+   "description": "Score contributed per matched pattern (Industry classification scoring).",
+   "fieldname": "weight",
+   "fieldtype": "Float",
+   "label": "Weight"
+  },
+  {
+   "default": "Full Text",
+   "description": "Where the patterns are matched.",
+   "fieldname": "match_scope",
+   "fieldtype": "Select",
+   "label": "Match Scope",
+   "options": "Headline\nFull Text\nHTML\nHeaders\nURL"
+  },
+  {
+   "fieldname": "patterns_section",
+   "fieldtype": "Section Break",
+   "label": "Patterns"
+  },
+  {
+   "fieldname": "patterns",
+   "fieldtype": "Table",
+   "label": "Patterns",
+   "options": "CRM Enrichment Rule Pattern"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2026-06-19 14:45:44.721670",
+ "modified_by": "Administrator",
+ "module": "Domain Enrichment",
+ "name": "CRM Enrichment Rule",
+ "naming_rule": "Expression",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Sales Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "row_format": "Dynamic",
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

crm/domain_enrichment/doctype/crm_enrichment_rule/crm_enrichment_rule.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class CRMEnrichmentRule(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		from crm.domain_enrichment.doctype.crm_enrichment_rule_pattern.crm_enrichment_rule_pattern import (
+			CRMEnrichmentRulePattern,
+		)
+
+		enabled: DF.Check
+		industry: DF.Link | None
+		match_scope: DF.Literal["Headline", "Full Text", "HTML", "Headers", "URL"]
+		patterns: DF.Table[CRMEnrichmentRulePattern]
+		rule_name: DF.Data
+		rule_type: DF.Literal["Industry", "Social"]
+		target_value: DF.Data | None
+		weight: DF.Float
+	# end: auto-generated types
+
+	pass

crm/domain_enrichment/doctype/crm_enrichment_rule/test_crm_enrichment_rule.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests import IntegrationTestCase
+
+# On IntegrationTestCase, the doctype test records and all
+# link-field test record dependencies are recursively loaded
+# Use these module variables to add/remove to/from that list
+EXTRA_TEST_RECORD_DEPENDENCIES = []  # eg. ["User"]
+IGNORE_TEST_RECORD_DEPENDENCIES = []  # eg. ["User"]
+
+
+class IntegrationTestCRMEnrichmentRule(IntegrationTestCase):
+	"""
+	Integration tests for CRMEnrichmentRule.
+	Use this class for testing interactions between multiple components.
+	"""
+
+	pass

crm/domain_enrichment/doctype/crm_enrichment_rule_pattern/crm_enrichment_rule_pattern.json

@@ -0,0 +1,43 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2026-06-17 00:00:00.000000",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "pattern",
+  "is_regex"
+ ],
+ "fields": [
+  {
+   "description": "A substring (default) or a regular expression matched against the rule's match scope.",
+   "fieldname": "pattern",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Pattern",
+   "reqd": 1
+  },
+  {
+   "default": "0",
+   "description": "Treat the pattern as a regular expression instead of a plain substring.",
+   "fieldname": "is_regex",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Is Regex"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2026-06-17 00:00:00.000000",
+ "modified_by": "Administrator",
+ "module": "Domain Enrichment",
+ "name": "CRM Enrichment Rule Pattern",
+ "owner": "Administrator",
+ "permissions": [],
+ "row_format": "Dynamic",
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

crm/domain_enrichment/doctype/crm_enrichment_rule_pattern/crm_enrichment_rule_pattern.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class CRMEnrichmentRulePattern(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		is_regex: DF.Check
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+		pattern: DF.Data
+	# end: auto-generated types
+
+	pass

crm/domain_enrichment/doctype/crm_enrichment_run/crm_enrichment_run.js

@@ -0,0 +1,64 @@
+// Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+// CRM Enrichment Run is a read-only audit log -- it is never created by hand.
+// Each run gets a "Retry" button that re-runs the enrichment for its linked
+// Lead / Deal / Organization. A retry produces a NEW run row (the log is
+// append-only), so on completion we point the user at the fresh run.
+
+frappe.ui.form.on("CRM Enrichment Run", {
+	refresh(frm) {
+		if (frm.is_new()) return;
+		if (!frm.doc.reference_doctype || !frm.doc.reference_name) return;
+
+		frm.add_custom_button(__("Retry Enrichment"), () => {
+			crm_enrichment_run_retry(frm);
+		});
+	},
+});
+
+function crm_enrichment_run_retry(frm) {
+	const total = 7; // pipeline.PROGRESS_STEPS
+	frappe.show_progress(__("Domain Enrichment"), 0, total, __("Queued…"));
+
+	const event = "domain_enrichment_progress";
+	const handler = (data) => {
+		if (
+			!data ||
+			data.reference_doctype !== frm.doc.reference_doctype ||
+			data.reference_name !== frm.doc.reference_name
+		) {
+			return; // not for this run's record
+		}
+
+		frappe.show_progress(__("Domain Enrichment"), data.step, data.total, data.message);
+
+		if (data.status === "completed") {
+			frappe.realtime.off(event, handler);
+			frappe.hide_progress();
+			frappe.show_alert(
+				{ message: __("Enrichment complete — a new run was created"), indicator: "green" },
+				7
+			);
+		} else if (data.status === "error") {
+			frappe.realtime.off(event, handler);
+			frappe.hide_progress();
+			frappe.msgprint({
+				title: __("Enrichment Failed"),
+				message: data.message || __("See Error Log."),
+				indicator: "red",
+			});
+		}
+	};
+
+	frappe.realtime.on(event, handler);
+
+	frappe.call({
+		method: "crm.domain_enrichment.api.retry",
+		args: { run: frm.doc.name },
+		error: () => {
+			frappe.realtime.off(event, handler);
+			frappe.hide_progress();
+		},
+	});
+}

crm/domain_enrichment/doctype/crm_enrichment_run/crm_enrichment_run.json

@@ -0,0 +1,198 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "autoname": "hash",
+ "creation": "2026-06-17 00:00:00.000000",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "reference_doctype",
+  "reference_name",
+  "column_break_ref",
+  "status",
+  "source_website",
+  "timing_section",
+  "started_on",
+  "finished_on",
+  "summary_section",
+  "company_name",
+  "industry",
+  "industry_confidence",
+  "column_break_summary",
+  "emails_found",
+  "phones_found",
+  "social_profiles",
+  "raw_section",
+  "raw_json",
+  "notes"
+ ],
+ "fields": [
+  {
+   "fieldname": "reference_doctype",
+   "fieldtype": "Link",
+   "in_standard_filter": 1,
+   "label": "Reference Doctype",
+   "options": "DocType",
+   "read_only": 1
+  },
+  {
+   "fieldname": "reference_name",
+   "fieldtype": "Dynamic Link",
+   "in_standard_filter": 1,
+   "label": "Reference Name",
+   "options": "reference_doctype",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_ref",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "Queued",
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Status",
+   "options": "Queued\nRunning\nCompleted\nFailed",
+   "read_only": 1
+  },
+  {
+   "fieldname": "source_website",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Source Website",
+   "read_only": 1
+  },
+  {
+   "fieldname": "timing_section",
+   "fieldtype": "Section Break",
+   "label": "Timing"
+  },
+  {
+   "fieldname": "started_on",
+   "fieldtype": "Datetime",
+   "label": "Started On",
+   "read_only": 1
+  },
+  {
+   "fieldname": "finished_on",
+   "fieldtype": "Datetime",
+   "label": "Finished On",
+   "read_only": 1
+  },
+  {
+   "fieldname": "summary_section",
+   "fieldtype": "Section Break",
+   "label": "Summary"
+  },
+  {
+   "fieldname": "company_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Company Name",
+   "read_only": 1
+  },
+  {
+   "fieldname": "industry",
+   "fieldtype": "Data",
+   "label": "Industry",
+   "read_only": 1
+  },
+  {
+   "fieldname": "industry_confidence",
+   "fieldtype": "Float",
+   "label": "Industry Confidence",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_summary",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "fieldname": "emails_found",
+   "fieldtype": "Int",
+   "label": "Emails Found",
+   "read_only": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "phones_found",
+   "fieldtype": "Int",
+   "label": "Phones Found",
+   "read_only": 1
+  },
+  {
+   "fieldname": "social_profiles",
+   "fieldtype": "Small Text",
+   "label": "Social Profiles",
+   "read_only": 1
+  },
+  {
+   "fieldname": "raw_section",
+   "fieldtype": "Section Break",
+   "label": "Raw"
+  },
+  {
+   "fieldname": "raw_json",
+   "fieldtype": "Code",
+   "label": "Raw JSON",
+   "options": "JSON",
+   "read_only": 1
+  },
+  {
+   "fieldname": "notes",
+   "fieldtype": "Small Text",
+   "label": "Notes",
+   "read_only": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2026-06-17 00:00:00.000000",
+ "modified_by": "Administrator",
+ "module": "Domain Enrichment",
+ "name": "CRM Enrichment Run",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 0,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Sales Manager",
+   "share": 1
+  },
+  {
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Sales User",
+   "share": 1
+  }
+ ],
+ "row_format": "Dynamic",
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "title_field": "source_website"
+}

crm/domain_enrichment/doctype/crm_enrichment_run/crm_enrichment_run.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class CRMEnrichmentRun(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		company_name: DF.Data | None
+		emails_found: DF.Int
+		finished_on: DF.Datetime | None
+		industry: DF.Data | None
+		industry_confidence: DF.Float
+		notes: DF.SmallText | None
+		phones_found: DF.Int
+		raw_json: DF.Code | None
+		reference_doctype: DF.Link | None
+		reference_name: DF.DynamicLink | None
+		social_profiles: DF.SmallText | None
+		source_website: DF.Data | None
+		started_on: DF.Datetime | None
+		status: DF.Literal["Queued", "Running", "Completed", "Failed"]
+	# end: auto-generated types
+
+	pass

crm/domain_enrichment/doctype/crm_enrichment_settings/crm_enrichment_settings.json

@@ -0,0 +1,200 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2026-06-17 00:00:00.000000",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "general_tab",
+  "enabled",
+  "automation_section",
+  "auto_enrich",
+  "crawl_tab",
+  "max_pages",
+  "max_depth",
+  "use_sitemap",
+  "request_timeout",
+  "column_break_crawl",
+  "max_download_bytes",
+  "retry_count",
+  "user_agent",
+  "tuning_section",
+  "link_priority_order",
+  "column_break_tuning",
+  "skip_patterns",
+  "ssrf_guard_tab",
+  "allowed_domains_section",
+  "allowed_domains",
+  "blocked_domains_section",
+  "blocked_domains"
+ ],
+ "fields": [
+  {
+   "fieldname": "general_tab",
+   "fieldtype": "Tab Break",
+   "label": "General"
+  },
+  {
+   "default": "1",
+   "description": "Master switch for the Domain Enrichment feature.",
+   "fieldname": "enabled",
+   "fieldtype": "Check",
+   "label": "Enabled"
+  },
+  {
+   "fieldname": "automation_section",
+   "fieldtype": "Section Break",
+   "label": "Automation"
+  },
+  {
+   "default": "0",
+   "description": "Automatically enrich a new CRM Lead, Deal or Organization in the background as soon as it is created (requires a website on the record). Only applies to the doctypes enabled above. When off, enrichment is triggered manually with the Enrich button.",
+   "fieldname": "auto_enrich",
+   "fieldtype": "Check",
+   "label": "Auto-enrich new records"
+  },
+  {
+   "fieldname": "crawl_tab",
+   "fieldtype": "Tab Break",
+   "label": "Crawl"
+  },
+  {
+   "default": "10",
+   "description": "Maximum number of pages crawled per enrichment.",
+   "fieldname": "max_pages",
+   "fieldtype": "Int",
+   "label": "Max Pages"
+  },
+  {
+   "default": "2",
+   "description": "Maximum link depth followed from the homepage.",
+   "fieldname": "max_depth",
+   "fieldtype": "Int",
+   "label": "Max Depth"
+  },
+  {
+   "default": "10",
+   "description": "Per-request timeout in seconds.",
+   "fieldname": "request_timeout",
+   "fieldtype": "Int",
+   "label": "Request Timeout (s)"
+  },
+  {
+   "default": "1",
+   "description": "Seed the crawl queue from the site's sitemap (declared in robots.txt, or the conventional /sitemap.xml) so pages not reachable within Max Depth still get a chance to be crawled. This is an addition on top of ordinary link-following, not a replacement -- a site with no sitemap falls straight through to link-following with no error.",
+   "fieldname": "use_sitemap",
+   "fieldtype": "Check",
+   "label": "Use Sitemap"
+  },
+  {
+   "fieldname": "column_break_crawl",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "3000000",
+   "description": "Maximum bytes downloaded per page (the stream is capped).",
+   "fieldname": "max_download_bytes",
+   "fieldtype": "Int",
+   "label": "Max Download Bytes"
+  },
+  {
+   "default": "2",
+   "description": "Number of retries on a failed request.",
+   "fieldname": "retry_count",
+   "fieldtype": "Int",
+   "label": "Retry Count"
+  },
+  {
+   "default": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+   "fieldname": "user_agent",
+   "fieldtype": "Data",
+   "label": "User Agent"
+  },
+  {
+   "fieldname": "tuning_section",
+   "fieldtype": "Section Break",
+   "label": "Crawl Tuning"
+  },
+  {
+   "description": "Crawl-ordering hints (not extraction). Links whose URL path or anchor text contain one of these are fetched first; higher weight = sooner. Lets the crawler spend its page budget on company pages (about/contact/team) before the cap. Defaults are seeded; edit or remove as needed.",
+   "fieldname": "link_priority_order",
+   "fieldtype": "Table",
+   "label": "Link Priority Order",
+   "options": "CRM Enrichment Link Priority"
+  },
+  {
+   "fieldname": "column_break_tuning",
+   "fieldtype": "Column Break"
+  },
+  {
+   "description": "Substrings that exclude a candidate link from the crawl.",
+   "fieldname": "skip_patterns",
+   "fieldtype": "Table",
+   "label": "Skip Patterns",
+   "options": "CRM Enrichment Skip Pattern"
+  },
+  {
+   "fieldname": "ssrf_guard_tab",
+   "fieldtype": "Tab Break",
+   "label": "SSRF Guard"
+  },
+  {
+   "fieldname": "allowed_domains_section",
+   "fieldtype": "Section Break",
+   "label": "Allowed Domains"
+  },
+  {
+   "description": "Allowlist (subdomain-aware). If you add any domains here, the crawler may ONLY visit those domains and their subdomains — everything else is rejected by the SSRF guard. Leave empty (the normal case) to allow any public site; this is for locked-down deployments that should reach only specific domains.",
+   "fieldname": "allowed_domains",
+   "fieldtype": "Table",
+   "label": "Allowed Domains",
+   "options": "CRM Enrichment Domain"
+  },
+  {
+   "fieldname": "blocked_domains_section",
+   "fieldtype": "Section Break",
+   "label": "Blocked Domains"
+  },
+  {
+   "description": "Denylist (subdomain-aware). A host that equals or is a subdomain of any entry is always rejected by the SSRF guard — e.g. blocking \"acme.com\" also blocks \"shop.acme.com\". Use it to permanently exclude internal domains, known-bad hosts, or link shorteners. Always wins over the allowlist.",
+   "fieldname": "blocked_domains",
+   "fieldtype": "Table",
+   "label": "Blocked Domains",
+   "options": "CRM Enrichment Domain"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "issingle": 1,
+ "links": [],
+ "modified": "2026-07-14 00:00:00.000000",
+ "modified_by": "Administrator",
+ "module": "Domain Enrichment",
+ "name": "CRM Enrichment Settings",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "Sales Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "row_format": "Dynamic",
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

crm/domain_enrichment/doctype/crm_enrichment_settings/crm_enrichment_settings.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class CRMEnrichmentSettings(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		from crm.domain_enrichment.doctype.crm_enrichment_domain.crm_enrichment_domain import (
+			CRMEnrichmentDomain,
+		)
+		from crm.domain_enrichment.doctype.crm_enrichment_link_priority.crm_enrichment_link_priority import (
+			CRMEnrichmentLinkPriority,
+		)
+		from crm.domain_enrichment.doctype.crm_enrichment_skip_pattern.crm_enrichment_skip_pattern import (
+			CRMEnrichmentSkipPattern,
+		)
+
+		allowed_domains: DF.Table[CRMEnrichmentDomain]
+		auto_enrich: DF.Check
+		blocked_domains: DF.Table[CRMEnrichmentDomain]
+		enabled: DF.Check
+		link_priority_order: DF.Table[CRMEnrichmentLinkPriority]
+		max_depth: DF.Int
+		max_download_bytes: DF.Int
+		max_pages: DF.Int
+		request_timeout: DF.Int
+		retry_count: DF.Int
+		skip_patterns: DF.Table[CRMEnrichmentSkipPattern]
+		user_agent: DF.Data | None
+	# end: auto-generated types
+
+	pass

crm/domain_enrichment/doctype/crm_enrichment_skip_pattern/crm_enrichment_skip_pattern.json

@@ -0,0 +1,34 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2026-06-17 00:00:00.000000",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "pattern"
+ ],
+ "fields": [
+  {
+   "description": "Substring; a candidate URL containing it is skipped during the crawl (e.g. \"/blog/\", \".pdf\", \"?\").",
+   "fieldname": "pattern",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Pattern",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2026-06-17 00:00:00.000000",
+ "modified_by": "Administrator",
+ "module": "Domain Enrichment",
+ "name": "CRM Enrichment Skip Pattern",
+ "owner": "Administrator",
+ "permissions": [],
+ "row_format": "Dynamic",
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

crm/domain_enrichment/doctype/crm_enrichment_skip_pattern/crm_enrichment_skip_pattern.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class CRMEnrichmentSkipPattern(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+		pattern: DF.Data
+	# end: auto-generated types
+
+	pass

crm/domain_enrichment/extractors.py

@@ -0,0 +1,700 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Generic, config-driven extractors.
+
+No literal keyword/industry/social tables here: classifiers iterate over
+``cfg.rules_by_type[...]``, honoring each rule's ``match_scope``, ``weight`` and
+compiled patterns. JSON-LD parsing, the favicon scorer, readability diagnosis,
+email/phone regex, and company-name cleaning stay as plain mechanics -- not
+tunable knowledge, so not pushed into doctypes.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from urllib.parse import urljoin, urlparse
+
+from .config import INDUSTRY_MIN_CONFIDENCE, INDUSTRY_MIN_SCORE
+from .result import (
+	Email,
+	Field,
+	Method,
+	Phone,
+	SocialProfile,
+)
+
+# --------------------------------------------------------------------------- #
+# Regexes (mechanics)
+# --------------------------------------------------------------------------- #
+EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
+
+PHONE_RE = re.compile(
+	r"""(?<!\w)
+    (\+?\d{1,3}[\s.-]?)?      # optional country code
+    (\(?\d{2,4}\)?[\s.-]?)    # area / first group
+    \d{3,4}[\s.-]?\d{3,4}     # subscriber number
+    (?!\w)""",
+	re.VERBOSE,
+)
+
+# Emails that are almost always noise (asset hashes, example domains, images).
+EMAIL_BLOCKLIST_SUBSTR = (
+	"example.com",
+	"sentry.io",
+	"@2x",
+	"@3x",
+	".png",
+	".jpg",
+	".gif",
+	".svg",
+	".webp",
+	"your-email",
+	"email@",
+)
+
+# Generic landing pages we should not treat as a social profile.
+SOCIAL_NEGATIVE = re.compile(r"/(sharer|share|intent|home|login|signup|privacy|tos)\b", re.I)
+
+
+# --------------------------------------------------------------------------- #
+# Generic rule executors
+# --------------------------------------------------------------------------- #
+SCOPE_HEADLINE = "Headline"
+SCOPE_FULL_TEXT = "Full Text"
+SCOPE_HTML = "HTML"
+SCOPE_HEADERS = "Headers"
+SCOPE_URL = "URL"
+
+
+def apply_keyword_rules(text_by_scope: dict, rules: list) -> dict:
+	"""Returns ``{label: weighted_score}`` -- ``label`` is the rule's industry
+	(Industry rules) or target_value (everything else)."""
+	scores: dict = {}
+	for rule in rules:
+		text = text_by_scope.get(rule.match_scope, "")
+		hits = rule.matches(text)
+		if hits:
+			scores[rule.label] = scores.get(rule.label, 0.0) + hits * (rule.weight or 1.0)
+	return scores
+
+
+# --------------------------------------------------------------------------- #
+# JSON-LD (mechanics)
+# --------------------------------------------------------------------------- #
+def parse_json_ld(soup):
+	blocks = []
+	for tag in soup.find_all("script", attrs={"type": "application/ld+json"}):
+		raw = tag.string or tag.get_text() or ""
+		if not raw.strip():
+			continue
+		try:
+			data = json.loads(raw)
+		except (ValueError, TypeError):
+			continue
+		if isinstance(data, dict):
+			if "@graph" in data and isinstance(data["@graph"], list):
+				blocks.extend([d for d in data["@graph"] if isinstance(d, dict)])
+			else:
+				blocks.append(data)
+		elif isinstance(data, list):
+			blocks.extend([d for d in data if isinstance(d, dict)])
+	return blocks
+
+
+def _ld_type_matches(block, wanted):
+	t = block.get("@type", "")
+	types = t if isinstance(t, list) else [t]
+	return any(str(x).lower() in wanted for x in types)
+
+
+# --------------------------------------------------------------------------- #
+# Readability diagnosis (mechanics)
+# --------------------------------------------------------------------------- #
+WAF_MARKERS_STRONG = (
+	"_incapsula_resource",
+	"incapsula",
+	"cf-browser-verification",
+	"just a moment",
+	"attention required",
+	"ddos protection",
+	"are you a human",
+)
+WAF_MARKERS_WEAK = (
+	"captcha",
+	"please enable javascript",
+	"enable cookies to continue",
+	"access denied",
+)
+WAF_WEAK_TEXT_THRESHOLD = 1000
+
+READABILITY_MESSAGES = {
+	"unreachable": "Couldn't reach this website (it timed out or refused the connection).",
+	"blocked": (
+		"This website is behind bot protection, so its content can't be read. "
+		"Try the company's corporate domain (e.g. acme.com vs shop.acme.com)."
+	),
+	"empty": (
+		"This site's main content is JavaScript-rendered; only page metadata "
+		"(title/description/logo) could be read, so some fields may be blank."
+	),
+}
+
+
+def diagnose_readability(pages):
+	"""Judge the homepage: '' (readable), 'blocked', 'empty', or 'unreachable'."""
+	if not pages:
+		return "unreachable"
+	home = pages[0]
+	if home.error or not home.status_code:
+		return "unreachable"
+	if home.status_code in (401, 403, 429):
+		return "blocked"
+	if home.status_code >= 500:
+		return "unreachable"
+	blob = (home.html or "")[:5000].lower()
+	text_len = len(home.text or "")
+	if any(marker in blob for marker in WAF_MARKERS_STRONG):
+		return "blocked"
+	if text_len < WAF_WEAK_TEXT_THRESHOLD and any(marker in blob for marker in WAF_MARKERS_WEAK):
+		return "blocked"
+	if text_len < 200:
+		return "empty"
+	return ""
+
+
+# --------------------------------------------------------------------------- #
+# Company information (mechanics)
+# --------------------------------------------------------------------------- #
+def _meta_with_method(soup, *names):
+	for name in names:
+		tag = soup.find("meta", attrs={"property": name}) or soup.find("meta", attrs={"name": name})
+		if tag and tag.get("content"):
+			return tag["content"].strip(), name
+	return "", ""
+
+
+# Below this, a description tag is a stub (a bare company name) rather than copy.
+_MIN_META_DESC_LEN = 30
+
+
+def _is_description_copy(content):
+	"""False for a stub or a verification token. Scripts that do not space
+	their words (CJK, Thai) are never ASCII-only, so they still count as copy."""
+	if len(content) < _MIN_META_DESC_LEN:
+		return False
+	return " " in content or not content.isascii()
+
+
+def _meta_description(soup):
+	fallback = ""
+	for name in ("og:description", "description"):
+		content, _name = _meta_with_method(soup, name)
+		if not content:
+			continue
+		if _is_description_copy(content):
+			return content
+		fallback = fallback or content
+	return fallback
+
+
+# A company name is usually the first chunk of a page <title>, before a separator.
+_NAME_SPLIT_RE = re.compile(r"\s*\|\s*|\s+[-–—•·]\s+|\s*:\s")
+_NAME_TAGLINE_RE = re.compile(
+	r"\s+(?:has|have|is|are|was|were|offers?|provides?|delivers?|helps?|"
+	r"enables?|builds?|makes?|creates?|specializes?|brings?|"
+	r"empowers?|powers?|leads?|connects?)\b",
+	re.I,
+)
+_NAME_MAX_WORDS = 6
+
+
+def _domain_brand(url):
+	netloc = (urlparse(url or "").netloc or "").lower().split(":")[0]
+	if netloc.startswith("www."):
+		netloc = netloc[4:]
+	labels = [label for label in netloc.split(".") if label]
+	return labels[-2] if len(labels) >= 2 else (labels[0] if labels else "")
+
+
+def _compact(text):
+	return re.sub(r"[^a-z0-9]", "", (text or "").lower())
+
+
+def _clean_company_name(raw, site_url=""):
+	"""Trim a page title / og:title down to just the company name."""
+	raw = (raw or "").strip()
+	if not raw:
+		return ""
+	chunks = [c.strip() for c in _NAME_SPLIT_RE.split(raw) if c.strip()]
+	name = chunks[0] if chunks else raw
+
+	brand = _compact(_domain_brand(site_url)) if site_url else ""
+	if brand and len(chunks) > 1:
+		for chunk in chunks:
+			c = _compact(chunk)
+			if c and (c == brand or (len(min(c, brand, key=len)) >= 4 and (c in brand or brand in c))):
+				name = chunk
+				break
+
+	m = _NAME_TAGLINE_RE.search(name)
+	if m and m.start() > 0:
+		name = name[: m.start()].strip()
+	if len(name.split()) > _NAME_MAX_WORDS:
+		name = name.split(",")[0].strip()
+	if len(name.split()) > _NAME_MAX_WORDS:
+		name = " ".join(name.split()[:_NAME_MAX_WORDS])
+	return name or raw
+
+
+# apple-touch-icons are 180px by convention when no explicit size is declared.
+_APPLE_TOUCH_PX = 180
+
+
+def _safe_url(base_url, raw):
+	"""Resolve ``raw`` against ``base_url``; return it only if it's a non-empty
+	``http(s)`` URL. Rejects ``javascript:``/``data:`` schemes a crawled site could
+	inject into a stored logo/image/icon field (stored-XSS defense)."""
+	raw = (raw or "").strip()
+	if not raw:
+		return ""
+	# urljoin() leaves javascript:/data: schemes as-is, so check the raw scheme
+	# first, then re-check the resolved URL for relative inputs.
+	scheme = urlparse(raw).scheme.lower()
+	if scheme and scheme not in ("http", "https"):
+		return ""
+	resolved = urljoin(base_url, raw)
+	return resolved if urlparse(resolved).scheme in ("http", "https") else ""
+
+
+def _best_icon(soup, base_url):
+	"""Best ``<link rel=icon>`` candidate: scalable SVG beats any raster, then
+	largest declared size wins."""
+	candidates = []
+	for tag in soup.find_all("link", href=True):
+		rel = tag.get("rel") or []
+		rel = " ".join(rel).lower() if isinstance(rel, list) else str(rel).lower()
+		if "icon" not in rel:
+			continue
+		href = tag["href"].strip()
+		safe = _safe_url(base_url, href)
+		if not safe:
+			continue
+		is_svg = (
+			"svg" in (tag.get("type") or "").lower()
+			or href.lower().split("?")[0].endswith(".svg")
+			or "mask-icon" in rel
+		)
+		m = re.search(r"(\d+)x\d+", tag.get("sizes", "") or "")
+		px = int(m.group(1)) if m else (_APPLE_TOUCH_PX if "apple-touch" in rel else 0)
+		candidates.append(((1 if is_svg else 0, px), safe))
+	if not candidates:
+		return ""
+	candidates.sort(key=lambda c: c[0], reverse=True)
+	return candidates[0][1]
+
+
+def extract_logo(soup, base_url):
+	"""The company's link icon -- a crisp, square brand mark, not the wider
+	social-share image (``og:image`` is usually a banner; see :func:`extract_image`).
+	Priority: SVG > largest declared raster/apple-touch > ``/favicon.ico`` fallback.
+	"""
+	icon_url = _best_icon(soup, base_url)
+	if icon_url:
+		return Field(icon_url, base_url, Method.FAVICON)
+
+	parsed = urlparse(base_url)
+	if parsed.scheme and parsed.netloc:
+		return Field(f"{parsed.scheme}://{parsed.netloc}/favicon.ico", base_url, Method.FAVICON)
+	return Field()
+
+
+def extract_image(soup, base_url):
+	"""The larger brand/social image (JSON-LD ``Organization.logo`` -> ``og:image``/
+	``twitter:image``), NOT the company logo -- see :func:`extract_logo`. Returns a
+	:class:`Field`, empty if none declared."""
+	for block in parse_json_ld(soup):
+		if not _ld_type_matches(block, _ORG_TYPES):
+			continue
+		logo = block.get("logo")
+		url = logo.get("url") if isinstance(logo, dict) else (logo if isinstance(logo, str) else "")
+		safe = _safe_url(base_url, url)
+		if safe:
+			return Field(safe, base_url, Method.JSON_LD)
+
+	og_content, _name = _meta_with_method(soup, "og:image", "og:image:url", "twitter:image")
+	safe = _safe_url(base_url, og_content)
+	if safe:
+		return Field(safe, base_url, Method.META_TAG)
+	return Field()
+
+
+# schema.org @types that denote a company/organization.
+_ORG_TYPES = {
+	"organization",
+	"corporation",
+	"localbusiness",
+	"ngo",
+	"educationalorganization",
+	"softwareapplication",
+}
+
+# Full-segment match, not a prefix: "about-our-reporting" (an ESG page) must not
+# qualify just because it starts with "about-".
+_ABOUT_SEGMENT_RE = re.compile(
+	r"^(about|about-us|about_us|aboutus|company|who-we-are|our-story|overview|mission)$",
+	re.I,
+)
+_NON_ABOUT_WORDS = (
+	"privacy",
+	"career",
+	"job",
+	"partner",
+	"webinar",
+	"policy",
+	"policies",
+	"accessibility",
+	"conduct",
+	"terms",
+	"cookie",
+	"legal",
+	"press",
+	"news",
+	"blog",
+	"event",
+	"investor",
+	"support",
+	"contact",
+	"sitemap",
+	"login",
+	"award",
+)
+_COMMERCE_RE = re.compile(r"^\s*(shop|buy|order|browse)\b|add to cart|shop now|buy now", re.I)
+
+
+def _last_path_segment(url):
+	path = urlparse(url).path.rstrip("/")
+	return path.rsplit("/", 1)[-1].lower() if path else ""
+
+
+def _is_about_page(url):
+	seg = _last_path_segment(url)
+	if not seg or any(w in seg for w in _NON_ABOUT_WORDS):
+		return False
+	return bool(_ABOUT_SEGMENT_RE.match(seg))
+
+
+def extract_company_info(homepage, soup):
+	url = homepage.url
+	info = {
+		"company_name": Field(),
+		"description": Field(),
+		"logo": Field(),
+		"image": Field(),
+		"social_links": [],
+	}
+
+	for block in parse_json_ld(soup):
+		if not _ld_type_matches(block, _ORG_TYPES):
+			continue
+		if not info["company_name"].value and block.get("name"):
+			info["company_name"] = Field(block["name"].strip(), url, Method.JSON_LD)
+		if not info["description"].value and block.get("description"):
+			info["description"] = Field(block["description"].strip(), url, Method.JSON_LD)
+		same_as = block.get("sameAs")
+		if isinstance(same_as, list):
+			info["social_links"].extend([s for s in same_as if isinstance(s, str)])
+		elif isinstance(same_as, str):
+			info["social_links"].append(same_as)
+		break  # first matching org block wins
+
+	if not info["company_name"].value:
+		content, _name = _meta_with_method(soup, "og:site_name", "og:title")
+		if content:
+			info["company_name"] = Field(_clean_company_name(content, url), url, Method.META_TAG)
+		elif homepage.title:
+			info["company_name"] = Field(_clean_company_name(homepage.title, url), url, Method.TITLE_TAG)
+
+	if not info["description"].value:
+		content = _meta_description(soup)
+		if content:
+			info["description"] = Field(content, url, Method.META_TAG)
+
+	info["logo"] = extract_logo(soup, url)
+	info["image"] = extract_image(soup, url)
+
+	return info
+
+
+_NON_CONTENT_PARENTS = {"nav", "header", "footer", "aside", "form"}
+_MIN_PARAGRAPH_LEN = 80
+_MAX_PARAGRAPHS_SCANNED = 6
+
+# Company copy speaks for itself ("we"/"our"). A staff profile is written in the
+# third person: a career phrase, a job title plus a pronoun, or repeated pronouns.
+_FIRST_PERSON_RE = re.compile(r"\b(?:we|our|ours|us)\b", re.I)
+_PRONOUN_RE = re.compile(r"\b(?:he|she|his|her)\b", re.I)
+_CAREER_RE = re.compile(r"\bbegan (?:his|her) career\b|\bprior to (?:joining|founding)\b", re.I)
+_TITLE_RE = re.compile(
+	r"\bis the (?:founder|co-founder|ceo|cto|cfo|chief|managing director|president)\b", re.I
+)
+_CTA_RE = re.compile(
+	r"^\s*(?:explore|discover|learn|find out|contact|call|browse|shop|buy|order|"
+	r"read more|get in touch|see|check out|sign up|subscribe|join)\b",
+	re.I,
+)
+
+
+def _company_name_hits(text, company_name):
+	name = (company_name or "").strip()
+	if not name:
+		return 0
+	return len(re.findall(rf"\b{re.escape(name)}\b", text, re.IGNORECASE))
+
+
+def _is_staff_profile(text):
+	if _CAREER_RE.search(text):
+		return True
+	if _FIRST_PERSON_RE.search(text):
+		return False
+	pronouns = len(_PRONOUN_RE.findall(text))
+	return pronouns >= 2 or (pronouns >= 1 and bool(_TITLE_RE.search(text)))
+
+
+def _describes_company(text, company_name):
+	"""True when the paragraph talks about the company, not something else."""
+	return bool(_FIRST_PERSON_RE.search(text)) or _company_name_hits(text, company_name) > 0
+
+
+def first_paragraph(soup, industry_rules=None, company_name=""):
+	"""Best substantial body paragraph on a page.
+
+	Reads up to ``_MAX_PARAGRAPHS_SCANNED`` qualifying ``<p>`` tags, drops the ones
+	that describe a person or tell the reader what to do next, and ranks the rest by
+	Industry-rule keyword hits (the same corpus ``classify_industry`` uses). Talking
+	about the company and naming it only break ties, never outrank keyword hits. If
+	every paragraph was dropped they are all reconsidered, so a page of nothing but
+	staff profiles still returns something. Read-only -- does not mutate ``soup``.
+	"""
+	candidates = []
+	for p in soup.find_all("p"):
+		if any(parent.name in _NON_CONTENT_PARENTS for parent in p.parents):
+			continue
+		text = " ".join((p.get_text(" ") or "").split())
+		if len(text) < _MIN_PARAGRAPH_LEN or " " not in text:
+			continue
+		candidates.append(text)
+		if len(candidates) >= _MAX_PARAGRAPHS_SCANNED:
+			break
+
+	if not candidates:
+		return ""
+	candidates = [
+		text for text in candidates if not (_is_staff_profile(text) or _CTA_RE.match(text))
+	] or candidates
+
+	# max() keeps the first-encountered element on ties, so document order still
+	# wins when nothing scores higher (covers the "all zero hits" case for free).
+	# The trailing slots can only break ties, never add to the industry-hit count.
+	return max(
+		candidates,
+		key=lambda text: (
+			sum(rule.matches(text) for rule in industry_rules or []),
+			_describes_company(text, company_name),
+			_company_name_hits(text, company_name),
+		),
+	)
+
+
+def select_description(pages, soups_by_url, industry_rules=None, company_name=""):
+	"""Pick the best *company* description across crawled pages.
+
+	Priority: JSON-LD ``Organization.description`` -> About-page body paragraph ->
+	head ``og:description``/``<meta description>`` -> home-page body paragraph
+	(last resort). ``industry_rules``/``company_name`` steer the body-paragraph
+	fallbacks (see ``first_paragraph``).
+	"""
+	home_url = pages[0].url if pages else ""
+	best_key = None
+	best_field = None
+
+	def consider(field_obj, score):
+		nonlocal best_key, best_field
+		if not field_obj.value:
+			return
+		key = (score, min(len(field_obj.value), 300))
+		if best_key is None or key > best_key:
+			best_key, best_field = key, field_obj
+
+	for page in pages:
+		soup = soups_by_url.get(page.url)
+		if soup is None:
+			continue
+
+		is_home = page.url == home_url
+		is_about = _is_about_page(page.url)
+
+		for block in parse_json_ld(soup):
+			if _ld_type_matches(block, _ORG_TYPES) and block.get("description"):
+				consider(Field(block["description"].strip(), page.url, Method.JSON_LD), 6)
+				break
+
+		if is_about:
+			consider(
+				Field(first_paragraph(soup, industry_rules, company_name), page.url, Method.BODY_TEXT), 5
+			)
+
+		if is_home or is_about:
+			content = _meta_description(soup)
+			if content:
+				score = 2 if (is_home and _COMMERCE_RE.search(content)) else 4
+				consider(Field(content, page.url, Method.META_TAG), score)
+
+		if is_home:
+			consider(
+				Field(first_paragraph(soup, industry_rules, company_name), page.url, Method.BODY_TEXT), 1
+			)
+
+	return best_field or Field()
+
+
+# --------------------------------------------------------------------------- #
+# Contacts: emails, phones (mechanics)
+# --------------------------------------------------------------------------- #
+def _clean_email(email):
+	email = email.strip().strip(".").lower()
+	if any(b in email for b in EMAIL_BLOCKLIST_SUBSTR):
+		return None
+	return email
+
+
+def extract_emails(pages):
+	found = {}
+	for page in pages:
+		if not page.html:
+			continue
+		for match in EMAIL_RE.findall(page.html):
+			cleaned = _clean_email(match)
+			if cleaned and cleaned not in found:
+				found[cleaned] = Email(value=cleaned, source=page.url)
+	return list(found.values())
+
+
+# TODO: Need to make this robust by adding proper mechanics or using a
+# thirdparty library like google/libphonenumbers
+def _valid_phone(raw):
+	# Requires an explicit "+" country-code prefix: a bare digit string is either
+	# ambiguous local formatting or noise (stats, reference numbers) that happens
+	# to look phone-shaped -- precision over recall.
+	if not raw.strip().startswith("+"):
+		return False
+	digits = re.sub(r"\D", "", raw)
+	return 7 <= len(digits) <= 15
+
+
+def extract_phones(pages):
+	found = {}
+	for page in pages:
+		if not page.html:
+			continue
+		for m in re.findall(r"tel:([+0-9()\s.\-]{7,})", page.html):
+			raw = m.strip()
+			digits = re.sub(r"\D", "", raw)
+			if _valid_phone(raw) and digits not in found:
+				found[digits] = Phone(
+					value="+" + digits if raw.strip().startswith("+") else digits,
+					raw=raw,
+					source=page.url,
+					method=Method.TEL_LINK,
+				)
+		for match in PHONE_RE.finditer(page.text or ""):
+			raw = match.group(0).strip()
+			if not raw:
+				continue
+			digits = re.sub(r"\D", "", raw)
+			if _valid_phone(raw) and digits not in found:
+				value = ("+" + digits) if raw.startswith("+") else digits
+				found[digits] = Phone(value=value, raw=raw, source=page.url, method=Method.REGEX)
+	return list(found.values())
+
+
+# --------------------------------------------------------------------------- #
+# Social profiles (config-driven via Social rules)
+# --------------------------------------------------------------------------- #
+def extract_social_profiles(pages, soups_by_url, social_rules, extra_links=None):
+	"""Return dict network -> SocialProfile(value, source, method); each Social
+	rule's ``target_value`` is the network name."""
+	profiles = {rule.target_value: SocialProfile() for rule in social_rules}
+
+	home_url = pages[0].url if pages else ""
+	candidates = [(link, home_url, Method.JSON_LD) for link in (extra_links or [])]
+	for page in pages:
+		soup = soups_by_url.get(page.url)
+		if soup is None:
+			continue
+		for a in soup.find_all("a", href=True):
+			candidates.append((a["href"].strip(), page.url, Method.SOCIAL_RULE))
+
+	for href, source, method in candidates:
+		if not href or SOCIAL_NEGATIVE.search(href):
+			continue
+		url = urljoin(source, href.strip())
+		for rule in social_rules:
+			network = rule.target_value
+			if profiles[network].value:
+				continue
+			if any(rx.search(url) for rx, _raw in rule.patterns):
+				profiles[network] = SocialProfile(value=url, source=source, method=method)
+	return profiles
+
+
+# --------------------------------------------------------------------------- #
+# Industry classification (config-driven via Industry rules)
+# --------------------------------------------------------------------------- #
+def classify_industry(pages, company_info, industry_rules):
+	"""Rule-based industry from the homepage's (and, when crawled, the About page's)
+	headline corpus. Industry rules are typically seeded with ``match_scope='Headline'``
+	(company name/description/title/headings, never body/footer). The homepage alone
+	is often thin (stylized brand copy, no meta description), so a crawled About
+	page's title/headings are folded in too. Returns (industry, confidence 0-1), or
+	("", 0.0) when the signal is too weak.
+	"""
+	if not industry_rules:
+		return "", 0.0
+
+	def _val(key):
+		f = company_info.get(key)
+		return getattr(f, "value", f) or ""
+
+	headline_parts = [_val("company_name"), _val("description")]
+	if pages:
+		home = pages[0]
+		headline_parts.append(home.title or "")
+		headline_parts.append(" ".join(home.headings))
+		about = next((p for p in pages[1:] if _is_about_page(p.url)), None)
+		if about:
+			headline_parts.append(about.title or "")
+			headline_parts.append(" ".join(about.headings))
+	headline = " ".join(p for p in headline_parts if p).lower()
+
+	# Same corpus under multiple scope keys so admins can re-scope rules without
+	# code changes; default seed uses Headline.
+	full_text = " ".join((p.text or "") for p in pages).lower()
+	text_by_scope = {
+		SCOPE_HEADLINE: headline,
+		SCOPE_FULL_TEXT: full_text,
+		SCOPE_HTML: full_text,
+	}
+
+	scores = apply_keyword_rules(text_by_scope, industry_rules)
+	total = sum(scores.values())
+	if not total:
+		return "", 0.0
+	best = max(scores, key=scores.get)
+	if scores[best] < INDUSTRY_MIN_SCORE:
+		return "", 0.0
+	confidence = scores[best] / total
+	if confidence < INDUSTRY_MIN_CONFIDENCE:
+		return "", 0.0
+	return best, round(confidence, 2)

crm/domain_enrichment/http.py

@@ -0,0 +1,382 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""HTTP fetch layer for the crawler, built on framework helpers, with an SSRF guard.
+
+Built on ``frappe.utils.get_request_session`` (so retry/adapter behavior is the
+framework's), this module adds the crawler-specific behavior the framework does NOT
+provide:
+
+* a hard download cap (``max_download_bytes``) via streamed reads,
+* an HTML-only content-type filter,
+* the never-raise ``(status_code, html, error)`` contract, and
+* a mandatory **SSRF guard** that resolves the hostname and rejects loopback /
+  private / link-local / reserved addresses, honors the Settings allow/block lists,
+  and **re-validates the resolved IP after every redirect** (it does not blindly
+  follow redirects). Every connection is **pinned to the IP the guard validated**
+  (Host header and TLS SNI/verification stay on the original hostname), so a
+  DNS-rebinding host cannot pass validation with one address and connect to another.
+
+Limits (timeout, byte cap, retries, user-agent) come from config, not constants.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+import socket
+from urllib.parse import urlparse
+
+import frappe
+from frappe.utils import get_request_session
+from requests.adapters import HTTPAdapter, Retry
+
+HTML_CONTENT_TYPES = ("text/html", "application/xhtml")
+RETRY_STATUS_FORCELIST = (429, 500, 502, 503, 504)
+# Cap redirect chases ourselves (we follow manually to re-check each hop).
+MAX_REDIRECTS = 5
+# Default crawler UA: a current desktop-Chrome string so bot walls that reject bare
+# clients serve HTML. Overridable via the CRM Enrichment Settings ``user_agent`` field.
+DEFAULT_USER_AGENT = (
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+	"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
+)
+
+
+class SSRFError(Exception):
+	"""Raised internally when a URL fails the SSRF guard. Never escapes ``fetch``."""
+
+
+# --------------------------------------------------------------------------- #
+# SSRF guard
+# --------------------------------------------------------------------------- #
+def _domain_in_list(host: str, domains: list) -> bool:
+	"""True if ``host`` equals or is a subdomain of any entry in ``domains``.
+
+	The trailing dot of a fully-qualified name (``evil.com.``) is stripped from both
+	sides -- it resolves identically in DNS, so leaving it on would let ``evil.com.``
+	slip past a ``evil.com`` block/allow rule.
+	"""
+	host = (host or "").lower().split(":")[0].rstrip(".")
+	for d in domains:
+		d = (d or "").lower().strip().rstrip(".")
+		if not d:
+			continue
+		if host == d or host.endswith("." + d):
+			return True
+	return False
+
+
+def _is_blocked_ip(ip: str) -> bool:
+	"""True if ``ip`` is not a public (globally-routable) address.
+
+	Mirrors the framework's own SSRF check (``make_safe_get_request`` in
+	frappe.utils.safe_exec): ``not is_global`` rejects loopback, private, link-local,
+	reserved and unspecified addresses -- and also shared/CGNAT ranges (100.64.0.0/10,
+	0.0.0.0/8) that an explicit flag enumeration tends to miss. ``is_global`` does NOT
+	cover multicast, so keep an explicit reject for it (the framework helper misses it).
+	"""
+	try:
+		addr = ipaddress.ip_address(ip)
+	except ValueError:
+		return True  # unparseable -> reject
+	return not addr.is_global or addr.is_multicast
+
+
+def _resolve_ips(host: str) -> list:
+	"""Resolve ``host`` to its IP strings via ``socket.getaddrinfo``."""
+	infos = socket.getaddrinfo(host, None)
+	# index 4 is sockaddr; element 0 is the IP for both AF_INET / AF_INET6.
+	return list({info[4][0] for info in infos})
+
+
+def validate_url(url: str, cfg) -> str:
+	"""SSRF guard entry point. Returns the validated URL or raises ``SSRFError``.
+
+	Validates the scheme, then resolves the hostname and rejects any URL that
+	resolves to a loopback/private/link-local/reserved address. Honors the
+	Settings allow/block domain lists: a blocked domain is always rejected; if an
+	allow list exists the host must be on it. The private-network rejection is
+	unconditional -- there is no setting to disable it (that would be a pure SSRF
+	footgun for a crawler that fetches user-supplied URLs).
+	"""
+	_validated_ips(url, cfg)
+	return url
+
+
+def _validated_ips(url: str, cfg) -> list:
+	"""Run the full SSRF checks for ``url`` and return the resolved, validated IPs.
+
+	The caller connects to one of these exact addresses (see ``_pinned_get``), so
+	the IP that was checked is the IP that serves the request. Raises ``SSRFError``
+	on any rejection.
+	"""
+	parsed = urlparse(url)
+	if parsed.scheme not in ("http", "https"):
+		raise SSRFError(f"unsupported URL scheme: {parsed.scheme or '(none)'}")
+
+	# Strip the trailing dot of a fully-qualified name at the source, so the SAME
+	# canonical host feeds the allow/block check AND DNS resolution -- ``evil.com.``
+	# resolves identically to ``evil.com`` and must not bypass either.
+	host = (parsed.hostname or "").strip().rstrip(".")
+	if not host:
+		raise SSRFError("URL has no host")
+
+	blocked = cfg.blocked_domains if cfg else []
+	allowed = cfg.allowed_domains if cfg else []
+	if _domain_in_list(host, blocked):
+		raise SSRFError(f"host is on the blocked-domains list: {host}")
+	if allowed and not _domain_in_list(host, allowed):
+		raise SSRFError(f"host is not on the allowed-domains list: {host}")
+
+	try:
+		ips = _resolve_ips(host)
+	except socket.gaierror as exc:
+		raise SSRFError(f"could not resolve host {host}: {exc}") from exc
+	if not ips:
+		raise SSRFError(f"could not resolve host {host}")
+	for ip in ips:
+		if _is_blocked_ip(ip):
+			raise SSRFError(f"host {host} resolves to a non-public address: {ip}")
+
+	# Deterministic order, IPv4 first: the caller pins the first entry, and unlike
+	# requests' resolver it won't fall back across address families, so an IPv6
+	# answer must not shadow a reachable IPv4 one on v4-only networks.
+	return sorted(ips, key=lambda ip: (":" in ip, ip))
+
+
+# --------------------------------------------------------------------------- #
+# Session
+# --------------------------------------------------------------------------- #
+def build_session(cfg=None):
+	"""Build the crawler session on the framework helper.
+
+	Sets a browser-like header set (User-Agent from Settings + Accept / Sec-Fetch /
+	client-hint headers a real browser sends) so servers that reject bare clients
+	still serve HTML, and customizes the retry status_forcelist for 429/5xx. Note:
+	this only spoofs headers -- the TLS fingerprint is still that of ``requests``
+	(see the TLS note in CONTEXT.md).
+	"""
+	retries = int(cfg.setting("retry_count")) if cfg else 2
+	user_agent = cfg.setting("user_agent") if cfg else DEFAULT_USER_AGENT
+	session = get_request_session(max_retries=retries)
+	# get_request_session only forces 500; widen to the usual transient set.
+	try:
+		adapter = HTTPAdapter(max_retries=_retry_policy(retries))
+		session.mount("http://", adapter)
+		session.mount("https://", adapter)
+	except Exception:  # pragma: no cover - keep the framework default on any issue
+		pass
+	session.headers.update(
+		{
+			"User-Agent": user_agent,
+			"Accept": (
+				"text/html,application/xhtml+xml,application/xml;q=0.9,"
+				"image/avif,image/webp,image/apng,*/*;q=0.8"
+			),
+			"Accept-Language": "en-US,en;q=0.9",
+			# Browser-ish request context so bot walls that check these serve HTML.
+			# (Accept-Encoding is left to requests so it only advertises codecs it can
+			# actually decode.)
+			"Upgrade-Insecure-Requests": "1",
+			"Sec-Fetch-Dest": "document",
+			"Sec-Fetch-Mode": "navigate",
+			"Sec-Fetch-Site": "none",
+			"Sec-Fetch-User": "?1",
+			"Sec-CH-UA": '"Chromium";v="125", "Not.A/Brand";v="24", "Google Chrome";v="125"',
+			"Sec-CH-UA-Mobile": "?0",
+			"Sec-CH-UA-Platform": '"Windows"',
+		}
+	)
+	return session
+
+
+def _retry_policy(retries: int) -> Retry:
+	return Retry(
+		total=retries,
+		backoff_factor=0.3,
+		status_forcelist=RETRY_STATUS_FORCELIST,
+		allowed_methods=frozenset(["GET", "HEAD"]),
+	)
+
+
+# --------------------------------------------------------------------------- #
+# Pinned connections (DNS-rebinding defense)
+# --------------------------------------------------------------------------- #
+class _PinnedIPAdapter(HTTPAdapter):
+	"""Transport adapter for URLs whose netloc was rewritten to a pre-validated IP.
+
+	``server_hostname`` keeps TLS SNI on the original hostname, and urllib3 also
+	matches the certificate against it (it falls back to ``server_hostname`` when
+	``assert_hostname`` is unset), so pinning does not weaken TLS verification.
+	urllib3 strips ``server_hostname`` for plain-HTTP pools, so the same adapter
+	serves both schemes.
+	"""
+
+	def __init__(self, hostname: str, **kwargs):
+		self._server_hostname = hostname
+		super().__init__(**kwargs)
+
+	def init_poolmanager(self, *args, **kwargs):
+		kwargs["server_hostname"] = self._server_hostname
+		super().init_poolmanager(*args, **kwargs)
+
+
+def _pinned_adapter(session, hostname: str, retries: int) -> _PinnedIPAdapter:
+	"""Per-hostname adapter cache on the session, so keep-alive pooling survives
+	across the pages of a crawl instead of reconnecting for every request."""
+	cache = getattr(session, "_pinned_adapters", None)
+	if cache is None:
+		cache = {}
+		session._pinned_adapters = cache
+	adapter = cache.get(hostname)
+	if adapter is None:
+		adapter = _PinnedIPAdapter(hostname, max_retries=_retry_policy(retries))
+		cache[hostname] = adapter
+	return adapter
+
+
+def _pinned_get(session, url: str, pinned_ip: str, timeout: int, retries: int):
+	"""GET ``url`` connecting to ``pinned_ip`` -- an address the SSRF guard just
+	validated -- so a DNS answer that changes between validation and connection
+	(rebinding) cannot steer the socket somewhere else. Only the socket target is
+	rewritten: the Host header, TLS SNI and certificate verification all stay on
+	the original hostname.
+	"""
+	import requests
+
+	parsed = urlparse(url)
+	host = parsed.hostname or ""
+	ip_netloc = f"[{pinned_ip}]" if ":" in pinned_ip else pinned_ip
+	if parsed.port:
+		ip_netloc = f"{ip_netloc}:{parsed.port}"
+	host_header = f"{host}:{parsed.port}" if parsed.port else host
+
+	prepared = session.prepare_request(
+		requests.Request(
+			"GET",
+			parsed._replace(netloc=ip_netloc).geturl(),
+			headers={"Host": host_header},
+		)
+	)
+	return _pinned_adapter(session, host, retries).send(
+		prepared,
+		timeout=timeout,
+		stream=True,
+		verify=session.verify,
+		cert=session.cert,
+	)
+
+
+# --------------------------------------------------------------------------- #
+# Fetch
+# --------------------------------------------------------------------------- #
+_META_CHARSET_RE = re.compile(rb"""charset=["']?\s*([a-zA-Z0-9_\-]+)""", re.IGNORECASE)
+
+
+def _sniff_html_charset(raw: bytes) -> str | None:
+	"""Charset declared in a <meta> tag (handles both `<meta charset=...>` and
+	`<meta http-equiv content="...; charset=...">`); None if absent."""
+	match = _META_CHARSET_RE.search(raw[:4096])
+	if match:
+		try:
+			return match.group(1).decode("ascii")
+		except (UnicodeDecodeError, AttributeError):
+			return None
+	return None
+
+
+def _read_capped(resp, max_bytes: int) -> str:
+	chunks = []
+	total = 0
+	for chunk in resp.iter_content(chunk_size=16_384, decode_unicode=False):
+		if not chunk:
+			continue
+		total += len(chunk)
+		chunks.append(chunk)
+		if total >= max_bytes:
+			break
+	raw = b"".join(chunks)
+	# requests defaults text/* without an explicit charset to ISO-8859-1, which
+	# mojibakes UTF-8 pages that only declare their charset in a <meta> tag. When the
+	# header carries no charset, trust the document's own declaration instead.
+	if "charset=" in resp.headers.get("Content-Type", "").lower():
+		encoding = resp.encoding or "utf-8"
+	else:
+		encoding = _sniff_html_charset(raw) or "utf-8"
+	try:
+		return raw.decode(encoding, errors="replace")
+	except (LookupError, TypeError):
+		return raw.decode("utf-8", errors="replace")
+
+
+def fetch(url: str, cfg, session=None, html_only: bool = True):
+	"""Fetch a URL and return ``(status_code, html, error, final_url)``.
+
+	Never raises -- any failure (SSRF rejection, timeout, non-HTML, oversized) is
+	reported as a non-empty ``error`` string with ``status_code`` 0 (or the actual
+	status for content-type skips). Redirects are followed manually so the resolved
+	IP of every hop is re-validated by the SSRF guard; ``final_url`` is the URL that
+	actually served the body (post-redirect), so callers resolve relative links and
+	record provenance against the right host. Pass ``html_only=False`` to accept
+	non-HTML bodies (e.g. ``text/plain`` robots.txt).
+
+	SSRF note: each hop connects to the exact IP the guard validated for it
+	(``_pinned_get``), so a DNS-rebinding host (short TTL, alternating answers)
+	cannot pass validation with a public address and serve the request from a
+	private one.
+	"""
+	import requests
+
+	timeout = int(cfg.setting("request_timeout")) if cfg else 10
+	max_bytes = int(cfg.setting("max_download_bytes")) if cfg else 3_000_000
+	retries = int(cfg.setting("retry_count")) if cfg else 2
+
+	own_session = session is None
+	session = session or build_session(cfg)
+	current = url
+	try:
+		for _hop in range(MAX_REDIRECTS + 1):
+			try:
+				ips = _validated_ips(current, cfg)
+			except SSRFError as exc:
+				return 0, "", f"blocked by SSRF guard: {exc}", current
+
+			# Redirects are followed manually (each hop re-validated and re-pinned).
+			resp = _pinned_get(session, current, ips[0], timeout, retries)
+
+			if resp.is_redirect or resp.is_permanent_redirect:
+				location = resp.headers.get("Location")
+				resp.close()
+				if not location:
+					return resp.status_code, "", "redirect without Location header", current
+				current = requests.compat.urljoin(current, location)
+				continue
+
+			content_type = resp.headers.get("Content-Type", "").lower()
+			if html_only and content_type and not any(ct in content_type for ct in HTML_CONTENT_TYPES):
+				status = resp.status_code
+				resp.close()
+				return status, "", f"skipped non-HTML content-type: {content_type}", current
+
+			html = _read_capped(resp, max_bytes)
+			status = resp.status_code
+			resp.close()
+			return status, html, "", current
+
+		return 0, "", "too many redirects", current
+	except requests.exceptions.Timeout:
+		return 0, "", f"timeout after {timeout}s", current
+	except requests.exceptions.TooManyRedirects:
+		return 0, "", "too many redirects", current
+	except requests.exceptions.SSLError as exc:
+		return 0, "", f"ssl error: {exc}", current
+	except requests.exceptions.ConnectionError as exc:
+		return 0, "", f"connection error: {exc}", current
+	except requests.exceptions.RequestException as exc:
+		return 0, "", f"request error: {exc}", current
+	except Exception as exc:  # pragma: no cover - last-resort guard
+		return 0, "", f"unexpected error: {exc}", current
+	finally:
+		if own_session:
+			session.close()

crm/domain_enrichment/install.py

@@ -0,0 +1,397 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""One-time code-to-data migration of the enrichment knowledge base.
+
+The POC baked its keyword / signature / mapping tables into Python constants.
+This seeder translates them into editable desk records (CRM Enrichment Rule +
+Pattern, CRM Enrichment Field Mapping) so a sales-ops admin owns the knowledge.
+
+Everything here is idempotent (skip-if-exists) and wired into both ``after_install``
+and ``after_migrate`` so a fresh install and every migrate land the defaults
+without clobbering admin edits.
+"""
+
+import frappe
+
+# --------------------------------------------------------------------------- #
+# Rule knowledge (recovered from the POC's extractors.py constant tables)
+# --------------------------------------------------------------------------- #
+
+# industry -> keywords (matched against headline text, weight 1 each)
+INDUSTRY_KEYWORDS = {
+	"ERP": [
+		"erp",
+		"enterprise resource planning",
+		"procurement",
+		"supply chain management",
+		"inventory management",
+	],
+	"CRM": [
+		"crm",
+		"customer relationship management",
+		"sales pipeline",
+		"lead management",
+		"contact management",
+	],
+	"Ecommerce": [
+		"ecommerce",
+		"e-commerce",
+		"online shopping",
+		"online store",
+		"marketplace",
+		"shopping cart",
+		"add to cart",
+		"add to bag",
+		"storefront",
+		"checkout",
+		"shop",
+		"free shipping",
+		"free returns",
+		"new arrivals",
+		"best sellers",
+		"bestsellers",
+		"size guide",
+		"apparel",
+		"clothing brand",
+		"direct-to-consumer",
+		"quick commerce",
+		"q-commerce",
+		"online grocery",
+		"online groceries",
+		"grocery delivery",
+		"grocery app",
+		"delivery app",
+		"online shop",
+	],
+	"SaaS": ["saas", "software as a service", "subscription software", "cloud platform", "free trial"],
+	"Technology": [
+		"software",
+		"developer",
+		"developers",
+		"api",
+		"open source",
+		"cloud computing",
+		"it services",
+		"information technology",
+		"artificial intelligence",
+		"machine learning",
+	],
+	"Semiconductors": [
+		"semiconductor",
+		"gpu",
+		"graphics card",
+		"processor",
+		"chipset",
+		"supercomputer",
+		"microprocessor",
+	],
+	"Manufacturing": [
+		"manufacturing",
+		"factory",
+		"production line",
+		"assembly line",
+		"fabrication",
+		"industrial equipment",
+		"3d printing",
+		"additive manufacturing",
+		"cnc machining",
+		"rapid prototyping",
+		"injection molding",
+		"prototyping",
+	],
+	"Healthcare": [
+		"healthcare",
+		"health care",
+		"clinic",
+		"hospital",
+		"patient care",
+		"medical",
+		"telehealth",
+		"pharmaceutical",
+		"pharma",
+	],
+	"Education": [
+		"education",
+		"e-learning",
+		"students",
+		"curriculum",
+		"university",
+		"online courses",
+		"courses",
+		"course",
+		"edtech",
+		"learning platform",
+	],
+	"Consulting": [
+		"consulting",
+		"advisory services",
+		"consultants",
+		"professional services",
+		"strategy consulting",
+	],
+	"Marketing": [
+		"marketing agency",
+		"digital marketing",
+		"advertising agency",
+		"seo services",
+		"ad campaigns",
+		"media buying",
+	],
+	"Finance": [
+		"fintech",
+		"banking",
+		"payments",
+		"lending",
+		"insurance",
+		"financial services",
+		"trading",
+		"wealth management",
+	],
+	"Logistics": [
+		"logistics",
+		"freight",
+		"warehousing",
+		"fulfillment",
+		"shipping carrier",
+		"last-mile delivery",
+	],
+	"Food & Beverage": [
+		"food and beverage",
+		"beverage",
+		"beverages",
+		"snacks",
+		"snack",
+		"packaged food",
+		"fmcg",
+		"consumer goods",
+		"sparkling water",
+		"iced tea",
+		"energy drink",
+		"soft drink",
+		"drink",
+		"drinks",
+		"soda",
+		"kombucha",
+		"cereal",
+		"coffee",
+		"chocolate",
+		"protein bar",
+	],
+	"Retail": ["retailer", "department store", "brick and mortar"],
+	"Media": [
+		"entertainment",
+		"streaming service",
+		"video game",
+		"game studio",
+		"publishing house",
+		"news media",
+		"broadcasting",
+		"journalism",
+		"journalists",
+		"media company",
+		"newsroom",
+		"newsletter",
+		"reported articles",
+		"investigative",
+		"editorial",
+		"magazine",
+		"podcast",
+		"reporting on",
+	],
+	"Real Estate": [
+		"real estate",
+		"property management",
+		"commercial property",
+		"interior design",
+		"home design",
+		"home improvement",
+		"remodeling",
+		"decorating",
+		"home renovation",
+	],
+	"Travel": ["hospitality", "tourism", "flight booking", "travel agency"],
+	"Automotive": ["automotive", "electric vehicle", "car manufacturer"],
+	"Telecom": ["telecommunications", "broadband", "network operator"],
+	"Energy": ["renewable energy", "solar power", "oil and gas"],
+	"Legal": [
+		"law firm",
+		"legal services",
+		"litigation",
+		"legal help",
+		"legal documents",
+		"legal advice",
+		"attorney",
+		"attorneys",
+		"lawyer",
+		"lawyers",
+	],
+	"Nonprofit": ["nonprofit", "non-profit", "ngo", "charity"],
+}
+
+# social network -> regex (matched against HTML; stored as regex patterns)
+SOCIAL_PATTERNS = {
+	"linkedin": [r"linkedin\.com/(company|in|school)/"],
+	"twitter": [r"(twitter\.com|x\.com)/[A-Za-z0-9_]+"],
+	"github": [r"github\.com/[A-Za-z0-9_.-]+"],
+	"facebook": [r"facebook\.com/[A-Za-z0-9_.\-/]+"],
+	"instagram": [r"instagram\.com/[A-Za-z0-9_.]+"],
+	"youtube": [r"youtube\.com/(channel/|c/|user/|@)[A-Za-z0-9_.\-]+"],
+}
+
+# --------------------------------------------------------------------------- #
+# Field-mapping knowledge (recovered from the POC's api/website_intelligence.py)
+# Translates FIELD_MAP / OVERRIDABLE_DEFAULTS / ALWAYS_REFRESH into records.
+# (source_key, target_doctype, target_fieldname, write_policy, default_values, create_missing_link)
+# --------------------------------------------------------------------------- #
+_LEAD = "CRM Lead"
+_DEAL = "CRM Deal"
+_ORG = "CRM Organization"
+
+# Per-doctype the org "name" lands on a different field: Lead.organization,
+# Deal.organization_name (free text), Org.organization_name.
+FIELD_MAPPINGS = [
+	# company name
+	("company_name", _LEAD, "organization", "Fill if empty", None, 0),
+	("company_name", _DEAL, "organization_name", "Fill if empty", None, 0),
+	("company_name", _ORG, "organization_name", "Fill if empty", None, 0),
+	# logo / description (shared)
+	*[
+		(k, dt, fn, "Fill if empty", None, 0)
+		for dt in (_LEAD, _DEAL, _ORG)
+		for k, fn in (("logo", "organization_logo"), ("description", "company_description"))
+	],
+	# industry — link, opt-in auto-create of missing CRM Industry
+	*[("industry", dt, "industry", "Fill if empty", None, 1) for dt in (_LEAD, _DEAL, _ORG)],
+	# social links (shared, fill-empty)
+	*[
+		(k, dt, fn, "Fill if empty", None, 0)
+		for dt in (_LEAD, _DEAL, _ORG)
+		for k, fn in (("linkedin", "linkedin"), ("twitter", "twitter"), ("facebook", "facebook"))
+	],
+]
+
+
+def seed_default_rules_and_mappings():
+	"""Idempotently create the default enrichment rules + field mappings.
+
+	Safe to run repeatedly (after_install + after_migrate): each record is keyed
+	on a stable name and skipped if it already exists, so admin edits are never
+	clobbered.
+	"""
+	_seed_industry_rules()
+	_seed_social_rules()
+	_seed_field_mappings()
+	_seed_settings_defaults()
+
+
+def _seed_settings_defaults():
+	"""Embed the default Link Priority Order rows into Settings.
+
+	Idempotent: only populated when the table is empty, so admin edits/removals are
+	never clobbered. The default list lives in ``config.DEFAULT_LINK_PRIORITY``.
+	"""
+	from crm.domain_enrichment.config import DEFAULT_LINK_PRIORITY
+
+	settings = frappe.get_single("CRM Enrichment Settings")
+	if settings.link_priority_order:
+		return
+	for keyword, weight in DEFAULT_LINK_PRIORITY:
+		settings.append("link_priority_order", {"keyword": keyword, "weight": weight})
+	settings.save(ignore_permissions=True)
+
+
+def _ensure_industry(industry: str):
+	if not frappe.db.exists("CRM Industry", industry):
+		frappe.get_doc({"doctype": "CRM Industry", "industry": industry}).insert(ignore_permissions=True)
+
+
+def _make_rule(
+	rule_name,
+	rule_type,
+	patterns,
+	*,
+	target_value=None,
+	industry=None,
+	weight=1,
+	match_scope="Full Text",
+	is_regex=False,
+):
+	# Match on the rule_name field, not the document name: the autoname
+	# ("{rule_type}.-.####") means name != rule_name, so a bare-string exists()
+	# never matches a seeded rule and re-seeding would collide with the unique
+	# rule_name constraint (breaking bench migrate). rule_name stays out of the
+	# name because it is free text (colons/spaces) that a naming series rejects.
+	if frappe.db.exists("CRM Enrichment Rule", {"rule_name": rule_name}):
+		return
+	doc = frappe.get_doc(
+		{
+			"doctype": "CRM Enrichment Rule",
+			"rule_name": rule_name,
+			"rule_type": rule_type,
+			"target_value": target_value,
+			"industry": industry,
+			"weight": weight,
+			"match_scope": match_scope,
+			"enabled": 1,
+			"patterns": [{"pattern": p, "is_regex": 1 if is_regex else 0} for p in patterns],
+		}
+	)
+	doc.insert(ignore_permissions=True)
+
+
+def _seed_industry_rules():
+	for industry, keywords in INDUSTRY_KEYWORDS.items():
+		_ensure_industry(industry)
+		_make_rule(
+			f"Industry: {industry}",
+			"Industry",
+			keywords,
+			industry=industry,
+			match_scope="Headline",
+		)
+
+
+def _seed_social_rules():
+	for network, patterns in SOCIAL_PATTERNS.items():
+		_make_rule(
+			f"Social: {network}",
+			"Social",
+			patterns,
+			target_value=network,
+			match_scope="HTML",
+			is_regex=True,
+		)
+
+
+def _seed_field_mappings():
+	for (
+		source_key,
+		target_doctype,
+		target_fieldname,
+		write_policy,
+		default_values,
+		create_link,
+	) in FIELD_MAPPINGS:
+		exists = frappe.db.exists(
+			"CRM Enrichment Field Mapping",
+			{
+				"source_key": source_key,
+				"target_doctype": target_doctype,
+				"target_fieldname": target_fieldname,
+			},
+		)
+		if exists:
+			continue
+		frappe.get_doc(
+			{
+				"doctype": "CRM Enrichment Field Mapping",
+				"enabled": 1,
+				"source_key": source_key,
+				"target_doctype": target_doctype,
+				"target_fieldname": target_fieldname,
+				"write_policy": write_policy,
+				"default_values": default_values,
+				"create_missing_link": create_link,
+			}
+		).insert(ignore_permissions=True)

crm/domain_enrichment/mapper.py

@@ -0,0 +1,255 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Single source of result -> CRM field logic, driven by Field Mapping records.
+
+``apply_to_document`` is the ONLY place that translates an ``EnrichmentResult``
+into CRM document fields. Phase 5's link-time Organization -> Lead/Deal copy reuses
+it, so all the write-policy / has-field / never-overwrite guards live here once.
+
+The mapping itself is data: each ``CRM Enrichment Field Mapping`` record names a
+``source_key`` (one of the 13 frozen in Phase 2), a ``target_fieldname``, a
+``write_policy`` (Fill if empty / Always refresh / Override defaults) and optional
+``default_values`` / ``create_missing_link``. This module knows how to resolve a
+``source_key`` against the result and how to apply each policy -- nothing else.
+"""
+
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+import frappe
+from frappe import _
+
+from .crawler import registrable_domain
+
+
+def _source_key_labels() -> dict:
+	"""Human labels for the realtime "filled: ..." toast, keyed by source_key. Falls
+	back to the field's meta label, then the fieldname, when not listed here. Built
+	at call time so ``_()`` translates in the requesting user's language -- a
+	module-level dict would evaluate at import and cache one language for all
+	sites/users (semgrep: frappe-breaks-multitenancy)."""
+	return {
+		"company_name": _("Organization Name"),
+		"description": _("Company Description"),
+		"logo": _("Logo"),
+		"industry": _("Industry"),
+		"primary_email": _("Email"),
+		"primary_phone": _("Phone"),
+		"secondary_phone": _("Phone"),
+		"linkedin": _("LinkedIn"),
+		"twitter": _("X (Twitter)"),
+		"github": _("GitHub"),
+		"facebook": _("Facebook"),
+		"instagram": _("Instagram"),
+		"youtube": _("YouTube"),
+	}
+
+
+# source_keys backed by a social_profiles[<network>] entry on the result.
+_SOCIAL_KEYS = ("linkedin", "twitter", "github", "facebook", "instagram", "youtube")
+
+# Write-policy constants (must match the CRM Enrichment Field Mapping Select options).
+POLICY_FILL_IF_EMPTY = "Fill if empty"
+POLICY_ALWAYS_REFRESH = "Always refresh"
+POLICY_OVERRIDE_DEFAULTS = "Override defaults"
+
+
+def _first_email(result) -> str:
+	"""Prefer an email on the company's own registrable domain, else the first.
+
+	Mirrors the POC behaviour: a sales address on the company's own domain is a
+	better company contact than a random third-party address picked up elsewhere.
+	"""
+	if not result.emails:
+		return ""
+	registrable = registrable_domain(urlparse(result.website or "").netloc)
+	if registrable:
+		for e in result.emails:
+			domain = e.value.lower().split("@")[-1]
+			# Exact registrable domain or a subdomain of it -- not a lookalike like
+			# "notacme.com" endswith "acme.com".
+			if domain == registrable or domain.endswith("." + registrable):
+				return e.value
+	return result.emails[0].value
+
+
+def _phone_at(result, index: int) -> str:
+	"""The raw (else cleaned) phone at ``index``; falls back to the primary."""
+	if not result.phones:
+		return ""
+	if index >= len(result.phones):
+		index = 0
+	p = result.phones[index]
+	return p.raw or p.value
+
+
+def get_value_for_source_key(result, source_key: str):
+	"""Resolve one of the 13 frozen source_keys against an ``EnrichmentResult``.
+
+	Returns a plain scalar (``""`` means "nothing to fill"). Scalar provenance
+	fields expose ``.value``; emails/phones/contacts are read off the result's
+	lists; social profiles off ``result.social_profiles[<network>]``.
+	"""
+	if source_key == "company_name":
+		return result.company_name.value or ""
+	if source_key == "description":
+		return result.description.value or ""
+	if source_key == "logo":
+		return result.logo.value or ""
+	if source_key == "image":
+		return result.image.value or ""
+	if source_key == "industry":
+		return result.industry.value or ""
+	if source_key == "primary_email":
+		return _first_email(result)
+	if source_key == "primary_phone":
+		return _phone_at(result, 0)
+	if source_key == "secondary_phone":
+		return _phone_at(result, 1)
+	if source_key in _SOCIAL_KEYS:
+		profile = result.social_profiles.get(source_key)
+		return profile.value if profile else ""
+	return ""
+
+
+def _ensure_link_target(target_doctype: str, target_fieldname: str, value: str):
+	"""For a Link field with ``create_missing_link`` set, auto-create the linked
+	master if it is missing. Explicit, opt-in -- never a hidden side effect, and
+	permission-respecting: the master is only created if the enriching user could
+	create it themselves (no ``ignore_permissions`` escalation from scraped data).
+
+	Returns the value to write (the linked name) or ``None`` to skip the field --
+	when the master is missing and the user lacks create rights, or on any error
+	(the field is skipped, the run is not aborted).
+	"""
+	meta = frappe.get_meta(target_doctype)
+	df = meta.get_field(target_fieldname)
+	if not df or df.fieldtype != "Link" or not df.options:
+		# Not a Link (or unknown field) -- nothing to create; write as-is.
+		return value
+	link_doctype = df.options
+	if frappe.db.exists(link_doctype, value):
+		return value
+	# Auto-create respects the enriching user's rights: the worker runs as the user
+	# who triggered enrichment (frappe.set_user in execute_job), so a user who cannot
+	# create the master must not be able to insert one via a crafted website. Skip the
+	# field rather than write a dangling link.
+	if not frappe.has_permission(link_doctype, ptype="create"):
+		return None
+	try:
+		link_meta = frappe.get_meta(link_doctype)
+		new_doc = {"doctype": link_doctype}
+		# Populate the field the master autonames from (e.g. CRM Industry.industry),
+		# falling back to the title field, so the created record's name == value.
+		autoname = link_meta.autoname or ""
+		if autoname.startswith("field:"):
+			new_doc[autoname.split(":", 1)[1]] = value
+		elif link_meta.get_title_field():
+			new_doc[link_meta.get_title_field()] = value
+		else:
+			new_doc["__newname"] = value
+		# Permission-respecting insert -- the create right was checked above.
+		frappe.get_doc(new_doc).insert()
+		return value
+	except Exception:
+		frappe.log_error(
+			title="Domain Enrichment: could not create link master",
+			message=f"{link_doctype}={value!r} for {target_doctype}.{target_fieldname}",
+		)
+		return None
+
+
+def _label_for(doc, source_key: str, target_fieldname: str) -> str:
+	label = _source_key_labels().get(source_key)
+	if label:
+		return label
+	df = doc.meta.get_field(target_fieldname)
+	return df.label if df and df.label else target_fieldname
+
+
+def _overridable_defaults(mapping) -> set:
+	"""The set of "really unset" default values for an Override-defaults mapping
+	(newline-separated in ``default_values``), e.g. {"", "1-10"}."""
+	defaults = {""}
+	for line in (mapping.default_values or "").splitlines():
+		token = line.strip()
+		if token:
+			defaults.add(token)
+	return defaults
+
+
+def apply_to_document(doc, result, cfg, fill_empty_only: bool = False) -> list[str]:
+	"""Populate mappable fields on a Lead / Deal / Organization from ``result``.
+
+	Driven by ``cfg.mappings_by_doctype[doc.doctype]`` (the enabled Field Mapping
+	records). For each mapping the value is resolved by ``source_key`` and written
+	according to ``write_policy``:
+
+	* **Fill if empty** -- set only when the doc field is currently empty.
+	* **Always refresh** -- overwrite with a fresh non-empty value on re-enrichment;
+	  never clears a stored value to empty (enrichment must not destroy data).
+	* **Override defaults** -- treat the mapping's ``default_values`` as empty and
+	  fill over them, else respect a real user value.
+
+	When ``fill_empty_only`` is True every mapping is forced to fill-empty semantics
+	regardless of its configured ``write_policy`` (and empty source values are never
+	written). This is the link-time Organization -> Lead/Deal copy path (Phase 5),
+	which must never overwrite or clear user-entered data.
+
+	Never overwrites real user-entered data except per the policies above. Does NOT
+	save the document -- the caller controls persistence/permissions. Returns the
+	human labels of the fields that were changed.
+	"""
+	filled: list[str] = []
+	mappings = cfg.mappings_by_doctype.get(doc.doctype, [])
+
+	for mapping in mappings:
+		fieldname = mapping.target_fieldname
+		if not doc.meta.has_field(fieldname):
+			continue
+		value = get_value_for_source_key(result, mapping.source_key)
+		label = _label_for(doc, mapping.source_key, fieldname)
+		current = doc.get(fieldname)
+		policy = POLICY_FILL_IF_EMPTY if fill_empty_only else (mapping.write_policy or POLICY_FILL_IF_EMPTY)
+
+		if policy == POLICY_ALWAYS_REFRESH:
+			# Overwrite with a fresh value, but never clear a stored value to empty:
+			# enrichment must not destroy data a site temporarily stopped exposing.
+			if not value or current == value:
+				continue
+			if mapping.create_missing_link:
+				value = _ensure_link_target(doc.doctype, fieldname, value)
+				if value is None:
+					continue
+			doc.set(fieldname, value)
+			if label not in filled:
+				filled.append(label)
+			continue
+
+		# Fill-if-empty / Override-defaults both require a non-empty fresh value.
+		if not value:
+			continue
+
+		if policy == POLICY_OVERRIDE_DEFAULTS:
+			overridable = _overridable_defaults(mapping)
+			if current and current not in overridable:
+				continue  # a real user value -- respect it
+		else:  # Fill if empty
+			if current:
+				continue
+
+		if current == value:
+			continue
+
+		if mapping.create_missing_link:
+			value = _ensure_link_target(doc.doctype, fieldname, value)
+			if value is None:
+				continue
+
+		doc.set(fieldname, value)
+		if label not in filled:
+			filled.append(label)
+
+	return filled

crm/domain_enrichment/pipeline.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Orchestration: load config, crawl, run extractors, run enabled sources, assemble
+an ``EnrichmentResult``.
+
+``run()`` accepts an optional ``progress`` callback ``fn(step_index, message)`` so
+Phase 4's ``tasks.py`` can surface the progress steps over realtime without the
+engine depending on Frappe. The pipeline never writes to the DB or CRM docs (that
+is Phase 4) and never raises for fetch/source failures -- they are captured as
+errors/notes on the result.
+"""
+
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+from . import extractors
+from .config import EnrichmentConfig, get_config
+from .crawler import crawl, probe_about_pages
+from .http import build_session
+from .result import EnrichmentResult, Field, Method
+
+PROGRESS_STEPS = [
+	"Discovering pages",
+	"Crawling website",
+	"Extracting company information",
+	"Extracting contacts",
+	"Classifying industry",
+	"Saving results",
+	"Completed",
+]
+
+
+def _normalize_website(url: str) -> str:
+	url = (url or "").strip()
+	if not url:
+		raise ValueError("website URL is required")
+	if not urlparse(url).scheme:
+		url = "https://" + url
+	parsed = urlparse(url)
+	if parsed.scheme not in ("http", "https"):
+		raise ValueError(f"unsupported URL scheme: {parsed.scheme}")
+	if not parsed.netloc:
+		raise ValueError(f"invalid URL: {url}")
+	return url
+
+
+def run(website: str, cfg: EnrichmentConfig = None, progress=None) -> EnrichmentResult:
+	"""Run the full enrichment engine for ``website`` and return the result.
+
+	:param website: the site to enrich (scheme optional; https assumed).
+	:param cfg: an :class:`EnrichmentConfig`; loaded via ``get_config()`` if omitted.
+	:param progress: optional ``fn(step_index, message)`` callback (0-based index
+	    into ``PROGRESS_STEPS``); exceptions in it are swallowed.
+	"""
+	website = _normalize_website(website)
+	cfg = cfg or get_config()
+	result = EnrichmentResult(website=website)
+
+	def emit(step_index, message=""):
+		if progress:
+			try:
+				progress(step_index, message or PROGRESS_STEPS[step_index])
+			except Exception:
+				pass  # progress reporting must never break enrichment
+
+	session = build_session(cfg)
+	try:
+		# 1 + 2. Discover + crawl (BFS does both; reported together).
+		emit(0)
+		crawled = crawl(website, cfg, session=session, progress=lambda msg: emit(1, msg))
+		pages = [page for page, _soup in crawled]
+		soups_by_url = {page.url: soup for page, soup in crawled}
+		result.pages_crawled = [{"url": p.url, "status": p.status_code, "error": p.error} for p in pages]
+		result.errors = [{"url": p.url, "error": p.error} for p in pages if p.error]
+
+		# Short-circuit on a blocked / unreachable homepage with a clear reason.
+		reason = extractors.diagnose_readability(pages)
+		if reason in ("blocked", "unreachable"):
+			result.notes.append(extractors.READABILITY_MESSAGES[reason])
+			emit(6)
+			return result
+		if reason == "empty":
+			# JS-rendered body: <head> metadata may still be usable -- fall through.
+			result.notes.append(extractors.READABILITY_MESSAGES["empty"])
+
+		homepage = pages[0]
+		home_soup = soups_by_url.get(homepage.url)
+
+		# If the BFS didn't surface an About page, probe the common paths -- it's the
+		# best source of a company description (and helps industry/contacts too). Only
+		# when the page budget allows more fetches.
+		max_pages = int(cfg.setting("max_pages", 10) or 10)
+		if home_soup and len(pages) < max_pages and not any(extractors._is_about_page(p.url) for p in pages):
+			for page, soup in probe_about_pages(
+				homepage.url, cfg, session=session, skip_urls=[p.url for p in pages]
+			):
+				pages.append(page)
+				soups_by_url[page.url] = soup
+				result.pages_crawled.append(
+					{"url": page.url, "status": page.status_code, "error": page.error}
+				)
+
+		# 3. Company information (each field carries provenance).
+		emit(2)
+		company = extractors.extract_company_info(homepage, home_soup) if home_soup else {}
+		result.company_name = company.get("company_name") or Field()
+		result.logo = company.get("logo") or Field()
+		result.image = company.get("image") or Field()
+		result.description = (
+			extractors.select_description(
+				pages,
+				soups_by_url,
+				industry_rules=cfg.rules("Industry"),
+				company_name=result.company_name.value,
+			)
+			or company.get("description")
+			or Field()
+		)
+
+		# 4. Contacts: emails, phones, social.
+		emit(3)
+		result.emails = extractors.extract_emails(pages)
+		result.phones = extractors.extract_phones(pages)
+		result.social_profiles = extractors.extract_social_profiles(
+			pages,
+			soups_by_url,
+			cfg.rules("Social"),
+			extra_links=company.get("social_links"),
+		)
+
+		# 5. Industry.
+		emit(4)
+		industry, confidence = extractors.classify_industry(pages, company, cfg.rules("Industry"))
+		result.industry = Field(industry, website, Method.KEYWORD_CLASSIFIER) if industry else Field()
+		result.industry_confidence = confidence
+
+		# 6 + 7. Done (saving handled by the caller in Phase 4).
+		emit(5)
+		emit(6)
+		return result
+	finally:
+		session.close()

crm/domain_enrichment/result.py

@@ -0,0 +1,150 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""EnrichmentResult schema. Preserves the {value, source, method} provenance shape.
+
+Every piece of evidence keeps a ``source`` URL so any field can be traced back to
+the page it was extracted from. Dataclasses serialize to plain dicts/JSON via
+``to_dict()`` so the result can be returned by the engine and stored by the mapper /
+run-writer (Phase 4) without any framework coupling. This module imports no framework
+services on purpose -- it is the pure, unit-testable result container.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+
+
+# Canonical extraction-method labels -- every extracted value records which one
+# produced it, so a reviewer can judge how much to trust it.
+class Method:
+	JSON_LD = "JSON-LD"
+	META_TAG = "Meta Tag"
+	TITLE_TAG = "Title Tag"
+	BODY_TEXT = "Body Text"
+	FAVICON = "Favicon"
+	REGEX = "Regex"
+	TEL_LINK = "tel: link"
+	TEXT_HEURISTIC = "Text Heuristic"
+	KEYWORD_CLASSIFIER = "Keyword Classifier"
+	SOCIAL_RULE = "Social Rule"
+
+
+@dataclass
+class Email:
+	value: str
+	source: str = ""
+	method: str = Method.REGEX
+
+	def to_dict(self):
+		return {"value": self.value, "source": self.source, "method": self.method}
+
+
+@dataclass
+class Phone:
+	value: str  # normalized / cleaned
+	raw: str = ""  # exactly as found on the page
+	source: str = ""
+	method: str = Method.REGEX
+
+	def to_dict(self):
+		return {
+			"value": self.value,
+			"raw": self.raw,
+			"source": self.source,
+			"method": self.method,
+		}
+
+
+@dataclass
+class Field:
+	"""A scalar value plus where/how it came from (company_name, description,
+	logo, industry, ...)."""
+
+	value: object = ""
+	source: str = ""
+	method: str = ""
+
+	def to_dict(self):
+		return {"value": self.value, "source": self.source, "method": self.method}
+
+
+@dataclass
+class SocialProfile:
+	value: str = ""
+	source: str = ""
+	method: str = Method.SOCIAL_RULE
+
+	def to_dict(self):
+		return {"value": self.value, "source": self.source, "method": self.method}
+
+
+@dataclass
+class EnrichmentResult:
+	website: str = ""
+	# Scalar fields carry their own provenance via Field.
+	company_name: Field = field(default_factory=Field)
+	description: Field = field(default_factory=Field)
+	logo: Field = field(default_factory=Field)  # the company link icon (favicon / apple-touch)
+	image: Field = field(default_factory=Field)  # larger social/JSON-LD image; kept, not the logo
+	industry: Field = field(default_factory=Field)
+	industry_confidence: float = 0.0
+
+	emails: list = field(default_factory=list)  # list[Email]
+	phones: list = field(default_factory=list)  # list[Phone]
+	social_profiles: dict = field(default_factory=dict)  # network -> SocialProfile
+
+	# Diagnostics -- not part of the required schema but invaluable for debugging.
+	pages_crawled: list = field(default_factory=list)
+	errors: list = field(default_factory=list)
+	notes: list = field(default_factory=list)  # human-readable warnings
+
+	def to_dict(self):
+		"""Render the canonical JSON schema. Every field is explainable: scalar
+		fields expose {value, source, method}; list items embed source + method;
+		social profiles are {value, source, method} per network."""
+		return {
+			"company_name": self.company_name.to_dict(),
+			"description": self.description.to_dict(),
+			"logo": self.logo.to_dict(),
+			"image": self.image.to_dict(),
+			"industry": self.industry.to_dict(),
+			"industry_confidence": round(self.industry_confidence, 2),
+			"emails": [e.to_dict() for e in self.emails],
+			"phones": [p.to_dict() for p in self.phones],
+			"social_profiles": {k: v.to_dict() for k, v in self.social_profiles.items()},
+			"_meta": {
+				"website": self.website,
+				"pages_crawled": self.pages_crawled,
+				"errors": self.errors,
+				"notes": self.notes,
+			},
+		}
+
+	def flat(self):
+		"""Convenience: collapse to plain scalar values (no provenance), useful
+		for storing the headline fields on a Frappe document."""
+		return {
+			"company_name": self.company_name.value,
+			"description": self.description.value,
+			"logo": self.logo.value,
+			"industry": self.industry.value,
+			"industry_confidence": round(self.industry_confidence, 2),
+			"social_profiles": {k: v.value for k, v in self.social_profiles.items()},
+		}
+
+
+@dataclass
+class CrawledPage:
+	"""A single fetched page and its parsed artifacts."""
+
+	url: str
+	status_code: int = 0
+	html: str = ""
+	text: str = ""
+	title: str = ""
+	headings: list = field(default_factory=list)
+	error: str = ""
+
+	def to_dict(self):
+		return asdict(self)

crm/domain_enrichment/tasks.py

@@ -0,0 +1,265 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Background worker + the single run-writer.
+
+``run_enrichment`` is the enqueued job: it runs the pipeline (streaming progress
+over realtime), writes a ``CRM Enrichment Run`` history record, applies the mapper
+to the origin document, and publishes a terminal event. It NEVER raises to the
+worker -- every failure is recorded on the Run, logged via ``frappe.log_error`` and
+reported over realtime.
+
+``write_run`` is the ONLY place that persists run history, so the storage choice
+(standalone Run doctype today) stays swappable.
+"""
+
+from __future__ import annotations
+
+import frappe
+from frappe.utils.telemetry import capture
+
+from .config import _setting, auto_enrich_enabled_for, get_config, get_settings
+from .mapper import apply_to_document
+from .pipeline import PROGRESS_STEPS
+from .pipeline import run as run_pipeline
+
+# Realtime event the frontend (Phase 6) subscribes to.
+PROGRESS_EVENT = "domain_enrichment_progress"
+
+# Total number of pipeline progress steps (mirrors pipeline.PROGRESS_STEPS).
+TOTAL_STEPS = len(PROGRESS_STEPS)
+
+
+def _publish(reference_doctype, reference_name, status, message="", step=0, payload=None, user=None):
+	"""Emit a single progress/terminal event. Mirrors crm/api/whatsapp.py shape:
+	always carries the reference, plus step/total/message/status and an optional
+	payload (terminal events put filled_fields/notes/result.flat() in the payload).
+	Never raises -- realtime is best-effort."""
+	try:
+		frappe.publish_realtime(
+			PROGRESS_EVENT,
+			{
+				"reference_doctype": reference_doctype,
+				"reference_name": reference_name,
+				"step": step,
+				"total": TOTAL_STEPS,
+				"message": message,
+				"status": status,
+				"payload": payload or {},
+			},
+			user=user,
+		)
+	except Exception:
+		pass
+
+
+def write_run(
+	reference_doctype: str,
+	reference_name: str,
+	website: str,
+	status: str,
+	result=None,
+	started_on=None,
+	notes: str = "",
+):
+	"""Persist exactly one ``CRM Enrichment Run`` from an ``EnrichmentResult``.
+
+	The single point of run-history writing -- storage stays swappable behind it.
+	When ``result`` is given the summary fields + ``raw_json`` (full ``to_dict()``)
+	are populated; otherwise a bare Run (status/website) is written.
+	"""
+	doc = frappe.new_doc("CRM Enrichment Run")
+	doc.reference_doctype = reference_doctype
+	doc.reference_name = reference_name
+	doc.source_website = website
+	doc.status = status
+	doc.started_on = started_on or frappe.utils.now_datetime()
+	if status in ("Completed", "Failed"):
+		doc.finished_on = frappe.utils.now_datetime()
+	if notes:
+		doc.notes = notes
+
+	if result is not None:
+		# extract_social_profiles pre-seeds an empty entry per configured rule; only
+		# summarise the networks that were actually found.
+		social = ", ".join(sorted(k for k, v in result.social_profiles.items() if v.value))
+		doc.company_name = result.company_name.value or ""
+		doc.industry = result.industry.value or ""
+		doc.industry_confidence = result.industry_confidence or 0
+		doc.emails_found = len(result.emails)
+		doc.phones_found = len(result.phones)
+		doc.social_profiles = social
+		doc.raw_json = frappe.as_json(result.to_dict())
+		if not notes and result.notes:
+			doc.notes = "\n".join(result.notes)
+
+	doc.insert(ignore_permissions=True)
+	return doc.name
+
+
+def enqueue_enrichment(
+	reference_doctype: str, reference_name: str, website: str, user: str, trigger: str = "manual"
+) -> dict:
+	"""Enqueue one ``run_enrichment`` job (long queue, per-doc ``job_id`` +
+	``deduplicate``, after commit). The single place the job is enqueued -- shared by
+	the manual (``api.enrich``/``api.retry``) and auto (``after_insert``) paths so a
+	manual click and an auto-fire never double-run, and the enqueue options stay in
+	one spot. ``enqueue_after_commit`` matters for the ``after_insert`` caller, whose
+	transaction has not committed yet; it is harmless for the already-saved paths.
+
+	Reads only the Settings Single for the timeout bound (cheap, framework-cached) --
+	it never assembles the full config.
+	"""
+	s = get_settings()
+	timeout = int(_setting(s, "request_timeout")) * int(_setting(s, "max_pages")) + 60
+	job_id = f"domain-enrich-{reference_doctype}-{reference_name}"
+	frappe.enqueue(
+		"crm.domain_enrichment.tasks.run_enrichment",
+		queue="long",
+		timeout=timeout,
+		job_id=job_id,
+		deduplicate=True,
+		enqueue_after_commit=True,
+		reference_doctype=reference_doctype,
+		reference_name=reference_name,
+		website=website,
+		user=user,
+		trigger=trigger,
+	)
+	return {"queued": True, "job_id": job_id, "website": website}
+
+
+def auto_enrich_on_create(doc, method=None):
+	"""Auto-enqueue enrichment for a new CRM record.
+
+	Called from the ``after_insert`` of the CRM Lead / CRM Deal / CRM Organization
+	controllers (the trigger stays visible where the record lives, rather than as a
+	doc_events hook). Best-effort and never raises into the save. Fires only when the
+	feature is enabled, ``auto_enrich`` is on, the doctype is enabled, and the record
+	has a website. Reuses the same per-doc ``job_id``
+	+ ``deduplicate`` as the manual ``api.enrich`` path, so a manual click and the
+	auto-fire never double-run. A new Deal created with a website is therefore enriched
+	alongside its Organization -- each crawls independently and writes its own fields.
+	"""
+	try:
+		# Cheap Settings-only gate -- runs on every Lead/Deal/Org insert, so it must
+		# not assemble the full config (Rules/Mappings). The worker builds that later.
+		if not auto_enrich_enabled_for(doc.doctype):
+			return
+
+		website = (doc.get("website") or "").strip()
+		if not website:
+			return
+
+		enqueue_enrichment(doc.doctype, doc.name, website, frappe.session.user, trigger="auto")
+	except Exception:
+		frappe.log_error(title="Domain Enrichment: auto_enrich_on_create failed")
+
+
+def run_enrichment(
+	reference_doctype: str,
+	reference_name: str,
+	website: str,
+	user: str | None = None,
+	trigger: str = "manual",
+):
+	"""Enqueued worker: crawl, map onto the origin doc, write a Run, stream progress.
+
+	Never raises to the worker. On success the mapped origin doc is saved (a normal
+	permission-respecting save -- it is the doc the user triggered enrichment on; no
+	related records are touched, that is Phase 5). On any exception the Run is marked
+	Failed, logged, and an error event is published.
+	"""
+	user = user or frappe.session.user
+	started_on = frappe.utils.now_datetime()
+
+	def progress(step_index, message=""):
+		_publish(
+			reference_doctype,
+			reference_name,
+			status="running",
+			message=message,
+			step=step_index,
+			user=user,
+		)
+
+	try:
+		cfg = get_config()
+		_publish(reference_doctype, reference_name, status="running", message="Starting", step=0, user=user)
+
+		result = run_pipeline(website, cfg=cfg, progress=progress)
+
+		doc = frappe.get_doc(reference_doctype, reference_name)
+		doc.check_permission("write")
+		filled_fields = apply_to_document(doc, result, cfg)
+		if filled_fields:
+			doc.save()
+
+		write_run(
+			reference_doctype,
+			reference_name,
+			website,
+			status="Completed",
+			result=result,
+			started_on=started_on,
+		)
+
+		_publish(
+			reference_doctype,
+			reference_name,
+			status="completed",
+			message="Completed",
+			step=TOTAL_STEPS - 1,
+			payload={
+				"filled_fields": filled_fields,
+				"notes": result.notes,
+				**result.flat(),
+			},
+			user=user,
+		)
+		capture(
+			"enrichment_run_completed",
+			"crm",
+			properties={
+				"doctype": reference_doctype,
+				"trigger": trigger,
+				"status": "success",
+			},
+		)
+	except Exception:
+		# Discard any partial writes (e.g. a doc.save() that fired side effects before a
+		# later step threw). execute_job commits when this function returns normally, so
+		# without the rollback those partials would be committed alongside the Failed run.
+		frappe.db.rollback()
+		frappe.log_error(
+			title="Domain Enrichment: run_enrichment failed",
+			message=frappe.get_traceback(),
+		)
+		try:
+			write_run(
+				reference_doctype,
+				reference_name,
+				website,
+				status="Failed",
+				started_on=started_on,
+				notes=frappe.utils.cstr(frappe.get_traceback())[:1000],
+			)
+		except Exception:
+			frappe.log_error(title="Domain Enrichment: could not write Failed run")
+		_publish(
+			reference_doctype,
+			reference_name,
+			status="error",
+			message=frappe._("Enrichment failed. Check the error log."),
+			step=0,
+			user=user,
+		)
+		capture(
+			"enrichment_run_completed",
+			"crm",
+			properties={
+				"doctype": reference_doctype,
+				"trigger": trigger,
+				"status": "failure",
+			},
+		)

crm/domain_enrichment/tests/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Tests for the domain enrichment engine (added in Phase 7)."""

crm/domain_enrichment/tests/fixtures.py

@@ -0,0 +1,319 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Offline HTML fixtures + in-memory config builders for network-free tests.
+
+The HTML fixtures are deterministic snapshots used by the pure unit tests so the
+crawler/extractor mechanics can be exercised without touching the network. The
+config builders assemble ``Rule`` / ``Mapping`` / ``EnrichmentConfig`` objects in
+memory (no DB) so the rule executors can be tested rule-driven, exactly as the
+engine would see them after ``get_config()`` — but without hitting Frappe.
+"""
+
+from __future__ import annotations
+
+import re
+
+from bs4 import BeautifulSoup
+
+from crm.domain_enrichment.config import EnrichmentConfig, Mapping, Rule
+from crm.domain_enrichment.crawler import extract_content
+from crm.domain_enrichment.result import CrawledPage
+
+# --------------------------------------------------------------------------- #
+# HTML fixtures
+# --------------------------------------------------------------------------- #
+HOMEPAGE = """
+<!doctype html>
+<html>
+<head>
+  <title>Acme Analytics | AI powered SaaS</title>
+  <meta name="description" content="Acme is an AI powered analytics platform.">
+  <meta property="og:site_name" content="Acme Analytics">
+  <meta property="og:description" content="Real-time analytics for modern teams.">
+  <meta property="og:image" content="https://acme.example/logo.png">
+  <script type="application/ld+json">
+  {
+    "@context": "https://schema.org",
+    "@type": "Organization",
+    "name": "Acme Analytics Inc.",
+    "description": "AI powered analytics SaaS for product teams.",
+    "logo": {"url": "https://acme.example/ld-logo.png"},
+    "sameAs": [
+      "https://www.linkedin.com/company/acme-analytics",
+      "https://twitter.com/acmeanalytics",
+      "https://github.com/acme"
+    ]
+  }
+  </script>
+  <script src="/_next/static/chunk.js"></script>
+  <script src="https://www.googletagmanager.com/gtag/js?id=G-XXededXX"></script>
+</head>
+<body>
+  <h1>AI powered analytics</h1>
+  <h2>We are hiring across engineering</h2>
+  <nav>
+    <a href="/about">About</a>
+    <a href="/contact">Contact</a>
+    <a href="/team">Team</a>
+    <a href="/careers">Careers</a>
+    <a href="https://facebook.com/sharer?u=x">Share</a>
+    <a href="https://instagram.com/acmeanalytics">Instagram</a>
+    <a href="https://external-site.com/page">External</a>
+  </nav>
+  <p>Contact us at hello@acme.example or call <a href="tel:+1 415 555 0142">us</a>.</p>
+  <p>Our subscription pricing plans start free. Customer relationship tooling.</p>
+</body>
+</html>
+"""
+
+ABOUT = """
+<!doctype html>
+<html><head><title>About Acme</title></head>
+<body>
+  <h1>About Acme</h1>
+  <p>We raised a Series A round and are backed by great investors.</p>
+  <div class="team">
+    <div class="member"><div class="name">Jane Doe</div><div class="role">CEO &amp; Co-Founder</div></div>
+    <div class="member"><div class="name">John Smith</div><div class="role">CTO</div></div>
+    <div class="member"><div class="name">Senior Vice President</div><div class="role">VP</div></div>
+    <div class="member"><div class="name">About Acme</div><div class="role">Director</div></div>
+  </div>
+  <p>Reach finance at billing@acme.example.</p>
+</body></html>
+"""
+
+CONTACT = """
+<!doctype html>
+<html><head><title>Contact Acme</title></head>
+<body>
+  <h1>Contact</h1>
+  <p>Head office: 100 Market St, San Francisco, CA, 94103, United States</p>
+  <p>Email: sales@acme.example</p>
+  <p>Phone: +1 (415) 555-0199</p>
+</body></html>
+"""
+
+WORDPRESS = """
+<html><head><meta name="generator" content="WordPress 6.4"></head>
+<body><img src="/wp-content/uploads/x.png"><script src="/wp-json/index.js"></script></body></html>
+"""
+
+EMPTY = "<html><head></head><body></body></html>"
+
+# A JavaScript-only SPA shell (e.g. a Vite/React build). The body is an empty mount
+# point; everything meaningful is rendered client-side, so requests + BeautifulSoup
+# see only the build-tool-default title and favicon -- no description, og, or JSON-LD.
+JS_SPA_SHELL = """
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite App</title>
+    <script type="module" crossorigin src="/assets/index-5d175413.js"></script>
+    <link rel="stylesheet" href="/assets/index-57b7cd59.css">
+    <script type="module">window.onerror = () => {};</script>
+  </head>
+  <body>
+    <div id="root"></div>
+  </body>
+</html>
+"""
+
+BROKEN_JSON_LD = """
+<html><head>
+<script type="application/ld+json">{ this is not valid json ]</script>
+<title>Broken Co</title>
+</head><body><h1>Broken</h1></body></html>
+"""
+
+# --------------------------------------------------------------------------- #
+# Sitemap fixtures
+# --------------------------------------------------------------------------- #
+URLSET_XML = """<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://acme.example/about</loc></url>
+  <url><loc>https://acme.example/contact</loc></url>
+  <url><loc>https://acme.example/blog/post-1</loc></url>
+</urlset>"""
+
+SITEMAP_INDEX_XML = """<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap><loc>https://acme.example/sitemap-pages.xml</loc></sitemap>
+  <sitemap><loc>https://acme.example/sitemap-blog.xml</loc></sitemap>
+</sitemapindex>"""
+
+# A sitemap trying to smuggle a DTD entity-expansion payload -- must be rejected
+# outright by _parse_sitemap before ET.fromstring ever sees it.
+MALICIOUS_DOCTYPE_XML = """<?xml version="1.0"?>
+<!DOCTYPE foo [<!ENTITY xxe "boom">]>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://acme.example/&xxe;</loc></url>
+</urlset>"""
+
+
+def fake_fetch(responses):
+	"""Build a ``fetch(url, cfg, session=None, html_only=True)`` stub keyed by exact URL.
+
+	``responses`` maps url -> (status, body) for a 200-shaped response. A URL absent
+	from the map behaves like a 404 (mirrors a missing sitemap/robots.txt), so tests
+	only need to declare the endpoints they care about.
+	"""
+
+	def fake(url, cfg, session=None, html_only=True):
+		entry = responses.get(url)
+		if entry is None:
+			return 404, "", "not found", url
+		status, body = entry
+		return status, body, "", url
+
+	return fake
+
+
+def fake_fetch_with_redirects(responses, redirects):
+	"""Like ``fake_fetch``, but ``redirects`` maps a requested URL to the resolved
+	(post-redirect) URL returned as ``final_url`` -- simulates a 301/302 hop without
+	modeling the actual redirect chain, since callers only ever look at the result.
+	"""
+	base = fake_fetch(responses)
+
+	def fake(url, cfg, session=None, html_only=True):
+		status, body, error, final_url = base(url, cfg, session=session, html_only=html_only)
+		return status, body, error, redirects.get(url, final_url)
+
+	return fake
+
+
+# --------------------------------------------------------------------------- #
+# Page helper
+# --------------------------------------------------------------------------- #
+def make_page(url, html, status_code=200):
+	"""Build a (CrawledPage, soup) pair from raw HTML, exactly as the crawler does.
+
+	``extract_content`` mutates its soup, so it is given a throwaway parse and the
+	returned ``soup`` is a fresh, unmutated DOM (mirrors ``crawler.crawl_page``).
+	"""
+	soup = BeautifulSoup(html, "html.parser")
+	content = extract_content(BeautifulSoup(html, "html.parser"))
+	page = CrawledPage(
+		url=url,
+		status_code=status_code,
+		html=html,
+		text=content["text"],
+		title=content["title"],
+		headings=content["headings"],
+	)
+	return page, soup
+
+
+# --------------------------------------------------------------------------- #
+# In-memory rule builders (no DB) — mirror what config._build_rules() produces
+# --------------------------------------------------------------------------- #
+def _compile(pattern, is_regex):
+	if is_regex:
+		return re.compile(pattern, re.IGNORECASE)
+	return re.compile(re.escape(pattern), re.IGNORECASE)
+
+
+def keyword_rule(rule_type, patterns, *, target_value="", industry="", weight=1.0, match_scope="Full Text"):
+	"""Build a substring keyword Rule directly (is_regex=0 for each pattern)."""
+	compiled = [(_compile(p, False), p) for p in patterns]
+	return Rule(
+		rule_type=rule_type,
+		target_value=target_value,
+		industry=industry,
+		weight=weight,
+		match_scope=match_scope,
+		patterns=compiled,
+	)
+
+
+def regex_rule(rule_type, patterns, *, target_value="", weight=1.0, match_scope="HTML"):
+	"""Build a regex Rule directly (is_regex=1 for each pattern)."""
+	compiled = [(_compile(p, True), p) for p in patterns]
+	return Rule(
+		rule_type=rule_type,
+		target_value=target_value,
+		weight=weight,
+		match_scope=match_scope,
+		patterns=compiled,
+	)
+
+
+def industry_rules():
+	"""A small set of Industry rules (Headline scope) covering the fixtures."""
+	return [
+		keyword_rule(
+			"Industry",
+			["saas", "software as a service", "subscription"],
+			industry="SaaS",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["crm", "customer relationship"],
+			industry="CRM",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["shop", "free shipping", "apparel", "ecommerce"],
+			industry="Ecommerce",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["beverage", "sparkling water", "iced tea", "cereal"],
+			industry="Food & Beverage",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["3d printing", "additive manufacturing", "manufacturing"],
+			industry="Manufacturing",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["legal help", "legal documents", "legal advice", "law firm"],
+			industry="Legal",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["media company", "journalists", "newsroom"],
+			industry="Media",
+			match_scope="Headline",
+		),
+	]
+
+
+def social_rules():
+	return [
+		regex_rule("Social", [r"linkedin\.com/(company|in|school)/"], target_value="linkedin"),
+		regex_rule("Social", [r"(twitter\.com|x\.com)/[A-Za-z0-9_]+"], target_value="twitter"),
+		regex_rule("Social", [r"github\.com/[A-Za-z0-9_.-]+"], target_value="github"),
+		regex_rule("Social", [r"facebook\.com/[A-Za-z0-9_.\-/]+"], target_value="facebook"),
+		regex_rule("Social", [r"instagram\.com/[A-Za-z0-9_.]+"], target_value="instagram"),
+	]
+
+
+def make_config(settings=None, **kwargs) -> EnrichmentConfig:
+	"""Assemble an in-memory ``EnrichmentConfig`` for the SSRF / engine tests."""
+	cfg = EnrichmentConfig(settings=settings or {})
+	for key, value in kwargs.items():
+		setattr(cfg, key, value)
+	return cfg
+
+
+def make_mapping(source_key, target_doctype, target_fieldname, write_policy="Fill if empty", **kwargs):
+	return Mapping(
+		source_key=source_key,
+		target_doctype=target_doctype,
+		target_fieldname=target_fieldname,
+		write_policy=write_policy,
+		create_missing_link=kwargs.get("create_missing_link", 0),
+		default_values=kwargs.get("default_values", ""),
+	)

crm/domain_enrichment/tests/test_config.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure unit tests for config-layer pattern compilation (``config._compile_pattern``).
+
+Fully offline: no DB, no network. Covers the word-boundary fix -- an un-anchored
+short substring keyword like "erp" or "api" previously matched inside unrelated
+words ("waterproofing", "capital"), causing false industry-classification hits.
+"""
+
+from __future__ import annotations
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment.config import _compile_pattern
+
+
+class CompilePatternTest(UnitTestCase):
+	def test_substring_pattern_does_not_match_inside_unrelated_word(self):
+		erp_rx = _compile_pattern("erp", is_regex=0)
+		self.assertIsNone(erp_rx.search("Industrial Adhesives, Paints and Waterproofing Solutions"))
+		self.assertIsNone(erp_rx.search("enterprise value creation"))
+
+	def test_substring_pattern_still_matches_standalone_word(self):
+		erp_rx = _compile_pattern("erp", is_regex=0)
+		self.assertIsNotNone(erp_rx.search("Our ERP rollout finished on schedule"))
+
+	def test_api_pattern_does_not_match_inside_capital_or_rapid(self):
+		api_rx = _compile_pattern("api", is_regex=0)
+		self.assertIsNone(api_rx.search("venture capital and rapid growth"))
+		self.assertIsNotNone(api_rx.search("a public REST API for developers"))
+
+	def test_multi_word_phrase_pattern_still_matches(self):
+		rx = _compile_pattern("enterprise resource planning", is_regex=0)
+		self.assertIsNotNone(rx.search("our Enterprise Resource Planning suite"))
+
+	def test_is_regex_patterns_are_left_unanchored(self):
+		# An admin-authored regex (is_regex=1) is compiled as-is -- no word-boundary
+		# wrapping, so existing admin patterns (e.g. social-profile URL regexes) keep
+		# their exact intended behavior.
+		rx = _compile_pattern(r"linkedin\.com/(company|in|school)/", is_regex=1)
+		self.assertIsNotNone(rx.search("https://www.linkedin.com/company/acme"))

crm/domain_enrichment/tests/test_crawler.py

@@ -0,0 +1,467 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure unit tests for the crawler mechanics (no network, no DB).
+
+Covers URL normalization, the same-site/registrable-domain check, link-priority
+ordering (config-driven keywords), the crawlable filter (asset/scheme/skip-pattern
+rejection) and ``extract_content`` (title/headings/visible text).
+"""
+
+from __future__ import annotations
+
+from unittest import mock
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment import crawler
+from crm.domain_enrichment.crawler import (
+	_is_crawlable,
+	_link_priority,
+	extract_content,
+	normalize_url,
+	registrable_domain,
+	same_site,
+)
+from crm.domain_enrichment.tests import fixtures
+from crm.domain_enrichment.tests.fixtures import make_config
+
+# The default BFS priority keywords used by the engine when Settings has none.
+PRIORITY = [
+	("about", 1.0),
+	("contact", 1.0),
+	("team", 1.0),
+	("careers", 1.0),
+	("company", 1.0),
+]
+SKIP_PATTERNS = [r"/wp-admin", r"\?replytocom="]
+
+
+class NormalizeUrlTest(UnitTestCase):
+	def test_strips_fragment(self):
+		self.assertEqual(normalize_url("https://x.com/about/#team"), "https://x.com/about")
+
+	def test_strips_trailing_slash_on_path(self):
+		self.assertEqual(normalize_url("https://x.com/about/"), "https://x.com/about")
+
+	def test_keeps_root_slash(self):
+		# A bare root "/" must not be stripped (path length 1).
+		self.assertEqual(normalize_url("https://x.com/"), "https://x.com/")
+
+	def test_strips_query_string(self):
+		# Query-string variants of the same page (tracking params, product-variant
+		# params) must collapse to one dedupe key -- otherwise the same page gets
+		# queued and fetched multiple times under different query strings.
+		self.assertEqual(normalize_url("https://x.com/w/book/1?ean=123"), "https://x.com/w/book/1")
+		self.assertEqual(
+			normalize_url("https://x.com/w/book/1?terms=free-shipping"), "https://x.com/w/book/1"
+		)
+		self.assertEqual(normalize_url("https://x.com/w/book/1"), "https://x.com/w/book/1")
+
+
+class RegistrableDomainTest(UnitTestCase):
+	def test_strips_www_and_port(self):
+		self.assertEqual(registrable_domain("www.example.com:443"), "example.com")
+
+	def test_deep_subdomains_collapse_to_registrable(self):
+		self.assertEqual(registrable_domain("blog.shop.example.com"), "example.com")
+
+	def test_single_label(self):
+		self.assertEqual(registrable_domain("localhost"), "localhost")
+
+	def test_bare_ip_returns_itself(self):
+		self.assertEqual(registrable_domain("10.0.0.5"), "10.0.0.5")
+
+	def test_multi_label_public_suffix_keeps_registrable_label(self):
+		# The bug this guards: a last-two-labels rule returns "co.uk" here, so every
+		# *.co.uk site collapses into one same-site bucket. The PSL keeps the SLD.
+		self.assertEqual(registrable_domain("acme.co.uk"), "acme.co.uk")
+		self.assertEqual(registrable_domain("www.shop.acme.com.au"), "acme.com.au")
+
+	def test_distinct_orgs_under_a_public_suffix_are_distinct(self):
+		self.assertNotEqual(registrable_domain("acme.co.uk"), registrable_domain("evil.co.uk"))
+
+	def test_private_hosting_suffix_is_registrable(self):
+		# github.io is a PSL *private* suffix: two Pages tenants are different sites.
+		self.assertEqual(registrable_domain("acme.github.io"), "acme.github.io")
+		self.assertNotEqual(registrable_domain("acme.github.io"), registrable_domain("other.github.io"))
+
+
+class SameSiteTest(UnitTestCase):
+	def test_ignores_www_and_subdomains(self):
+		self.assertTrue(same_site("https://www.x.com/a", "x.com"))
+		self.assertTrue(same_site("https://blog.x.com/a", "x.com"))
+
+	def test_rejects_unrelated_org_sharing_a_public_suffix(self):
+		# Would be a false "same site" under the naive last-two-labels rule.
+		self.assertFalse(same_site("https://evil.co.uk/x", "acme.co.uk"))
+		self.assertFalse(same_site("https://other.github.io/x", "acme.github.io"))
+
+	def test_rejects_other_domain(self):
+		self.assertFalse(same_site("https://other.com/a", "x.com"))
+
+
+class LinkPriorityTest(UnitTestCase):
+	def test_priority_pages_rank_before_generic(self):
+		about = _link_priority("https://x.com/about", "About", PRIORITY)
+		blog = _link_priority("https://x.com/blog/post", "Read the post", PRIORITY)
+		self.assertLess(about, blog)
+
+	def test_anchor_text_counts(self):
+		# A generic URL with a priority keyword in the anchor still ranks high.
+		hit = _link_priority("https://x.com/p/42", "Our team", PRIORITY)
+		miss = _link_priority("https://x.com/p/43", "Read more", PRIORITY)
+		self.assertLess(hit, miss)
+
+	def test_non_priority_sorts_last(self):
+		generic = _link_priority("https://x.com/random", "", PRIORITY)
+		self.assertEqual(generic, len(PRIORITY) + 1)
+
+	def test_ancestor_segment_does_not_leak_priority_to_unrelated_leaf(self):
+		# A news article nested under "/about-us/..." must not inherit "about-us"'s
+		# priority from an ancestor path segment alone -- only the leaf segment counts.
+		news_article = _link_priority("https://x.com/about-us/corporate-news/2023/some-award", "", PRIORITY)
+		generic = _link_priority("https://x.com/blog/post", "", PRIORITY)
+		about_page = _link_priority("https://x.com/about-us", "", PRIORITY)
+		self.assertEqual(news_article, generic)
+		self.assertLess(about_page, news_article)
+
+
+class IsCrawlableTest(UnitTestCase):
+	def test_rejects_non_http_schemes(self):
+		self.assertFalse(_is_crawlable("mailto:a@b.com", []))
+		self.assertFalse(_is_crawlable("tel:+15555550142", []))
+		self.assertFalse(_is_crawlable("javascript:void(0)", []))
+
+	def test_rejects_asset_extensions(self):
+		self.assertFalse(_is_crawlable("https://x.com/file.pdf", []))
+		self.assertFalse(_is_crawlable("https://x.com/logo.png", []))
+		self.assertFalse(_is_crawlable("https://x.com/app.js", []))
+
+	def test_allows_normal_page(self):
+		self.assertTrue(_is_crawlable("https://x.com/about", []))
+
+	def test_rejects_configured_skip_pattern(self):
+		self.assertFalse(_is_crawlable("https://x.com/wp-admin/edit", SKIP_PATTERNS))
+		self.assertFalse(_is_crawlable("https://x.com/post?replytocom=5", SKIP_PATTERNS))
+		self.assertTrue(_is_crawlable("https://x.com/about", SKIP_PATTERNS))
+
+	def test_malformed_skip_pattern_treated_as_substring(self):
+		# An invalid regex must not crash; it falls back to substring match.
+		self.assertFalse(_is_crawlable("https://x.com/[bad", ["[bad"]))
+
+
+class ExtractContentTest(UnitTestCase):
+	def test_pulls_title_headings_and_text(self):
+		_page, soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+		content = extract_content(soup)
+		self.assertEqual(content["title"], "Acme Analytics | AI powered SaaS")
+		self.assertIn("AI powered analytics", content["headings"])
+		self.assertIn("hello@acme.example", content["text"])
+
+	def test_strips_script_and_style(self):
+		html = (
+			"<html><head><style>.x{}</style></head><body><script>var x=1</script><p>Visible</p></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://x.com", html)
+		content = extract_content(soup)
+		self.assertIn("Visible", content["text"])
+		self.assertNotIn("var x", content["text"])
+
+
+class ParseSitemapTest(UnitTestCase):
+	def test_parses_urlset(self):
+		kind, locs = crawler._parse_sitemap(fixtures.URLSET_XML)
+		self.assertEqual(kind, "urlset")
+		self.assertIn("https://acme.example/about", locs)
+		self.assertIn("https://acme.example/contact", locs)
+		self.assertIn("https://acme.example/blog/post-1", locs)
+
+	def test_parses_sitemapindex(self):
+		kind, locs = crawler._parse_sitemap(fixtures.SITEMAP_INDEX_XML)
+		self.assertEqual(kind, "sitemapindex")
+		self.assertEqual(
+			locs,
+			["https://acme.example/sitemap-pages.xml", "https://acme.example/sitemap-blog.xml"],
+		)
+
+	def test_malformed_xml_returns_none_kind(self):
+		kind, locs = crawler._parse_sitemap("<urlset><url><loc>unclosed")
+		self.assertIsNone(kind)
+		self.assertEqual(locs, [])
+
+	def test_doctype_rejected_before_parsing(self):
+		# Must never reach ET.fromstring with a DTD payload.
+		kind, locs = crawler._parse_sitemap(fixtures.MALICIOUS_DOCTYPE_XML)
+		self.assertIsNone(kind)
+		self.assertEqual(locs, [])
+
+	def test_unrelated_root_element_ignored(self):
+		kind, locs = crawler._parse_sitemap("<rss><channel></channel></rss>")
+		self.assertIsNone(kind)
+		self.assertEqual(locs, [])
+
+
+class DiscoverSitemapUrlsTest(UnitTestCase):
+	def setUp(self):
+		self.cfg = make_config()
+
+	def test_uses_robots_declared_sitemap(self):
+		robots = mock.Mock()
+		robots.site_maps.return_value = ["https://acme.example/my-sitemap.xml"]
+		responses = {"https://acme.example/my-sitemap.xml": (200, fixtures.URLSET_XML)}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, robots)
+		self.assertIn("https://acme.example/about", urls)
+		self.assertIn("https://acme.example/contact", urls)
+
+	def test_falls_back_to_conventional_path_when_robots_declares_none(self):
+		robots = mock.Mock()
+		robots.site_maps.return_value = None
+		responses = {"https://acme.example/sitemap.xml": (200, fixtures.URLSET_XML)}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, robots)
+		self.assertIn("https://acme.example/about", urls)
+
+	def test_falls_back_when_robots_is_none(self):
+		responses = {"https://acme.example/sitemap.xml": (200, fixtures.URLSET_XML)}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, None)
+		self.assertIn("https://acme.example/about", urls)
+
+	def test_missing_sitemap_returns_empty_without_raising(self):
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch({})):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, None)
+		self.assertEqual(urls, [])
+
+	def test_malformed_xml_returns_empty(self):
+		responses = {"https://acme.example/sitemap.xml": (200, "not xml at all")}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, None)
+		self.assertEqual(urls, [])
+
+	def test_follows_one_level_of_sitemap_index(self):
+		responses = {
+			"https://acme.example/sitemap.xml": (200, fixtures.SITEMAP_INDEX_XML),
+			"https://acme.example/sitemap-pages.xml": (200, fixtures.URLSET_XML),
+			"https://acme.example/sitemap-blog.xml": (
+				200,
+				'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+				"<url><loc>https://acme.example/blog/hello</loc></url></urlset>",
+			),
+		}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, None)
+		self.assertIn("https://acme.example/about", urls)
+		self.assertIn("https://acme.example/blog/hello", urls)
+
+	def test_rejects_cross_site_sitemap(self):
+		# The sitemap FILE lives on a different site than the one we're enriching --
+		# must never be fetched, regardless of what its <loc> entries claim. Uses
+		# real-style ".com" hosts (not "*.example") so registrable_domain() actually
+		# differentiates them -- ".example" isn't in the bundled Public Suffix List,
+		# so acme.example/evil.example would collapse to the same registrable domain.
+		robots = mock.Mock()
+		robots.site_maps.return_value = ["https://evil-corp.com/sitemap.xml"]
+		responses = {
+			"https://evil-corp.com/sitemap.xml": (
+				200,
+				'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+				"<url><loc>https://evil-corp.com/about</loc></url></urlset>",
+			)
+		}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme-corp.com", self.cfg, None, robots)
+		self.assertEqual(urls, [])
+
+
+class CrawlSitemapSeedingTest(UnitTestCase):
+	"""End-to-end: crawl() seeded from a robots.txt-declared sitemap.
+
+	Sitemap discovery must be a pure addition on top of the ordinary BFS: it should
+	surface a page that isn't linked from the homepage, without displacing the
+	homepage as the first crawled result, and it must disappear cleanly (no error)
+	when disabled or when the site has no sitemap at all.
+	"""
+
+	HOMEPAGE_HTML = '<html><head><title>Acme</title></head><body><a href="/contact">Contact</a></body></html>'
+	CONTACT_HTML = "<html><body>Contact us</body></html>"
+	HIDDEN_ABOUT_HTML = "<html><head><title>Hidden About</title></head><body><h1>About</h1></body></html>"
+	ROBOTS_TXT = "Sitemap: https://acme.example/sitemap.xml"
+	SITEMAP_XML = (
+		'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+		"<url><loc>https://acme.example/hidden-about</loc></url></urlset>"
+	)
+
+	def _responses(self, **overrides):
+		base = {
+			"https://acme.example": (200, self.HOMEPAGE_HTML),
+			"https://acme.example/robots.txt": (200, self.ROBOTS_TXT),
+			"https://acme.example/sitemap.xml": (200, self.SITEMAP_XML),
+			"https://acme.example/contact": (200, self.CONTACT_HTML),
+			"https://acme.example/hidden-about": (200, self.HIDDEN_ABOUT_HTML),
+		}
+		base.update(overrides)
+		return base
+
+	def test_sitemap_only_page_is_crawled_homepage_stays_first(self):
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 1},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(self._responses())):
+			results = crawler.crawl("https://acme.example", cfg)
+		urls = [page.url for page, _soup in results]
+		self.assertEqual(urls[0], "https://acme.example")
+		self.assertIn("https://acme.example/hidden-about", urls)
+		self.assertIn("https://acme.example/contact", urls)
+
+	def test_use_sitemap_disabled_never_fetches_sitemap(self):
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(self._responses())):
+			results = crawler.crawl("https://acme.example", cfg)
+		urls = [page.url for page, _soup in results]
+		self.assertEqual(urls[0], "https://acme.example")
+		self.assertNotIn("https://acme.example/hidden-about", urls)
+		self.assertIn("https://acme.example/contact", urls)
+
+	def test_no_sitemap_falls_through_to_plain_link_following(self):
+		# robots.txt exists but declares no sitemap, and /sitemap.xml 404s -- the
+		# crawl must proceed exactly as ordinary link-following, no error surfaced.
+		responses = self._responses(**{"https://acme.example/robots.txt": (200, "User-agent: *")})
+		del responses["https://acme.example/sitemap.xml"]
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 1},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			results = crawler.crawl("https://acme.example", cfg)
+		urls = [page.url for page, _soup in results]
+		self.assertEqual(urls[0], "https://acme.example")
+		self.assertIn("https://acme.example/contact", urls)
+		self.assertNotIn("https://acme.example/hidden-about", urls)
+		self.assertEqual(len(results), 2)  # homepage + contact only, no crash, no phantom pages
+
+
+class RedirectDedupTest(UnitTestCase):
+	"""A redirect can land two different *requested* URLs on the same resolved page
+	(e.g. /contact.html -> /contact) -- the pre-fetch `visited` check can't catch
+	this since the requested URLs genuinely differ. crawl() must collapse the
+	duplicate after the fact and not let it consume a page-budget slot.
+	"""
+
+	def test_redirect_collapsed_duplicate_does_not_consume_budget(self):
+		homepage_html = (
+			'<html><body><a href="/contact">Contact</a>'
+			'<a href="/contact.html">Contact (alt)</a>'
+			'<a href="/team">Team</a></body></html>'
+		)
+		responses = {
+			"https://acme.example": (200, homepage_html),
+			"https://acme.example/contact": (200, "<html><body>Contact page</body></html>"),
+			"https://acme.example/contact.html": (200, "<html><body>Contact page</body></html>"),
+			"https://acme.example/team": (200, "<html><body>Team page</body></html>"),
+		}
+		# /contact.html resolves (redirects) to the same page as /contact.
+		redirects = {"https://acme.example/contact.html": "https://acme.example/contact"}
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(
+			crawler, "fetch", side_effect=fixtures.fake_fetch_with_redirects(responses, redirects)
+		):
+			results = crawler.crawl("https://acme.example", cfg)
+		urls = [page.url for page, _soup in results]
+		self.assertEqual(urls.count("https://acme.example/contact"), 1)
+		# The freed budget slot means /team -- otherwise starved by the wasted
+		# duplicate fetch -- still gets crawled.
+		self.assertIn("https://acme.example/team", urls)
+
+
+class OffSiteRedirectScopeTest(UnitTestCase):
+	"""A same-site link that redirects off-domain must not be treated as an
+	in-scope crawled page: fetch()'s redirect loop only re-validates SSRF safety
+	per hop, never registrable-domain scope.
+	"""
+
+	def test_offsite_redirect_marked_out_of_scope(self):
+		homepage_html = (
+			'<html><body><a href="/investor-relations">IR</a><a href="/team">Team</a></body></html>'
+		)
+		responses = {
+			"https://jpm-test.com": (200, homepage_html),
+			"https://jpm-test.com/investor-relations": (200, "<html><body>redirected content</body></html>"),
+			"https://jpm-test.com/team": (200, "<html><body>Team page</body></html>"),
+		}
+		redirects = {
+			"https://jpm-test.com/investor-relations": "https://reports.jpmchase-test.com/ir/2022.htm"
+		}
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(
+			crawler, "fetch", side_effect=fixtures.fake_fetch_with_redirects(responses, redirects)
+		):
+			results = crawler.crawl("https://jpm-test.com", cfg)
+		offsite = next(
+			page for page, _soup in results if page.url == "https://reports.jpmchase-test.com/ir/2022.htm"
+		)
+		self.assertTrue(offsite.error)
+		self.assertEqual(offsite.html, "")
+		self.assertEqual(offsite.text, "")
+
+	def test_offsite_redirect_clears_title_and_headings(self):
+		# title/headings are parsed by crawl_page() before the off-site check runs,
+		# so clearing html/text/soup alone isn't enough -- extractors that read
+		# title/headings straight off the CrawledPage (e.g. classify_industry's
+		# About-page lookup) would otherwise still see off-site content.
+		homepage_html = '<html><body><a href="/about">About</a></body></html>'
+		responses = {
+			"https://jpm-test.com": (200, homepage_html),
+			"https://jpm-test.com/about": (
+				200,
+				"<html><head><title>Global Widget Manufacturing Co</title></head>"
+				"<body><h1>Leading widget manufacturer since 1900</h1></body></html>",
+			),
+		}
+		redirects = {"https://jpm-test.com/about": "https://widgetco-test.com/about"}
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(
+			crawler, "fetch", side_effect=fixtures.fake_fetch_with_redirects(responses, redirects)
+		):
+			results = crawler.crawl("https://jpm-test.com", cfg)
+		offsite = next(page for page, _soup in results if page.url == "https://widgetco-test.com/about")
+		self.assertTrue(offsite.error)
+		self.assertEqual(offsite.title, "")
+		self.assertEqual(offsite.headings, [])
+
+	def test_homepage_redirect_is_still_adopted_not_flagged(self):
+		# The existing "homepage moved to a new host" behavior must be unaffected --
+		# only NON-homepage pages get the off-site-scope check.
+		responses = {"https://old-home-test.com": (200, "<html><body>hi</body></html>")}
+		redirects = {"https://old-home-test.com": "https://new-home-test.com"}
+		cfg = make_config(
+			settings={"max_pages": 1, "max_depth": 0, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(
+			crawler, "fetch", side_effect=fixtures.fake_fetch_with_redirects(responses, redirects)
+		):
+			results = crawler.crawl("https://old-home-test.com", cfg)
+		self.assertEqual(results[0][0].url, "https://new-home-test.com")
+		self.assertFalse(results[0][0].error)

crm/domain_enrichment/tests/test_extractors.py

@@ -0,0 +1,862 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure, network-free unit tests for the config-driven extractors.
+
+The classifiers (industry / social) are exercised with **in-memory Rule objects**
+built directly (see ``tests.fixtures``), never the DB — this is the rule-driven
+rewrite of the POC's extractor tests. The mechanics (company-name cleaning, favicon
+scorer, JSON-LD parse, email/phone extraction, readability diagnosis) are tested
+standalone.
+"""
+
+from __future__ import annotations
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment import extractors
+from crm.domain_enrichment.result import CrawledPage, Method
+from crm.domain_enrichment.tests import fixtures
+
+
+def _pages(*specs):
+	"""specs: (url, html) tuples -> list[CrawledPage], soups_by_url dict."""
+	pages, soups = [], {}
+	for url, html in specs:
+		page, soup = fixtures.make_page(url, html)
+		pages.append(page)
+		soups[url] = soup
+	return pages, soups
+
+
+# --------------------------------------------------------------------------- #
+# Company information (mechanics)
+# --------------------------------------------------------------------------- #
+class CompanyInfoTest(UnitTestCase):
+	def setUp(self):
+		self.page, self.soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+
+	def test_prefers_json_ld_for_name(self):
+		info = extractors.extract_company_info(self.page, self.soup)
+		self.assertEqual(info["company_name"].value, "Acme Analytics Inc.")
+		self.assertEqual(info["company_name"].method, Method.JSON_LD)
+		self.assertEqual(info["company_name"].source, "https://acme.example")
+
+	def test_description_from_json_ld(self):
+		info = extractors.extract_company_info(self.page, self.soup)
+		self.assertIn("analytics", info["description"].value.lower())
+
+	def test_logo_is_link_icon_not_json_ld_image(self):
+		# The company logo is the link icon (HOMEPAGE declares none -> favicon.ico
+		# fallback), NOT the JSON-LD Organization.logo. That image is kept separately.
+		info = extractors.extract_company_info(self.page, self.soup)
+		self.assertEqual(info["logo"].method, Method.FAVICON)
+		self.assertEqual(info["logo"].value, "https://acme.example/favicon.ico")
+		self.assertEqual(info["image"].method, Method.JSON_LD)
+		self.assertEqual(info["image"].value, "https://acme.example/ld-logo.png")
+
+	def test_logo_prefers_declared_apple_touch_icon(self):
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<link rel='icon' href='/fav.png' sizes='16x16'>"
+			"<link rel='apple-touch-icon' href='/touch.png'>"
+			"</head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["logo"].value, "https://x.example/touch.png")
+
+	def test_meta_fallback_when_no_json_ld(self):
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head><title>Foo</title><meta property='og:site_name' content='Foo Inc'></head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["company_name"].value, "Foo Inc")
+		self.assertEqual(info["company_name"].method, Method.META_TAG)
+
+	def test_broken_json_ld_does_not_crash(self):
+		page, soup = fixtures.make_page("https://b.example", fixtures.BROKEN_JSON_LD)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["company_name"].method, Method.TITLE_TAG)
+
+	def test_degenerate_og_description_yields_to_plain_description(self):
+		# Broken markup can park a site-verification token on og:description while the
+		# plain description tag carries the real copy.
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<meta property='og:description' content='JIs-hfpDlO6niGZsMes79IWdxC0A9frop'>"
+			"<meta name='description' content='Acme Organics is a manufacturer and "
+			"exporter of aroma chemicals based in Mumbai.'>"
+			"</head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertTrue(info["description"].value.startswith("Acme Organics is a manufacturer"))
+
+	def test_og_description_still_preferred_when_usable(self):
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<meta property='og:description' content='Acme builds developer tools for teams.'>"
+			"<meta name='description' content='Some other blurb entirely for the page.'>"
+			"</head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["description"].value, "Acme builds developer tools for teams.")
+
+	def test_degenerate_description_still_used_when_nothing_better(self):
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head><meta name='description' content='Acme GmbH'></head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["description"].value, "Acme GmbH")
+
+	def test_unspaced_script_description_beats_token(self):
+		# Japanese copy has no ASCII spaces, so a space check alone would mistake it
+		# for a token and keep the og:description token instead.
+		copy = "株式会社アクメは東京を拠点に産業用ロボットの設計・製造・販売を手がける専門メーカーです。"
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<meta property='og:description' content='JIs-hfpDlO6niGZsMes79IWdxC0A9frop'>"
+			f"<meta name='description' content='{copy}'>"
+			"</head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["description"].value, copy)
+
+
+class LogoResolutionTest(UnitTestCase):
+	"""extract_logo returns the link icon only; the larger social/JSON-LD image is
+	captured by extract_image, never used as the logo."""
+
+	def test_scalable_svg_beats_small_png(self):
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<link rel='icon' type='image/png' href='/fav-32.png' sizes='32x32'>"
+			"<link rel='icon' type='image/svg+xml' href='/logo.svg'>"
+			"</head><body></body></html>",
+		)
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/logo.svg")
+		self.assertEqual(logo.method, Method.FAVICON)
+
+	def test_logo_is_link_icon_even_when_og_image_exists(self):
+		# A declared link icon is the logo; the bigger og:image is NOT the logo -- it
+		# is captured as the image instead.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<link rel='icon' href='/fav.ico' sizes='16x16'>"
+			"<meta property='og:image' content='https://x.example/social-1200.png'>"
+			"</head><body></body></html>",
+		)
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/fav.ico")
+		self.assertEqual(logo.method, Method.FAVICON)
+		image = extractors.extract_image(soup, "https://x.example")
+		self.assertEqual(image.value, "https://x.example/social-1200.png")
+		self.assertEqual(image.method, Method.META_TAG)
+
+	def test_falls_back_to_favicon_ico_ignoring_og_image(self):
+		# No link icon declared -> favicon.ico, even though an og:image exists.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<meta property='og:image' content='https://x.example/banner.png'>"
+			"</head><body></body></html>",
+		)
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/favicon.ico")
+		self.assertEqual(logo.method, Method.FAVICON)
+
+	def test_image_prefers_json_ld_logo_over_og_image(self):
+		# extract_image keeps the old "best big image" priority: curated JSON-LD logo
+		# outranks a social-share og:image.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			'<script type="application/ld+json">'
+			'{"@type":"Organization","logo":"https://x.example/brand.png"}</script>'
+			"<meta property='og:image' content='https://x.example/banner.png'>"
+			"</head><body></body></html>",
+		)
+		image = extractors.extract_image(soup, "https://x.example")
+		self.assertEqual(image.value, "https://x.example/brand.png")
+		self.assertEqual(image.method, Method.JSON_LD)
+
+	def test_image_empty_when_none_declared(self):
+		_p, soup = fixtures.make_page("https://x.example", "<html><head></head><body></body></html>")
+		self.assertEqual(extractors.extract_image(soup, "https://x.example").value, "")
+
+	def test_falls_back_to_favicon_ico(self):
+		_p, soup = fixtures.make_page("https://x.example", "<html><head></head><body></body></html>")
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/favicon.ico")
+		self.assertEqual(logo.method, Method.FAVICON)
+
+	def test_rejects_unsafe_scheme_in_icon(self):
+		# A crawled site injecting a javascript: icon href must not be stored as the
+		# logo -- it is skipped and the safe favicon.ico fallback is used.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head><link rel='icon' href='javascript:alert(1)'></head><body></body></html>",
+		)
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/favicon.ico")
+
+	def test_rejects_unsafe_scheme_in_image(self):
+		# javascript:/data: in JSON-LD logo or og:image is dropped, never stored.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			'<script type="application/ld+json">{"@type":"Organization","logo":"javascript:evil()"}</script>'
+			"<meta property='og:image' content='data:image/svg+xml;base64,PHN2Zz4='>"
+			"</head><body></body></html>",
+		)
+		self.assertEqual(extractors.extract_image(soup, "https://x.example").value, "")
+
+
+class CompanyNameCleaningTest(UnitTestCase):
+	def test_strips_tagline_after_separator(self):
+		self.assertEqual(extractors._clean_company_name("PostHog – dev tools"), "PostHog")
+
+	def test_trims_descriptive_sentence_title(self):
+		self.assertEqual(
+			extractors._clean_company_name(
+				"ITC Ltd has diversified presence in FMCG, Paperboards & Packaging"
+			),
+			"ITC Ltd",
+		)
+		self.assertEqual(extractors._clean_company_name("Acme Corp provides cloud security"), "Acme Corp")
+
+	def test_brand_at_end_matches_domain(self):
+		self.assertEqual(
+			extractors._clean_company_name(
+				"Innovative IT & Software Development Services | 8848 Digital",
+				"https://8848digital.com/",
+			),
+			"8848 Digital",
+		)
+
+	def test_without_domain_hint_falls_back_to_first_chunk(self):
+		self.assertEqual(
+			extractors._clean_company_name("Innovative IT & Software Development Services | 8848 Digital"),
+			"Innovative IT & Software Development Services",
+		)
+
+
+class JsonLdTest(UnitTestCase):
+	def test_parses_graph_blocks(self):
+		_page, soup = fixtures.make_page(
+			"https://x.com",
+			'<html><head><script type="application/ld+json">'
+			'{"@graph":[{"@type":"Organization","name":"A"},{"@type":"WebSite"}]}'
+			"</script></head><body></body></html>",
+		)
+		blocks = extractors.parse_json_ld(soup)
+		self.assertEqual(len(blocks), 2)
+		self.assertEqual(blocks[0]["name"], "A")
+
+	def test_ignores_invalid_json(self):
+		_page, soup = fixtures.make_page("https://x.com", fixtures.BROKEN_JSON_LD)
+		self.assertEqual(extractors.parse_json_ld(soup), [])
+
+
+class DescriptionSelectionTest(UnitTestCase):
+	def test_prefers_about_over_product_homepage(self):
+		pages, soups = _pages(
+			(
+				"https://shop.example",
+				"<html><head><meta property='og:description' content='Shop the latest gadgets.'></head><body></body></html>",
+			),
+			(
+				"https://shop.example/about",
+				"<html><head><meta name='description' content='Acme builds developer tools for teams.'></head><body></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertEqual(desc.value, "Acme builds developer tools for teams.")
+		self.assertEqual(desc.source, "https://shop.example/about")
+
+	def test_falls_back_to_homepage(self):
+		pages, soups = _pages(
+			(
+				"https://x.example",
+				"<html><head><meta property='og:description' content='We make great software.'></head><body></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertEqual(desc.value, "We make great software.")
+
+	def test_json_ld_is_authoritative(self):
+		pages, soups = _pages(
+			(
+				"https://s.example",
+				'<html><head><script type="application/ld+json">{"@type":"Organization","description":"Acme is a developer-tools company."}</script></head><body></body></html>',
+			),
+			(
+				"https://s.example/about",
+				"<html><head><meta name='description' content='Some about-page blurb.'></head><body></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertEqual(desc.method, Method.JSON_LD)
+
+	def test_about_body_paragraph_when_no_meta(self):
+		# No description metadata anywhere -> derive it from the About page's body copy.
+		about_body = (
+			"<main><p>Acme Robotics designs autonomous warehouse robots that help "
+			"retailers fulfil orders faster and at lower cost.</p></main>"
+		)
+		pages, soups = _pages(
+			("https://acme.example", "<html><head><title>Acme</title></head><body></body></html>"),
+			("https://acme.example/about", f"<html><body>{about_body}</body></html>"),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertIn("autonomous warehouse robots", desc.value)
+		self.assertEqual(desc.method, Method.BODY_TEXT)
+		self.assertEqual(desc.source, "https://acme.example/about")
+
+	def test_head_meta_beats_home_body_paragraph(self):
+		# With an og:description present, head meta wins over a homepage body paragraph.
+		pages, soups = _pages(
+			(
+				"https://acme.example",
+				"<html><head><meta name='description' content='Acme makes developer tools.'>"
+				"</head><body><main><p>Some long homepage marketing paragraph that is "
+				"definitely over the minimum length threshold for a body paragraph.</p>"
+				"</main></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertEqual(desc.value, "Acme makes developer tools.")
+		self.assertEqual(desc.method, Method.META_TAG)
+
+	def test_industry_keyword_density_steers_about_paragraph_choice(self):
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		pages, soups = _pages(
+			(
+				"https://acmebank.example",
+				"<html><head><title>Acme Bank</title></head><body></body></html>",
+			),
+			(
+				"https://acmebank.example/about",
+				"<html><body><main>"
+				"<p>Putting our long-tenured investment teams on the line to earn the "
+				"trust of institutional investors across every market we serve.</p>"
+				"<p>Acme Bank is a full-service financial institution serving corporations, "
+				"institutions and private clients across commercial and investment banking.</p>"
+				"</main></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups, industry_rules=finance_rules)
+		self.assertTrue(desc.value.startswith("Acme Bank is a full-service"))
+
+
+class FirstParagraphTest(UnitTestCase):
+	def test_skips_nav_and_returns_first_substantial_paragraph(self):
+		html = (
+			"<html><body><nav><p>Home About Contact Login Signup Pricing Docs Blog "
+			"Resources Careers Partners Support Status Community Downloads Enterprise</p></nav>"
+			"<main><p>Short.</p><p>Globex Corp is a logistics platform that moves freight "
+			"across the country for enterprise shippers.</p></main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://globex.example", html)
+		result = extractors.first_paragraph(soup)
+		self.assertIn("logistics platform", result)
+		self.assertNotIn("Login", result)  # nav paragraph skipped despite being long
+
+	def test_returns_empty_when_no_substantial_paragraph(self):
+		_page, soup = fixtures.make_page("https://x.example", "<html><body><p>Hi.</p></body></html>")
+		self.assertEqual(extractors.first_paragraph(soup), "")
+
+	def test_prefers_paragraph_with_more_keyword_hits_over_first_qualifying_one(self):
+		# A narrow pull-quote with zero industry-keyword hits precedes a genuine
+		# descriptive paragraph -- the keyword-denser paragraph should win.
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>Putting our long-tenured investment teams on the line to earn the "
+			"trust of institutional investors across every market we serve.</p>"
+			"<p>Acme Bank is a full-service financial institution serving corporations, "
+			"institutions and private clients across commercial and investment banking.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Acme Bank is a full-service"))
+
+	def test_falls_back_to_first_paragraph_when_no_keyword_hits(self):
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>Putting our long-tenured investment teams on the line to earn the "
+			"trust of institutional investors across every market we serve.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Putting our long-tenured"))
+
+	def test_falls_back_to_first_paragraph_when_no_industry_rules_given(self):
+		# No rules configured at all (industry_rules=None or []) -- with nothing else
+		# to separate the candidates, document order still decides.
+		html = (
+			"<html><body><main>"
+			"<p>Putting our long-tenured investment teams on the line to earn the "
+			"trust of institutional investors across every market we serve.</p>"
+			"<p>Acme Bank is a full-service financial institution serving corporations, "
+			"institutions and private clients across commercial and investment banking.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Putting our long-tenured"))
+		self.assertTrue(
+			extractors.first_paragraph(soup, industry_rules=[]).startswith("Putting our long-tenured")
+		)
+		self.assertTrue(
+			extractors.first_paragraph(soup, industry_rules=None).startswith("Putting our long-tenured")
+		)
+
+	def test_company_name_breaks_tie_between_equal_keyword_hits(self):
+		# Both paragraphs mention "banking" exactly once (tied on industry hits) --
+		# only the one that also names the company should win.
+		finance_rules = [fixtures.keyword_rule("Industry", ["banking"], industry="Finance")]
+		html = (
+			"<html><body><main>"
+			"<p>Our roots trace back decades of steady growth in commercial banking "
+			"and long-term client relationships across every region we serve.</p>"
+			"<p>Acme Bank has led the market in retail banking with award-winning "
+			"service for individuals, families and small businesses nationwide.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules, company_name="Acme Bank")
+		self.assertTrue(result.startswith("Acme Bank has led the market"))
+
+	def test_leadership_bio_never_wins_over_company_description(self):
+		# An About page's founder bio is dense in industry keywords (it lists former
+		# employers and sectors), so keyword scoring alone picks it over the real
+		# description. Bios must be rejected outright.
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["insurance", "asset management"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>Acme Capital is a registered portfolio manager focused on uncovering "
+			"long-term opportunities for clients across public equity markets.</p>"
+			"<p>Priya Nair is the Founder and Chief Investment Officer of Acme Capital, "
+			"where she previously led insurance and asset management mandates.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmecapital.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Acme Capital is a registered portfolio manager"))
+
+	def test_call_to_action_paragraph_never_wins(self):
+		finance_rules = [fixtures.keyword_rule("Industry", ["hvac"], industry="Construction")]
+		html = (
+			"<html><body><main>"
+			"<p>Kishika Temperatures has been a leader in air conditioning maintenance "
+			"and HVAC solutions for over 30 years, serving homes across the region.</p>"
+			"<p>Explore our services to see why our HVAC and HVAC-adjacent teams are the "
+			"preferred choice for leading developers, hotels and residential projects.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://kishika.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Kishika Temperatures has been a leader"))
+
+	def test_overview_mentioning_founder_once_is_not_a_profile(self):
+		# A company overview may name its founder with a single pronoun. That is
+		# not a staff profile and must still win over a generic sector essay.
+		html = (
+			"<html><body><main>"
+			"<p>Over the past decade, advances in warehouse robotics have reshaped how "
+			"goods move between suppliers and consumers around the world.</p>"
+			"<p>Acme Robotics builds warehouse automation for retailers. Founded by "
+			"Jane Doe in 2015, she still leads our engineering team today.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmerobotics.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Robotics builds"))
+
+	def test_company_described_as_founder_of_something_is_not_a_profile(self):
+		html = (
+			"<html><body><main>"
+			"<p>Over the past decade, advances in open hardware have reshaped how "
+			"engineers share designs and collaborate across company lines.</p>"
+			"<p>Acme Labs is the founder of the Open Hardware Alliance, and we design "
+			"reference boards used by hundreds of member companies worldwide.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmelabs.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Labs is the founder"))
+
+	def test_overview_speaking_for_the_company_is_never_a_profile(self):
+		# Two pronouns in copy that says "our" is still the company talking about
+		# itself, not a staff profile.
+		html = (
+			"<html><body><main>"
+			"<p>Over the past decade, advances in warehouse robotics have reshaped how "
+			"goods move between suppliers and consumers around the world.</p>"
+			"<p>Acme Robotics builds warehouse automation. Jane Doe founded it in 2015; "
+			"she leads our engineering team and her designs power every product.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmerobotics.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Robotics builds"))
+
+	def test_repeated_pronouns_mark_a_profile(self):
+		html = (
+			"<html><body><main>"
+			"<p>Priya Nair joined the firm in 2010. She oversees every client mandate "
+			"and her research has been cited across the industry for years.</p>"
+			"<p>Acme Capital is a registered portfolio manager focused on uncovering "
+			"long-term opportunities for clients across public equity markets.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmecapital.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Capital is a registered"))
+
+	def test_falls_back_to_rejected_paragraphs_when_nothing_else_qualifies(self):
+		# A page of nothing but bios must still yield a paragraph -- rejecting every
+		# candidate should not silently drop the description.
+		html = (
+			"<html><body><main>"
+			"<p>Priya Nair is the Founder of Acme Capital, where she leads the firm's "
+			"investment committee and oversees portfolio construction end to end.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmecapital.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Priya Nair is the Founder"))
+
+	def test_speaking_for_the_company_breaks_a_zero_hit_tie(self):
+		# Neither paragraph scores an industry hit: a generic essay about the sector
+		# sits first, the company's own copy second. Document order must not decide.
+		finance_rules = [fixtures.keyword_rule("Industry", ["banking"], industry="Finance")]
+		html = (
+			"<html><body><main>"
+			"<p>Over the past decade, advances in concrete technology have allowed "
+			"engineers to design mega structures that redefine infrastructure.</p>"
+			"<p>Capital RMC, established in 2016, stands at the forefront of the "
+			"construction sector, and we deliver world-class ready-mix concrete.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://capitalrmc.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Capital RMC, established in 2016"))
+
+	def test_speaking_for_the_company_cannot_outrank_more_industry_hits(self):
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>A full-service financial institution built on decades of trusted "
+			"commercial and retail banking relationships across every market.</p>"
+			"<p>We are proud to serve local businesses across the region with "
+			"award-winning service and community-first values that endure.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("A full-service financial institution"))
+
+	def test_company_name_mentions_cannot_outrank_more_industry_hits(self):
+		# The first paragraph scores 2 industry hits and never names the company; the
+		# second scores only 1 industry hit despite repeating the company name three
+		# times. Industry hits must still decide -- company name is tiebreak-only.
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>A full-service financial institution built on decades of trusted "
+			"commercial and retail banking relationships across every market.</p>"
+			"<p>Acme Bank, Acme Bank, Acme Bank -- proud to serve local businesses "
+			"with award-winning retail banking and community-first values.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules, company_name="Acme Bank")
+		self.assertTrue(result.startswith("A full-service financial institution"))
+
+
+class AboutPageDetectionTest(UnitTestCase):
+	def test_canonical_about_slugs_match(self):
+		for path in (
+			"/about",
+			"/about-us",
+			"/about_us",
+			"/aboutus",
+			"/company",
+			"/our-story",
+			"/en/about-us",
+		):
+			self.assertTrue(extractors._is_about_page(f"https://x.example{path}"), path)
+
+	def test_prefix_match_no_longer_qualifies(self):
+		# Regression guard: "about-our-reporting" (an ESG page) starts with "about-"
+		# but is not itself a canonical About-page slug.
+		self.assertFalse(extractors._is_about_page("https://x.example/esg-topics/about-our-reporting"))
+		self.assertFalse(extractors._is_about_page("https://x.example/about-cookies"))
+
+	def test_non_about_words_still_rejected(self):
+		self.assertFalse(extractors._is_about_page("https://x.example/about/careers"))
+		self.assertFalse(extractors._is_about_page("https://x.example/about-us/contact"))
+
+
+# --------------------------------------------------------------------------- #
+# Contacts: emails + phones (mechanics)
+# --------------------------------------------------------------------------- #
+class EmailExtractionTest(UnitTestCase):
+	def setUp(self):
+		self.pages, _ = _pages(
+			("https://acme.example", fixtures.HOMEPAGE),
+			("https://acme.example/about", fixtures.ABOUT),
+			("https://acme.example/contact", fixtures.CONTACT),
+		)
+
+	def test_emails_deduped_and_sourced(self):
+		emails = extractors.extract_emails(self.pages)
+		values = {e.value for e in emails}
+		self.assertIn("hello@acme.example", values)
+		self.assertIn("sales@acme.example", values)
+		self.assertEqual(len(values), len(emails))  # deduped
+
+	def test_blocklisted_noise_emails_rejected(self):
+		pages, _ = _pages(
+			("https://x.com", "<html><body>logo@2x.png test@example.com real@x.com</body></html>"),
+		)
+		values = {e.value for e in extractors.extract_emails(pages)}
+		self.assertIn("real@x.com", values)
+		self.assertNotIn("test@example.com", values)
+
+
+class PhoneExtractionTest(UnitTestCase):
+	def setUp(self):
+		self.pages, _ = _pages(
+			("https://acme.example", fixtures.HOMEPAGE),
+			("https://acme.example/contact", fixtures.CONTACT),
+		)
+
+	def test_tel_link_marked(self):
+		phones = extractors.extract_phones(self.pages)
+		self.assertIn(Method.TEL_LINK, {p.method for p in phones})
+		self.assertTrue(any("4155550142" in p.value.replace("+", "") for p in phones))
+
+	def test_international_number_captured(self):
+		pages, _ = _pages(
+			("https://x.example/contact", "<html><body><p>Call us: +612 9003 8888</p></body></html>")
+		)
+		phones = extractors.extract_phones(pages)
+		self.assertTrue(any("61290038888" in p.value.replace("+", "") for p in phones))
+
+	def test_number_without_country_code_rejected(self):
+		# Deliberate precision-over-recall call: a number with no "+" is either a
+		# local/ambiguous format or noise (reference numbers, stats) that happens to
+		# fall in a phone-shaped digit range -- only accept explicit international
+		# numbers, even though this means dropping some real local-format numbers.
+		pages, _ = _pages(
+			(
+				"https://x.example/contact",
+				"<html><body><p>Local line: 2165703. Toll-free: 1800-266-6066.</p>"
+				'<a href="tel:18002666066">Call</a></body></html>',
+			)
+		)
+		phones = extractors.extract_phones(pages)
+		self.assertEqual(phones, [])
+
+
+# --------------------------------------------------------------------------- #
+# Social profiles (config-driven via Social rules)
+# --------------------------------------------------------------------------- #
+class SocialTest(UnitTestCase):
+	def test_detects_networks_and_skips_share_links(self):
+		page, soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+		company = extractors.extract_company_info(page, soup)
+		profiles = extractors.extract_social_profiles(
+			[page], {page.url: soup}, fixtures.social_rules(), extra_links=company["social_links"]
+		)
+		self.assertTrue(profiles["linkedin"].value)
+		self.assertTrue(profiles["github"].value)
+		self.assertTrue(profiles["instagram"].value)
+		# JSON-LD sameAs links win and are marked as such.
+		self.assertEqual(profiles["linkedin"].method, Method.JSON_LD)
+		# A facebook 'sharer' link must be ignored.
+		self.assertEqual(profiles["facebook"].value, "")
+
+	def test_protocol_relative_links_made_absolute(self):
+		page, soup = fixtures.make_page(
+			"https://bluedart.com",
+			"<html><body><a href='//www.linkedin.com/company/bluedart/'>li</a>"
+			"<a href='//twitter.com/BlueDart_'>tw</a></body></html>",
+		)
+		profiles = extractors.extract_social_profiles([page], {page.url: soup}, fixtures.social_rules())
+		self.assertEqual(profiles["linkedin"].value, "https://www.linkedin.com/company/bluedart/")
+		self.assertEqual(profiles["twitter"].value, "https://twitter.com/BlueDart_")
+
+
+# --------------------------------------------------------------------------- #
+# Industry classification (config-driven via Industry rules)
+# --------------------------------------------------------------------------- #
+class IndustryTest(UnitTestCase):
+	def _classify(self, name, description, extra_html=""):
+		page, _ = fixtures.make_page(
+			"https://x.example",
+			f"<html><head><title>{name}</title></head><body>{extra_html}</body></html>",
+		)
+		company = {"company_name": name, "description": description}
+		return extractors.classify_industry([page], company, fixtures.industry_rules())
+
+	def test_classifies_saas_or_crm_with_confidence(self):
+		page, soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+		company = extractors.extract_company_info(page, soup)
+		industry, conf = extractors.classify_industry([page], company, fixtures.industry_rules())
+		self.assertIn(industry, ("SaaS", "CRM"))
+		self.assertGreater(conf, 0.0)
+		self.assertLessEqual(conf, 1.0)
+
+	def test_empty_site_returns_no_industry(self):
+		page, _ = fixtures.make_page("https://e.example", fixtures.EMPTY)
+		industry, conf = extractors.classify_industry([page], {}, fixtures.industry_rules())
+		self.assertEqual(industry, "")
+		self.assertEqual(conf, 0.0)
+
+	def test_no_rules_returns_empty(self):
+		page, _ = fixtures.make_page("https://e.example", fixtures.HOMEPAGE)
+		self.assertEqual(extractors.classify_industry([page], {}, []), ("", 0.0))
+
+	def test_d2c_brand_maps_to_ecommerce(self):
+		self.assertEqual(self._classify("Gymshark", "Shop gym clothing and workout apparel.")[0], "Ecommerce")
+
+	def test_beverage_brand_maps_to_food_and_beverage(self):
+		self.assertEqual(
+			self._classify("Liquid Death", "A beverage company with sparkling water and iced tea.")[0],
+			"Food & Beverage",
+		)
+
+	def test_legal_help_beats_stray_keyword(self):
+		self.assertEqual(
+			self._classify(
+				"Rocket Lawyer", "Rocket Lawyer makes legal help simple — legal documents and advice."
+			)[0],
+			"Legal",
+		)
+
+	def test_3d_printing_maps_to_manufacturing(self):
+		# A "shop" nav link must not beat the core business signal in the headline.
+		self.assertEqual(
+			self._classify(
+				"Desktop Metal",
+				"Metal 3D printing and additive manufacturing for engineers.",
+				"<nav>Shop</nav>",
+			)[0],
+			"Manufacturing",
+		)
+
+	def test_about_page_headline_supplements_thin_homepage(self):
+		# A homepage with no meta description and abstract brand-voice headings
+		# carries no classification signal on its own -- a crawled About page's
+		# title/headings should still let the classifier find a match.
+		finance_rules = [
+			fixtures.keyword_rule(
+				"Industry",
+				["banking", "financial services"],
+				industry="Finance",
+				match_scope="Headline",
+			)
+		]
+		home, _ = fixtures.make_page(
+			"https://acmebank.example",
+			"<html><head><title>Acme | Official Website</title></head>"
+			"<body><h1>Acme home</h1><h2>What problem can we solve together?</h2></body></html>",
+		)
+		about, _ = fixtures.make_page(
+			"https://acmebank.example/about",
+			"<html><head><title>About Acme</title></head>"
+			"<body><h2>A leader in financial services and banking</h2></body></html>",
+		)
+		company = {"company_name": "Acme", "description": ""}
+		industry, conf = extractors.classify_industry([home, about], company, finance_rules)
+		self.assertEqual(industry, "Finance")
+		self.assertGreater(conf, 0.0)
+
+	def test_homepage_only_still_returns_nothing_when_thin(self):
+		finance_rules = [
+			fixtures.keyword_rule(
+				"Industry",
+				["banking", "financial services"],
+				industry="Finance",
+				match_scope="Headline",
+			)
+		]
+		home, _ = fixtures.make_page(
+			"https://acmebank.example",
+			"<html><head><title>Acme | Official Website</title></head>"
+			"<body><h1>Acme home</h1><h2>What problem can we solve together?</h2></body></html>",
+		)
+		company = {"company_name": "Acme", "description": ""}
+		industry, conf = extractors.classify_industry([home], company, finance_rules)
+		self.assertEqual(industry, "")
+		self.assertEqual(conf, 0.0)
+
+
+# --------------------------------------------------------------------------- #
+# Generic rule executors directly
+# --------------------------------------------------------------------------- #
+class RuleExecutorTest(UnitTestCase):
+	def test_apply_keyword_rules_weights_and_sums(self):
+		rule = fixtures.keyword_rule(
+			"Industry", ["saas", "subscription"], industry="SaaS", weight=2.0, match_scope="Headline"
+		)
+		scores = extractors.apply_keyword_rules({"Headline": "a saas subscription saas"}, [rule])
+		# 3 hits * weight 2.0 = 6.0
+		self.assertEqual(scores["SaaS"], 6.0)
+
+
+# --------------------------------------------------------------------------- #
+# Readability diagnosis (mechanics)
+# --------------------------------------------------------------------------- #
+class ReadabilityTest(UnitTestCase):
+	def test_blocked_waf_challenge(self):
+		page, _ = fixtures.make_page(
+			"https://www.pepsi.com",
+			'<html><head><script src="/_Incapsula_Resource?SWJIYLWA=1"></script></head><body></body></html>',
+		)
+		self.assertEqual(extractors.diagnose_readability([page]), "blocked")
+
+	def test_empty_js_only_shell(self):
+		page, _ = fixtures.make_page(
+			"https://spa.example", "<html><head></head><body><div id='root'></div></body></html>"
+		)
+		self.assertEqual(extractors.diagnose_readability([page]), "empty")
+
+	def test_unreachable_when_failed(self):
+		page = CrawledPage(url="https://x.example", status_code=0, error="timeout after 10s")
+		self.assertEqual(extractors.diagnose_readability([page]), "unreachable")
+
+	def test_http_403_is_blocked(self):
+		page = CrawledPage(
+			url="https://www.pepperfry.com",
+			status_code=403,
+			html="<html><head><title>Access Denied</title></head></html>",
+			text="Access Denied",
+			title="Access Denied",
+		)
+		self.assertEqual(extractors.diagnose_readability([page]), "blocked")
+
+	def test_readable_site_returns_empty_reason(self):
+		page, _ = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+		self.assertEqual(extractors.diagnose_readability([page]), "")

crm/domain_enrichment/tests/test_integration.py

@@ -0,0 +1,301 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""IntegrationTestCase tests for the Frappe layer (real test DB).
+
+Covers: seeder idempotency, the mapper write-policies against real CRM
+Lead/Deal/Organization docs, the single Run writer, API permission/allow-list
+enforcement + rate-limit, and the link-time Organization -> Lead/Deal copy.
+
+All tests are network-free: wherever a crawl would happen the pipeline / fetch is
+monkeypatched to return a canned EnrichmentResult, so nothing leaves the box.
+"""
+
+from __future__ import annotations
+
+from unittest import mock
+
+import frappe
+from frappe.tests import IntegrationTestCase
+
+from crm.domain_enrichment import config, install, tasks
+from crm.domain_enrichment.config import get_config
+from crm.domain_enrichment.cross_record import copy_enrichment_from_organization
+from crm.domain_enrichment.mapper import apply_to_document
+from crm.domain_enrichment.result import EnrichmentResult, Field, SocialProfile
+from crm.domain_enrichment.tests import fixtures
+
+
+def canned_result(website="https://acme.example") -> EnrichmentResult:
+	"""A fully-populated EnrichmentResult — no network, deterministic."""
+	r = EnrichmentResult(website=website)
+	r.company_name = Field("Acme Analytics Inc.", website, "JSON-LD")
+	r.description = Field("AI powered analytics SaaS for product teams.", website, "Meta Tag")
+	r.logo = Field("https://acme.example/favicon.ico", website, "Favicon")
+	r.industry = Field("SaaS", website, "Keyword Classifier")
+	r.industry_confidence = 0.8
+	r.social_profiles = {
+		"linkedin": SocialProfile("https://www.linkedin.com/company/acme", website),
+		"twitter": SocialProfile("https://twitter.com/acme", website),
+	}
+	return r
+
+
+class SeederIdempotencyTest(IntegrationTestCase):
+	def tearDown(self):
+		frappe.db.rollback()
+
+	def test_seeder_is_a_no_op_on_second_run(self):
+		# The seeder runs on install/migrate, so records already exist. Running it
+		# again must not create duplicate Rule / Field Mapping rows.
+		install.seed_default_rules_and_mappings()
+		rules_before = frappe.db.count("CRM Enrichment Rule")
+		mappings_before = frappe.db.count("CRM Enrichment Field Mapping")
+
+		install.seed_default_rules_and_mappings()
+
+		self.assertEqual(frappe.db.count("CRM Enrichment Rule"), rules_before)
+		self.assertEqual(frappe.db.count("CRM Enrichment Field Mapping"), mappings_before)
+		self.assertGreater(rules_before, 0)
+		self.assertGreater(mappings_before, 0)
+
+
+class MapperWritePolicyTest(IntegrationTestCase):
+	def setUp(self):
+		self.cfg = get_config()
+		self.result = canned_result()
+
+	def tearDown(self):
+		frappe.db.rollback()
+
+	def _new_org(self, **kwargs):
+		doc = frappe.new_doc("CRM Organization")
+		doc.organization_name = kwargs.pop("organization_name", "Mapper Test Org")
+		for k, v in kwargs.items():
+			doc.set(k, v)
+		return doc
+
+	def test_fill_if_empty_fills_only_empty_fields(self):
+		org = self._new_org(company_description="")
+		apply_to_document(org, self.result, self.cfg)
+		self.assertEqual(org.company_description, "AI powered analytics SaaS for product teams.")
+
+	def test_fill_if_empty_never_clobbers_user_data(self):
+		org = self._new_org(company_description="Hand-written by sales.")
+		apply_to_document(org, self.result, self.cfg)
+		self.assertEqual(org.company_description, "Hand-written by sales.")
+
+	def test_create_missing_link_creates_industry(self):
+		industry = "Quantum Widgets " + frappe.generate_hash(length=6)
+		self.assertFalse(frappe.db.exists("CRM Industry", industry))
+		result = canned_result()
+		result.industry = Field(industry, "https://x.example", "Keyword Classifier")
+		org = self._new_org(industry="")
+		apply_to_document(org, result, self.cfg)
+		self.assertTrue(frappe.db.exists("CRM Industry", industry))
+		self.assertEqual(org.industry, industry)
+
+	def test_create_missing_link_respects_user_create_permission(self):
+		# A user without CRM Industry create rights must not be able to auto-create a
+		# master from a scraped value: the master is not inserted and the field is
+		# skipped (no dangling link), rather than an ignore_permissions escalation.
+		industry = "Contraband Industry " + frappe.generate_hash(length=6)
+		result = canned_result()
+		result.industry = Field(industry, "https://x.example", "Keyword Classifier")
+		org = self._new_org(industry="")
+
+		user = _make_minimal_user()
+		self.addCleanup(frappe.set_user, "Administrator")
+		frappe.set_user(user)
+		apply_to_document(org, result, self.cfg)
+
+		self.assertFalse(frappe.db.exists("CRM Industry", industry))
+		self.assertFalse(org.industry)
+
+
+class RunWriterTest(IntegrationTestCase):
+	def tearDown(self):
+		frappe.db.rollback()
+
+	def test_writes_exactly_one_run_with_summary(self):
+		org = frappe.new_doc("CRM Organization")
+		org.organization_name = "Run Writer Org " + frappe.generate_hash(length=4)
+		org.website = "https://acme.example"
+		org.insert()
+
+		before = frappe.db.count("CRM Enrichment Run", {"reference_name": org.name})
+		run_name = tasks.write_run(
+			"CRM Organization",
+			org.name,
+			"https://acme.example",
+			status="Completed",
+			result=canned_result(),
+		)
+		after = frappe.db.count("CRM Enrichment Run", {"reference_name": org.name})
+		self.assertEqual(after - before, 1)
+
+		run = frappe.get_doc("CRM Enrichment Run", run_name)
+		self.assertEqual(run.status, "Completed")
+		self.assertEqual(run.company_name, "Acme Analytics Inc.")
+		self.assertEqual(run.industry, "SaaS")
+		self.assertTrue(run.finished_on)
+		self.assertIn("Acme Analytics", run.raw_json)
+
+
+class ApiPermissionTest(IntegrationTestCase):
+	def tearDown(self):
+		frappe.db.rollback()
+		frappe.set_user("Administrator")
+
+	def test_enrich_throws_on_disallowed_doctype(self):
+		from crm.domain_enrichment.api import enrich
+
+		with self.assertRaises(frappe.ValidationError):
+			enrich("ToDo", "anything")
+
+	def test_enrich_requires_write_permission(self):
+		from crm.domain_enrichment.api import enrich
+
+		org = frappe.new_doc("CRM Organization")
+		org.organization_name = "Perm Org " + frappe.generate_hash(length=4)
+		org.website = "https://acme.example"
+		org.insert()
+
+		# A user with no role on CRM Organization must be blocked by check_permission.
+		user = _make_minimal_user()
+		frappe.set_user(user)
+		with self.assertRaises(frappe.PermissionError):
+			enrich("CRM Organization", org.name)
+
+	def test_enrich_routes_are_rate_limited(self):
+		# enrich / retry each carry rate_limit(limit=ENRICH_RATE_LIMIT).
+		from crm.domain_enrichment import api
+
+		for fn in (api.enrich, api.retry):
+			self.assertTrue(hasattr(fn, "__wrapped__"), fn.__name__)
+
+
+class CrossRecordCopyTest(IntegrationTestCase):
+	def setUp(self):
+		self.org = frappe.new_doc("CRM Organization")
+		self.org.organization_name = "Enriched Org " + frappe.generate_hash(length=4)
+		self.org.website = "https://acme.example"
+		self.org.company_description = "AI powered analytics SaaS."
+		self.org.linkedin = "https://www.linkedin.com/company/acme"
+		self.org.insert()
+		# Mark it enriched via a completed Run (the cleanest "is enriched" signal).
+		tasks.write_run(
+			"CRM Organization",
+			self.org.name,
+			self.org.website,
+			status="Completed",
+			result=canned_result(),
+		)
+
+	def tearDown(self):
+		frappe.db.rollback()
+
+	def test_copies_empty_fields_from_enriched_org(self):
+		deal = frappe.new_doc("CRM Deal")
+		deal.organization = self.org.name
+		filled = copy_enrichment_from_organization(deal)
+		self.assertTrue(filled)
+		self.assertEqual(deal.company_description, "AI powered analytics SaaS.")
+		self.assertEqual(deal.linkedin, "https://www.linkedin.com/company/acme")
+
+	def test_does_not_clobber_user_set_fields(self):
+		deal = frappe.new_doc("CRM Deal")
+		deal.organization = self.org.name
+		deal.company_description = "User wrote this."
+		copy_enrichment_from_organization(deal)
+		self.assertEqual(deal.company_description, "User wrote this.")
+
+	def test_non_enriched_org_is_a_noop(self):
+		plain = frappe.new_doc("CRM Organization")
+		plain.organization_name = "Plain Org " + frappe.generate_hash(length=4)
+		plain.website = "https://plain.example"
+		plain.insert()
+
+		deal = frappe.new_doc("CRM Deal")
+		deal.organization = plain.name
+		filled = copy_enrichment_from_organization(deal)
+		self.assertEqual(filled, [])
+		self.assertFalse(deal.company_description)
+
+
+class AutoEnrichOnCreateTest(IntegrationTestCase):
+	"""The after_insert auto-enrich hook (Lead, Deal, Organization), gated by Settings."""
+
+	def _settings(self, **overrides):
+		s = {
+			"enabled": 1,
+			"auto_enrich": 1,
+			"request_timeout": 10,
+			"max_pages": 10,
+		}
+		s.update(overrides)
+		return frappe._dict(s)
+
+	def _doc(self, doctype, website="https://acme.example", name="X1"):
+		return frappe._dict(doctype=doctype, name=name, website=website)
+
+	def _run(self, doc, settings):
+		# auto_enrich_on_create now does a Settings-only gate (config.get_settings), and
+		# enqueue_enrichment reads its timeout the same way -- patch both to the fake
+		# Settings and assert on frappe.enqueue.
+		with (
+			mock.patch.object(config, "get_settings", return_value=settings),
+			mock.patch.object(tasks, "get_settings", return_value=settings),
+			mock.patch.object(tasks.frappe, "enqueue") as enq,
+		):
+			tasks.auto_enrich_on_create(doc)
+		return enq
+
+	def test_enqueues_for_new_lead(self):
+		enq = self._run(self._doc("CRM Lead", name="L1"), self._settings())
+		enq.assert_called_once()
+		kwargs = enq.call_args.kwargs
+		self.assertEqual(kwargs["reference_doctype"], "CRM Lead")
+		self.assertEqual(kwargs["job_id"], "domain-enrich-CRM Lead-L1")
+
+	def test_enqueues_for_new_deal(self):
+		enq = self._run(self._doc("CRM Deal", name="D1"), self._settings())
+		enq.assert_called_once()
+		kwargs = enq.call_args.kwargs
+		self.assertEqual(kwargs["reference_doctype"], "CRM Deal")
+		self.assertEqual(kwargs["job_id"], "domain-enrich-CRM Deal-D1")
+		self.assertTrue(kwargs["deduplicate"])
+
+	def test_enqueues_for_new_organization(self):
+		enq = self._run(self._doc("CRM Organization", name="O1"), self._settings())
+		enq.assert_called_once()
+		self.assertEqual(enq.call_args.kwargs["reference_doctype"], "CRM Organization")
+
+	def test_skips_when_auto_enrich_off(self):
+		enq = self._run(self._doc("CRM Deal"), self._settings(auto_enrich=0))
+		enq.assert_not_called()
+
+	def test_skips_when_feature_disabled(self):
+		enq = self._run(self._doc("CRM Deal"), self._settings(enabled=0))
+		enq.assert_not_called()
+
+	def test_skips_doctype_outside_the_allow_list(self):
+		enq = self._run(self._doc("CRM Task"), self._settings())
+		enq.assert_not_called()
+
+	def test_skips_without_website(self):
+		enq = self._run(self._doc("CRM Deal", website=""), self._settings())
+		enq.assert_not_called()
+
+
+def _make_minimal_user():
+	"""Create (or reuse) a user with no CRM Organization role for permission tests."""
+	email = "de-noperm@example.com"
+	if not frappe.db.exists("User", email):
+		user = frappe.new_doc("User")
+		user.email = email
+		user.first_name = "No"
+		user.last_name = "Perm"
+		user.send_welcome_email = 0
+		user.insert(ignore_permissions=True)
+	return email

crm/domain_enrichment/tests/test_pipeline.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure, network-free pipeline tests.
+
+The crawl is stubbed (``pipeline.crawl`` patched to return canned offline pages), so
+the whole pipeline runs with zero network. Asserts the assembled ``EnrichmentResult``,
+the explainable provenance schema (``to_dict``/``flat``) the mapper depends on, and
+the progress-step contract.
+"""
+
+from __future__ import annotations
+
+from unittest import mock
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment import pipeline as pipeline_mod
+from crm.domain_enrichment.pipeline import PROGRESS_STEPS, _normalize_website
+from crm.domain_enrichment.pipeline import run as run_pipeline
+from crm.domain_enrichment.result import EnrichmentResult
+from crm.domain_enrichment.tests import fixtures
+
+# The schema the CRM field mapper relies on. If extractors change the shape these
+# assertions fail loudly instead of the mapper silently reading missing keys.
+EXPECTED_TOP_KEYS = {
+	"company_name",
+	"description",
+	"logo",
+	"image",
+	"industry",
+	"industry_confidence",
+	"emails",
+	"phones",
+	"social_profiles",
+	"_meta",
+}
+EXPECTED_FLAT_KEYS = {
+	"company_name",
+	"description",
+	"logo",
+	"industry",
+	"industry_confidence",
+	"social_profiles",
+}
+SCALAR_SUBKEYS = {"value", "source", "method"}
+
+
+def _full_config():
+	"""An in-memory config wired with all the rule types the fixtures exercise."""
+	return fixtures.make_config(
+		settings={"max_pages": 5, "max_depth": 1, "request_timeout": 5},
+		rules_by_type={
+			"Industry": fixtures.industry_rules(),
+			"Social": fixtures.social_rules(),
+		},
+	)
+
+
+def _canned_crawl(*_args, **_kwargs):
+	"""Return canned (CrawledPage, soup) tuples instead of crawling the network."""
+	return [
+		fixtures.make_page("https://acme.example", fixtures.HOMEPAGE),
+		fixtures.make_page("https://acme.example/about", fixtures.ABOUT),
+		fixtures.make_page("https://acme.example/contact", fixtures.CONTACT),
+	]
+
+
+class NormalizeWebsiteTest(UnitTestCase):
+	def test_adds_https_scheme(self):
+		self.assertEqual(_normalize_website("frappe.io"), "https://frappe.io")
+
+	def test_rejects_empty(self):
+		with self.assertRaises(ValueError):
+			_normalize_website("")
+
+	def test_rejects_bad_scheme(self):
+		with self.assertRaises(ValueError):
+			_normalize_website("ftp://x.com")
+
+
+class PipelineRunTest(UnitTestCase):
+	def setUp(self):
+		self.cfg = _full_config()
+
+	def test_full_result_is_explainable(self):
+		steps = []
+		with mock.patch.object(pipeline_mod, "crawl", _canned_crawl):
+			result = run_pipeline(
+				"https://acme.example",
+				cfg=self.cfg,
+				progress=lambda i, m: steps.append((i, m)),
+			)
+		out = result.to_dict()
+
+		self.assertEqual(out["company_name"]["value"], "Acme Analytics Inc.")
+		self.assertTrue(out["company_name"]["source"])
+		self.assertTrue(out["company_name"]["method"])
+
+		# Every list item carries provenance.
+		for item in out["emails"] + out["phones"]:
+			self.assertTrue(item["source"], f"missing source: {item}")
+			self.assertTrue(item["method"], f"missing method: {item}")
+
+		# Progress emitted, ending at the final step.
+		emitted = [i for i, _ in steps]
+		self.assertEqual(emitted[-1], len(PROGRESS_STEPS) - 1)
+
+	def test_flat_view(self):
+		with mock.patch.object(pipeline_mod, "crawl", _canned_crawl):
+			result = run_pipeline("https://acme.example", cfg=self.cfg)
+		flat = result.flat()
+		self.assertEqual(flat["company_name"], "Acme Analytics Inc.")
+		self.assertIsInstance(flat["social_profiles"], dict)
+
+	def test_unreachable_site_returns_empty_without_raising(self):
+		from crm.domain_enrichment.result import CrawledPage
+
+		def broken_crawl(*_a, **_k):
+			return [(CrawledPage(url="https://x.example", status_code=0, error="timeout after 5s"), None)]
+
+		with mock.patch.object(pipeline_mod, "crawl", broken_crawl):
+			result = run_pipeline("https://x.example", cfg=self.cfg)
+		out = result.to_dict()
+		self.assertEqual(out["company_name"]["value"], "")
+		self.assertTrue(out["_meta"]["errors"])
+		self.assertTrue(result.notes)  # carries the "couldn't reach" note
+
+
+class JsSpaSiteTest(UnitTestCase):
+	"""Enriching a JavaScript-only SPA (empty body, client-rendered content).
+
+	Without a render step the engine sees only the build-tool-default `<head>` metadata
+	and an empty `<body>`. It must (a) flag the page as JS-rendered, (b) not fabricate
+	a description/industry/contacts, and (c) still surface whatever metadata exists
+	(title-tag company name + favicon), all without raising.
+	"""
+
+	def _spa_crawl(self, *_a, **_k):
+		return [fixtures.make_page("https://spa.example", fixtures.JS_SPA_SHELL)]
+
+	def test_js_only_site_yields_metadata_only(self):
+		cfg = _full_config()
+		with (
+			mock.patch.object(pipeline_mod, "crawl", self._spa_crawl),
+			# No About page is crawled; the probe must not hit the real network.
+			mock.patch.object(pipeline_mod, "probe_about_pages", return_value=[]),
+		):
+			result = run_pipeline("https://spa.example", cfg=cfg)
+		out = result.to_dict()
+
+		# Flagged as JS-rendered, and nothing fabricated.
+		self.assertTrue(any("JavaScript-rendered" in n for n in result.notes))
+		self.assertEqual(out["description"]["value"], "")
+		self.assertEqual(out["industry"]["value"], "")
+		self.assertEqual(out["emails"], [])
+		self.assertEqual(out["phones"], [])
+		self.assertTrue(all(p.value == "" for p in result.social_profiles.values()))
+
+		# Only the metadata the shell actually exposes is surfaced.
+		self.assertEqual(out["company_name"]["value"], "Vite App")
+		self.assertEqual(out["company_name"]["method"], "Title Tag")
+		self.assertTrue(out["logo"]["value"].endswith("/vite.svg"))
+
+	def test_readability_diagnosed_empty(self):
+		from crm.domain_enrichment import extractors
+
+		page, _soup = fixtures.make_page("https://spa.example", fixtures.JS_SPA_SHELL)
+		self.assertEqual(extractors.diagnose_readability([page]), "empty")
+
+
+class SchemaStabilityTest(UnitTestCase):
+	"""The result schema must be identical regardless of what a site exposes."""
+
+	def _assert_invariants(self, out, flat):
+		self.assertEqual(set(out), EXPECTED_TOP_KEYS)
+		self.assertEqual(set(flat), EXPECTED_FLAT_KEYS)
+		for key in ("company_name", "description", "logo", "industry"):
+			self.assertEqual(set(out[key]), SCALAR_SUBKEYS)
+
+	def test_empty_result_has_full_schema(self):
+		result = EnrichmentResult()
+		self._assert_invariants(result.to_dict(), result.flat())
+
+	def test_populated_result_has_same_schema(self):
+		with mock.patch.object(pipeline_mod, "crawl", _canned_crawl):
+			result = run_pipeline("https://acme.example", cfg=_full_config())
+		out = result.to_dict()
+		self._assert_invariants(out, result.flat())
+		for e in out["emails"]:
+			self.assertEqual(set(e), {"value", "source", "method"})
+		for p in out["phones"]:
+			self.assertEqual(set(p), {"value", "raw", "source", "method"})

crm/domain_enrichment/tests/test_ssrf.py

@@ -0,0 +1,214 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure unit tests for the SSRF guard (``http.validate_url`` / ``http._is_blocked_ip``).
+
+These are fully offline: ``socket.getaddrinfo`` is monkeypatched so DNS resolution
+is deterministic and no network call is ever made. The guard must reject loopback,
+link-local, private and reserved addresses (incl. the cloud-metadata endpoint
+169.254.169.254), reject non-http(s) schemes, honor allow/block domain lists, and
+allow a normal public host. The private-network rejection is unconditional --
+there is no setting to disable it.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from unittest import mock
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment import http
+from crm.domain_enrichment.http import SSRFError, validate_url
+from crm.domain_enrichment.tests.fixtures import make_config
+
+
+@contextmanager
+def _resolves_to(ip_map):
+	"""Patch socket.getaddrinfo so a host resolves to the given IP(s).
+
+	``ip_map`` maps hostname -> ip string (or list of ip strings). An unknown host
+	raises gaierror, mimicking a real resolver failure.
+	"""
+	import socket
+
+	def fake_getaddrinfo(host, *args, **kwargs):
+		if host not in ip_map:
+			raise socket.gaierror(f"unknown host {host}")
+		ips = ip_map[host]
+		if isinstance(ips, str):
+			ips = [ips]
+		# getaddrinfo returns 5-tuples; element 4 is the sockaddr (ip, port, ...).
+		return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 0)) for ip in ips]
+
+	with mock.patch.object(http.socket, "getaddrinfo", side_effect=fake_getaddrinfo):
+		yield
+
+
+class SSRFGuardTest(UnitTestCase):
+	def setUp(self):
+		self.cfg = make_config()  # default: no allow/block lists
+
+	# -- rejection of internal / dangerous addresses ----------------------- #
+	def test_rejects_loopback_127(self):
+		with _resolves_to({"evil.example": "127.0.0.1"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://evil.example/", self.cfg)
+
+	def test_rejects_cloud_metadata_link_local(self):
+		# 169.254.169.254 is the AWS/GCP metadata endpoint — a classic SSRF target.
+		with _resolves_to({"metadata.example": "169.254.169.254"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://metadata.example/latest/meta-data/", self.cfg)
+
+	def test_rejects_private_10_range(self):
+		with _resolves_to({"intranet.example": "10.0.0.5"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://intranet.example/", self.cfg)
+
+	def test_rejects_localhost(self):
+		with _resolves_to({"localhost": "127.0.0.1"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://localhost:8000/", self.cfg)
+
+	def test_rejects_ipv6_loopback(self):
+		with _resolves_to({"v6.example": "::1"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://v6.example/", self.cfg)
+
+	# -- scheme validation ------------------------------------------------- #
+	def test_rejects_non_http_scheme(self):
+		with self.assertRaises(SSRFError):
+			validate_url("ftp://example.com/x", self.cfg)
+		with self.assertRaises(SSRFError):
+			validate_url("file:///etc/passwd", self.cfg)
+		with self.assertRaises(SSRFError):
+			validate_url("gopher://example.com/", self.cfg)
+
+	def test_rejects_url_without_host(self):
+		with self.assertRaises(SSRFError):
+			validate_url("http:///nohost", self.cfg)
+
+	def test_rejects_unresolvable_host(self):
+		with _resolves_to({}):  # nothing resolves
+			with self.assertRaises(SSRFError):
+				validate_url("http://does-not-resolve.example/", self.cfg)
+
+	# -- the happy path ---------------------------------------------------- #
+	def test_allows_public_host(self):
+		with _resolves_to({"frappe.io": "104.21.0.1"}):
+			self.assertEqual(validate_url("https://frappe.io/", self.cfg), "https://frappe.io/")
+
+	def test_rejects_when_any_resolved_ip_is_private(self):
+		# DNS-rebinding style: one public + one private IP -> must reject.
+		with _resolves_to({"mixed.example": ["8.8.8.8", "10.1.2.3"]}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://mixed.example/", self.cfg)
+
+	# -- the private-network guard is unconditional ------------------------ #
+	def test_private_network_guard_cannot_be_disabled(self):
+		# There is no setting to allow private networks -- a would-be bypass flag
+		# is inert, the guard still rejects a private address.
+		cfg = make_config(settings={"allow_private_networks": 1})
+		with _resolves_to({"intranet.example": "10.0.0.5"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://intranet.example/", cfg)
+
+	# -- allow / block lists ----------------------------------------------- #
+	def test_blocked_domain_is_always_rejected(self):
+		cfg = make_config(blocked_domains=["evil.com"])
+		with _resolves_to({"evil.com": "8.8.8.8"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://evil.com/", cfg)
+		# Subdomains of a blocked domain are blocked too.
+		with _resolves_to({"sub.evil.com": "8.8.8.8"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://sub.evil.com/", cfg)
+
+	def test_trailing_dot_fqdn_does_not_bypass_block_list(self):
+		# evil.com. resolves the same as evil.com in DNS, so the trailing dot must not
+		# slip past the block rule (the block check fires before resolution).
+		cfg = make_config(blocked_domains=["evil.com"])
+		with self.assertRaises(SSRFError):
+			validate_url("http://evil.com./", cfg)
+		with self.assertRaises(SSRFError):
+			validate_url("http://sub.evil.com./", cfg)
+		# And a block entry that itself carries a trailing dot still matches.
+		self.assertTrue(http._domain_in_list("evil.com", ["evil.com."]))
+
+	def test_trailing_dot_host_is_normalized_for_allow_list_and_resolution(self):
+		# The stripped host feeds both the allow-list check and DNS resolution: the
+		# resolver only knows "good.com" (no dot), so passing "good.com." proves the
+		# trailing dot was stripped before both steps.
+		cfg = make_config(allowed_domains=["good.com"])
+		with _resolves_to({"good.com": "8.8.8.8"}):
+			self.assertEqual(validate_url("http://good.com./", cfg), "http://good.com./")
+
+	def test_allow_list_restricts_to_listed_hosts(self):
+		cfg = make_config(allowed_domains=["good.com"])
+		with _resolves_to({"good.com": "8.8.8.8", "other.com": "8.8.8.8"}):
+			self.assertEqual(validate_url("http://good.com/", cfg), "http://good.com/")
+			with self.assertRaises(SSRFError):
+				validate_url("http://other.com/", cfg)
+
+	# -- direct IP-classification unit ------------------------------------- #
+	def test_is_blocked_ip_classification(self):
+		for ip in ("127.0.0.1", "10.0.0.5", "172.16.0.1", "192.168.1.1", "169.254.169.254", "::1", "0.0.0.0"):
+			self.assertTrue(http._is_blocked_ip(ip), ip)
+		for ip in ("8.8.8.8", "104.21.0.1", "1.1.1.1"):
+			self.assertFalse(http._is_blocked_ip(ip), ip)
+		# Unparseable -> rejected.
+		self.assertTrue(http._is_blocked_ip("not-an-ip"))
+
+
+class PinnedConnectionTest(UnitTestCase):
+	"""DNS-rebinding defense: the socket must go to the IP the guard validated.
+
+	``_pinned_get`` rewrites the URL netloc to the validated IP and keeps the
+	original hostname in the Host header / TLS SNI, so a resolver answer that
+	changes after validation can no longer steer the connection.
+	"""
+
+	def _capture_send(self, captured):
+		import requests
+
+		def fake_send(adapter, request, **kwargs):
+			captured["url"] = request.url
+			captured["host_header"] = request.headers.get("Host")
+			raise requests.exceptions.ConnectionError("offline test: no socket")
+
+		return mock.patch.object(http._PinnedIPAdapter, "send", autospec=True, side_effect=fake_send)
+
+	def test_pinned_get_connects_to_ip_with_hostname_host_header(self):
+		import requests
+
+		captured = {}
+		with self._capture_send(captured), self.assertRaises(requests.exceptions.ConnectionError):
+			http._pinned_get(requests.Session(), "https://example.com/about", "93.184.216.34", 5, 2)
+		self.assertEqual(captured["url"], "https://93.184.216.34/about")
+		self.assertEqual(captured["host_header"], "example.com")
+
+	def test_pinned_get_brackets_ipv6_and_keeps_explicit_port(self):
+		import requests
+
+		captured = {}
+		with self._capture_send(captured), self.assertRaises(requests.exceptions.ConnectionError):
+			http._pinned_get(requests.Session(), "https://example.com:8443/x", "2606:2800:220:1::", 5, 2)
+		self.assertEqual(captured["url"], "https://[2606:2800:220:1::]:8443/x")
+		self.assertEqual(captured["host_header"], "example.com:8443")
+
+	def test_pinned_adapter_keeps_tls_verification_on_hostname(self):
+		adapter = http._PinnedIPAdapter("example.com")
+		# urllib3 uses server_hostname for SNI and falls back to it for certificate
+		# matching, so verification stays against the hostname, not the IP literal.
+		self.assertEqual(adapter.poolmanager.connection_pool_kw.get("server_hostname"), "example.com")
+
+	def test_fetch_connects_to_the_validated_ip(self):
+		import requests
+
+		captured = {}
+		with _resolves_to({"rebind.example": "8.8.8.8"}), self._capture_send(captured):
+			status, _html, error, _final = http.fetch("http://rebind.example/", make_config())
+		self.assertEqual(captured["url"], "http://8.8.8.8/")
+		self.assertEqual(status, 0)
+		self.assertIn("connection error", error)

crm/extends/notification_log.py

@@ -0,0 +1,49 @@
+import frappe
+
+# CRM doctype -> frontend list route. Records live at /crm/<route>/<name>.
+CRM_ROUTES = {
+	"CRM Lead": "leads",
+	"CRM Deal": "deals",
+}
+
+
+def before_insert(doc, method=None):
+	"""Point CRM notification emails at the CRM UI instead of the Desk form.
+
+	Notification Log's email builder uses `doc.link` when set, otherwise it
+	falls back to a Desk (`/app/...`) form URL. CRM users often have Desk
+	access restricted, so build a CRM frontend link for CRM documents.
+	See https://github.com/frappe/crm/issues/705.
+	"""
+	if doc.link:
+		return
+
+	route = get_crm_route(doc.document_type, doc.document_name)
+	if route:
+		doc.link = frappe.utils.get_url(route)
+
+
+def get_crm_route(doctype, name):
+	"""Frontend path for a CRM record, or None for non-CRM documents.
+
+	Tasks have no standalone page in the CRM UI, so link to their parent
+	Lead/Deal's Tasks tab, mirroring the in-app notification behaviour.
+	"""
+	if not name:
+		return None
+
+	suffix = ""
+	if doctype == "CRM Task":
+		parent = frappe.db.get_value(
+			"CRM Task", name, ["reference_doctype", "reference_docname"], as_dict=True
+		)
+		if not parent or not parent.reference_docname:
+			return None
+		doctype, name = parent.reference_doctype, parent.reference_docname
+		suffix = "#tasks"
+
+	list_route = CRM_ROUTES.get(doctype)
+	if not list_route:
+		return None
+
+	return f"/crm/{list_route}/{name}{suffix}"

crm/fcrm/doctype/crm_call_log/crm_call_log.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+// frappe.ui.form.on("CRM Call Log", {
+// 	refresh(frm) {
+
+// 	},
+// });

crm/fcrm/doctype/crm_call_log/crm_call_log.json

@@ -0,0 +1,212 @@
+{
+ "actions": [],
+ "allow_import": 1,
+ "allow_rename": 1,
+ "autoname": "field:id",
+ "creation": "2023-08-28 00:23:36.229137",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "telephony_medium",
+  "section_break_gyqe",
+  "id",
+  "from",
+  "status",
+  "duration",
+  "medium",
+  "start_time",
+  "reference_doctype",
+  "reference_docname",
+  "column_break_ufnp",
+  "to",
+  "type",
+  "receiver",
+  "caller",
+  "recording_url",
+  "end_time",
+  "note",
+  "section_break_kebz",
+  "links"
+ ],
+ "fields": [
+  {
+   "fieldname": "id",
+   "fieldtype": "Data",
+   "label": "ID",
+   "unique": 1
+  },
+  {
+   "fieldname": "from",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "From Number",
+   "options": "Phone",
+   "reqd": 1
+  },
+  {
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "in_standard_filter": 1,
+   "label": "Status",
+   "options": "Initiated\nRinging\nIn Progress\nCompleted\nFailed\nBusy\nNo Answer\nQueued\nCanceled",
+   "reqd": 1
+  },
+  {
+   "fieldname": "start_time",
+   "fieldtype": "Datetime",
+   "label": "Start Time"
+  },
+  {
+   "fieldname": "medium",
+   "fieldtype": "Data",
+   "label": "Medium"
+  },
+  {
+   "fieldname": "column_break_ufnp",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "type",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Type",
+   "options": "Incoming\nOutgoing",
+   "reqd": 1
+  },
+  {
+   "fieldname": "to",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "To Number",
+   "options": "Phone",
+   "reqd": 1
+  },
+  {
+   "fieldname": "duration",
+   "fieldtype": "Duration",
+   "in_list_view": 1,
+   "label": "Duration"
+  },
+  {
+   "fieldname": "recording_url",
+   "fieldtype": "Small Text",
+   "label": "Recording URL"
+  },
+  {
+   "fieldname": "end_time",
+   "fieldtype": "Datetime",
+   "label": "End Time"
+  },
+  {
+   "fieldname": "note",
+   "fieldtype": "Link",
+   "label": "Note",
+   "options": "FCRM Note"
+  },
+  {
+   "depends_on": "eval:doc.type == 'Incoming'",
+   "fieldname": "receiver",
+   "fieldtype": "Link",
+   "label": "Call Received By",
+   "options": "User"
+  },
+  {
+   "depends_on": "eval:doc.type == 'Outgoing'",
+   "fieldname": "caller",
+   "fieldtype": "Link",
+   "label": "Caller",
+   "options": "User"
+  },
+  {
+   "default": "CRM Lead",
+   "fieldname": "reference_doctype",
+   "fieldtype": "Link",
+   "label": "Reference Document Type",
+   "options": "DocType"
+  },
+  {
+   "fieldname": "reference_docname",
+   "fieldtype": "Dynamic Link",
+   "label": "Reference Name",
+   "options": "reference_doctype"
+  },
+  {
+   "fieldname": "section_break_kebz",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "links",
+   "fieldtype": "Table",
+   "label": "Links",
+   "options": "Dynamic Link"
+  },
+  {
+   "fieldname": "telephony_medium",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Telephony Medium",
+   "options": "\nManual\nTwilio\nExotel",
+   "read_only": 1
+  },
+  {
+   "fieldname": "section_break_gyqe",
+   "fieldtype": "Section Break"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2026-07-30 12:31:44.974374",
+ "modified_by": "Administrator",
+ "module": "FCRM",
+ "name": "CRM Call Log",
+ "naming_rule": "By fieldname",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Sales Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Sales User",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "row_format": "Dynamic",
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

crm/fcrm/doctype/crm_call_log/crm_call_log.py

@@ -0,0 +1,303 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+import frappe
+from frappe import _, generate_hash
+from frappe.model.document import Document
+
+from crm.integrations.api import get_contact_by_phone_number
+from crm.utils import seconds_to_duration
+
+
+class CRMCallLog(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.core.doctype.dynamic_link.dynamic_link import DynamicLink
+		from frappe.types import DF
+
+		caller: DF.Link | None
+		duration: DF.Duration | None
+		end_time: DF.Datetime | None
+		id: DF.Data | None
+		links: DF.Table[DynamicLink]
+		medium: DF.Data | None
+		note: DF.Link | None
+		receiver: DF.Link | None
+		recording_url: DF.SmallText | None
+		reference_docname: DF.DynamicLink | None
+		reference_doctype: DF.Link | None
+		start_time: DF.Datetime | None
+		status: DF.Literal[
+			"Initiated",
+			"Ringing",
+			"In Progress",
+			"Completed",
+			"Failed",
+			"Busy",
+			"No Answer",
+			"Queued",
+			"Canceled",
+		]
+		telephony_medium: DF.Literal["", "Manual", "Twilio", "Exotel"]
+		to: DF.Data
+		type: DF.Literal["Incoming", "Outgoing"]
+	# end: auto-generated types
+
+	def before_insert(self):
+		if not self.id:
+			self.id = generate_hash(length=12)
+		if not self.telephony_medium:
+			self.telephony_medium = "Manual"
+
+	@staticmethod
+	def default_list_data():
+		columns = [
+			{
+				"label": "Caller",
+				"type": "Link",
+				"key": "caller",
+				"options": "User",
+				"width": "9rem",
+			},
+			{
+				"label": "Receiver",
+				"type": "Link",
+				"key": "receiver",
+				"options": "User",
+				"width": "9rem",
+			},
+			{
+				"label": "Type",
+				"type": "Select",
+				"key": "type",
+				"width": "9rem",
+			},
+			{
+				"label": "Status",
+				"type": "Select",
+				"key": "status",
+				"width": "9rem",
+			},
+			{
+				"label": "Duration",
+				"type": "Duration",
+				"key": "duration",
+				"width": "6rem",
+			},
+			{
+				"label": "From (number)",
+				"type": "Data",
+				"key": "from",
+				"width": "9rem",
+			},
+			{
+				"label": "To (number)",
+				"type": "Data",
+				"key": "to",
+				"width": "9rem",
+			},
+			{
+				"label": "Created On",
+				"type": "Datetime",
+				"key": "creation",
+				"width": "8rem",
+			},
+		]
+		rows = [
+			"name",
+			"caller",
+			"receiver",
+			"type",
+			"status",
+			"duration",
+			"from",
+			"to",
+			"note",
+			"recording_url",
+			"reference_doctype",
+			"reference_docname",
+			"creation",
+		]
+		return {"columns": columns, "rows": rows}
+
+	def parse_list_data(calls):
+		return [parse_call_log(call) for call in calls] if calls else []
+
+	def has_link(self, doctype, name):
+		for link in self.links:
+			if link.link_doctype == doctype and link.link_name == name:
+				return True
+
+	def link_with_reference_doc(self, reference_doctype, reference_name):
+		if self.has_link(reference_doctype, reference_name):
+			return
+
+		self.append("links", {"link_doctype": reference_doctype, "link_name": reference_name})
+
+	def as_dict(self, *args, **kwargs):
+		d = super().as_dict(*args, **kwargs)
+		if d.get("recording_url"):
+			d["recording_url_path"] = (
+				f"/api/method/crm.integrations.api.get_recording_url?call_log_name={d.get('name')}"
+			)
+		return d
+
+
+def parse_call_log(call):
+	call["show_recording"] = False
+	call["_duration"] = seconds_to_duration(call.get("duration"))
+	if call.get("type") == "Incoming":
+		call["activity_type"] = "incoming_call"
+		contact = get_contact_by_phone_number(call.get("from"))
+		receiver = (
+			frappe.db.get_values("User", call.get("receiver"), ["full_name", "user_image"])[0]
+			if call.get("receiver")
+			else [None, None]
+		)
+		call["_caller"] = {
+			"label": contact.get("full_name", "Unknown"),
+			"image": contact.get("image"),
+		}
+		call["_receiver"] = {
+			"label": receiver[0],
+			"image": receiver[1],
+		}
+	elif call.get("type") == "Outgoing":
+		call["activity_type"] = "outgoing_call"
+		contact = get_contact_by_phone_number(call.get("to"))
+		caller = (
+			frappe.db.get_values("User", call.get("caller"), ["full_name", "user_image"])[0]
+			if call.get("caller")
+			else [None, None]
+		)
+		call["_caller"] = {
+			"label": caller[0],
+			"image": caller[1],
+		}
+		call["_receiver"] = {
+			"label": contact.get("full_name", "Unknown"),
+			"image": contact.get("image"),
+		}
+
+	return call
+
+
+@frappe.whitelist()
+def get_call_log(name: str):
+	if not frappe.has_permission("CRM Call Log", "read", name):
+		frappe.throw(_("Not permitted"), frappe.PermissionError)
+
+	call = frappe.get_cached_doc(
+		"CRM Call Log",
+		name,
+		fields=[
+			"name",
+			"caller",
+			"receiver",
+			"duration",
+			"type",
+			"status",
+			"from",
+			"to",
+			"note",
+			"recording_url",
+			"recording_url_path",
+			"reference_doctype",
+			"reference_docname",
+			"creation",
+		],
+	).as_dict()
+
+	call = parse_call_log(call)
+
+	note_names = [call["note"]] if call.get("note") else []
+	task_names = []
+	references = {}
+
+	if call.get("reference_doctype") in ("CRM Lead", "CRM Deal") and call.get("reference_docname"):
+		references[call["reference_doctype"]] = call["reference_docname"]
+
+	for link in call.get("links") or []:
+		link_doctype, link_name = link.get("link_doctype"), link.get("link_name")
+		if link_doctype == "CRM Task":
+			task_names.append(link_name)
+		elif link_doctype == "FCRM Note":
+			note_names.append(link_name)
+		elif link_doctype in ("CRM Lead", "CRM Deal"):
+			references[link_doctype] = link_name
+
+	call["_notes"] = get_permitted_docs("FCRM Note", note_names)
+	call["_tasks"] = get_permitted_docs("CRM Task", task_names)
+
+	lead, deal = references.get("CRM Lead"), references.get("CRM Deal")
+	if lead and frappe.has_permission("CRM Lead", "read", lead):
+		call["_lead"] = lead
+	if deal and frappe.has_permission("CRM Deal", "read", deal):
+		call["_deal"] = deal
+
+	return call
+
+
+def get_permitted_docs(doctype: str, names: list[str]) -> list[dict]:
+	if not names or not frappe.has_permission(doctype, "read"):
+		return []
+
+	docs = frappe.get_list(doctype, filters={"name": ("in", names)}, fields=["*"], limit=len(names))
+	return sorted(docs, key=lambda doc: names.index(str(doc.name)))
+
+
+@frappe.whitelist()
+def create_lead_from_call_log(call_log: str | dict, lead_details: str | dict | None = None):
+	call_log_data = frappe.parse_json(call_log or {})
+
+	if isinstance(call_log_data, str):
+		call_log_name = call_log_data
+	elif isinstance(call_log_data, dict):
+		call_log_name = call_log_data.get("name")
+	else:
+		call_log_name = None
+
+	if not call_log_name:
+		frappe.throw(_("A valid call log is required."), frappe.ValidationError)
+
+	call_doc = frappe.get_doc("CRM Call Log", call_log_name)
+
+	if not call_doc.has_permission("write"):
+		frappe.throw(_("You are not permitted to update this call log."), frappe.PermissionError)
+
+	if not frappe.has_permission("CRM Lead", "create"):
+		frappe.throw(_("You are not permitted to create leads."), frappe.PermissionError)
+
+	lead_details_data = frappe.parse_json(lead_details or {})
+	if lead_details_data and not isinstance(lead_details_data, dict):
+		frappe.throw(_("Invalid lead details supplied."), frappe.ValidationError)
+
+	lead = frappe.new_doc("CRM Lead")
+	meta = frappe.get_meta("CRM Lead")
+	valid_fieldnames = [df.fieldname for df in meta.fields]
+
+	sanitized_details = {
+		key: value for key, value in (lead_details_data or {}).items() if key in valid_fieldnames
+	}
+
+	if "lead_owner" in valid_fieldnames and not sanitized_details.get("lead_owner"):
+		sanitized_details["lead_owner"] = frappe.session.user
+
+	if "mobile_no" in valid_fieldnames and not sanitized_details.get("mobile_no"):
+		sanitized_details["mobile_no"] = call_doc.get("from") or ""
+
+	if "first_name" in valid_fieldnames and not sanitized_details.get("first_name"):
+		reference_label = sanitized_details.get("mobile_no") or call_doc.name
+		sanitized_details["first_name"] = _("Lead from call {0}").format(reference_label)
+
+	lead.update(sanitized_details)
+	lead.insert()
+
+	call_doc.link_with_reference_doc("CRM Lead", lead.name)
+	call_doc.save()
+
+	return lead.name

crm/fcrm/doctype/crm_call_log/test_crm_call_log.py

@@ -0,0 +1,562 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+
+from unittest.mock import MagicMock, patch
+
+import frappe
+from frappe.tests import IntegrationTestCase
+
+from crm.fcrm.doctype.crm_call_log.crm_call_log import (
+	create_lead_from_call_log,
+	get_call_log,
+	parse_call_log,
+)
+from crm.integrations.api import _get_recording_credentials
+
+
+class TestCRMCallLog(IntegrationTestCase):
+	def tearDown(self):
+		frappe.db.rollback()
+
+	def test_call_log_creation_incoming(self):
+		"""Test creating an incoming call log"""
+		call = create_test_call_log(
+			type="Incoming",
+			status="Completed",
+			caller=None,
+			receiver="Administrator",
+		)
+
+		self.assertEqual(call.type, "Incoming")
+		self.assertEqual(call.status, "Completed")
+		self.assertEqual(call.receiver, "Administrator")
+
+	def test_call_log_creation_outgoing(self):
+		"""Test creating an outgoing call log"""
+		call = create_test_call_log(
+			type="Outgoing",
+			status="Completed",
+			caller="Administrator",
+			receiver=None,
+		)
+
+		self.assertEqual(call.type, "Outgoing")
+		self.assertEqual(call.status, "Completed")
+		self.assertEqual(call.caller, "Administrator")
+
+	def test_call_log_with_duration(self):
+		"""Test call log with duration field"""
+		call = create_test_call_log(
+			type="Incoming",
+			status="Completed",
+			duration=120,  # 2 minutes
+		)
+
+		self.assertEqual(call.duration, 120)
+
+	def test_call_log_with_recording_url(self):
+		"""Test call log with recording URL"""
+		recording_url = "https://example.com/recording.mp3"
+		call = create_test_call_log(
+			type="Outgoing",
+			status="Completed",
+			recording_url=recording_url,
+		)
+
+		self.assertEqual(call.recording_url, recording_url)
+
+	def test_has_link_method(self):
+		"""Test has_link method to check if document link exists"""
+		# Create a lead for linking
+		lead = frappe.get_doc(
+			{
+				"doctype": "CRM Lead",
+				"first_name": "Test Lead",
+				"lead_owner": "Administrator",
+			}
+		).insert()
+
+		call = create_test_call_log()
+
+		# Initially no link should exist
+		self.assertFalse(call.has_link("CRM Lead", lead.name))
+
+		# Add link
+		call.link_with_reference_doc("CRM Lead", lead.name)
+
+		# Now link should exist
+		self.assertTrue(call.has_link("CRM Lead", lead.name))
+
+	def test_link_with_reference_doc(self):
+		"""Test linking call log with reference document"""
+		# Create a deal for linking
+		org = frappe.get_doc(
+			{
+				"doctype": "CRM Organization",
+				"organization_name": "Test Org for Call Log",
+			}
+		).insert()
+
+		deal = frappe.get_doc(
+			{
+				"doctype": "CRM Deal",
+				"organization": org.name,
+				"deal_owner": "Administrator",
+			}
+		).insert()
+
+		call = create_test_call_log()
+
+		# Link with deal
+		call.link_with_reference_doc("CRM Deal", deal.name)
+		call.save()
+
+		# Verify link was created
+		self.assertTrue(call.has_link("CRM Deal", deal.name))
+		self.assertEqual(len(call.links), 1)
+		self.assertEqual(call.links[0].link_doctype, "CRM Deal")
+		self.assertEqual(call.links[0].link_name, deal.name)
+
+	def test_link_with_reference_doc_duplicate_prevention(self):
+		"""Test that linking same document twice doesn't create duplicate links"""
+		lead = frappe.get_doc(
+			{
+				"doctype": "CRM Lead",
+				"first_name": "Test Lead",
+				"lead_owner": "Administrator",
+			}
+		).insert()
+
+		call = create_test_call_log()
+
+		# Link twice
+		call.link_with_reference_doc("CRM Lead", lead.name)
+		call.link_with_reference_doc("CRM Lead", lead.name)
+		call.save()
+
+		# Should only have one link
+		self.assertEqual(len(call.links), 1)
+
+	def test_default_list_data_returns_actual_logs(self):
+		"""Test default_list_data returns actual call log data"""
+		from crm.fcrm.doctype.crm_call_log.crm_call_log import CRMCallLog
+
+		# Create test call logs with valid users
+		create_test_call_log(
+			type="Incoming",
+			caller=None,
+			receiver="Administrator",
+			status="Completed",
+		)
+		create_test_call_log(
+			type="Outgoing",
+			caller="Administrator",
+			receiver=None,
+			status="Failed",
+		)
+
+		data = CRMCallLog.default_list_data()
+
+		# Verify structure
+		self.assertIn("columns", data)
+		self.assertIn("rows", data)
+		self.assertIsInstance(data["columns"], list)
+		self.assertIsInstance(data["rows"], list)
+
+		# Verify columns contain required fields
+		self.assertTrue(len(data["columns"]) > 0)
+		column_keys = [col["key"] for col in data["columns"]]
+		self.assertIn("caller", column_keys)
+		self.assertIn("receiver", column_keys)
+		self.assertIn("type", column_keys)
+		self.assertIn("status", column_keys)
+
+		# Verify rows contain data fields
+		self.assertIn("name", data["rows"])
+		self.assertIn("caller", data["rows"])
+		self.assertIn("receiver", data["rows"])
+		self.assertIn("type", data["rows"])
+		self.assertIn("status", data["rows"])
+
+	def test_parse_call_log_incoming(self):
+		"""Test parse_call_log function with incoming call"""
+		# Create a user for testing
+		if not frappe.db.exists("User", "test@example.com"):
+			frappe.get_doc(
+				{
+					"doctype": "User",
+					"email": "test@example.com",
+					"first_name": "Test",
+				}
+			).insert()
+
+		call_data = {
+			"type": "Incoming",
+			"from": "+1234567890",
+			"to": "+0987654321",
+			"receiver": "test@example.com",
+			"duration": 120,
+		}
+
+		parsed = parse_call_log(call_data)
+
+		# Verify parsed data with actual values
+		self.assertEqual(parsed["activity_type"], "incoming_call")
+		self.assertEqual(parsed["_duration"], "2m")
+		# _caller and _receiver are dicts with label and image
+		self.assertIsInstance(parsed["_caller"], dict)
+		self.assertIn("label", parsed["_caller"])
+		self.assertEqual(parsed["_caller"]["label"], "Unknown")  # Phone number resolves to Unknown
+		self.assertIsInstance(parsed["_receiver"], dict)
+		self.assertIn("label", parsed["_receiver"])
+		self.assertEqual(parsed["from"], "+1234567890")
+		self.assertEqual(parsed["to"], "+0987654321")
+
+	def test_parse_call_log_outgoing(self):
+		"""Test parse_call_log function with outgoing call"""
+		call_data = {
+			"type": "Outgoing",
+			"from": "+1234567890",
+			"to": "+0987654321",
+			"caller": "Administrator",
+			"duration": 180,
+		}
+
+		parsed = parse_call_log(call_data)
+
+		# Verify parsed data with actual values
+		self.assertEqual(parsed["activity_type"], "outgoing_call")
+		self.assertEqual(parsed["_duration"], "3m")
+		# _caller and _receiver are dicts with label and image
+		self.assertIsInstance(parsed["_caller"], dict)
+		self.assertEqual(parsed["_caller"]["label"], "Administrator")
+		self.assertIsInstance(parsed["_receiver"], dict)
+		self.assertEqual(parsed["_receiver"]["label"], "Unknown")  # Phone number resolves to Unknown
+		self.assertEqual(parsed["from"], "+1234567890")
+		self.assertEqual(parsed["to"], "+0987654321")
+
+	def test_get_call_log_api(self):
+		"""Test get_call_log API function"""
+		call = create_test_call_log(
+			type="Outgoing",
+			status="Completed",
+			duration=60,
+		)
+
+		# Call the API
+		result = get_call_log(call.name)
+
+		# Verify result
+		self.assertEqual(result["name"], call.name)
+		self.assertEqual(result["type"], "Outgoing")
+		self.assertEqual(result["status"], "Completed")
+		self.assertIn("_duration", result)
+		self.assertIn("_tasks", result)
+		self.assertIn("_notes", result)
+
+	def test_get_call_log_denies_user_without_read_access(self):
+		"""A logged in user with no call log access must not be able to read one"""
+		call = create_test_call_log(type="Outgoing", status="Completed")
+
+		if not frappe.db.exists("User", "no-roles-user@example.com"):
+			frappe.get_doc(
+				{
+					"doctype": "User",
+					"email": "no-roles-user@example.com",
+					"first_name": "No Roles",
+				}
+			).insert(ignore_permissions=True)
+
+		frappe.set_user("no-roles-user@example.com")
+		try:
+			with self.assertRaises(frappe.PermissionError):
+				get_call_log(call.name)
+		finally:
+			frappe.set_user("Administrator")
+
+	def test_get_call_log_hides_lead_user_cannot_read(self):
+		"""A user who can read the call log but not its lead must not receive the lead id"""
+		lead = frappe.get_doc(
+			{
+				"doctype": "CRM Lead",
+				"first_name": "Restricted Lead",
+				"lead_owner": "Administrator",
+			}
+		).insert()
+		call = create_test_call_log(reference_doctype="CRM Lead", reference_docname=lead.name)
+
+		if not frappe.db.exists("User", "sales-user@example.com"):
+			frappe.get_doc(
+				{
+					"doctype": "User",
+					"email": "sales-user@example.com",
+					"first_name": "Sales",
+					"roles": [{"role": "Sales User"}],
+				}
+			).insert(ignore_permissions=True)
+
+		frappe.set_user("sales-user@example.com")
+		try:
+			result = get_call_log(call.name)
+		finally:
+			frappe.set_user("Administrator")
+
+		self.assertEqual(result["name"], call.name)
+		self.assertNotIn("_lead", result)
+
+	def test_get_call_log_with_reference_lead(self):
+		"""Test get_call_log API with reference to CRM Lead"""
+		lead = frappe.get_doc(
+			{
+				"doctype": "CRM Lead",
+				"first_name": "Test Lead",
+				"lead_owner": "Administrator",
+			}
+		).insert()
+
+		call = create_test_call_log(
+			reference_doctype="CRM Lead",
+			reference_docname=lead.name,
+		)
+
+		result = get_call_log(call.name)
+
+		# Verify lead reference
+		self.assertEqual(result.get("_lead"), lead.name)
+
+	def test_get_call_log_with_reference_deal(self):
+		"""Test get_call_log API with reference to CRM Deal"""
+		org = frappe.get_doc(
+			{
+				"doctype": "CRM Organization",
+				"organization_name": "Test Org for Call",
+			}
+		).insert()
+
+		deal = frappe.get_doc(
+			{
+				"doctype": "CRM Deal",
+				"organization": org.name,
+				"deal_owner": "Administrator",
+			}
+		).insert()
+
+		call = create_test_call_log(
+			reference_doctype="CRM Deal",
+			reference_docname=deal.name,
+		)
+
+		result = get_call_log(call.name)
+
+		# Verify deal reference
+		self.assertEqual(result.get("_deal"), deal.name)
+
+	def test_get_call_log_with_linked_task(self):
+		"""Test get_call_log API with linked CRM Task"""
+		call = create_test_call_log()
+
+		# Create and link a task
+		task = frappe.get_doc(
+			{
+				"doctype": "CRM Task",
+				"title": "Follow up call",
+				"assigned_to": "Administrator",
+			}
+		).insert()
+
+		call.link_with_reference_doc("CRM Task", task.name)
+		call.save()
+
+		result = get_call_log(call.name)
+
+		# Verify task is in results
+		self.assertEqual(len(result["_tasks"]), 1)
+		self.assertEqual(result["_tasks"][0]["name"], task.name)
+
+	def test_get_call_log_returns_all_linked_tasks_beyond_default_page_length(self):
+		"""Linked records must not be cut off by the framework's default list page length"""
+		call = create_test_call_log()
+		task_count = 25
+
+		for i in range(task_count):
+			task = frappe.get_doc(
+				{
+					"doctype": "CRM Task",
+					"title": f"Follow up {i}",
+					"assigned_to": "Administrator",
+				}
+			).insert()
+			call.link_with_reference_doc("CRM Task", task.name)
+		call.save()
+
+		result = get_call_log(call.name)
+
+		self.assertEqual(len(result["_tasks"]), task_count)
+
+	def test_create_lead_from_call_log_basic(self):
+		"""Test creating a lead from call log"""
+		call = create_test_call_log(
+			type="Incoming",
+			from_number="+1234567890",
+		)
+
+		# Create lead from call log
+		lead_name = create_lead_from_call_log(
+			call_log=frappe.as_json({"name": call.name}),
+			lead_details=frappe.as_json({"first_name": "John", "last_name": "Doe"}),
+		)
+
+		# Verify lead was created
+		self.assertTrue(frappe.db.exists("CRM Lead", lead_name))
+		lead = frappe.get_doc("CRM Lead", lead_name)
+		self.assertEqual(lead.first_name, "John")
+		self.assertEqual(lead.last_name, "Doe")
+		self.assertEqual(lead.mobile_no, "+1234567890")
+
+		# Verify call log was linked
+		call.reload()
+		self.assertTrue(call.has_link("CRM Lead", lead_name))
+
+	def test_create_lead_from_call_log_with_owner(self):
+		"""Test creating lead from call log with specific owner"""
+		call = create_test_call_log(
+			type="Outgoing",
+			to="+9876543210",
+		)
+
+		lead_name = create_lead_from_call_log(
+			call_log=frappe.as_json({"name": call.name}),
+			lead_details=frappe.as_json({"first_name": "Jane", "lead_owner": "Administrator"}),
+		)
+
+		lead = frappe.get_doc("CRM Lead", lead_name)
+		self.assertEqual(lead.lead_owner, "Administrator")
+
+	def test_create_lead_from_call_log_no_details(self):
+		"""Test creating lead from call log without lead details"""
+		call = create_test_call_log(
+			type="Incoming",
+			from_number="+1112223333",
+		)
+
+		lead_name = create_lead_from_call_log(call_log=frappe.as_json({"name": call.name}))
+
+		# Verify lead was created with default name
+		lead = frappe.get_doc("CRM Lead", lead_name)
+		self.assertTrue(lead.first_name.startswith("Lead from call"))
+		self.assertEqual(lead.mobile_no, "+1112223333")
+
+	def test_create_lead_from_call_log_invalid_call_log(self):
+		"""Test that invalid call log throws error"""
+		with self.assertRaises(frappe.DoesNotExistError):
+			create_lead_from_call_log(call_log=frappe.as_json({"name": "invalid_name"}))
+
+	def test_create_lead_from_call_log_permission_check(self):
+		"""Test that permission is checked when creating lead from call log"""
+		create_test_call_log()
+
+		# Verify that empty call log throws error
+		with self.assertRaises(frappe.ValidationError):
+			create_lead_from_call_log(call_log=frappe.as_json({}))
+
+	def test_call_log_status_filtering(self):
+		"""Test that call logs can be filtered by status"""
+		# Create calls with different statuses
+		completed_call = create_test_call_log(status="Completed", type="Incoming")
+		failed_call = create_test_call_log(status="Failed", type="Outgoing")
+		busy_call = create_test_call_log(status="Busy", type="Incoming")
+
+		# Verify statuses are set correctly
+		self.assertEqual(completed_call.status, "Completed")
+		self.assertEqual(failed_call.status, "Failed")
+		self.assertEqual(busy_call.status, "Busy")
+
+		# Test filtering by status
+		completed_logs = frappe.get_all("CRM Call Log", filters={"status": "Completed"})
+		failed_logs = frappe.get_all("CRM Call Log", filters={"status": "Failed"})
+
+		self.assertGreaterEqual(len(completed_logs), 1)
+		self.assertGreaterEqual(len(failed_logs), 1)
+
+		# Verify our specific calls are in the right filtered lists
+		completed_names = [log.name for log in completed_logs]
+		failed_names = [log.name for log in failed_logs]
+
+		self.assertIn(completed_call.name, completed_names)
+		self.assertIn(failed_call.name, failed_names)
+		self.assertNotIn(busy_call.name, completed_names)
+		self.assertNotIn(busy_call.name, failed_names)
+
+	def test_call_log_with_telephony_medium(self):
+		"""Test call log with different telephony mediums"""
+		call = create_test_call_log(telephony_medium="Twilio")
+		self.assertEqual(call.telephony_medium, "Twilio")
+
+		call2 = create_test_call_log(telephony_medium="Exotel")
+		self.assertEqual(call2.telephony_medium, "Exotel")
+
+	def test_recording_credentials_manual_medium_needs_no_auth(self):
+		"""A manually added recording is fetched as-is, with no provider auth."""
+		self.assertIsNone(_get_recording_credentials("Manual"))
+
+	def test_recording_credentials_unknown_medium_returns_none(self):
+		"""An empty/unrecognized medium must not raise; it just skips auth."""
+		self.assertIsNone(_get_recording_credentials(""))
+		self.assertIsNone(_get_recording_credentials(None))
+
+	def test_recording_credentials_twilio_unconfigured_returns_none(self):
+		"""Twilio without a configured secret falls back to no auth instead of raising.
+
+		Regression: get_password used to raise when the secret was unset, which made
+		the recording proxy 500 and the UI show "Recording not available".
+		"""
+		settings = MagicMock()
+		settings.api_key = "ACxxxxxxxx"
+		settings.get_password.return_value = None
+		with patch("crm.integrations.api.frappe.get_single", return_value=settings):
+			self.assertIsNone(_get_recording_credentials("Twilio"))
+
+	def test_recording_credentials_twilio_configured_returns_tuple(self):
+		"""Twilio with both key and secret set yields the auth pair."""
+		settings = MagicMock()
+		settings.api_key = "ACxxxxxxxx"
+		settings.get_password.return_value = "twilio_secret"
+		with patch("crm.integrations.api.frappe.get_single", return_value=settings):
+			self.assertEqual(_get_recording_credentials("Twilio"), ("ACxxxxxxxx", "twilio_secret"))
+
+	def test_recording_credentials_exotel_configured_returns_tuple(self):
+		"""Exotel with both key and token set yields the auth pair."""
+		settings = MagicMock()
+		settings.api_key = "exotel_key"
+		settings.get_password.return_value = "exotel_token"
+		with patch("crm.integrations.api.frappe.get_single", return_value=settings):
+			self.assertEqual(_get_recording_credentials("Exotel"), ("exotel_key", "exotel_token"))
+
+
+def create_test_call_log(**kwargs):
+	"""Helper function to create a CRM Call Log for testing"""
+	import uuid
+
+	# Generate unique ID if not provided (required for autoname)
+	unique_id = kwargs.pop("id", str(uuid.uuid4())[:10])
+
+	# Handle special field name mapping
+	if "from_number" in kwargs:
+		kwargs["from"] = kwargs.pop("from_number")
+
+	# Set defaults for required fields
+	data = {
+		"doctype": "CRM Call Log",
+		"id": unique_id,
+		"type": "Incoming",
+		"status": "Completed",
+		"to": "+1234567890",
+		"from": "+0987654321",
+	}
+
+	# Update with all provided kwargs
+	data.update(kwargs)
+
+	call_log = frappe.get_doc(data)
+	call_log.insert()
+	return call_log

crm/fcrm/doctype/crm_communication_status/crm_communication_status.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+// frappe.ui.form.on("CRM Communication Status", {
+// 	refresh(frm) {
+
+// 	},
+// });

crm/fcrm/doctype/crm_communication_status/crm_communication_status.json

@@ -0,0 +1,72 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "autoname": "field:status",
+ "creation": "2023-12-13 13:25:07.213100",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "status"
+ ],
+ "fields": [
+  {
+   "fieldname": "status",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Status",
+   "reqd": 1,
+   "unique": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2026-01-28 17:10:00.827558",
+ "modified_by": "shariq@frappe.io",
+ "module": "FCRM",
+ "name": "CRM Communication Status",
+ "naming_rule": "By fieldname",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Sales Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Sales User",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "row_format": "Dynamic",
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

crm/fcrm/doctype/crm_communication_status/crm_communication_status.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class CRMCommunicationStatus(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		status: DF.Data
+	# end: auto-generated types
+
+	pass

crm/fcrm/doctype/crm_communication_status/test_crm_communication_status.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests import UnitTestCase
+
+
+class TestCRMCommunicationStatus(UnitTestCase):
+	pass

crm/fcrm/doctype/crm_contacts/crm_contacts.json

@@ -0,0 +1,92 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2023-08-25 19:08:36.356133",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "contact",
+  "full_name",
+  "email",
+  "column_break_uvny",
+  "gender",
+  "mobile_no",
+  "phone",
+  "is_primary"
+ ],
+ "fields": [
+  {
+   "fieldname": "contact",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Contact",
+   "options": "Contact"
+  },
+  {
+   "fetch_from": "contact.full_name",
+   "fieldname": "full_name",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Full Name",
+   "read_only": 1
+  },
+  {
+   "fetch_from": "contact.email_id",
+   "fieldname": "email",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Email",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_uvny",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fetch_from": "contact.mobile_no",
+   "fieldname": "mobile_no",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Mobile No.",
+   "options": "Phone",
+   "read_only": 1
+  },
+  {
+   "fetch_from": "contact.phone",
+   "fieldname": "phone",
+   "fieldtype": "Data",
+   "label": "Phone",
+   "options": "Phone",
+   "read_only": 1
+  },
+  {
+   "fetch_from": "contact.gender",
+   "fieldname": "gender",
+   "fieldtype": "Link",
+   "label": "Gender",
+   "options": "Gender",
+   "read_only": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "is_primary",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Is Primary"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2026-02-23 12:25:24.755041",
+ "modified_by": "Administrator",
+ "module": "FCRM",
+ "name": "CRM Contacts",
+ "owner": "Administrator",
+ "permissions": [],
+ "row_format": "Dynamic",
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

crm/fcrm/doctype/crm_contacts/crm_contacts.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class CRMContacts(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		contact: DF.Link | None
+		email: DF.Data | None
+		full_name: DF.Data | None
+		gender: DF.Link | None
+		is_primary: DF.Check
+		mobile_no: DF.Data | None
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+		phone: DF.Data | None
+	# end: auto-generated types
+
+	pass