snapshot of https://github.com/frappe/crm.git (develop, 2026-09-18; blobs > 131072B omitted)

crm/domain_enrichment/config.py

@@ -0,0 +1,261 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Loads enrichment configuration (Settings + Rules + Field Mappings) from the desk.
+
+The engine is rule-agnostic: it loads admin-edited config here and executes it.
+``get_config()`` assembles an ``EnrichmentConfig`` from the config doctypes on demand
+(not cached -- see the note on ``get_config``). Hot paths that only need the Settings
+toggles use ``get_settings`` / ``auto_enrich_enabled_for`` instead of the full build.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+import frappe
+
+# The doctypes enrichment runs on. Single source of truth shared by the manual (api)
+# and auto-enrich (tasks) paths, and the allow-list the whitelisted API checks a
+# caller-supplied doctype against.
+ENRICHABLE_DOCTYPES = ("CRM Lead", "CRM Deal", "CRM Organization")
+
+
+# Sensible fallbacks applied when the Single doctype has not been saved yet (the
+# JSON field defaults only populate a freshly-created row, which may not exist).
+DEFAULT_SETTINGS = {
+	"enabled": 1,
+	"auto_enrich": 0,
+	"max_pages": 10,
+	"max_depth": 2,
+	"use_sitemap": 1,
+	"request_timeout": 10,
+	"max_download_bytes": 3_000_000,
+	"retry_count": 2,
+	"user_agent": (
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+		"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
+	),
+}
+
+
+def _setting(doc, key):
+	"""One Settings field, falling back to DEFAULT_SETTINGS when unset."""
+	val = doc.get(key)
+	return DEFAULT_SETTINGS.get(key) if val in (None, "") else val
+
+
+def get_settings():
+	"""The CRM Enrichment Settings Single, framework-cached via ``get_cached_doc``.
+
+	Cheap: reads only the Single, never assembling Rules/Mappings. For hot paths --
+	the per-insert auto-enrich flag check and the enqueue timeout -- that must not
+	build the full config on every record creation.
+	"""
+	return frappe.get_cached_doc("CRM Enrichment Settings")
+
+
+def auto_enrich_enabled_for(doctype: str) -> bool:
+	"""True if auto-enrich-on-create should fire for ``doctype``. A cheap,
+	Settings-only check (feature enabled + auto_enrich on + an enrichable doctype) --
+	no Rules/Mappings assembled."""
+	s = get_settings()
+	return bool(_setting(s, "enabled") and _setting(s, "auto_enrich") and doctype in ENRICHABLE_DOCTYPES)
+
+
+# Industry classifier thresholds. These are mechanics (how confident the winner
+# must be), not tunable knowledge, so they stay constants. If an admin ever needs
+# to tune them they can be promoted to Settings fields without touching callers.
+INDUSTRY_MIN_SCORE = 2  # at least one headline keyword hit (weighted)
+INDUSTRY_MIN_CONFIDENCE = 0.34  # the winner must clearly lead
+
+# Default link-priority ordering (BFS) -- seeded into Settings.link_priority_order
+# on install, and used as a fallback when that table is empty.
+DEFAULT_LINK_PRIORITY = [
+	("about", 1.0),
+	("contact", 1.0),
+	("team", 1.0),
+	("leadership", 1.0),
+	("careers", 1.0),
+	("company", 1.0),
+	("people", 1.0),
+	("founders", 1.0),
+	("management", 1.0),
+	("jobs", 1.0),
+]
+
+
+@dataclass
+class Rule:
+	"""A single config-driven rule with its compiled patterns.
+
+	``patterns`` is a list of (compiled_regex, raw_pattern) tuples. Substring
+	patterns (is_regex=0) are compiled as escaped, case-insensitive regexes so the
+	executors can use one uniform match path.
+	"""
+
+	rule_type: str
+	target_value: str = ""
+	industry: str = ""
+	weight: float = 1.0
+	match_scope: str = "Full Text"
+	patterns: list = field(default_factory=list)
+
+	@property
+	def label(self) -> str:
+		"""The value a match yields: industry name for Industry rules, else
+		target_value (tech name / network)."""
+		return self.industry if self.rule_type == "Industry" else self.target_value
+
+	def matches(self, text: str) -> int:
+		"""Total number of pattern hits across this rule's patterns in ``text``."""
+		if not text:
+			return 0
+		return sum(len(rx.findall(text)) for rx, _raw in self.patterns)
+
+
+@dataclass
+class Mapping:
+	source_key: str
+	target_doctype: str
+	target_fieldname: str
+	write_policy: str = "Fill if empty"
+	create_missing_link: int = 0
+	default_values: str = ""
+
+
+@dataclass
+class EnrichmentConfig:
+	settings: dict = field(default_factory=dict)
+	rules_by_type: dict = field(default_factory=dict)  # rule_type -> list[Rule]
+	mappings_by_doctype: dict = field(default_factory=dict)  # doctype -> list[Mapping]
+	link_priority: list = field(default_factory=list)  # list[(keyword, weight)]
+	skip_patterns: list = field(default_factory=list)  # list[str]
+	allowed_domains: list = field(default_factory=list)
+	blocked_domains: list = field(default_factory=list)
+
+	# Convenience accessors with fallbacks --------------------------------- #
+	def setting(self, key, default=None):
+		val = self.settings.get(key)
+		if val in (None, ""):
+			return DEFAULT_SETTINGS.get(key, default)
+		return val
+
+	def rules(self, rule_type: str) -> list:
+		return self.rules_by_type.get(rule_type, [])
+
+
+def _compile_pattern(pattern: str, is_regex: int):
+	try:
+		if is_regex:
+			return re.compile(pattern, re.IGNORECASE)
+		# Word-boundary-anchor substring patterns: an un-anchored short keyword like
+		# "erp" or "api" matches inside completely unrelated words ("waterproofing",
+		# "capital"), causing false industry-classification hits. An admin who wants
+		# unanchored substring matching can still opt in via is_regex=1.
+		return re.compile(rf"\b{re.escape(pattern)}\b", re.IGNORECASE)
+	except re.error:
+		# A malformed admin-entered regex must not break the whole config build.
+		frappe.log_error(
+			title="Domain Enrichment: invalid rule pattern",
+			message=f"Could not compile pattern: {pattern!r}",
+		)
+		return None
+
+
+def _build_rules() -> dict:
+	rules_by_type: dict = {}
+	names = frappe.get_all("CRM Enrichment Rule", filters={"enabled": 1}, pluck="name")
+	for name in names:
+		doc = frappe.get_doc("CRM Enrichment Rule", name)
+		compiled = []
+		for pat in doc.patterns or []:
+			rx = _compile_pattern(pat.pattern, pat.is_regex)
+			if rx is not None:
+				compiled.append((rx, pat.pattern))
+		if not compiled:
+			continue
+		rule = Rule(
+			rule_type=doc.rule_type,
+			target_value=doc.target_value or "",
+			industry=doc.industry or "",
+			weight=doc.weight or 1.0,
+			match_scope=doc.match_scope or "Full Text",
+			patterns=compiled,
+		)
+		rules_by_type.setdefault(doc.rule_type, []).append(rule)
+	return rules_by_type
+
+
+def _build_mappings() -> dict:
+	mappings_by_doctype: dict = {}
+	rows = frappe.get_all(
+		"CRM Enrichment Field Mapping",
+		filters={"enabled": 1},
+		fields=[
+			"source_key",
+			"target_doctype",
+			"target_fieldname",
+			"write_policy",
+			"create_missing_link",
+			"default_values",
+		],
+	)
+	for row in rows:
+		mapping = Mapping(
+			source_key=row.source_key,
+			target_doctype=row.target_doctype,
+			target_fieldname=row.target_fieldname,
+			write_policy=row.write_policy or "Fill if empty",
+			create_missing_link=row.create_missing_link or 0,
+			default_values=row.default_values or "",
+		)
+		mappings_by_doctype.setdefault(row.target_doctype, []).append(mapping)
+	return mappings_by_doctype
+
+
+def _build_settings(settings_doc) -> dict:
+	out = {}
+	for key in DEFAULT_SETTINGS:
+		out[key] = settings_doc.get(key)
+	return out
+
+
+def _build_config() -> EnrichmentConfig:
+	settings_doc = frappe.get_cached_doc("CRM Enrichment Settings")
+
+	link_priority = [
+		(kw.keyword.lower(), kw.weight or 1.0)
+		for kw in (settings_doc.link_priority_order or [])
+		if kw.keyword
+	] or list(DEFAULT_LINK_PRIORITY)
+
+	skip_patterns = [sp.pattern for sp in (settings_doc.skip_patterns or []) if sp.pattern]
+
+	allowed_domains = [d.domain.lower().strip() for d in (settings_doc.allowed_domains or []) if d.domain]
+	blocked_domains = [d.domain.lower().strip() for d in (settings_doc.blocked_domains or []) if d.domain]
+
+	return EnrichmentConfig(
+		settings=_build_settings(settings_doc),
+		rules_by_type=_build_rules(),
+		mappings_by_doctype=_build_mappings(),
+		link_priority=link_priority,
+		skip_patterns=skip_patterns,
+		allowed_domains=allowed_domains,
+		blocked_domains=blocked_domains,
+	)
+
+
+def get_config() -> EnrichmentConfig:
+	"""Assemble the full enrichment config (Settings + Rules + Field Mappings) fresh.
+
+	Deliberately NOT cached. It is built at most a couple of times per enrichment run,
+	and a run is a multi-second, network-bound crawl -- so the handful of small queries
+	here is noise. A cache, by contrast, buys nothing measurable and adds a real
+	stale-config surface (any write that bypasses ``on_update`` -- a bulk ``db_set``,
+	direct SQL -- would serve stale rules until a restart). The per-insert hot path does
+	NOT call this; it uses ``auto_enrich_enabled_for`` (Settings-only). Do not re-add a
+	cache here without a measured hot-path caller that justifies it.
+	"""
+	return _build_config()