snapshot of https://github.com/frappe/crm.git (develop, 2026-09-18; blobs > 131072B omitted)

crm/domain_enrichment/tests/test_pipeline.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure, network-free pipeline tests.
+
+The crawl is stubbed (``pipeline.crawl`` patched to return canned offline pages), so
+the whole pipeline runs with zero network. Asserts the assembled ``EnrichmentResult``,
+the explainable provenance schema (``to_dict``/``flat``) the mapper depends on, and
+the progress-step contract.
+"""
+
+from __future__ import annotations
+
+from unittest import mock
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment import pipeline as pipeline_mod
+from crm.domain_enrichment.pipeline import PROGRESS_STEPS, _normalize_website
+from crm.domain_enrichment.pipeline import run as run_pipeline
+from crm.domain_enrichment.result import EnrichmentResult
+from crm.domain_enrichment.tests import fixtures
+
+# The schema the CRM field mapper relies on. If extractors change the shape these
+# assertions fail loudly instead of the mapper silently reading missing keys.
+EXPECTED_TOP_KEYS = {
+	"company_name",
+	"description",
+	"logo",
+	"image",
+	"industry",
+	"industry_confidence",
+	"emails",
+	"phones",
+	"social_profiles",
+	"_meta",
+}
+EXPECTED_FLAT_KEYS = {
+	"company_name",
+	"description",
+	"logo",
+	"industry",
+	"industry_confidence",
+	"social_profiles",
+}
+SCALAR_SUBKEYS = {"value", "source", "method"}
+
+
+def _full_config():
+	"""An in-memory config wired with all the rule types the fixtures exercise."""
+	return fixtures.make_config(
+		settings={"max_pages": 5, "max_depth": 1, "request_timeout": 5},
+		rules_by_type={
+			"Industry": fixtures.industry_rules(),
+			"Social": fixtures.social_rules(),
+		},
+	)
+
+
+def _canned_crawl(*_args, **_kwargs):
+	"""Return canned (CrawledPage, soup) tuples instead of crawling the network."""
+	return [
+		fixtures.make_page("https://acme.example", fixtures.HOMEPAGE),
+		fixtures.make_page("https://acme.example/about", fixtures.ABOUT),
+		fixtures.make_page("https://acme.example/contact", fixtures.CONTACT),
+	]
+
+
+class NormalizeWebsiteTest(UnitTestCase):
+	def test_adds_https_scheme(self):
+		self.assertEqual(_normalize_website("frappe.io"), "https://frappe.io")
+
+	def test_rejects_empty(self):
+		with self.assertRaises(ValueError):
+			_normalize_website("")
+
+	def test_rejects_bad_scheme(self):
+		with self.assertRaises(ValueError):
+			_normalize_website("ftp://x.com")
+
+
+class PipelineRunTest(UnitTestCase):
+	def setUp(self):
+		self.cfg = _full_config()
+
+	def test_full_result_is_explainable(self):
+		steps = []
+		with mock.patch.object(pipeline_mod, "crawl", _canned_crawl):
+			result = run_pipeline(
+				"https://acme.example",
+				cfg=self.cfg,
+				progress=lambda i, m: steps.append((i, m)),
+			)
+		out = result.to_dict()
+
+		self.assertEqual(out["company_name"]["value"], "Acme Analytics Inc.")
+		self.assertTrue(out["company_name"]["source"])
+		self.assertTrue(out["company_name"]["method"])
+
+		# Every list item carries provenance.
+		for item in out["emails"] + out["phones"]:
+			self.assertTrue(item["source"], f"missing source: {item}")
+			self.assertTrue(item["method"], f"missing method: {item}")
+
+		# Progress emitted, ending at the final step.
+		emitted = [i for i, _ in steps]
+		self.assertEqual(emitted[-1], len(PROGRESS_STEPS) - 1)
+
+	def test_flat_view(self):
+		with mock.patch.object(pipeline_mod, "crawl", _canned_crawl):
+			result = run_pipeline("https://acme.example", cfg=self.cfg)
+		flat = result.flat()
+		self.assertEqual(flat["company_name"], "Acme Analytics Inc.")
+		self.assertIsInstance(flat["social_profiles"], dict)
+
+	def test_unreachable_site_returns_empty_without_raising(self):
+		from crm.domain_enrichment.result import CrawledPage
+
+		def broken_crawl(*_a, **_k):
+			return [(CrawledPage(url="https://x.example", status_code=0, error="timeout after 5s"), None)]
+
+		with mock.patch.object(pipeline_mod, "crawl", broken_crawl):
+			result = run_pipeline("https://x.example", cfg=self.cfg)
+		out = result.to_dict()
+		self.assertEqual(out["company_name"]["value"], "")
+		self.assertTrue(out["_meta"]["errors"])
+		self.assertTrue(result.notes)  # carries the "couldn't reach" note
+
+
+class JsSpaSiteTest(UnitTestCase):
+	"""Enriching a JavaScript-only SPA (empty body, client-rendered content).
+
+	Without a render step the engine sees only the build-tool-default `<head>` metadata
+	and an empty `<body>`. It must (a) flag the page as JS-rendered, (b) not fabricate
+	a description/industry/contacts, and (c) still surface whatever metadata exists
+	(title-tag company name + favicon), all without raising.
+	"""
+
+	def _spa_crawl(self, *_a, **_k):
+		return [fixtures.make_page("https://spa.example", fixtures.JS_SPA_SHELL)]
+
+	def test_js_only_site_yields_metadata_only(self):
+		cfg = _full_config()
+		with (
+			mock.patch.object(pipeline_mod, "crawl", self._spa_crawl),
+			# No About page is crawled; the probe must not hit the real network.
+			mock.patch.object(pipeline_mod, "probe_about_pages", return_value=[]),
+		):
+			result = run_pipeline("https://spa.example", cfg=cfg)
+		out = result.to_dict()
+
+		# Flagged as JS-rendered, and nothing fabricated.
+		self.assertTrue(any("JavaScript-rendered" in n for n in result.notes))
+		self.assertEqual(out["description"]["value"], "")
+		self.assertEqual(out["industry"]["value"], "")
+		self.assertEqual(out["emails"], [])
+		self.assertEqual(out["phones"], [])
+		self.assertTrue(all(p.value == "" for p in result.social_profiles.values()))
+
+		# Only the metadata the shell actually exposes is surfaced.
+		self.assertEqual(out["company_name"]["value"], "Vite App")
+		self.assertEqual(out["company_name"]["method"], "Title Tag")
+		self.assertTrue(out["logo"]["value"].endswith("/vite.svg"))
+
+	def test_readability_diagnosed_empty(self):
+		from crm.domain_enrichment import extractors
+
+		page, _soup = fixtures.make_page("https://spa.example", fixtures.JS_SPA_SHELL)
+		self.assertEqual(extractors.diagnose_readability([page]), "empty")
+
+
+class SchemaStabilityTest(UnitTestCase):
+	"""The result schema must be identical regardless of what a site exposes."""
+
+	def _assert_invariants(self, out, flat):
+		self.assertEqual(set(out), EXPECTED_TOP_KEYS)
+		self.assertEqual(set(flat), EXPECTED_FLAT_KEYS)
+		for key in ("company_name", "description", "logo", "industry"):
+			self.assertEqual(set(out[key]), SCALAR_SUBKEYS)
+
+	def test_empty_result_has_full_schema(self):
+		result = EnrichmentResult()
+		self._assert_invariants(result.to_dict(), result.flat())
+
+	def test_populated_result_has_same_schema(self):
+		with mock.patch.object(pipeline_mod, "crawl", _canned_crawl):
+			result = run_pipeline("https://acme.example", cfg=_full_config())
+		out = result.to_dict()
+		self._assert_invariants(out, result.flat())
+		for e in out["emails"]:
+			self.assertEqual(set(e), {"value", "source", "method"})
+		for p in out["phones"]:
+			self.assertEqual(set(p), {"value", "raw", "source", "method"})