snapshot of https://github.com/frappe/crm.git (develop, 2026-09-18; blobs > 131072B omitted)

crm/domain_enrichment/tests/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Tests for the domain enrichment engine (added in Phase 7)."""

crm/domain_enrichment/tests/fixtures.py

@@ -0,0 +1,319 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Offline HTML fixtures + in-memory config builders for network-free tests.
+
+The HTML fixtures are deterministic snapshots used by the pure unit tests so the
+crawler/extractor mechanics can be exercised without touching the network. The
+config builders assemble ``Rule`` / ``Mapping`` / ``EnrichmentConfig`` objects in
+memory (no DB) so the rule executors can be tested rule-driven, exactly as the
+engine would see them after ``get_config()`` — but without hitting Frappe.
+"""
+
+from __future__ import annotations
+
+import re
+
+from bs4 import BeautifulSoup
+
+from crm.domain_enrichment.config import EnrichmentConfig, Mapping, Rule
+from crm.domain_enrichment.crawler import extract_content
+from crm.domain_enrichment.result import CrawledPage
+
+# --------------------------------------------------------------------------- #
+# HTML fixtures
+# --------------------------------------------------------------------------- #
+HOMEPAGE = """
+<!doctype html>
+<html>
+<head>
+  <title>Acme Analytics | AI powered SaaS</title>
+  <meta name="description" content="Acme is an AI powered analytics platform.">
+  <meta property="og:site_name" content="Acme Analytics">
+  <meta property="og:description" content="Real-time analytics for modern teams.">
+  <meta property="og:image" content="https://acme.example/logo.png">
+  <script type="application/ld+json">
+  {
+    "@context": "https://schema.org",
+    "@type": "Organization",
+    "name": "Acme Analytics Inc.",
+    "description": "AI powered analytics SaaS for product teams.",
+    "logo": {"url": "https://acme.example/ld-logo.png"},
+    "sameAs": [
+      "https://www.linkedin.com/company/acme-analytics",
+      "https://twitter.com/acmeanalytics",
+      "https://github.com/acme"
+    ]
+  }
+  </script>
+  <script src="/_next/static/chunk.js"></script>
+  <script src="https://www.googletagmanager.com/gtag/js?id=G-XXededXX"></script>
+</head>
+<body>
+  <h1>AI powered analytics</h1>
+  <h2>We are hiring across engineering</h2>
+  <nav>
+    <a href="/about">About</a>
+    <a href="/contact">Contact</a>
+    <a href="/team">Team</a>
+    <a href="/careers">Careers</a>
+    <a href="https://facebook.com/sharer?u=x">Share</a>
+    <a href="https://instagram.com/acmeanalytics">Instagram</a>
+    <a href="https://external-site.com/page">External</a>
+  </nav>
+  <p>Contact us at hello@acme.example or call <a href="tel:+1 415 555 0142">us</a>.</p>
+  <p>Our subscription pricing plans start free. Customer relationship tooling.</p>
+</body>
+</html>
+"""
+
+ABOUT = """
+<!doctype html>
+<html><head><title>About Acme</title></head>
+<body>
+  <h1>About Acme</h1>
+  <p>We raised a Series A round and are backed by great investors.</p>
+  <div class="team">
+    <div class="member"><div class="name">Jane Doe</div><div class="role">CEO &amp; Co-Founder</div></div>
+    <div class="member"><div class="name">John Smith</div><div class="role">CTO</div></div>
+    <div class="member"><div class="name">Senior Vice President</div><div class="role">VP</div></div>
+    <div class="member"><div class="name">About Acme</div><div class="role">Director</div></div>
+  </div>
+  <p>Reach finance at billing@acme.example.</p>
+</body></html>
+"""
+
+CONTACT = """
+<!doctype html>
+<html><head><title>Contact Acme</title></head>
+<body>
+  <h1>Contact</h1>
+  <p>Head office: 100 Market St, San Francisco, CA, 94103, United States</p>
+  <p>Email: sales@acme.example</p>
+  <p>Phone: +1 (415) 555-0199</p>
+</body></html>
+"""
+
+WORDPRESS = """
+<html><head><meta name="generator" content="WordPress 6.4"></head>
+<body><img src="/wp-content/uploads/x.png"><script src="/wp-json/index.js"></script></body></html>
+"""
+
+EMPTY = "<html><head></head><body></body></html>"
+
+# A JavaScript-only SPA shell (e.g. a Vite/React build). The body is an empty mount
+# point; everything meaningful is rendered client-side, so requests + BeautifulSoup
+# see only the build-tool-default title and favicon -- no description, og, or JSON-LD.
+JS_SPA_SHELL = """
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite App</title>
+    <script type="module" crossorigin src="/assets/index-5d175413.js"></script>
+    <link rel="stylesheet" href="/assets/index-57b7cd59.css">
+    <script type="module">window.onerror = () => {};</script>
+  </head>
+  <body>
+    <div id="root"></div>
+  </body>
+</html>
+"""
+
+BROKEN_JSON_LD = """
+<html><head>
+<script type="application/ld+json">{ this is not valid json ]</script>
+<title>Broken Co</title>
+</head><body><h1>Broken</h1></body></html>
+"""
+
+# --------------------------------------------------------------------------- #
+# Sitemap fixtures
+# --------------------------------------------------------------------------- #
+URLSET_XML = """<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://acme.example/about</loc></url>
+  <url><loc>https://acme.example/contact</loc></url>
+  <url><loc>https://acme.example/blog/post-1</loc></url>
+</urlset>"""
+
+SITEMAP_INDEX_XML = """<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap><loc>https://acme.example/sitemap-pages.xml</loc></sitemap>
+  <sitemap><loc>https://acme.example/sitemap-blog.xml</loc></sitemap>
+</sitemapindex>"""
+
+# A sitemap trying to smuggle a DTD entity-expansion payload -- must be rejected
+# outright by _parse_sitemap before ET.fromstring ever sees it.
+MALICIOUS_DOCTYPE_XML = """<?xml version="1.0"?>
+<!DOCTYPE foo [<!ENTITY xxe "boom">]>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://acme.example/&xxe;</loc></url>
+</urlset>"""
+
+
+def fake_fetch(responses):
+	"""Build a ``fetch(url, cfg, session=None, html_only=True)`` stub keyed by exact URL.
+
+	``responses`` maps url -> (status, body) for a 200-shaped response. A URL absent
+	from the map behaves like a 404 (mirrors a missing sitemap/robots.txt), so tests
+	only need to declare the endpoints they care about.
+	"""
+
+	def fake(url, cfg, session=None, html_only=True):
+		entry = responses.get(url)
+		if entry is None:
+			return 404, "", "not found", url
+		status, body = entry
+		return status, body, "", url
+
+	return fake
+
+
+def fake_fetch_with_redirects(responses, redirects):
+	"""Like ``fake_fetch``, but ``redirects`` maps a requested URL to the resolved
+	(post-redirect) URL returned as ``final_url`` -- simulates a 301/302 hop without
+	modeling the actual redirect chain, since callers only ever look at the result.
+	"""
+	base = fake_fetch(responses)
+
+	def fake(url, cfg, session=None, html_only=True):
+		status, body, error, final_url = base(url, cfg, session=session, html_only=html_only)
+		return status, body, error, redirects.get(url, final_url)
+
+	return fake
+
+
+# --------------------------------------------------------------------------- #
+# Page helper
+# --------------------------------------------------------------------------- #
+def make_page(url, html, status_code=200):
+	"""Build a (CrawledPage, soup) pair from raw HTML, exactly as the crawler does.
+
+	``extract_content`` mutates its soup, so it is given a throwaway parse and the
+	returned ``soup`` is a fresh, unmutated DOM (mirrors ``crawler.crawl_page``).
+	"""
+	soup = BeautifulSoup(html, "html.parser")
+	content = extract_content(BeautifulSoup(html, "html.parser"))
+	page = CrawledPage(
+		url=url,
+		status_code=status_code,
+		html=html,
+		text=content["text"],
+		title=content["title"],
+		headings=content["headings"],
+	)
+	return page, soup
+
+
+# --------------------------------------------------------------------------- #
+# In-memory rule builders (no DB) — mirror what config._build_rules() produces
+# --------------------------------------------------------------------------- #
+def _compile(pattern, is_regex):
+	if is_regex:
+		return re.compile(pattern, re.IGNORECASE)
+	return re.compile(re.escape(pattern), re.IGNORECASE)
+
+
+def keyword_rule(rule_type, patterns, *, target_value="", industry="", weight=1.0, match_scope="Full Text"):
+	"""Build a substring keyword Rule directly (is_regex=0 for each pattern)."""
+	compiled = [(_compile(p, False), p) for p in patterns]
+	return Rule(
+		rule_type=rule_type,
+		target_value=target_value,
+		industry=industry,
+		weight=weight,
+		match_scope=match_scope,
+		patterns=compiled,
+	)
+
+
+def regex_rule(rule_type, patterns, *, target_value="", weight=1.0, match_scope="HTML"):
+	"""Build a regex Rule directly (is_regex=1 for each pattern)."""
+	compiled = [(_compile(p, True), p) for p in patterns]
+	return Rule(
+		rule_type=rule_type,
+		target_value=target_value,
+		weight=weight,
+		match_scope=match_scope,
+		patterns=compiled,
+	)
+
+
+def industry_rules():
+	"""A small set of Industry rules (Headline scope) covering the fixtures."""
+	return [
+		keyword_rule(
+			"Industry",
+			["saas", "software as a service", "subscription"],
+			industry="SaaS",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["crm", "customer relationship"],
+			industry="CRM",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["shop", "free shipping", "apparel", "ecommerce"],
+			industry="Ecommerce",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["beverage", "sparkling water", "iced tea", "cereal"],
+			industry="Food & Beverage",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["3d printing", "additive manufacturing", "manufacturing"],
+			industry="Manufacturing",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["legal help", "legal documents", "legal advice", "law firm"],
+			industry="Legal",
+			match_scope="Headline",
+		),
+		keyword_rule(
+			"Industry",
+			["media company", "journalists", "newsroom"],
+			industry="Media",
+			match_scope="Headline",
+		),
+	]
+
+
+def social_rules():
+	return [
+		regex_rule("Social", [r"linkedin\.com/(company|in|school)/"], target_value="linkedin"),
+		regex_rule("Social", [r"(twitter\.com|x\.com)/[A-Za-z0-9_]+"], target_value="twitter"),
+		regex_rule("Social", [r"github\.com/[A-Za-z0-9_.-]+"], target_value="github"),
+		regex_rule("Social", [r"facebook\.com/[A-Za-z0-9_.\-/]+"], target_value="facebook"),
+		regex_rule("Social", [r"instagram\.com/[A-Za-z0-9_.]+"], target_value="instagram"),
+	]
+
+
+def make_config(settings=None, **kwargs) -> EnrichmentConfig:
+	"""Assemble an in-memory ``EnrichmentConfig`` for the SSRF / engine tests."""
+	cfg = EnrichmentConfig(settings=settings or {})
+	for key, value in kwargs.items():
+		setattr(cfg, key, value)
+	return cfg
+
+
+def make_mapping(source_key, target_doctype, target_fieldname, write_policy="Fill if empty", **kwargs):
+	return Mapping(
+		source_key=source_key,
+		target_doctype=target_doctype,
+		target_fieldname=target_fieldname,
+		write_policy=write_policy,
+		create_missing_link=kwargs.get("create_missing_link", 0),
+		default_values=kwargs.get("default_values", ""),
+	)

crm/domain_enrichment/tests/test_config.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure unit tests for config-layer pattern compilation (``config._compile_pattern``).
+
+Fully offline: no DB, no network. Covers the word-boundary fix -- an un-anchored
+short substring keyword like "erp" or "api" previously matched inside unrelated
+words ("waterproofing", "capital"), causing false industry-classification hits.
+"""
+
+from __future__ import annotations
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment.config import _compile_pattern
+
+
+class CompilePatternTest(UnitTestCase):
+	def test_substring_pattern_does_not_match_inside_unrelated_word(self):
+		erp_rx = _compile_pattern("erp", is_regex=0)
+		self.assertIsNone(erp_rx.search("Industrial Adhesives, Paints and Waterproofing Solutions"))
+		self.assertIsNone(erp_rx.search("enterprise value creation"))
+
+	def test_substring_pattern_still_matches_standalone_word(self):
+		erp_rx = _compile_pattern("erp", is_regex=0)
+		self.assertIsNotNone(erp_rx.search("Our ERP rollout finished on schedule"))
+
+	def test_api_pattern_does_not_match_inside_capital_or_rapid(self):
+		api_rx = _compile_pattern("api", is_regex=0)
+		self.assertIsNone(api_rx.search("venture capital and rapid growth"))
+		self.assertIsNotNone(api_rx.search("a public REST API for developers"))
+
+	def test_multi_word_phrase_pattern_still_matches(self):
+		rx = _compile_pattern("enterprise resource planning", is_regex=0)
+		self.assertIsNotNone(rx.search("our Enterprise Resource Planning suite"))
+
+	def test_is_regex_patterns_are_left_unanchored(self):
+		# An admin-authored regex (is_regex=1) is compiled as-is -- no word-boundary
+		# wrapping, so existing admin patterns (e.g. social-profile URL regexes) keep
+		# their exact intended behavior.
+		rx = _compile_pattern(r"linkedin\.com/(company|in|school)/", is_regex=1)
+		self.assertIsNotNone(rx.search("https://www.linkedin.com/company/acme"))

crm/domain_enrichment/tests/test_crawler.py

@@ -0,0 +1,467 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure unit tests for the crawler mechanics (no network, no DB).
+
+Covers URL normalization, the same-site/registrable-domain check, link-priority
+ordering (config-driven keywords), the crawlable filter (asset/scheme/skip-pattern
+rejection) and ``extract_content`` (title/headings/visible text).
+"""
+
+from __future__ import annotations
+
+from unittest import mock
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment import crawler
+from crm.domain_enrichment.crawler import (
+	_is_crawlable,
+	_link_priority,
+	extract_content,
+	normalize_url,
+	registrable_domain,
+	same_site,
+)
+from crm.domain_enrichment.tests import fixtures
+from crm.domain_enrichment.tests.fixtures import make_config
+
+# The default BFS priority keywords used by the engine when Settings has none.
+PRIORITY = [
+	("about", 1.0),
+	("contact", 1.0),
+	("team", 1.0),
+	("careers", 1.0),
+	("company", 1.0),
+]
+SKIP_PATTERNS = [r"/wp-admin", r"\?replytocom="]
+
+
+class NormalizeUrlTest(UnitTestCase):
+	def test_strips_fragment(self):
+		self.assertEqual(normalize_url("https://x.com/about/#team"), "https://x.com/about")
+
+	def test_strips_trailing_slash_on_path(self):
+		self.assertEqual(normalize_url("https://x.com/about/"), "https://x.com/about")
+
+	def test_keeps_root_slash(self):
+		# A bare root "/" must not be stripped (path length 1).
+		self.assertEqual(normalize_url("https://x.com/"), "https://x.com/")
+
+	def test_strips_query_string(self):
+		# Query-string variants of the same page (tracking params, product-variant
+		# params) must collapse to one dedupe key -- otherwise the same page gets
+		# queued and fetched multiple times under different query strings.
+		self.assertEqual(normalize_url("https://x.com/w/book/1?ean=123"), "https://x.com/w/book/1")
+		self.assertEqual(
+			normalize_url("https://x.com/w/book/1?terms=free-shipping"), "https://x.com/w/book/1"
+		)
+		self.assertEqual(normalize_url("https://x.com/w/book/1"), "https://x.com/w/book/1")
+
+
+class RegistrableDomainTest(UnitTestCase):
+	def test_strips_www_and_port(self):
+		self.assertEqual(registrable_domain("www.example.com:443"), "example.com")
+
+	def test_deep_subdomains_collapse_to_registrable(self):
+		self.assertEqual(registrable_domain("blog.shop.example.com"), "example.com")
+
+	def test_single_label(self):
+		self.assertEqual(registrable_domain("localhost"), "localhost")
+
+	def test_bare_ip_returns_itself(self):
+		self.assertEqual(registrable_domain("10.0.0.5"), "10.0.0.5")
+
+	def test_multi_label_public_suffix_keeps_registrable_label(self):
+		# The bug this guards: a last-two-labels rule returns "co.uk" here, so every
+		# *.co.uk site collapses into one same-site bucket. The PSL keeps the SLD.
+		self.assertEqual(registrable_domain("acme.co.uk"), "acme.co.uk")
+		self.assertEqual(registrable_domain("www.shop.acme.com.au"), "acme.com.au")
+
+	def test_distinct_orgs_under_a_public_suffix_are_distinct(self):
+		self.assertNotEqual(registrable_domain("acme.co.uk"), registrable_domain("evil.co.uk"))
+
+	def test_private_hosting_suffix_is_registrable(self):
+		# github.io is a PSL *private* suffix: two Pages tenants are different sites.
+		self.assertEqual(registrable_domain("acme.github.io"), "acme.github.io")
+		self.assertNotEqual(registrable_domain("acme.github.io"), registrable_domain("other.github.io"))
+
+
+class SameSiteTest(UnitTestCase):
+	def test_ignores_www_and_subdomains(self):
+		self.assertTrue(same_site("https://www.x.com/a", "x.com"))
+		self.assertTrue(same_site("https://blog.x.com/a", "x.com"))
+
+	def test_rejects_unrelated_org_sharing_a_public_suffix(self):
+		# Would be a false "same site" under the naive last-two-labels rule.
+		self.assertFalse(same_site("https://evil.co.uk/x", "acme.co.uk"))
+		self.assertFalse(same_site("https://other.github.io/x", "acme.github.io"))
+
+	def test_rejects_other_domain(self):
+		self.assertFalse(same_site("https://other.com/a", "x.com"))
+
+
+class LinkPriorityTest(UnitTestCase):
+	def test_priority_pages_rank_before_generic(self):
+		about = _link_priority("https://x.com/about", "About", PRIORITY)
+		blog = _link_priority("https://x.com/blog/post", "Read the post", PRIORITY)
+		self.assertLess(about, blog)
+
+	def test_anchor_text_counts(self):
+		# A generic URL with a priority keyword in the anchor still ranks high.
+		hit = _link_priority("https://x.com/p/42", "Our team", PRIORITY)
+		miss = _link_priority("https://x.com/p/43", "Read more", PRIORITY)
+		self.assertLess(hit, miss)
+
+	def test_non_priority_sorts_last(self):
+		generic = _link_priority("https://x.com/random", "", PRIORITY)
+		self.assertEqual(generic, len(PRIORITY) + 1)
+
+	def test_ancestor_segment_does_not_leak_priority_to_unrelated_leaf(self):
+		# A news article nested under "/about-us/..." must not inherit "about-us"'s
+		# priority from an ancestor path segment alone -- only the leaf segment counts.
+		news_article = _link_priority("https://x.com/about-us/corporate-news/2023/some-award", "", PRIORITY)
+		generic = _link_priority("https://x.com/blog/post", "", PRIORITY)
+		about_page = _link_priority("https://x.com/about-us", "", PRIORITY)
+		self.assertEqual(news_article, generic)
+		self.assertLess(about_page, news_article)
+
+
+class IsCrawlableTest(UnitTestCase):
+	def test_rejects_non_http_schemes(self):
+		self.assertFalse(_is_crawlable("mailto:a@b.com", []))
+		self.assertFalse(_is_crawlable("tel:+15555550142", []))
+		self.assertFalse(_is_crawlable("javascript:void(0)", []))
+
+	def test_rejects_asset_extensions(self):
+		self.assertFalse(_is_crawlable("https://x.com/file.pdf", []))
+		self.assertFalse(_is_crawlable("https://x.com/logo.png", []))
+		self.assertFalse(_is_crawlable("https://x.com/app.js", []))
+
+	def test_allows_normal_page(self):
+		self.assertTrue(_is_crawlable("https://x.com/about", []))
+
+	def test_rejects_configured_skip_pattern(self):
+		self.assertFalse(_is_crawlable("https://x.com/wp-admin/edit", SKIP_PATTERNS))
+		self.assertFalse(_is_crawlable("https://x.com/post?replytocom=5", SKIP_PATTERNS))
+		self.assertTrue(_is_crawlable("https://x.com/about", SKIP_PATTERNS))
+
+	def test_malformed_skip_pattern_treated_as_substring(self):
+		# An invalid regex must not crash; it falls back to substring match.
+		self.assertFalse(_is_crawlable("https://x.com/[bad", ["[bad"]))
+
+
+class ExtractContentTest(UnitTestCase):
+	def test_pulls_title_headings_and_text(self):
+		_page, soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+		content = extract_content(soup)
+		self.assertEqual(content["title"], "Acme Analytics | AI powered SaaS")
+		self.assertIn("AI powered analytics", content["headings"])
+		self.assertIn("hello@acme.example", content["text"])
+
+	def test_strips_script_and_style(self):
+		html = (
+			"<html><head><style>.x{}</style></head><body><script>var x=1</script><p>Visible</p></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://x.com", html)
+		content = extract_content(soup)
+		self.assertIn("Visible", content["text"])
+		self.assertNotIn("var x", content["text"])
+
+
+class ParseSitemapTest(UnitTestCase):
+	def test_parses_urlset(self):
+		kind, locs = crawler._parse_sitemap(fixtures.URLSET_XML)
+		self.assertEqual(kind, "urlset")
+		self.assertIn("https://acme.example/about", locs)
+		self.assertIn("https://acme.example/contact", locs)
+		self.assertIn("https://acme.example/blog/post-1", locs)
+
+	def test_parses_sitemapindex(self):
+		kind, locs = crawler._parse_sitemap(fixtures.SITEMAP_INDEX_XML)
+		self.assertEqual(kind, "sitemapindex")
+		self.assertEqual(
+			locs,
+			["https://acme.example/sitemap-pages.xml", "https://acme.example/sitemap-blog.xml"],
+		)
+
+	def test_malformed_xml_returns_none_kind(self):
+		kind, locs = crawler._parse_sitemap("<urlset><url><loc>unclosed")
+		self.assertIsNone(kind)
+		self.assertEqual(locs, [])
+
+	def test_doctype_rejected_before_parsing(self):
+		# Must never reach ET.fromstring with a DTD payload.
+		kind, locs = crawler._parse_sitemap(fixtures.MALICIOUS_DOCTYPE_XML)
+		self.assertIsNone(kind)
+		self.assertEqual(locs, [])
+
+	def test_unrelated_root_element_ignored(self):
+		kind, locs = crawler._parse_sitemap("<rss><channel></channel></rss>")
+		self.assertIsNone(kind)
+		self.assertEqual(locs, [])
+
+
+class DiscoverSitemapUrlsTest(UnitTestCase):
+	def setUp(self):
+		self.cfg = make_config()
+
+	def test_uses_robots_declared_sitemap(self):
+		robots = mock.Mock()
+		robots.site_maps.return_value = ["https://acme.example/my-sitemap.xml"]
+		responses = {"https://acme.example/my-sitemap.xml": (200, fixtures.URLSET_XML)}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, robots)
+		self.assertIn("https://acme.example/about", urls)
+		self.assertIn("https://acme.example/contact", urls)
+
+	def test_falls_back_to_conventional_path_when_robots_declares_none(self):
+		robots = mock.Mock()
+		robots.site_maps.return_value = None
+		responses = {"https://acme.example/sitemap.xml": (200, fixtures.URLSET_XML)}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, robots)
+		self.assertIn("https://acme.example/about", urls)
+
+	def test_falls_back_when_robots_is_none(self):
+		responses = {"https://acme.example/sitemap.xml": (200, fixtures.URLSET_XML)}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, None)
+		self.assertIn("https://acme.example/about", urls)
+
+	def test_missing_sitemap_returns_empty_without_raising(self):
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch({})):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, None)
+		self.assertEqual(urls, [])
+
+	def test_malformed_xml_returns_empty(self):
+		responses = {"https://acme.example/sitemap.xml": (200, "not xml at all")}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, None)
+		self.assertEqual(urls, [])
+
+	def test_follows_one_level_of_sitemap_index(self):
+		responses = {
+			"https://acme.example/sitemap.xml": (200, fixtures.SITEMAP_INDEX_XML),
+			"https://acme.example/sitemap-pages.xml": (200, fixtures.URLSET_XML),
+			"https://acme.example/sitemap-blog.xml": (
+				200,
+				'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+				"<url><loc>https://acme.example/blog/hello</loc></url></urlset>",
+			),
+		}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme.example", self.cfg, None, None)
+		self.assertIn("https://acme.example/about", urls)
+		self.assertIn("https://acme.example/blog/hello", urls)
+
+	def test_rejects_cross_site_sitemap(self):
+		# The sitemap FILE lives on a different site than the one we're enriching --
+		# must never be fetched, regardless of what its <loc> entries claim. Uses
+		# real-style ".com" hosts (not "*.example") so registrable_domain() actually
+		# differentiates them -- ".example" isn't in the bundled Public Suffix List,
+		# so acme.example/evil.example would collapse to the same registrable domain.
+		robots = mock.Mock()
+		robots.site_maps.return_value = ["https://evil-corp.com/sitemap.xml"]
+		responses = {
+			"https://evil-corp.com/sitemap.xml": (
+				200,
+				'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+				"<url><loc>https://evil-corp.com/about</loc></url></urlset>",
+			)
+		}
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			urls = crawler.discover_sitemap_urls("https://acme-corp.com", self.cfg, None, robots)
+		self.assertEqual(urls, [])
+
+
+class CrawlSitemapSeedingTest(UnitTestCase):
+	"""End-to-end: crawl() seeded from a robots.txt-declared sitemap.
+
+	Sitemap discovery must be a pure addition on top of the ordinary BFS: it should
+	surface a page that isn't linked from the homepage, without displacing the
+	homepage as the first crawled result, and it must disappear cleanly (no error)
+	when disabled or when the site has no sitemap at all.
+	"""
+
+	HOMEPAGE_HTML = '<html><head><title>Acme</title></head><body><a href="/contact">Contact</a></body></html>'
+	CONTACT_HTML = "<html><body>Contact us</body></html>"
+	HIDDEN_ABOUT_HTML = "<html><head><title>Hidden About</title></head><body><h1>About</h1></body></html>"
+	ROBOTS_TXT = "Sitemap: https://acme.example/sitemap.xml"
+	SITEMAP_XML = (
+		'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+		"<url><loc>https://acme.example/hidden-about</loc></url></urlset>"
+	)
+
+	def _responses(self, **overrides):
+		base = {
+			"https://acme.example": (200, self.HOMEPAGE_HTML),
+			"https://acme.example/robots.txt": (200, self.ROBOTS_TXT),
+			"https://acme.example/sitemap.xml": (200, self.SITEMAP_XML),
+			"https://acme.example/contact": (200, self.CONTACT_HTML),
+			"https://acme.example/hidden-about": (200, self.HIDDEN_ABOUT_HTML),
+		}
+		base.update(overrides)
+		return base
+
+	def test_sitemap_only_page_is_crawled_homepage_stays_first(self):
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 1},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(self._responses())):
+			results = crawler.crawl("https://acme.example", cfg)
+		urls = [page.url for page, _soup in results]
+		self.assertEqual(urls[0], "https://acme.example")
+		self.assertIn("https://acme.example/hidden-about", urls)
+		self.assertIn("https://acme.example/contact", urls)
+
+	def test_use_sitemap_disabled_never_fetches_sitemap(self):
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(self._responses())):
+			results = crawler.crawl("https://acme.example", cfg)
+		urls = [page.url for page, _soup in results]
+		self.assertEqual(urls[0], "https://acme.example")
+		self.assertNotIn("https://acme.example/hidden-about", urls)
+		self.assertIn("https://acme.example/contact", urls)
+
+	def test_no_sitemap_falls_through_to_plain_link_following(self):
+		# robots.txt exists but declares no sitemap, and /sitemap.xml 404s -- the
+		# crawl must proceed exactly as ordinary link-following, no error surfaced.
+		responses = self._responses(**{"https://acme.example/robots.txt": (200, "User-agent: *")})
+		del responses["https://acme.example/sitemap.xml"]
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 1},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(crawler, "fetch", side_effect=fixtures.fake_fetch(responses)):
+			results = crawler.crawl("https://acme.example", cfg)
+		urls = [page.url for page, _soup in results]
+		self.assertEqual(urls[0], "https://acme.example")
+		self.assertIn("https://acme.example/contact", urls)
+		self.assertNotIn("https://acme.example/hidden-about", urls)
+		self.assertEqual(len(results), 2)  # homepage + contact only, no crash, no phantom pages
+
+
+class RedirectDedupTest(UnitTestCase):
+	"""A redirect can land two different *requested* URLs on the same resolved page
+	(e.g. /contact.html -> /contact) -- the pre-fetch `visited` check can't catch
+	this since the requested URLs genuinely differ. crawl() must collapse the
+	duplicate after the fact and not let it consume a page-budget slot.
+	"""
+
+	def test_redirect_collapsed_duplicate_does_not_consume_budget(self):
+		homepage_html = (
+			'<html><body><a href="/contact">Contact</a>'
+			'<a href="/contact.html">Contact (alt)</a>'
+			'<a href="/team">Team</a></body></html>'
+		)
+		responses = {
+			"https://acme.example": (200, homepage_html),
+			"https://acme.example/contact": (200, "<html><body>Contact page</body></html>"),
+			"https://acme.example/contact.html": (200, "<html><body>Contact page</body></html>"),
+			"https://acme.example/team": (200, "<html><body>Team page</body></html>"),
+		}
+		# /contact.html resolves (redirects) to the same page as /contact.
+		redirects = {"https://acme.example/contact.html": "https://acme.example/contact"}
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(
+			crawler, "fetch", side_effect=fixtures.fake_fetch_with_redirects(responses, redirects)
+		):
+			results = crawler.crawl("https://acme.example", cfg)
+		urls = [page.url for page, _soup in results]
+		self.assertEqual(urls.count("https://acme.example/contact"), 1)
+		# The freed budget slot means /team -- otherwise starved by the wasted
+		# duplicate fetch -- still gets crawled.
+		self.assertIn("https://acme.example/team", urls)
+
+
+class OffSiteRedirectScopeTest(UnitTestCase):
+	"""A same-site link that redirects off-domain must not be treated as an
+	in-scope crawled page: fetch()'s redirect loop only re-validates SSRF safety
+	per hop, never registrable-domain scope.
+	"""
+
+	def test_offsite_redirect_marked_out_of_scope(self):
+		homepage_html = (
+			'<html><body><a href="/investor-relations">IR</a><a href="/team">Team</a></body></html>'
+		)
+		responses = {
+			"https://jpm-test.com": (200, homepage_html),
+			"https://jpm-test.com/investor-relations": (200, "<html><body>redirected content</body></html>"),
+			"https://jpm-test.com/team": (200, "<html><body>Team page</body></html>"),
+		}
+		redirects = {
+			"https://jpm-test.com/investor-relations": "https://reports.jpmchase-test.com/ir/2022.htm"
+		}
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(
+			crawler, "fetch", side_effect=fixtures.fake_fetch_with_redirects(responses, redirects)
+		):
+			results = crawler.crawl("https://jpm-test.com", cfg)
+		offsite = next(
+			page for page, _soup in results if page.url == "https://reports.jpmchase-test.com/ir/2022.htm"
+		)
+		self.assertTrue(offsite.error)
+		self.assertEqual(offsite.html, "")
+		self.assertEqual(offsite.text, "")
+
+	def test_offsite_redirect_clears_title_and_headings(self):
+		# title/headings are parsed by crawl_page() before the off-site check runs,
+		# so clearing html/text/soup alone isn't enough -- extractors that read
+		# title/headings straight off the CrawledPage (e.g. classify_industry's
+		# About-page lookup) would otherwise still see off-site content.
+		homepage_html = '<html><body><a href="/about">About</a></body></html>'
+		responses = {
+			"https://jpm-test.com": (200, homepage_html),
+			"https://jpm-test.com/about": (
+				200,
+				"<html><head><title>Global Widget Manufacturing Co</title></head>"
+				"<body><h1>Leading widget manufacturer since 1900</h1></body></html>",
+			),
+		}
+		redirects = {"https://jpm-test.com/about": "https://widgetco-test.com/about"}
+		cfg = make_config(
+			settings={"max_pages": 10, "max_depth": 1, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(
+			crawler, "fetch", side_effect=fixtures.fake_fetch_with_redirects(responses, redirects)
+		):
+			results = crawler.crawl("https://jpm-test.com", cfg)
+		offsite = next(page for page, _soup in results if page.url == "https://widgetco-test.com/about")
+		self.assertTrue(offsite.error)
+		self.assertEqual(offsite.title, "")
+		self.assertEqual(offsite.headings, [])
+
+	def test_homepage_redirect_is_still_adopted_not_flagged(self):
+		# The existing "homepage moved to a new host" behavior must be unaffected --
+		# only NON-homepage pages get the off-site-scope check.
+		responses = {"https://old-home-test.com": (200, "<html><body>hi</body></html>")}
+		redirects = {"https://old-home-test.com": "https://new-home-test.com"}
+		cfg = make_config(
+			settings={"max_pages": 1, "max_depth": 0, "use_sitemap": 0},
+			link_priority=PRIORITY,
+			skip_patterns=[],
+		)
+		with mock.patch.object(
+			crawler, "fetch", side_effect=fixtures.fake_fetch_with_redirects(responses, redirects)
+		):
+			results = crawler.crawl("https://old-home-test.com", cfg)
+		self.assertEqual(results[0][0].url, "https://new-home-test.com")
+		self.assertFalse(results[0][0].error)

crm/domain_enrichment/tests/test_extractors.py

@@ -0,0 +1,862 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure, network-free unit tests for the config-driven extractors.
+
+The classifiers (industry / social) are exercised with **in-memory Rule objects**
+built directly (see ``tests.fixtures``), never the DB — this is the rule-driven
+rewrite of the POC's extractor tests. The mechanics (company-name cleaning, favicon
+scorer, JSON-LD parse, email/phone extraction, readability diagnosis) are tested
+standalone.
+"""
+
+from __future__ import annotations
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment import extractors
+from crm.domain_enrichment.result import CrawledPage, Method
+from crm.domain_enrichment.tests import fixtures
+
+
+def _pages(*specs):
+	"""specs: (url, html) tuples -> list[CrawledPage], soups_by_url dict."""
+	pages, soups = [], {}
+	for url, html in specs:
+		page, soup = fixtures.make_page(url, html)
+		pages.append(page)
+		soups[url] = soup
+	return pages, soups
+
+
+# --------------------------------------------------------------------------- #
+# Company information (mechanics)
+# --------------------------------------------------------------------------- #
+class CompanyInfoTest(UnitTestCase):
+	def setUp(self):
+		self.page, self.soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+
+	def test_prefers_json_ld_for_name(self):
+		info = extractors.extract_company_info(self.page, self.soup)
+		self.assertEqual(info["company_name"].value, "Acme Analytics Inc.")
+		self.assertEqual(info["company_name"].method, Method.JSON_LD)
+		self.assertEqual(info["company_name"].source, "https://acme.example")
+
+	def test_description_from_json_ld(self):
+		info = extractors.extract_company_info(self.page, self.soup)
+		self.assertIn("analytics", info["description"].value.lower())
+
+	def test_logo_is_link_icon_not_json_ld_image(self):
+		# The company logo is the link icon (HOMEPAGE declares none -> favicon.ico
+		# fallback), NOT the JSON-LD Organization.logo. That image is kept separately.
+		info = extractors.extract_company_info(self.page, self.soup)
+		self.assertEqual(info["logo"].method, Method.FAVICON)
+		self.assertEqual(info["logo"].value, "https://acme.example/favicon.ico")
+		self.assertEqual(info["image"].method, Method.JSON_LD)
+		self.assertEqual(info["image"].value, "https://acme.example/ld-logo.png")
+
+	def test_logo_prefers_declared_apple_touch_icon(self):
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<link rel='icon' href='/fav.png' sizes='16x16'>"
+			"<link rel='apple-touch-icon' href='/touch.png'>"
+			"</head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["logo"].value, "https://x.example/touch.png")
+
+	def test_meta_fallback_when_no_json_ld(self):
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head><title>Foo</title><meta property='og:site_name' content='Foo Inc'></head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["company_name"].value, "Foo Inc")
+		self.assertEqual(info["company_name"].method, Method.META_TAG)
+
+	def test_broken_json_ld_does_not_crash(self):
+		page, soup = fixtures.make_page("https://b.example", fixtures.BROKEN_JSON_LD)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["company_name"].method, Method.TITLE_TAG)
+
+	def test_degenerate_og_description_yields_to_plain_description(self):
+		# Broken markup can park a site-verification token on og:description while the
+		# plain description tag carries the real copy.
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<meta property='og:description' content='JIs-hfpDlO6niGZsMes79IWdxC0A9frop'>"
+			"<meta name='description' content='Acme Organics is a manufacturer and "
+			"exporter of aroma chemicals based in Mumbai.'>"
+			"</head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertTrue(info["description"].value.startswith("Acme Organics is a manufacturer"))
+
+	def test_og_description_still_preferred_when_usable(self):
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<meta property='og:description' content='Acme builds developer tools for teams.'>"
+			"<meta name='description' content='Some other blurb entirely for the page.'>"
+			"</head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["description"].value, "Acme builds developer tools for teams.")
+
+	def test_degenerate_description_still_used_when_nothing_better(self):
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head><meta name='description' content='Acme GmbH'></head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["description"].value, "Acme GmbH")
+
+	def test_unspaced_script_description_beats_token(self):
+		# Japanese copy has no ASCII spaces, so a space check alone would mistake it
+		# for a token and keep the og:description token instead.
+		copy = "株式会社アクメは東京を拠点に産業用ロボットの設計・製造・販売を手がける専門メーカーです。"
+		page, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<meta property='og:description' content='JIs-hfpDlO6niGZsMes79IWdxC0A9frop'>"
+			f"<meta name='description' content='{copy}'>"
+			"</head><body></body></html>",
+		)
+		info = extractors.extract_company_info(page, soup)
+		self.assertEqual(info["description"].value, copy)
+
+
+class LogoResolutionTest(UnitTestCase):
+	"""extract_logo returns the link icon only; the larger social/JSON-LD image is
+	captured by extract_image, never used as the logo."""
+
+	def test_scalable_svg_beats_small_png(self):
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<link rel='icon' type='image/png' href='/fav-32.png' sizes='32x32'>"
+			"<link rel='icon' type='image/svg+xml' href='/logo.svg'>"
+			"</head><body></body></html>",
+		)
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/logo.svg")
+		self.assertEqual(logo.method, Method.FAVICON)
+
+	def test_logo_is_link_icon_even_when_og_image_exists(self):
+		# A declared link icon is the logo; the bigger og:image is NOT the logo -- it
+		# is captured as the image instead.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<link rel='icon' href='/fav.ico' sizes='16x16'>"
+			"<meta property='og:image' content='https://x.example/social-1200.png'>"
+			"</head><body></body></html>",
+		)
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/fav.ico")
+		self.assertEqual(logo.method, Method.FAVICON)
+		image = extractors.extract_image(soup, "https://x.example")
+		self.assertEqual(image.value, "https://x.example/social-1200.png")
+		self.assertEqual(image.method, Method.META_TAG)
+
+	def test_falls_back_to_favicon_ico_ignoring_og_image(self):
+		# No link icon declared -> favicon.ico, even though an og:image exists.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			"<meta property='og:image' content='https://x.example/banner.png'>"
+			"</head><body></body></html>",
+		)
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/favicon.ico")
+		self.assertEqual(logo.method, Method.FAVICON)
+
+	def test_image_prefers_json_ld_logo_over_og_image(self):
+		# extract_image keeps the old "best big image" priority: curated JSON-LD logo
+		# outranks a social-share og:image.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			'<script type="application/ld+json">'
+			'{"@type":"Organization","logo":"https://x.example/brand.png"}</script>'
+			"<meta property='og:image' content='https://x.example/banner.png'>"
+			"</head><body></body></html>",
+		)
+		image = extractors.extract_image(soup, "https://x.example")
+		self.assertEqual(image.value, "https://x.example/brand.png")
+		self.assertEqual(image.method, Method.JSON_LD)
+
+	def test_image_empty_when_none_declared(self):
+		_p, soup = fixtures.make_page("https://x.example", "<html><head></head><body></body></html>")
+		self.assertEqual(extractors.extract_image(soup, "https://x.example").value, "")
+
+	def test_falls_back_to_favicon_ico(self):
+		_p, soup = fixtures.make_page("https://x.example", "<html><head></head><body></body></html>")
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/favicon.ico")
+		self.assertEqual(logo.method, Method.FAVICON)
+
+	def test_rejects_unsafe_scheme_in_icon(self):
+		# A crawled site injecting a javascript: icon href must not be stored as the
+		# logo -- it is skipped and the safe favicon.ico fallback is used.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head><link rel='icon' href='javascript:alert(1)'></head><body></body></html>",
+		)
+		logo = extractors.extract_logo(soup, "https://x.example")
+		self.assertEqual(logo.value, "https://x.example/favicon.ico")
+
+	def test_rejects_unsafe_scheme_in_image(self):
+		# javascript:/data: in JSON-LD logo or og:image is dropped, never stored.
+		_p, soup = fixtures.make_page(
+			"https://x.example",
+			"<html><head>"
+			'<script type="application/ld+json">{"@type":"Organization","logo":"javascript:evil()"}</script>'
+			"<meta property='og:image' content='data:image/svg+xml;base64,PHN2Zz4='>"
+			"</head><body></body></html>",
+		)
+		self.assertEqual(extractors.extract_image(soup, "https://x.example").value, "")
+
+
+class CompanyNameCleaningTest(UnitTestCase):
+	def test_strips_tagline_after_separator(self):
+		self.assertEqual(extractors._clean_company_name("PostHog – dev tools"), "PostHog")
+
+	def test_trims_descriptive_sentence_title(self):
+		self.assertEqual(
+			extractors._clean_company_name(
+				"ITC Ltd has diversified presence in FMCG, Paperboards & Packaging"
+			),
+			"ITC Ltd",
+		)
+		self.assertEqual(extractors._clean_company_name("Acme Corp provides cloud security"), "Acme Corp")
+
+	def test_brand_at_end_matches_domain(self):
+		self.assertEqual(
+			extractors._clean_company_name(
+				"Innovative IT & Software Development Services | 8848 Digital",
+				"https://8848digital.com/",
+			),
+			"8848 Digital",
+		)
+
+	def test_without_domain_hint_falls_back_to_first_chunk(self):
+		self.assertEqual(
+			extractors._clean_company_name("Innovative IT & Software Development Services | 8848 Digital"),
+			"Innovative IT & Software Development Services",
+		)
+
+
+class JsonLdTest(UnitTestCase):
+	def test_parses_graph_blocks(self):
+		_page, soup = fixtures.make_page(
+			"https://x.com",
+			'<html><head><script type="application/ld+json">'
+			'{"@graph":[{"@type":"Organization","name":"A"},{"@type":"WebSite"}]}'
+			"</script></head><body></body></html>",
+		)
+		blocks = extractors.parse_json_ld(soup)
+		self.assertEqual(len(blocks), 2)
+		self.assertEqual(blocks[0]["name"], "A")
+
+	def test_ignores_invalid_json(self):
+		_page, soup = fixtures.make_page("https://x.com", fixtures.BROKEN_JSON_LD)
+		self.assertEqual(extractors.parse_json_ld(soup), [])
+
+
+class DescriptionSelectionTest(UnitTestCase):
+	def test_prefers_about_over_product_homepage(self):
+		pages, soups = _pages(
+			(
+				"https://shop.example",
+				"<html><head><meta property='og:description' content='Shop the latest gadgets.'></head><body></body></html>",
+			),
+			(
+				"https://shop.example/about",
+				"<html><head><meta name='description' content='Acme builds developer tools for teams.'></head><body></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertEqual(desc.value, "Acme builds developer tools for teams.")
+		self.assertEqual(desc.source, "https://shop.example/about")
+
+	def test_falls_back_to_homepage(self):
+		pages, soups = _pages(
+			(
+				"https://x.example",
+				"<html><head><meta property='og:description' content='We make great software.'></head><body></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertEqual(desc.value, "We make great software.")
+
+	def test_json_ld_is_authoritative(self):
+		pages, soups = _pages(
+			(
+				"https://s.example",
+				'<html><head><script type="application/ld+json">{"@type":"Organization","description":"Acme is a developer-tools company."}</script></head><body></body></html>',
+			),
+			(
+				"https://s.example/about",
+				"<html><head><meta name='description' content='Some about-page blurb.'></head><body></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertEqual(desc.method, Method.JSON_LD)
+
+	def test_about_body_paragraph_when_no_meta(self):
+		# No description metadata anywhere -> derive it from the About page's body copy.
+		about_body = (
+			"<main><p>Acme Robotics designs autonomous warehouse robots that help "
+			"retailers fulfil orders faster and at lower cost.</p></main>"
+		)
+		pages, soups = _pages(
+			("https://acme.example", "<html><head><title>Acme</title></head><body></body></html>"),
+			("https://acme.example/about", f"<html><body>{about_body}</body></html>"),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertIn("autonomous warehouse robots", desc.value)
+		self.assertEqual(desc.method, Method.BODY_TEXT)
+		self.assertEqual(desc.source, "https://acme.example/about")
+
+	def test_head_meta_beats_home_body_paragraph(self):
+		# With an og:description present, head meta wins over a homepage body paragraph.
+		pages, soups = _pages(
+			(
+				"https://acme.example",
+				"<html><head><meta name='description' content='Acme makes developer tools.'>"
+				"</head><body><main><p>Some long homepage marketing paragraph that is "
+				"definitely over the minimum length threshold for a body paragraph.</p>"
+				"</main></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups)
+		self.assertEqual(desc.value, "Acme makes developer tools.")
+		self.assertEqual(desc.method, Method.META_TAG)
+
+	def test_industry_keyword_density_steers_about_paragraph_choice(self):
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		pages, soups = _pages(
+			(
+				"https://acmebank.example",
+				"<html><head><title>Acme Bank</title></head><body></body></html>",
+			),
+			(
+				"https://acmebank.example/about",
+				"<html><body><main>"
+				"<p>Putting our long-tenured investment teams on the line to earn the "
+				"trust of institutional investors across every market we serve.</p>"
+				"<p>Acme Bank is a full-service financial institution serving corporations, "
+				"institutions and private clients across commercial and investment banking.</p>"
+				"</main></body></html>",
+			),
+		)
+		desc = extractors.select_description(pages, soups, industry_rules=finance_rules)
+		self.assertTrue(desc.value.startswith("Acme Bank is a full-service"))
+
+
+class FirstParagraphTest(UnitTestCase):
+	def test_skips_nav_and_returns_first_substantial_paragraph(self):
+		html = (
+			"<html><body><nav><p>Home About Contact Login Signup Pricing Docs Blog "
+			"Resources Careers Partners Support Status Community Downloads Enterprise</p></nav>"
+			"<main><p>Short.</p><p>Globex Corp is a logistics platform that moves freight "
+			"across the country for enterprise shippers.</p></main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://globex.example", html)
+		result = extractors.first_paragraph(soup)
+		self.assertIn("logistics platform", result)
+		self.assertNotIn("Login", result)  # nav paragraph skipped despite being long
+
+	def test_returns_empty_when_no_substantial_paragraph(self):
+		_page, soup = fixtures.make_page("https://x.example", "<html><body><p>Hi.</p></body></html>")
+		self.assertEqual(extractors.first_paragraph(soup), "")
+
+	def test_prefers_paragraph_with_more_keyword_hits_over_first_qualifying_one(self):
+		# A narrow pull-quote with zero industry-keyword hits precedes a genuine
+		# descriptive paragraph -- the keyword-denser paragraph should win.
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>Putting our long-tenured investment teams on the line to earn the "
+			"trust of institutional investors across every market we serve.</p>"
+			"<p>Acme Bank is a full-service financial institution serving corporations, "
+			"institutions and private clients across commercial and investment banking.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Acme Bank is a full-service"))
+
+	def test_falls_back_to_first_paragraph_when_no_keyword_hits(self):
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>Putting our long-tenured investment teams on the line to earn the "
+			"trust of institutional investors across every market we serve.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Putting our long-tenured"))
+
+	def test_falls_back_to_first_paragraph_when_no_industry_rules_given(self):
+		# No rules configured at all (industry_rules=None or []) -- with nothing else
+		# to separate the candidates, document order still decides.
+		html = (
+			"<html><body><main>"
+			"<p>Putting our long-tenured investment teams on the line to earn the "
+			"trust of institutional investors across every market we serve.</p>"
+			"<p>Acme Bank is a full-service financial institution serving corporations, "
+			"institutions and private clients across commercial and investment banking.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Putting our long-tenured"))
+		self.assertTrue(
+			extractors.first_paragraph(soup, industry_rules=[]).startswith("Putting our long-tenured")
+		)
+		self.assertTrue(
+			extractors.first_paragraph(soup, industry_rules=None).startswith("Putting our long-tenured")
+		)
+
+	def test_company_name_breaks_tie_between_equal_keyword_hits(self):
+		# Both paragraphs mention "banking" exactly once (tied on industry hits) --
+		# only the one that also names the company should win.
+		finance_rules = [fixtures.keyword_rule("Industry", ["banking"], industry="Finance")]
+		html = (
+			"<html><body><main>"
+			"<p>Our roots trace back decades of steady growth in commercial banking "
+			"and long-term client relationships across every region we serve.</p>"
+			"<p>Acme Bank has led the market in retail banking with award-winning "
+			"service for individuals, families and small businesses nationwide.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules, company_name="Acme Bank")
+		self.assertTrue(result.startswith("Acme Bank has led the market"))
+
+	def test_leadership_bio_never_wins_over_company_description(self):
+		# An About page's founder bio is dense in industry keywords (it lists former
+		# employers and sectors), so keyword scoring alone picks it over the real
+		# description. Bios must be rejected outright.
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["insurance", "asset management"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>Acme Capital is a registered portfolio manager focused on uncovering "
+			"long-term opportunities for clients across public equity markets.</p>"
+			"<p>Priya Nair is the Founder and Chief Investment Officer of Acme Capital, "
+			"where she previously led insurance and asset management mandates.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmecapital.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Acme Capital is a registered portfolio manager"))
+
+	def test_call_to_action_paragraph_never_wins(self):
+		finance_rules = [fixtures.keyword_rule("Industry", ["hvac"], industry="Construction")]
+		html = (
+			"<html><body><main>"
+			"<p>Kishika Temperatures has been a leader in air conditioning maintenance "
+			"and HVAC solutions for over 30 years, serving homes across the region.</p>"
+			"<p>Explore our services to see why our HVAC and HVAC-adjacent teams are the "
+			"preferred choice for leading developers, hotels and residential projects.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://kishika.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Kishika Temperatures has been a leader"))
+
+	def test_overview_mentioning_founder_once_is_not_a_profile(self):
+		# A company overview may name its founder with a single pronoun. That is
+		# not a staff profile and must still win over a generic sector essay.
+		html = (
+			"<html><body><main>"
+			"<p>Over the past decade, advances in warehouse robotics have reshaped how "
+			"goods move between suppliers and consumers around the world.</p>"
+			"<p>Acme Robotics builds warehouse automation for retailers. Founded by "
+			"Jane Doe in 2015, she still leads our engineering team today.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmerobotics.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Robotics builds"))
+
+	def test_company_described_as_founder_of_something_is_not_a_profile(self):
+		html = (
+			"<html><body><main>"
+			"<p>Over the past decade, advances in open hardware have reshaped how "
+			"engineers share designs and collaborate across company lines.</p>"
+			"<p>Acme Labs is the founder of the Open Hardware Alliance, and we design "
+			"reference boards used by hundreds of member companies worldwide.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmelabs.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Labs is the founder"))
+
+	def test_overview_speaking_for_the_company_is_never_a_profile(self):
+		# Two pronouns in copy that says "our" is still the company talking about
+		# itself, not a staff profile.
+		html = (
+			"<html><body><main>"
+			"<p>Over the past decade, advances in warehouse robotics have reshaped how "
+			"goods move between suppliers and consumers around the world.</p>"
+			"<p>Acme Robotics builds warehouse automation. Jane Doe founded it in 2015; "
+			"she leads our engineering team and her designs power every product.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmerobotics.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Robotics builds"))
+
+	def test_repeated_pronouns_mark_a_profile(self):
+		html = (
+			"<html><body><main>"
+			"<p>Priya Nair joined the firm in 2010. She oversees every client mandate "
+			"and her research has been cited across the industry for years.</p>"
+			"<p>Acme Capital is a registered portfolio manager focused on uncovering "
+			"long-term opportunities for clients across public equity markets.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmecapital.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Capital is a registered"))
+
+	def test_falls_back_to_rejected_paragraphs_when_nothing_else_qualifies(self):
+		# A page of nothing but bios must still yield a paragraph -- rejecting every
+		# candidate should not silently drop the description.
+		html = (
+			"<html><body><main>"
+			"<p>Priya Nair is the Founder of Acme Capital, where she leads the firm's "
+			"investment committee and oversees portfolio construction end to end.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmecapital.example", html)
+		self.assertTrue(extractors.first_paragraph(soup).startswith("Priya Nair is the Founder"))
+
+	def test_speaking_for_the_company_breaks_a_zero_hit_tie(self):
+		# Neither paragraph scores an industry hit: a generic essay about the sector
+		# sits first, the company's own copy second. Document order must not decide.
+		finance_rules = [fixtures.keyword_rule("Industry", ["banking"], industry="Finance")]
+		html = (
+			"<html><body><main>"
+			"<p>Over the past decade, advances in concrete technology have allowed "
+			"engineers to design mega structures that redefine infrastructure.</p>"
+			"<p>Capital RMC, established in 2016, stands at the forefront of the "
+			"construction sector, and we deliver world-class ready-mix concrete.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://capitalrmc.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("Capital RMC, established in 2016"))
+
+	def test_speaking_for_the_company_cannot_outrank_more_industry_hits(self):
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>A full-service financial institution built on decades of trusted "
+			"commercial and retail banking relationships across every market.</p>"
+			"<p>We are proud to serve local businesses across the region with "
+			"award-winning service and community-first values that endure.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
+		self.assertTrue(result.startswith("A full-service financial institution"))
+
+	def test_company_name_mentions_cannot_outrank_more_industry_hits(self):
+		# The first paragraph scores 2 industry hits and never names the company; the
+		# second scores only 1 industry hit despite repeating the company name three
+		# times. Industry hits must still decide -- company name is tiebreak-only.
+		finance_rules = [
+			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
+		]
+		html = (
+			"<html><body><main>"
+			"<p>A full-service financial institution built on decades of trusted "
+			"commercial and retail banking relationships across every market.</p>"
+			"<p>Acme Bank, Acme Bank, Acme Bank -- proud to serve local businesses "
+			"with award-winning retail banking and community-first values.</p>"
+			"</main></body></html>"
+		)
+		_page, soup = fixtures.make_page("https://acmebank.example", html)
+		result = extractors.first_paragraph(soup, industry_rules=finance_rules, company_name="Acme Bank")
+		self.assertTrue(result.startswith("A full-service financial institution"))
+
+
+class AboutPageDetectionTest(UnitTestCase):
+	def test_canonical_about_slugs_match(self):
+		for path in (
+			"/about",
+			"/about-us",
+			"/about_us",
+			"/aboutus",
+			"/company",
+			"/our-story",
+			"/en/about-us",
+		):
+			self.assertTrue(extractors._is_about_page(f"https://x.example{path}"), path)
+
+	def test_prefix_match_no_longer_qualifies(self):
+		# Regression guard: "about-our-reporting" (an ESG page) starts with "about-"
+		# but is not itself a canonical About-page slug.
+		self.assertFalse(extractors._is_about_page("https://x.example/esg-topics/about-our-reporting"))
+		self.assertFalse(extractors._is_about_page("https://x.example/about-cookies"))
+
+	def test_non_about_words_still_rejected(self):
+		self.assertFalse(extractors._is_about_page("https://x.example/about/careers"))
+		self.assertFalse(extractors._is_about_page("https://x.example/about-us/contact"))
+
+
+# --------------------------------------------------------------------------- #
+# Contacts: emails + phones (mechanics)
+# --------------------------------------------------------------------------- #
+class EmailExtractionTest(UnitTestCase):
+	def setUp(self):
+		self.pages, _ = _pages(
+			("https://acme.example", fixtures.HOMEPAGE),
+			("https://acme.example/about", fixtures.ABOUT),
+			("https://acme.example/contact", fixtures.CONTACT),
+		)
+
+	def test_emails_deduped_and_sourced(self):
+		emails = extractors.extract_emails(self.pages)
+		values = {e.value for e in emails}
+		self.assertIn("hello@acme.example", values)
+		self.assertIn("sales@acme.example", values)
+		self.assertEqual(len(values), len(emails))  # deduped
+
+	def test_blocklisted_noise_emails_rejected(self):
+		pages, _ = _pages(
+			("https://x.com", "<html><body>logo@2x.png test@example.com real@x.com</body></html>"),
+		)
+		values = {e.value for e in extractors.extract_emails(pages)}
+		self.assertIn("real@x.com", values)
+		self.assertNotIn("test@example.com", values)
+
+
+class PhoneExtractionTest(UnitTestCase):
+	def setUp(self):
+		self.pages, _ = _pages(
+			("https://acme.example", fixtures.HOMEPAGE),
+			("https://acme.example/contact", fixtures.CONTACT),
+		)
+
+	def test_tel_link_marked(self):
+		phones = extractors.extract_phones(self.pages)
+		self.assertIn(Method.TEL_LINK, {p.method for p in phones})
+		self.assertTrue(any("4155550142" in p.value.replace("+", "") for p in phones))
+
+	def test_international_number_captured(self):
+		pages, _ = _pages(
+			("https://x.example/contact", "<html><body><p>Call us: +612 9003 8888</p></body></html>")
+		)
+		phones = extractors.extract_phones(pages)
+		self.assertTrue(any("61290038888" in p.value.replace("+", "") for p in phones))
+
+	def test_number_without_country_code_rejected(self):
+		# Deliberate precision-over-recall call: a number with no "+" is either a
+		# local/ambiguous format or noise (reference numbers, stats) that happens to
+		# fall in a phone-shaped digit range -- only accept explicit international
+		# numbers, even though this means dropping some real local-format numbers.
+		pages, _ = _pages(
+			(
+				"https://x.example/contact",
+				"<html><body><p>Local line: 2165703. Toll-free: 1800-266-6066.</p>"
+				'<a href="tel:18002666066">Call</a></body></html>',
+			)
+		)
+		phones = extractors.extract_phones(pages)
+		self.assertEqual(phones, [])
+
+
+# --------------------------------------------------------------------------- #
+# Social profiles (config-driven via Social rules)
+# --------------------------------------------------------------------------- #
+class SocialTest(UnitTestCase):
+	def test_detects_networks_and_skips_share_links(self):
+		page, soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+		company = extractors.extract_company_info(page, soup)
+		profiles = extractors.extract_social_profiles(
+			[page], {page.url: soup}, fixtures.social_rules(), extra_links=company["social_links"]
+		)
+		self.assertTrue(profiles["linkedin"].value)
+		self.assertTrue(profiles["github"].value)
+		self.assertTrue(profiles["instagram"].value)
+		# JSON-LD sameAs links win and are marked as such.
+		self.assertEqual(profiles["linkedin"].method, Method.JSON_LD)
+		# A facebook 'sharer' link must be ignored.
+		self.assertEqual(profiles["facebook"].value, "")
+
+	def test_protocol_relative_links_made_absolute(self):
+		page, soup = fixtures.make_page(
+			"https://bluedart.com",
+			"<html><body><a href='//www.linkedin.com/company/bluedart/'>li</a>"
+			"<a href='//twitter.com/BlueDart_'>tw</a></body></html>",
+		)
+		profiles = extractors.extract_social_profiles([page], {page.url: soup}, fixtures.social_rules())
+		self.assertEqual(profiles["linkedin"].value, "https://www.linkedin.com/company/bluedart/")
+		self.assertEqual(profiles["twitter"].value, "https://twitter.com/BlueDart_")
+
+
+# --------------------------------------------------------------------------- #
+# Industry classification (config-driven via Industry rules)
+# --------------------------------------------------------------------------- #
+class IndustryTest(UnitTestCase):
+	def _classify(self, name, description, extra_html=""):
+		page, _ = fixtures.make_page(
+			"https://x.example",
+			f"<html><head><title>{name}</title></head><body>{extra_html}</body></html>",
+		)
+		company = {"company_name": name, "description": description}
+		return extractors.classify_industry([page], company, fixtures.industry_rules())
+
+	def test_classifies_saas_or_crm_with_confidence(self):
+		page, soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+		company = extractors.extract_company_info(page, soup)
+		industry, conf = extractors.classify_industry([page], company, fixtures.industry_rules())
+		self.assertIn(industry, ("SaaS", "CRM"))
+		self.assertGreater(conf, 0.0)
+		self.assertLessEqual(conf, 1.0)
+
+	def test_empty_site_returns_no_industry(self):
+		page, _ = fixtures.make_page("https://e.example", fixtures.EMPTY)
+		industry, conf = extractors.classify_industry([page], {}, fixtures.industry_rules())
+		self.assertEqual(industry, "")
+		self.assertEqual(conf, 0.0)
+
+	def test_no_rules_returns_empty(self):
+		page, _ = fixtures.make_page("https://e.example", fixtures.HOMEPAGE)
+		self.assertEqual(extractors.classify_industry([page], {}, []), ("", 0.0))
+
+	def test_d2c_brand_maps_to_ecommerce(self):
+		self.assertEqual(self._classify("Gymshark", "Shop gym clothing and workout apparel.")[0], "Ecommerce")
+
+	def test_beverage_brand_maps_to_food_and_beverage(self):
+		self.assertEqual(
+			self._classify("Liquid Death", "A beverage company with sparkling water and iced tea.")[0],
+			"Food & Beverage",
+		)
+
+	def test_legal_help_beats_stray_keyword(self):
+		self.assertEqual(
+			self._classify(
+				"Rocket Lawyer", "Rocket Lawyer makes legal help simple — legal documents and advice."
+			)[0],
+			"Legal",
+		)
+
+	def test_3d_printing_maps_to_manufacturing(self):
+		# A "shop" nav link must not beat the core business signal in the headline.
+		self.assertEqual(
+			self._classify(
+				"Desktop Metal",
+				"Metal 3D printing and additive manufacturing for engineers.",
+				"<nav>Shop</nav>",
+			)[0],
+			"Manufacturing",
+		)
+
+	def test_about_page_headline_supplements_thin_homepage(self):
+		# A homepage with no meta description and abstract brand-voice headings
+		# carries no classification signal on its own -- a crawled About page's
+		# title/headings should still let the classifier find a match.
+		finance_rules = [
+			fixtures.keyword_rule(
+				"Industry",
+				["banking", "financial services"],
+				industry="Finance",
+				match_scope="Headline",
+			)
+		]
+		home, _ = fixtures.make_page(
+			"https://acmebank.example",
+			"<html><head><title>Acme | Official Website</title></head>"
+			"<body><h1>Acme home</h1><h2>What problem can we solve together?</h2></body></html>",
+		)
+		about, _ = fixtures.make_page(
+			"https://acmebank.example/about",
+			"<html><head><title>About Acme</title></head>"
+			"<body><h2>A leader in financial services and banking</h2></body></html>",
+		)
+		company = {"company_name": "Acme", "description": ""}
+		industry, conf = extractors.classify_industry([home, about], company, finance_rules)
+		self.assertEqual(industry, "Finance")
+		self.assertGreater(conf, 0.0)
+
+	def test_homepage_only_still_returns_nothing_when_thin(self):
+		finance_rules = [
+			fixtures.keyword_rule(
+				"Industry",
+				["banking", "financial services"],
+				industry="Finance",
+				match_scope="Headline",
+			)
+		]
+		home, _ = fixtures.make_page(
+			"https://acmebank.example",
+			"<html><head><title>Acme | Official Website</title></head>"
+			"<body><h1>Acme home</h1><h2>What problem can we solve together?</h2></body></html>",
+		)
+		company = {"company_name": "Acme", "description": ""}
+		industry, conf = extractors.classify_industry([home], company, finance_rules)
+		self.assertEqual(industry, "")
+		self.assertEqual(conf, 0.0)
+
+
+# --------------------------------------------------------------------------- #
+# Generic rule executors directly
+# --------------------------------------------------------------------------- #
+class RuleExecutorTest(UnitTestCase):
+	def test_apply_keyword_rules_weights_and_sums(self):
+		rule = fixtures.keyword_rule(
+			"Industry", ["saas", "subscription"], industry="SaaS", weight=2.0, match_scope="Headline"
+		)
+		scores = extractors.apply_keyword_rules({"Headline": "a saas subscription saas"}, [rule])
+		# 3 hits * weight 2.0 = 6.0
+		self.assertEqual(scores["SaaS"], 6.0)
+
+
+# --------------------------------------------------------------------------- #
+# Readability diagnosis (mechanics)
+# --------------------------------------------------------------------------- #
+class ReadabilityTest(UnitTestCase):
+	def test_blocked_waf_challenge(self):
+		page, _ = fixtures.make_page(
+			"https://www.pepsi.com",
+			'<html><head><script src="/_Incapsula_Resource?SWJIYLWA=1"></script></head><body></body></html>',
+		)
+		self.assertEqual(extractors.diagnose_readability([page]), "blocked")
+
+	def test_empty_js_only_shell(self):
+		page, _ = fixtures.make_page(
+			"https://spa.example", "<html><head></head><body><div id='root'></div></body></html>"
+		)
+		self.assertEqual(extractors.diagnose_readability([page]), "empty")
+
+	def test_unreachable_when_failed(self):
+		page = CrawledPage(url="https://x.example", status_code=0, error="timeout after 10s")
+		self.assertEqual(extractors.diagnose_readability([page]), "unreachable")
+
+	def test_http_403_is_blocked(self):
+		page = CrawledPage(
+			url="https://www.pepperfry.com",
+			status_code=403,
+			html="<html><head><title>Access Denied</title></head></html>",
+			text="Access Denied",
+			title="Access Denied",
+		)
+		self.assertEqual(extractors.diagnose_readability([page]), "blocked")
+
+	def test_readable_site_returns_empty_reason(self):
+		page, _ = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
+		self.assertEqual(extractors.diagnose_readability([page]), "")

crm/domain_enrichment/tests/test_integration.py

@@ -0,0 +1,301 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""IntegrationTestCase tests for the Frappe layer (real test DB).
+
+Covers: seeder idempotency, the mapper write-policies against real CRM
+Lead/Deal/Organization docs, the single Run writer, API permission/allow-list
+enforcement + rate-limit, and the link-time Organization -> Lead/Deal copy.
+
+All tests are network-free: wherever a crawl would happen the pipeline / fetch is
+monkeypatched to return a canned EnrichmentResult, so nothing leaves the box.
+"""
+
+from __future__ import annotations
+
+from unittest import mock
+
+import frappe
+from frappe.tests import IntegrationTestCase
+
+from crm.domain_enrichment import config, install, tasks
+from crm.domain_enrichment.config import get_config
+from crm.domain_enrichment.cross_record import copy_enrichment_from_organization
+from crm.domain_enrichment.mapper import apply_to_document
+from crm.domain_enrichment.result import EnrichmentResult, Field, SocialProfile
+from crm.domain_enrichment.tests import fixtures
+
+
+def canned_result(website="https://acme.example") -> EnrichmentResult:
+	"""A fully-populated EnrichmentResult — no network, deterministic."""
+	r = EnrichmentResult(website=website)
+	r.company_name = Field("Acme Analytics Inc.", website, "JSON-LD")
+	r.description = Field("AI powered analytics SaaS for product teams.", website, "Meta Tag")
+	r.logo = Field("https://acme.example/favicon.ico", website, "Favicon")
+	r.industry = Field("SaaS", website, "Keyword Classifier")
+	r.industry_confidence = 0.8
+	r.social_profiles = {
+		"linkedin": SocialProfile("https://www.linkedin.com/company/acme", website),
+		"twitter": SocialProfile("https://twitter.com/acme", website),
+	}
+	return r
+
+
+class SeederIdempotencyTest(IntegrationTestCase):
+	def tearDown(self):
+		frappe.db.rollback()
+
+	def test_seeder_is_a_no_op_on_second_run(self):
+		# The seeder runs on install/migrate, so records already exist. Running it
+		# again must not create duplicate Rule / Field Mapping rows.
+		install.seed_default_rules_and_mappings()
+		rules_before = frappe.db.count("CRM Enrichment Rule")
+		mappings_before = frappe.db.count("CRM Enrichment Field Mapping")
+
+		install.seed_default_rules_and_mappings()
+
+		self.assertEqual(frappe.db.count("CRM Enrichment Rule"), rules_before)
+		self.assertEqual(frappe.db.count("CRM Enrichment Field Mapping"), mappings_before)
+		self.assertGreater(rules_before, 0)
+		self.assertGreater(mappings_before, 0)
+
+
+class MapperWritePolicyTest(IntegrationTestCase):
+	def setUp(self):
+		self.cfg = get_config()
+		self.result = canned_result()
+
+	def tearDown(self):
+		frappe.db.rollback()
+
+	def _new_org(self, **kwargs):
+		doc = frappe.new_doc("CRM Organization")
+		doc.organization_name = kwargs.pop("organization_name", "Mapper Test Org")
+		for k, v in kwargs.items():
+			doc.set(k, v)
+		return doc
+
+	def test_fill_if_empty_fills_only_empty_fields(self):
+		org = self._new_org(company_description="")
+		apply_to_document(org, self.result, self.cfg)
+		self.assertEqual(org.company_description, "AI powered analytics SaaS for product teams.")
+
+	def test_fill_if_empty_never_clobbers_user_data(self):
+		org = self._new_org(company_description="Hand-written by sales.")
+		apply_to_document(org, self.result, self.cfg)
+		self.assertEqual(org.company_description, "Hand-written by sales.")
+
+	def test_create_missing_link_creates_industry(self):
+		industry = "Quantum Widgets " + frappe.generate_hash(length=6)
+		self.assertFalse(frappe.db.exists("CRM Industry", industry))
+		result = canned_result()
+		result.industry = Field(industry, "https://x.example", "Keyword Classifier")
+		org = self._new_org(industry="")
+		apply_to_document(org, result, self.cfg)
+		self.assertTrue(frappe.db.exists("CRM Industry", industry))
+		self.assertEqual(org.industry, industry)
+
+	def test_create_missing_link_respects_user_create_permission(self):
+		# A user without CRM Industry create rights must not be able to auto-create a
+		# master from a scraped value: the master is not inserted and the field is
+		# skipped (no dangling link), rather than an ignore_permissions escalation.
+		industry = "Contraband Industry " + frappe.generate_hash(length=6)
+		result = canned_result()
+		result.industry = Field(industry, "https://x.example", "Keyword Classifier")
+		org = self._new_org(industry="")
+
+		user = _make_minimal_user()
+		self.addCleanup(frappe.set_user, "Administrator")
+		frappe.set_user(user)
+		apply_to_document(org, result, self.cfg)
+
+		self.assertFalse(frappe.db.exists("CRM Industry", industry))
+		self.assertFalse(org.industry)
+
+
+class RunWriterTest(IntegrationTestCase):
+	def tearDown(self):
+		frappe.db.rollback()
+
+	def test_writes_exactly_one_run_with_summary(self):
+		org = frappe.new_doc("CRM Organization")
+		org.organization_name = "Run Writer Org " + frappe.generate_hash(length=4)
+		org.website = "https://acme.example"
+		org.insert()
+
+		before = frappe.db.count("CRM Enrichment Run", {"reference_name": org.name})
+		run_name = tasks.write_run(
+			"CRM Organization",
+			org.name,
+			"https://acme.example",
+			status="Completed",
+			result=canned_result(),
+		)
+		after = frappe.db.count("CRM Enrichment Run", {"reference_name": org.name})
+		self.assertEqual(after - before, 1)
+
+		run = frappe.get_doc("CRM Enrichment Run", run_name)
+		self.assertEqual(run.status, "Completed")
+		self.assertEqual(run.company_name, "Acme Analytics Inc.")
+		self.assertEqual(run.industry, "SaaS")
+		self.assertTrue(run.finished_on)
+		self.assertIn("Acme Analytics", run.raw_json)
+
+
+class ApiPermissionTest(IntegrationTestCase):
+	def tearDown(self):
+		frappe.db.rollback()
+		frappe.set_user("Administrator")
+
+	def test_enrich_throws_on_disallowed_doctype(self):
+		from crm.domain_enrichment.api import enrich
+
+		with self.assertRaises(frappe.ValidationError):
+			enrich("ToDo", "anything")
+
+	def test_enrich_requires_write_permission(self):
+		from crm.domain_enrichment.api import enrich
+
+		org = frappe.new_doc("CRM Organization")
+		org.organization_name = "Perm Org " + frappe.generate_hash(length=4)
+		org.website = "https://acme.example"
+		org.insert()
+
+		# A user with no role on CRM Organization must be blocked by check_permission.
+		user = _make_minimal_user()
+		frappe.set_user(user)
+		with self.assertRaises(frappe.PermissionError):
+			enrich("CRM Organization", org.name)
+
+	def test_enrich_routes_are_rate_limited(self):
+		# enrich / retry each carry rate_limit(limit=ENRICH_RATE_LIMIT).
+		from crm.domain_enrichment import api
+
+		for fn in (api.enrich, api.retry):
+			self.assertTrue(hasattr(fn, "__wrapped__"), fn.__name__)
+
+
+class CrossRecordCopyTest(IntegrationTestCase):
+	def setUp(self):
+		self.org = frappe.new_doc("CRM Organization")
+		self.org.organization_name = "Enriched Org " + frappe.generate_hash(length=4)
+		self.org.website = "https://acme.example"
+		self.org.company_description = "AI powered analytics SaaS."
+		self.org.linkedin = "https://www.linkedin.com/company/acme"
+		self.org.insert()
+		# Mark it enriched via a completed Run (the cleanest "is enriched" signal).
+		tasks.write_run(
+			"CRM Organization",
+			self.org.name,
+			self.org.website,
+			status="Completed",
+			result=canned_result(),
+		)
+
+	def tearDown(self):
+		frappe.db.rollback()
+
+	def test_copies_empty_fields_from_enriched_org(self):
+		deal = frappe.new_doc("CRM Deal")
+		deal.organization = self.org.name
+		filled = copy_enrichment_from_organization(deal)
+		self.assertTrue(filled)
+		self.assertEqual(deal.company_description, "AI powered analytics SaaS.")
+		self.assertEqual(deal.linkedin, "https://www.linkedin.com/company/acme")
+
+	def test_does_not_clobber_user_set_fields(self):
+		deal = frappe.new_doc("CRM Deal")
+		deal.organization = self.org.name
+		deal.company_description = "User wrote this."
+		copy_enrichment_from_organization(deal)
+		self.assertEqual(deal.company_description, "User wrote this.")
+
+	def test_non_enriched_org_is_a_noop(self):
+		plain = frappe.new_doc("CRM Organization")
+		plain.organization_name = "Plain Org " + frappe.generate_hash(length=4)
+		plain.website = "https://plain.example"
+		plain.insert()
+
+		deal = frappe.new_doc("CRM Deal")
+		deal.organization = plain.name
+		filled = copy_enrichment_from_organization(deal)
+		self.assertEqual(filled, [])
+		self.assertFalse(deal.company_description)
+
+
+class AutoEnrichOnCreateTest(IntegrationTestCase):
+	"""The after_insert auto-enrich hook (Lead, Deal, Organization), gated by Settings."""
+
+	def _settings(self, **overrides):
+		s = {
+			"enabled": 1,
+			"auto_enrich": 1,
+			"request_timeout": 10,
+			"max_pages": 10,
+		}
+		s.update(overrides)
+		return frappe._dict(s)
+
+	def _doc(self, doctype, website="https://acme.example", name="X1"):
+		return frappe._dict(doctype=doctype, name=name, website=website)
+
+	def _run(self, doc, settings):
+		# auto_enrich_on_create now does a Settings-only gate (config.get_settings), and
+		# enqueue_enrichment reads its timeout the same way -- patch both to the fake
+		# Settings and assert on frappe.enqueue.
+		with (
+			mock.patch.object(config, "get_settings", return_value=settings),
+			mock.patch.object(tasks, "get_settings", return_value=settings),
+			mock.patch.object(tasks.frappe, "enqueue") as enq,
+		):
+			tasks.auto_enrich_on_create(doc)
+		return enq
+
+	def test_enqueues_for_new_lead(self):
+		enq = self._run(self._doc("CRM Lead", name="L1"), self._settings())
+		enq.assert_called_once()
+		kwargs = enq.call_args.kwargs
+		self.assertEqual(kwargs["reference_doctype"], "CRM Lead")
+		self.assertEqual(kwargs["job_id"], "domain-enrich-CRM Lead-L1")
+
+	def test_enqueues_for_new_deal(self):
+		enq = self._run(self._doc("CRM Deal", name="D1"), self._settings())
+		enq.assert_called_once()
+		kwargs = enq.call_args.kwargs
+		self.assertEqual(kwargs["reference_doctype"], "CRM Deal")
+		self.assertEqual(kwargs["job_id"], "domain-enrich-CRM Deal-D1")
+		self.assertTrue(kwargs["deduplicate"])
+
+	def test_enqueues_for_new_organization(self):
+		enq = self._run(self._doc("CRM Organization", name="O1"), self._settings())
+		enq.assert_called_once()
+		self.assertEqual(enq.call_args.kwargs["reference_doctype"], "CRM Organization")
+
+	def test_skips_when_auto_enrich_off(self):
+		enq = self._run(self._doc("CRM Deal"), self._settings(auto_enrich=0))
+		enq.assert_not_called()
+
+	def test_skips_when_feature_disabled(self):
+		enq = self._run(self._doc("CRM Deal"), self._settings(enabled=0))
+		enq.assert_not_called()
+
+	def test_skips_doctype_outside_the_allow_list(self):
+		enq = self._run(self._doc("CRM Task"), self._settings())
+		enq.assert_not_called()
+
+	def test_skips_without_website(self):
+		enq = self._run(self._doc("CRM Deal", website=""), self._settings())
+		enq.assert_not_called()
+
+
+def _make_minimal_user():
+	"""Create (or reuse) a user with no CRM Organization role for permission tests."""
+	email = "de-noperm@example.com"
+	if not frappe.db.exists("User", email):
+		user = frappe.new_doc("User")
+		user.email = email
+		user.first_name = "No"
+		user.last_name = "Perm"
+		user.send_welcome_email = 0
+		user.insert(ignore_permissions=True)
+	return email

crm/domain_enrichment/tests/test_pipeline.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure, network-free pipeline tests.
+
+The crawl is stubbed (``pipeline.crawl`` patched to return canned offline pages), so
+the whole pipeline runs with zero network. Asserts the assembled ``EnrichmentResult``,
+the explainable provenance schema (``to_dict``/``flat``) the mapper depends on, and
+the progress-step contract.
+"""
+
+from __future__ import annotations
+
+from unittest import mock
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment import pipeline as pipeline_mod
+from crm.domain_enrichment.pipeline import PROGRESS_STEPS, _normalize_website
+from crm.domain_enrichment.pipeline import run as run_pipeline
+from crm.domain_enrichment.result import EnrichmentResult
+from crm.domain_enrichment.tests import fixtures
+
+# The schema the CRM field mapper relies on. If extractors change the shape these
+# assertions fail loudly instead of the mapper silently reading missing keys.
+EXPECTED_TOP_KEYS = {
+	"company_name",
+	"description",
+	"logo",
+	"image",
+	"industry",
+	"industry_confidence",
+	"emails",
+	"phones",
+	"social_profiles",
+	"_meta",
+}
+EXPECTED_FLAT_KEYS = {
+	"company_name",
+	"description",
+	"logo",
+	"industry",
+	"industry_confidence",
+	"social_profiles",
+}
+SCALAR_SUBKEYS = {"value", "source", "method"}
+
+
+def _full_config():
+	"""An in-memory config wired with all the rule types the fixtures exercise."""
+	return fixtures.make_config(
+		settings={"max_pages": 5, "max_depth": 1, "request_timeout": 5},
+		rules_by_type={
+			"Industry": fixtures.industry_rules(),
+			"Social": fixtures.social_rules(),
+		},
+	)
+
+
+def _canned_crawl(*_args, **_kwargs):
+	"""Return canned (CrawledPage, soup) tuples instead of crawling the network."""
+	return [
+		fixtures.make_page("https://acme.example", fixtures.HOMEPAGE),
+		fixtures.make_page("https://acme.example/about", fixtures.ABOUT),
+		fixtures.make_page("https://acme.example/contact", fixtures.CONTACT),
+	]
+
+
+class NormalizeWebsiteTest(UnitTestCase):
+	def test_adds_https_scheme(self):
+		self.assertEqual(_normalize_website("frappe.io"), "https://frappe.io")
+
+	def test_rejects_empty(self):
+		with self.assertRaises(ValueError):
+			_normalize_website("")
+
+	def test_rejects_bad_scheme(self):
+		with self.assertRaises(ValueError):
+			_normalize_website("ftp://x.com")
+
+
+class PipelineRunTest(UnitTestCase):
+	def setUp(self):
+		self.cfg = _full_config()
+
+	def test_full_result_is_explainable(self):
+		steps = []
+		with mock.patch.object(pipeline_mod, "crawl", _canned_crawl):
+			result = run_pipeline(
+				"https://acme.example",
+				cfg=self.cfg,
+				progress=lambda i, m: steps.append((i, m)),
+			)
+		out = result.to_dict()
+
+		self.assertEqual(out["company_name"]["value"], "Acme Analytics Inc.")
+		self.assertTrue(out["company_name"]["source"])
+		self.assertTrue(out["company_name"]["method"])
+
+		# Every list item carries provenance.
+		for item in out["emails"] + out["phones"]:
+			self.assertTrue(item["source"], f"missing source: {item}")
+			self.assertTrue(item["method"], f"missing method: {item}")
+
+		# Progress emitted, ending at the final step.
+		emitted = [i for i, _ in steps]
+		self.assertEqual(emitted[-1], len(PROGRESS_STEPS) - 1)
+
+	def test_flat_view(self):
+		with mock.patch.object(pipeline_mod, "crawl", _canned_crawl):
+			result = run_pipeline("https://acme.example", cfg=self.cfg)
+		flat = result.flat()
+		self.assertEqual(flat["company_name"], "Acme Analytics Inc.")
+		self.assertIsInstance(flat["social_profiles"], dict)
+
+	def test_unreachable_site_returns_empty_without_raising(self):
+		from crm.domain_enrichment.result import CrawledPage
+
+		def broken_crawl(*_a, **_k):
+			return [(CrawledPage(url="https://x.example", status_code=0, error="timeout after 5s"), None)]
+
+		with mock.patch.object(pipeline_mod, "crawl", broken_crawl):
+			result = run_pipeline("https://x.example", cfg=self.cfg)
+		out = result.to_dict()
+		self.assertEqual(out["company_name"]["value"], "")
+		self.assertTrue(out["_meta"]["errors"])
+		self.assertTrue(result.notes)  # carries the "couldn't reach" note
+
+
+class JsSpaSiteTest(UnitTestCase):
+	"""Enriching a JavaScript-only SPA (empty body, client-rendered content).
+
+	Without a render step the engine sees only the build-tool-default `<head>` metadata
+	and an empty `<body>`. It must (a) flag the page as JS-rendered, (b) not fabricate
+	a description/industry/contacts, and (c) still surface whatever metadata exists
+	(title-tag company name + favicon), all without raising.
+	"""
+
+	def _spa_crawl(self, *_a, **_k):
+		return [fixtures.make_page("https://spa.example", fixtures.JS_SPA_SHELL)]
+
+	def test_js_only_site_yields_metadata_only(self):
+		cfg = _full_config()
+		with (
+			mock.patch.object(pipeline_mod, "crawl", self._spa_crawl),
+			# No About page is crawled; the probe must not hit the real network.
+			mock.patch.object(pipeline_mod, "probe_about_pages", return_value=[]),
+		):
+			result = run_pipeline("https://spa.example", cfg=cfg)
+		out = result.to_dict()
+
+		# Flagged as JS-rendered, and nothing fabricated.
+		self.assertTrue(any("JavaScript-rendered" in n for n in result.notes))
+		self.assertEqual(out["description"]["value"], "")
+		self.assertEqual(out["industry"]["value"], "")
+		self.assertEqual(out["emails"], [])
+		self.assertEqual(out["phones"], [])
+		self.assertTrue(all(p.value == "" for p in result.social_profiles.values()))
+
+		# Only the metadata the shell actually exposes is surfaced.
+		self.assertEqual(out["company_name"]["value"], "Vite App")
+		self.assertEqual(out["company_name"]["method"], "Title Tag")
+		self.assertTrue(out["logo"]["value"].endswith("/vite.svg"))
+
+	def test_readability_diagnosed_empty(self):
+		from crm.domain_enrichment import extractors
+
+		page, _soup = fixtures.make_page("https://spa.example", fixtures.JS_SPA_SHELL)
+		self.assertEqual(extractors.diagnose_readability([page]), "empty")
+
+
+class SchemaStabilityTest(UnitTestCase):
+	"""The result schema must be identical regardless of what a site exposes."""
+
+	def _assert_invariants(self, out, flat):
+		self.assertEqual(set(out), EXPECTED_TOP_KEYS)
+		self.assertEqual(set(flat), EXPECTED_FLAT_KEYS)
+		for key in ("company_name", "description", "logo", "industry"):
+			self.assertEqual(set(out[key]), SCALAR_SUBKEYS)
+
+	def test_empty_result_has_full_schema(self):
+		result = EnrichmentResult()
+		self._assert_invariants(result.to_dict(), result.flat())
+
+	def test_populated_result_has_same_schema(self):
+		with mock.patch.object(pipeline_mod, "crawl", _canned_crawl):
+			result = run_pipeline("https://acme.example", cfg=_full_config())
+		out = result.to_dict()
+		self._assert_invariants(out, result.flat())
+		for e in out["emails"]:
+			self.assertEqual(set(e), {"value", "source", "method"})
+		for p in out["phones"]:
+			self.assertEqual(set(p), {"value", "raw", "source", "method"})

crm/domain_enrichment/tests/test_ssrf.py

@@ -0,0 +1,214 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Pure unit tests for the SSRF guard (``http.validate_url`` / ``http._is_blocked_ip``).
+
+These are fully offline: ``socket.getaddrinfo`` is monkeypatched so DNS resolution
+is deterministic and no network call is ever made. The guard must reject loopback,
+link-local, private and reserved addresses (incl. the cloud-metadata endpoint
+169.254.169.254), reject non-http(s) schemes, honor allow/block domain lists, and
+allow a normal public host. The private-network rejection is unconditional --
+there is no setting to disable it.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from unittest import mock
+
+from frappe.tests import UnitTestCase
+
+from crm.domain_enrichment import http
+from crm.domain_enrichment.http import SSRFError, validate_url
+from crm.domain_enrichment.tests.fixtures import make_config
+
+
+@contextmanager
+def _resolves_to(ip_map):
+	"""Patch socket.getaddrinfo so a host resolves to the given IP(s).
+
+	``ip_map`` maps hostname -> ip string (or list of ip strings). An unknown host
+	raises gaierror, mimicking a real resolver failure.
+	"""
+	import socket
+
+	def fake_getaddrinfo(host, *args, **kwargs):
+		if host not in ip_map:
+			raise socket.gaierror(f"unknown host {host}")
+		ips = ip_map[host]
+		if isinstance(ips, str):
+			ips = [ips]
+		# getaddrinfo returns 5-tuples; element 4 is the sockaddr (ip, port, ...).
+		return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 0)) for ip in ips]
+
+	with mock.patch.object(http.socket, "getaddrinfo", side_effect=fake_getaddrinfo):
+		yield
+
+
+class SSRFGuardTest(UnitTestCase):
+	def setUp(self):
+		self.cfg = make_config()  # default: no allow/block lists
+
+	# -- rejection of internal / dangerous addresses ----------------------- #
+	def test_rejects_loopback_127(self):
+		with _resolves_to({"evil.example": "127.0.0.1"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://evil.example/", self.cfg)
+
+	def test_rejects_cloud_metadata_link_local(self):
+		# 169.254.169.254 is the AWS/GCP metadata endpoint — a classic SSRF target.
+		with _resolves_to({"metadata.example": "169.254.169.254"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://metadata.example/latest/meta-data/", self.cfg)
+
+	def test_rejects_private_10_range(self):
+		with _resolves_to({"intranet.example": "10.0.0.5"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://intranet.example/", self.cfg)
+
+	def test_rejects_localhost(self):
+		with _resolves_to({"localhost": "127.0.0.1"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://localhost:8000/", self.cfg)
+
+	def test_rejects_ipv6_loopback(self):
+		with _resolves_to({"v6.example": "::1"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://v6.example/", self.cfg)
+
+	# -- scheme validation ------------------------------------------------- #
+	def test_rejects_non_http_scheme(self):
+		with self.assertRaises(SSRFError):
+			validate_url("ftp://example.com/x", self.cfg)
+		with self.assertRaises(SSRFError):
+			validate_url("file:///etc/passwd", self.cfg)
+		with self.assertRaises(SSRFError):
+			validate_url("gopher://example.com/", self.cfg)
+
+	def test_rejects_url_without_host(self):
+		with self.assertRaises(SSRFError):
+			validate_url("http:///nohost", self.cfg)
+
+	def test_rejects_unresolvable_host(self):
+		with _resolves_to({}):  # nothing resolves
+			with self.assertRaises(SSRFError):
+				validate_url("http://does-not-resolve.example/", self.cfg)
+
+	# -- the happy path ---------------------------------------------------- #
+	def test_allows_public_host(self):
+		with _resolves_to({"frappe.io": "104.21.0.1"}):
+			self.assertEqual(validate_url("https://frappe.io/", self.cfg), "https://frappe.io/")
+
+	def test_rejects_when_any_resolved_ip_is_private(self):
+		# DNS-rebinding style: one public + one private IP -> must reject.
+		with _resolves_to({"mixed.example": ["8.8.8.8", "10.1.2.3"]}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://mixed.example/", self.cfg)
+
+	# -- the private-network guard is unconditional ------------------------ #
+	def test_private_network_guard_cannot_be_disabled(self):
+		# There is no setting to allow private networks -- a would-be bypass flag
+		# is inert, the guard still rejects a private address.
+		cfg = make_config(settings={"allow_private_networks": 1})
+		with _resolves_to({"intranet.example": "10.0.0.5"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://intranet.example/", cfg)
+
+	# -- allow / block lists ----------------------------------------------- #
+	def test_blocked_domain_is_always_rejected(self):
+		cfg = make_config(blocked_domains=["evil.com"])
+		with _resolves_to({"evil.com": "8.8.8.8"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://evil.com/", cfg)
+		# Subdomains of a blocked domain are blocked too.
+		with _resolves_to({"sub.evil.com": "8.8.8.8"}):
+			with self.assertRaises(SSRFError):
+				validate_url("http://sub.evil.com/", cfg)
+
+	def test_trailing_dot_fqdn_does_not_bypass_block_list(self):
+		# evil.com. resolves the same as evil.com in DNS, so the trailing dot must not
+		# slip past the block rule (the block check fires before resolution).
+		cfg = make_config(blocked_domains=["evil.com"])
+		with self.assertRaises(SSRFError):
+			validate_url("http://evil.com./", cfg)
+		with self.assertRaises(SSRFError):
+			validate_url("http://sub.evil.com./", cfg)
+		# And a block entry that itself carries a trailing dot still matches.
+		self.assertTrue(http._domain_in_list("evil.com", ["evil.com."]))
+
+	def test_trailing_dot_host_is_normalized_for_allow_list_and_resolution(self):
+		# The stripped host feeds both the allow-list check and DNS resolution: the
+		# resolver only knows "good.com" (no dot), so passing "good.com." proves the
+		# trailing dot was stripped before both steps.
+		cfg = make_config(allowed_domains=["good.com"])
+		with _resolves_to({"good.com": "8.8.8.8"}):
+			self.assertEqual(validate_url("http://good.com./", cfg), "http://good.com./")
+
+	def test_allow_list_restricts_to_listed_hosts(self):
+		cfg = make_config(allowed_domains=["good.com"])
+		with _resolves_to({"good.com": "8.8.8.8", "other.com": "8.8.8.8"}):
+			self.assertEqual(validate_url("http://good.com/", cfg), "http://good.com/")
+			with self.assertRaises(SSRFError):
+				validate_url("http://other.com/", cfg)
+
+	# -- direct IP-classification unit ------------------------------------- #
+	def test_is_blocked_ip_classification(self):
+		for ip in ("127.0.0.1", "10.0.0.5", "172.16.0.1", "192.168.1.1", "169.254.169.254", "::1", "0.0.0.0"):
+			self.assertTrue(http._is_blocked_ip(ip), ip)
+		for ip in ("8.8.8.8", "104.21.0.1", "1.1.1.1"):
+			self.assertFalse(http._is_blocked_ip(ip), ip)
+		# Unparseable -> rejected.
+		self.assertTrue(http._is_blocked_ip("not-an-ip"))
+
+
+class PinnedConnectionTest(UnitTestCase):
+	"""DNS-rebinding defense: the socket must go to the IP the guard validated.
+
+	``_pinned_get`` rewrites the URL netloc to the validated IP and keeps the
+	original hostname in the Host header / TLS SNI, so a resolver answer that
+	changes after validation can no longer steer the connection.
+	"""
+
+	def _capture_send(self, captured):
+		import requests
+
+		def fake_send(adapter, request, **kwargs):
+			captured["url"] = request.url
+			captured["host_header"] = request.headers.get("Host")
+			raise requests.exceptions.ConnectionError("offline test: no socket")
+
+		return mock.patch.object(http._PinnedIPAdapter, "send", autospec=True, side_effect=fake_send)
+
+	def test_pinned_get_connects_to_ip_with_hostname_host_header(self):
+		import requests
+
+		captured = {}
+		with self._capture_send(captured), self.assertRaises(requests.exceptions.ConnectionError):
+			http._pinned_get(requests.Session(), "https://example.com/about", "93.184.216.34", 5, 2)
+		self.assertEqual(captured["url"], "https://93.184.216.34/about")
+		self.assertEqual(captured["host_header"], "example.com")
+
+	def test_pinned_get_brackets_ipv6_and_keeps_explicit_port(self):
+		import requests
+
+		captured = {}
+		with self._capture_send(captured), self.assertRaises(requests.exceptions.ConnectionError):
+			http._pinned_get(requests.Session(), "https://example.com:8443/x", "2606:2800:220:1::", 5, 2)
+		self.assertEqual(captured["url"], "https://[2606:2800:220:1::]:8443/x")
+		self.assertEqual(captured["host_header"], "example.com:8443")
+
+	def test_pinned_adapter_keeps_tls_verification_on_hostname(self):
+		adapter = http._PinnedIPAdapter("example.com")
+		# urllib3 uses server_hostname for SNI and falls back to it for certificate
+		# matching, so verification stays against the hostname, not the IP literal.
+		self.assertEqual(adapter.poolmanager.connection_pool_kw.get("server_hostname"), "example.com")
+
+	def test_fetch_connects_to_the_validated_ip(self):
+		import requests
+
+		captured = {}
+		with _resolves_to({"rebind.example": "8.8.8.8"}), self._capture_send(captured):
+			status, _html, error, _final = http.fetch("http://rebind.example/", make_config())
+		self.assertEqual(captured["url"], "http://8.8.8.8/")
+		self.assertEqual(status, 0)
+		self.assertIn("connection error", error)