snapshot of https://github.com/frappe/crm.git (develop, 2026-09-18; blobs > 131072B omitted)

crm/domain_enrichment/extractors.py

@@ -0,0 +1,700 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+"""Generic, config-driven extractors.
+
+No literal keyword/industry/social tables here: classifiers iterate over
+``cfg.rules_by_type[...]``, honoring each rule's ``match_scope``, ``weight`` and
+compiled patterns. JSON-LD parsing, the favicon scorer, readability diagnosis,
+email/phone regex, and company-name cleaning stay as plain mechanics -- not
+tunable knowledge, so not pushed into doctypes.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from urllib.parse import urljoin, urlparse
+
+from .config import INDUSTRY_MIN_CONFIDENCE, INDUSTRY_MIN_SCORE
+from .result import (
+	Email,
+	Field,
+	Method,
+	Phone,
+	SocialProfile,
+)
+
+# --------------------------------------------------------------------------- #
+# Regexes (mechanics)
+# --------------------------------------------------------------------------- #
+EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
+
+PHONE_RE = re.compile(
+	r"""(?<!\w)
+    (\+?\d{1,3}[\s.-]?)?      # optional country code
+    (\(?\d{2,4}\)?[\s.-]?)    # area / first group
+    \d{3,4}[\s.-]?\d{3,4}     # subscriber number
+    (?!\w)""",
+	re.VERBOSE,
+)
+
+# Emails that are almost always noise (asset hashes, example domains, images).
+EMAIL_BLOCKLIST_SUBSTR = (
+	"example.com",
+	"sentry.io",
+	"@2x",
+	"@3x",
+	".png",
+	".jpg",
+	".gif",
+	".svg",
+	".webp",
+	"your-email",
+	"email@",
+)
+
+# Generic landing pages we should not treat as a social profile.
+SOCIAL_NEGATIVE = re.compile(r"/(sharer|share|intent|home|login|signup|privacy|tos)\b", re.I)
+
+
+# --------------------------------------------------------------------------- #
+# Generic rule executors
+# --------------------------------------------------------------------------- #
+SCOPE_HEADLINE = "Headline"
+SCOPE_FULL_TEXT = "Full Text"
+SCOPE_HTML = "HTML"
+SCOPE_HEADERS = "Headers"
+SCOPE_URL = "URL"
+
+
+def apply_keyword_rules(text_by_scope: dict, rules: list) -> dict:
+	"""Returns ``{label: weighted_score}`` -- ``label`` is the rule's industry
+	(Industry rules) or target_value (everything else)."""
+	scores: dict = {}
+	for rule in rules:
+		text = text_by_scope.get(rule.match_scope, "")
+		hits = rule.matches(text)
+		if hits:
+			scores[rule.label] = scores.get(rule.label, 0.0) + hits * (rule.weight or 1.0)
+	return scores
+
+
+# --------------------------------------------------------------------------- #
+# JSON-LD (mechanics)
+# --------------------------------------------------------------------------- #
+def parse_json_ld(soup):
+	blocks = []
+	for tag in soup.find_all("script", attrs={"type": "application/ld+json"}):
+		raw = tag.string or tag.get_text() or ""
+		if not raw.strip():
+			continue
+		try:
+			data = json.loads(raw)
+		except (ValueError, TypeError):
+			continue
+		if isinstance(data, dict):
+			if "@graph" in data and isinstance(data["@graph"], list):
+				blocks.extend([d for d in data["@graph"] if isinstance(d, dict)])
+			else:
+				blocks.append(data)
+		elif isinstance(data, list):
+			blocks.extend([d for d in data if isinstance(d, dict)])
+	return blocks
+
+
+def _ld_type_matches(block, wanted):
+	t = block.get("@type", "")
+	types = t if isinstance(t, list) else [t]
+	return any(str(x).lower() in wanted for x in types)
+
+
+# --------------------------------------------------------------------------- #
+# Readability diagnosis (mechanics)
+# --------------------------------------------------------------------------- #
+WAF_MARKERS_STRONG = (
+	"_incapsula_resource",
+	"incapsula",
+	"cf-browser-verification",
+	"just a moment",
+	"attention required",
+	"ddos protection",
+	"are you a human",
+)
+WAF_MARKERS_WEAK = (
+	"captcha",
+	"please enable javascript",
+	"enable cookies to continue",
+	"access denied",
+)
+WAF_WEAK_TEXT_THRESHOLD = 1000
+
+READABILITY_MESSAGES = {
+	"unreachable": "Couldn't reach this website (it timed out or refused the connection).",
+	"blocked": (
+		"This website is behind bot protection, so its content can't be read. "
+		"Try the company's corporate domain (e.g. acme.com vs shop.acme.com)."
+	),
+	"empty": (
+		"This site's main content is JavaScript-rendered; only page metadata "
+		"(title/description/logo) could be read, so some fields may be blank."
+	),
+}
+
+
+def diagnose_readability(pages):
+	"""Judge the homepage: '' (readable), 'blocked', 'empty', or 'unreachable'."""
+	if not pages:
+		return "unreachable"
+	home = pages[0]
+	if home.error or not home.status_code:
+		return "unreachable"
+	if home.status_code in (401, 403, 429):
+		return "blocked"
+	if home.status_code >= 500:
+		return "unreachable"
+	blob = (home.html or "")[:5000].lower()
+	text_len = len(home.text or "")
+	if any(marker in blob for marker in WAF_MARKERS_STRONG):
+		return "blocked"
+	if text_len < WAF_WEAK_TEXT_THRESHOLD and any(marker in blob for marker in WAF_MARKERS_WEAK):
+		return "blocked"
+	if text_len < 200:
+		return "empty"
+	return ""
+
+
+# --------------------------------------------------------------------------- #
+# Company information (mechanics)
+# --------------------------------------------------------------------------- #
+def _meta_with_method(soup, *names):
+	for name in names:
+		tag = soup.find("meta", attrs={"property": name}) or soup.find("meta", attrs={"name": name})
+		if tag and tag.get("content"):
+			return tag["content"].strip(), name
+	return "", ""
+
+
+# Below this, a description tag is a stub (a bare company name) rather than copy.
+_MIN_META_DESC_LEN = 30
+
+
+def _is_description_copy(content):
+	"""False for a stub or a verification token. Scripts that do not space
+	their words (CJK, Thai) are never ASCII-only, so they still count as copy."""
+	if len(content) < _MIN_META_DESC_LEN:
+		return False
+	return " " in content or not content.isascii()
+
+
+def _meta_description(soup):
+	fallback = ""
+	for name in ("og:description", "description"):
+		content, _name = _meta_with_method(soup, name)
+		if not content:
+			continue
+		if _is_description_copy(content):
+			return content
+		fallback = fallback or content
+	return fallback
+
+
+# A company name is usually the first chunk of a page <title>, before a separator.
+_NAME_SPLIT_RE = re.compile(r"\s*\|\s*|\s+[-–—•·]\s+|\s*:\s")
+_NAME_TAGLINE_RE = re.compile(
+	r"\s+(?:has|have|is|are|was|were|offers?|provides?|delivers?|helps?|"
+	r"enables?|builds?|makes?|creates?|specializes?|brings?|"
+	r"empowers?|powers?|leads?|connects?)\b",
+	re.I,
+)
+_NAME_MAX_WORDS = 6
+
+
+def _domain_brand(url):
+	netloc = (urlparse(url or "").netloc or "").lower().split(":")[0]
+	if netloc.startswith("www."):
+		netloc = netloc[4:]
+	labels = [label for label in netloc.split(".") if label]
+	return labels[-2] if len(labels) >= 2 else (labels[0] if labels else "")
+
+
+def _compact(text):
+	return re.sub(r"[^a-z0-9]", "", (text or "").lower())
+
+
+def _clean_company_name(raw, site_url=""):
+	"""Trim a page title / og:title down to just the company name."""
+	raw = (raw or "").strip()
+	if not raw:
+		return ""
+	chunks = [c.strip() for c in _NAME_SPLIT_RE.split(raw) if c.strip()]
+	name = chunks[0] if chunks else raw
+
+	brand = _compact(_domain_brand(site_url)) if site_url else ""
+	if brand and len(chunks) > 1:
+		for chunk in chunks:
+			c = _compact(chunk)
+			if c and (c == brand or (len(min(c, brand, key=len)) >= 4 and (c in brand or brand in c))):
+				name = chunk
+				break
+
+	m = _NAME_TAGLINE_RE.search(name)
+	if m and m.start() > 0:
+		name = name[: m.start()].strip()
+	if len(name.split()) > _NAME_MAX_WORDS:
+		name = name.split(",")[0].strip()
+	if len(name.split()) > _NAME_MAX_WORDS:
+		name = " ".join(name.split()[:_NAME_MAX_WORDS])
+	return name or raw
+
+
+# apple-touch-icons are 180px by convention when no explicit size is declared.
+_APPLE_TOUCH_PX = 180
+
+
+def _safe_url(base_url, raw):
+	"""Resolve ``raw`` against ``base_url``; return it only if it's a non-empty
+	``http(s)`` URL. Rejects ``javascript:``/``data:`` schemes a crawled site could
+	inject into a stored logo/image/icon field (stored-XSS defense)."""
+	raw = (raw or "").strip()
+	if not raw:
+		return ""
+	# urljoin() leaves javascript:/data: schemes as-is, so check the raw scheme
+	# first, then re-check the resolved URL for relative inputs.
+	scheme = urlparse(raw).scheme.lower()
+	if scheme and scheme not in ("http", "https"):
+		return ""
+	resolved = urljoin(base_url, raw)
+	return resolved if urlparse(resolved).scheme in ("http", "https") else ""
+
+
+def _best_icon(soup, base_url):
+	"""Best ``<link rel=icon>`` candidate: scalable SVG beats any raster, then
+	largest declared size wins."""
+	candidates = []
+	for tag in soup.find_all("link", href=True):
+		rel = tag.get("rel") or []
+		rel = " ".join(rel).lower() if isinstance(rel, list) else str(rel).lower()
+		if "icon" not in rel:
+			continue
+		href = tag["href"].strip()
+		safe = _safe_url(base_url, href)
+		if not safe:
+			continue
+		is_svg = (
+			"svg" in (tag.get("type") or "").lower()
+			or href.lower().split("?")[0].endswith(".svg")
+			or "mask-icon" in rel
+		)
+		m = re.search(r"(\d+)x\d+", tag.get("sizes", "") or "")
+		px = int(m.group(1)) if m else (_APPLE_TOUCH_PX if "apple-touch" in rel else 0)
+		candidates.append(((1 if is_svg else 0, px), safe))
+	if not candidates:
+		return ""
+	candidates.sort(key=lambda c: c[0], reverse=True)
+	return candidates[0][1]
+
+
+def extract_logo(soup, base_url):
+	"""The company's link icon -- a crisp, square brand mark, not the wider
+	social-share image (``og:image`` is usually a banner; see :func:`extract_image`).
+	Priority: SVG > largest declared raster/apple-touch > ``/favicon.ico`` fallback.
+	"""
+	icon_url = _best_icon(soup, base_url)
+	if icon_url:
+		return Field(icon_url, base_url, Method.FAVICON)
+
+	parsed = urlparse(base_url)
+	if parsed.scheme and parsed.netloc:
+		return Field(f"{parsed.scheme}://{parsed.netloc}/favicon.ico", base_url, Method.FAVICON)
+	return Field()
+
+
+def extract_image(soup, base_url):
+	"""The larger brand/social image (JSON-LD ``Organization.logo`` -> ``og:image``/
+	``twitter:image``), NOT the company logo -- see :func:`extract_logo`. Returns a
+	:class:`Field`, empty if none declared."""
+	for block in parse_json_ld(soup):
+		if not _ld_type_matches(block, _ORG_TYPES):
+			continue
+		logo = block.get("logo")
+		url = logo.get("url") if isinstance(logo, dict) else (logo if isinstance(logo, str) else "")
+		safe = _safe_url(base_url, url)
+		if safe:
+			return Field(safe, base_url, Method.JSON_LD)
+
+	og_content, _name = _meta_with_method(soup, "og:image", "og:image:url", "twitter:image")
+	safe = _safe_url(base_url, og_content)
+	if safe:
+		return Field(safe, base_url, Method.META_TAG)
+	return Field()
+
+
+# schema.org @types that denote a company/organization.
+_ORG_TYPES = {
+	"organization",
+	"corporation",
+	"localbusiness",
+	"ngo",
+	"educationalorganization",
+	"softwareapplication",
+}
+
+# Full-segment match, not a prefix: "about-our-reporting" (an ESG page) must not
+# qualify just because it starts with "about-".
+_ABOUT_SEGMENT_RE = re.compile(
+	r"^(about|about-us|about_us|aboutus|company|who-we-are|our-story|overview|mission)$",
+	re.I,
+)
+_NON_ABOUT_WORDS = (
+	"privacy",
+	"career",
+	"job",
+	"partner",
+	"webinar",
+	"policy",
+	"policies",
+	"accessibility",
+	"conduct",
+	"terms",
+	"cookie",
+	"legal",
+	"press",
+	"news",
+	"blog",
+	"event",
+	"investor",
+	"support",
+	"contact",
+	"sitemap",
+	"login",
+	"award",
+)
+_COMMERCE_RE = re.compile(r"^\s*(shop|buy|order|browse)\b|add to cart|shop now|buy now", re.I)
+
+
+def _last_path_segment(url):
+	path = urlparse(url).path.rstrip("/")
+	return path.rsplit("/", 1)[-1].lower() if path else ""
+
+
+def _is_about_page(url):
+	seg = _last_path_segment(url)
+	if not seg or any(w in seg for w in _NON_ABOUT_WORDS):
+		return False
+	return bool(_ABOUT_SEGMENT_RE.match(seg))
+
+
+def extract_company_info(homepage, soup):
+	url = homepage.url
+	info = {
+		"company_name": Field(),
+		"description": Field(),
+		"logo": Field(),
+		"image": Field(),
+		"social_links": [],
+	}
+
+	for block in parse_json_ld(soup):
+		if not _ld_type_matches(block, _ORG_TYPES):
+			continue
+		if not info["company_name"].value and block.get("name"):
+			info["company_name"] = Field(block["name"].strip(), url, Method.JSON_LD)
+		if not info["description"].value and block.get("description"):
+			info["description"] = Field(block["description"].strip(), url, Method.JSON_LD)
+		same_as = block.get("sameAs")
+		if isinstance(same_as, list):
+			info["social_links"].extend([s for s in same_as if isinstance(s, str)])
+		elif isinstance(same_as, str):
+			info["social_links"].append(same_as)
+		break  # first matching org block wins
+
+	if not info["company_name"].value:
+		content, _name = _meta_with_method(soup, "og:site_name", "og:title")
+		if content:
+			info["company_name"] = Field(_clean_company_name(content, url), url, Method.META_TAG)
+		elif homepage.title:
+			info["company_name"] = Field(_clean_company_name(homepage.title, url), url, Method.TITLE_TAG)
+
+	if not info["description"].value:
+		content = _meta_description(soup)
+		if content:
+			info["description"] = Field(content, url, Method.META_TAG)
+
+	info["logo"] = extract_logo(soup, url)
+	info["image"] = extract_image(soup, url)
+
+	return info
+
+
+_NON_CONTENT_PARENTS = {"nav", "header", "footer", "aside", "form"}
+_MIN_PARAGRAPH_LEN = 80
+_MAX_PARAGRAPHS_SCANNED = 6
+
+# Company copy speaks for itself ("we"/"our"). A staff profile is written in the
+# third person: a career phrase, a job title plus a pronoun, or repeated pronouns.
+_FIRST_PERSON_RE = re.compile(r"\b(?:we|our|ours|us)\b", re.I)
+_PRONOUN_RE = re.compile(r"\b(?:he|she|his|her)\b", re.I)
+_CAREER_RE = re.compile(r"\bbegan (?:his|her) career\b|\bprior to (?:joining|founding)\b", re.I)
+_TITLE_RE = re.compile(
+	r"\bis the (?:founder|co-founder|ceo|cto|cfo|chief|managing director|president)\b", re.I
+)
+_CTA_RE = re.compile(
+	r"^\s*(?:explore|discover|learn|find out|contact|call|browse|shop|buy|order|"
+	r"read more|get in touch|see|check out|sign up|subscribe|join)\b",
+	re.I,
+)
+
+
+def _company_name_hits(text, company_name):
+	name = (company_name or "").strip()
+	if not name:
+		return 0
+	return len(re.findall(rf"\b{re.escape(name)}\b", text, re.IGNORECASE))
+
+
+def _is_staff_profile(text):
+	if _CAREER_RE.search(text):
+		return True
+	if _FIRST_PERSON_RE.search(text):
+		return False
+	pronouns = len(_PRONOUN_RE.findall(text))
+	return pronouns >= 2 or (pronouns >= 1 and bool(_TITLE_RE.search(text)))
+
+
+def _describes_company(text, company_name):
+	"""True when the paragraph talks about the company, not something else."""
+	return bool(_FIRST_PERSON_RE.search(text)) or _company_name_hits(text, company_name) > 0
+
+
+def first_paragraph(soup, industry_rules=None, company_name=""):
+	"""Best substantial body paragraph on a page.
+
+	Reads up to ``_MAX_PARAGRAPHS_SCANNED`` qualifying ``<p>`` tags, drops the ones
+	that describe a person or tell the reader what to do next, and ranks the rest by
+	Industry-rule keyword hits (the same corpus ``classify_industry`` uses). Talking
+	about the company and naming it only break ties, never outrank keyword hits. If
+	every paragraph was dropped they are all reconsidered, so a page of nothing but
+	staff profiles still returns something. Read-only -- does not mutate ``soup``.
+	"""
+	candidates = []
+	for p in soup.find_all("p"):
+		if any(parent.name in _NON_CONTENT_PARENTS for parent in p.parents):
+			continue
+		text = " ".join((p.get_text(" ") or "").split())
+		if len(text) < _MIN_PARAGRAPH_LEN or " " not in text:
+			continue
+		candidates.append(text)
+		if len(candidates) >= _MAX_PARAGRAPHS_SCANNED:
+			break
+
+	if not candidates:
+		return ""
+	candidates = [
+		text for text in candidates if not (_is_staff_profile(text) or _CTA_RE.match(text))
+	] or candidates
+
+	# max() keeps the first-encountered element on ties, so document order still
+	# wins when nothing scores higher (covers the "all zero hits" case for free).
+	# The trailing slots can only break ties, never add to the industry-hit count.
+	return max(
+		candidates,
+		key=lambda text: (
+			sum(rule.matches(text) for rule in industry_rules or []),
+			_describes_company(text, company_name),
+			_company_name_hits(text, company_name),
+		),
+	)
+
+
+def select_description(pages, soups_by_url, industry_rules=None, company_name=""):
+	"""Pick the best *company* description across crawled pages.
+
+	Priority: JSON-LD ``Organization.description`` -> About-page body paragraph ->
+	head ``og:description``/``<meta description>`` -> home-page body paragraph
+	(last resort). ``industry_rules``/``company_name`` steer the body-paragraph
+	fallbacks (see ``first_paragraph``).
+	"""
+	home_url = pages[0].url if pages else ""
+	best_key = None
+	best_field = None
+
+	def consider(field_obj, score):
+		nonlocal best_key, best_field
+		if not field_obj.value:
+			return
+		key = (score, min(len(field_obj.value), 300))
+		if best_key is None or key > best_key:
+			best_key, best_field = key, field_obj
+
+	for page in pages:
+		soup = soups_by_url.get(page.url)
+		if soup is None:
+			continue
+
+		is_home = page.url == home_url
+		is_about = _is_about_page(page.url)
+
+		for block in parse_json_ld(soup):
+			if _ld_type_matches(block, _ORG_TYPES) and block.get("description"):
+				consider(Field(block["description"].strip(), page.url, Method.JSON_LD), 6)
+				break
+
+		if is_about:
+			consider(
+				Field(first_paragraph(soup, industry_rules, company_name), page.url, Method.BODY_TEXT), 5
+			)
+
+		if is_home or is_about:
+			content = _meta_description(soup)
+			if content:
+				score = 2 if (is_home and _COMMERCE_RE.search(content)) else 4
+				consider(Field(content, page.url, Method.META_TAG), score)
+
+		if is_home:
+			consider(
+				Field(first_paragraph(soup, industry_rules, company_name), page.url, Method.BODY_TEXT), 1
+			)
+
+	return best_field or Field()
+
+
+# --------------------------------------------------------------------------- #
+# Contacts: emails, phones (mechanics)
+# --------------------------------------------------------------------------- #
+def _clean_email(email):
+	email = email.strip().strip(".").lower()
+	if any(b in email for b in EMAIL_BLOCKLIST_SUBSTR):
+		return None
+	return email
+
+
+def extract_emails(pages):
+	found = {}
+	for page in pages:
+		if not page.html:
+			continue
+		for match in EMAIL_RE.findall(page.html):
+			cleaned = _clean_email(match)
+			if cleaned and cleaned not in found:
+				found[cleaned] = Email(value=cleaned, source=page.url)
+	return list(found.values())
+
+
+# TODO: Need to make this robust by adding proper mechanics or using a
+# thirdparty library like google/libphonenumbers
+def _valid_phone(raw):
+	# Requires an explicit "+" country-code prefix: a bare digit string is either
+	# ambiguous local formatting or noise (stats, reference numbers) that happens
+	# to look phone-shaped -- precision over recall.
+	if not raw.strip().startswith("+"):
+		return False
+	digits = re.sub(r"\D", "", raw)
+	return 7 <= len(digits) <= 15
+
+
+def extract_phones(pages):
+	found = {}
+	for page in pages:
+		if not page.html:
+			continue
+		for m in re.findall(r"tel:([+0-9()\s.\-]{7,})", page.html):
+			raw = m.strip()
+			digits = re.sub(r"\D", "", raw)
+			if _valid_phone(raw) and digits not in found:
+				found[digits] = Phone(
+					value="+" + digits if raw.strip().startswith("+") else digits,
+					raw=raw,
+					source=page.url,
+					method=Method.TEL_LINK,
+				)
+		for match in PHONE_RE.finditer(page.text or ""):
+			raw = match.group(0).strip()
+			if not raw:
+				continue
+			digits = re.sub(r"\D", "", raw)
+			if _valid_phone(raw) and digits not in found:
+				value = ("+" + digits) if raw.startswith("+") else digits
+				found[digits] = Phone(value=value, raw=raw, source=page.url, method=Method.REGEX)
+	return list(found.values())
+
+
+# --------------------------------------------------------------------------- #
+# Social profiles (config-driven via Social rules)
+# --------------------------------------------------------------------------- #
+def extract_social_profiles(pages, soups_by_url, social_rules, extra_links=None):
+	"""Return dict network -> SocialProfile(value, source, method); each Social
+	rule's ``target_value`` is the network name."""
+	profiles = {rule.target_value: SocialProfile() for rule in social_rules}
+
+	home_url = pages[0].url if pages else ""
+	candidates = [(link, home_url, Method.JSON_LD) for link in (extra_links or [])]
+	for page in pages:
+		soup = soups_by_url.get(page.url)
+		if soup is None:
+			continue
+		for a in soup.find_all("a", href=True):
+			candidates.append((a["href"].strip(), page.url, Method.SOCIAL_RULE))
+
+	for href, source, method in candidates:
+		if not href or SOCIAL_NEGATIVE.search(href):
+			continue
+		url = urljoin(source, href.strip())
+		for rule in social_rules:
+			network = rule.target_value
+			if profiles[network].value:
+				continue
+			if any(rx.search(url) for rx, _raw in rule.patterns):
+				profiles[network] = SocialProfile(value=url, source=source, method=method)
+	return profiles
+
+
+# --------------------------------------------------------------------------- #
+# Industry classification (config-driven via Industry rules)
+# --------------------------------------------------------------------------- #
+def classify_industry(pages, company_info, industry_rules):
+	"""Rule-based industry from the homepage's (and, when crawled, the About page's)
+	headline corpus. Industry rules are typically seeded with ``match_scope='Headline'``
+	(company name/description/title/headings, never body/footer). The homepage alone
+	is often thin (stylized brand copy, no meta description), so a crawled About
+	page's title/headings are folded in too. Returns (industry, confidence 0-1), or
+	("", 0.0) when the signal is too weak.
+	"""
+	if not industry_rules:
+		return "", 0.0
+
+	def _val(key):
+		f = company_info.get(key)
+		return getattr(f, "value", f) or ""
+
+	headline_parts = [_val("company_name"), _val("description")]
+	if pages:
+		home = pages[0]
+		headline_parts.append(home.title or "")
+		headline_parts.append(" ".join(home.headings))
+		about = next((p for p in pages[1:] if _is_about_page(p.url)), None)
+		if about:
+			headline_parts.append(about.title or "")
+			headline_parts.append(" ".join(about.headings))
+	headline = " ".join(p for p in headline_parts if p).lower()
+
+	# Same corpus under multiple scope keys so admins can re-scope rules without
+	# code changes; default seed uses Headline.
+	full_text = " ".join((p.text or "") for p in pages).lower()
+	text_by_scope = {
+		SCOPE_HEADLINE: headline,
+		SCOPE_FULL_TEXT: full_text,
+		SCOPE_HTML: full_text,
+	}
+
+	scores = apply_keyword_rules(text_by_scope, industry_rules)
+	total = sum(scores.values())
+	if not total:
+		return "", 0.0
+	best = max(scores, key=scores.get)
+	if scores[best] < INDUSTRY_MIN_SCORE:
+		return "", 0.0
+	confidence = scores[best] / total
+	if confidence < INDUSTRY_MIN_CONFIDENCE:
+		return "", 0.0
+	return best, round(confidence, 2)