upstream-frappe-crm/crm/domain_enrichment/tests/test_extractors.py

# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

"""Pure, network-free unit tests for the config-driven extractors.

The classifiers (industry / social) are exercised with **in-memory Rule objects**
built directly (see ``tests.fixtures``), never the DB — this is the rule-driven
rewrite of the POC's extractor tests. The mechanics (company-name cleaning, favicon
scorer, JSON-LD parse, email/phone extraction, readability diagnosis) are tested
standalone.
"""

from __future__ import annotations

from frappe.tests import UnitTestCase

from crm.domain_enrichment import extractors
from crm.domain_enrichment.result import CrawledPage, Method
from crm.domain_enrichment.tests import fixtures


def _pages(*specs):
	"""specs: (url, html) tuples -> list[CrawledPage], soups_by_url dict."""
	pages, soups = [], {}
	for url, html in specs:
		page, soup = fixtures.make_page(url, html)
		pages.append(page)
		soups[url] = soup
	return pages, soups


# --------------------------------------------------------------------------- #
# Company information (mechanics)
# --------------------------------------------------------------------------- #
class CompanyInfoTest(UnitTestCase):
	def setUp(self):
		self.page, self.soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)

	def test_prefers_json_ld_for_name(self):
		info = extractors.extract_company_info(self.page, self.soup)
		self.assertEqual(info["company_name"].value, "Acme Analytics Inc.")
		self.assertEqual(info["company_name"].method, Method.JSON_LD)
		self.assertEqual(info["company_name"].source, "https://acme.example")

	def test_description_from_json_ld(self):
		info = extractors.extract_company_info(self.page, self.soup)
		self.assertIn("analytics", info["description"].value.lower())

	def test_logo_is_link_icon_not_json_ld_image(self):
		# The company logo is the link icon (HOMEPAGE declares none -> favicon.ico
		# fallback), NOT the JSON-LD Organization.logo. That image is kept separately.
		info = extractors.extract_company_info(self.page, self.soup)
		self.assertEqual(info["logo"].method, Method.FAVICON)
		self.assertEqual(info["logo"].value, "https://acme.example/favicon.ico")
		self.assertEqual(info["image"].method, Method.JSON_LD)
		self.assertEqual(info["image"].value, "https://acme.example/ld-logo.png")

	def test_logo_prefers_declared_apple_touch_icon(self):
		page, soup = fixtures.make_page(
			"https://x.example",
			"<html><head>"
			"<link rel='icon' href='/fav.png' sizes='16x16'>"
			"<link rel='apple-touch-icon' href='/touch.png'>"
			"</head><body></body></html>",
		)
		info = extractors.extract_company_info(page, soup)
		self.assertEqual(info["logo"].value, "https://x.example/touch.png")

	def test_meta_fallback_when_no_json_ld(self):
		page, soup = fixtures.make_page(
			"https://x.example",
			"<html><head><title>Foo</title><meta property='og:site_name' content='Foo Inc'></head><body></body></html>",
		)
		info = extractors.extract_company_info(page, soup)
		self.assertEqual(info["company_name"].value, "Foo Inc")
		self.assertEqual(info["company_name"].method, Method.META_TAG)

	def test_broken_json_ld_does_not_crash(self):
		page, soup = fixtures.make_page("https://b.example", fixtures.BROKEN_JSON_LD)
		info = extractors.extract_company_info(page, soup)
		self.assertEqual(info["company_name"].method, Method.TITLE_TAG)

	def test_degenerate_og_description_yields_to_plain_description(self):
		# Broken markup can park a site-verification token on og:description while the
		# plain description tag carries the real copy.
		page, soup = fixtures.make_page(
			"https://x.example",
			"<html><head>"
			"<meta property='og:description' content='JIs-hfpDlO6niGZsMes79IWdxC0A9frop'>"
			"<meta name='description' content='Acme Organics is a manufacturer and "
			"exporter of aroma chemicals based in Mumbai.'>"
			"</head><body></body></html>",
		)
		info = extractors.extract_company_info(page, soup)
		self.assertTrue(info["description"].value.startswith("Acme Organics is a manufacturer"))

	def test_og_description_still_preferred_when_usable(self):
		page, soup = fixtures.make_page(
			"https://x.example",
			"<html><head>"
			"<meta property='og:description' content='Acme builds developer tools for teams.'>"
			"<meta name='description' content='Some other blurb entirely for the page.'>"
			"</head><body></body></html>",
		)
		info = extractors.extract_company_info(page, soup)
		self.assertEqual(info["description"].value, "Acme builds developer tools for teams.")

	def test_degenerate_description_still_used_when_nothing_better(self):
		page, soup = fixtures.make_page(
			"https://x.example",
			"<html><head><meta name='description' content='Acme GmbH'></head><body></body></html>",
		)
		info = extractors.extract_company_info(page, soup)
		self.assertEqual(info["description"].value, "Acme GmbH")

	def test_unspaced_script_description_beats_token(self):
		# Japanese copy has no ASCII spaces, so a space check alone would mistake it
		# for a token and keep the og:description token instead.
		copy = "株式会社アクメは東京を拠点に産業用ロボットの設計・製造・販売を手がける専門メーカーです。"
		page, soup = fixtures.make_page(
			"https://x.example",
			"<html><head>"
			"<meta property='og:description' content='JIs-hfpDlO6niGZsMes79IWdxC0A9frop'>"
			f"<meta name='description' content='{copy}'>"
			"</head><body></body></html>",
		)
		info = extractors.extract_company_info(page, soup)
		self.assertEqual(info["description"].value, copy)


class LogoResolutionTest(UnitTestCase):
	"""extract_logo returns the link icon only; the larger social/JSON-LD image is
	captured by extract_image, never used as the logo."""

	def test_scalable_svg_beats_small_png(self):
		_p, soup = fixtures.make_page(
			"https://x.example",
			"<html><head>"
			"<link rel='icon' type='image/png' href='/fav-32.png' sizes='32x32'>"
			"<link rel='icon' type='image/svg+xml' href='/logo.svg'>"
			"</head><body></body></html>",
		)
		logo = extractors.extract_logo(soup, "https://x.example")
		self.assertEqual(logo.value, "https://x.example/logo.svg")
		self.assertEqual(logo.method, Method.FAVICON)

	def test_logo_is_link_icon_even_when_og_image_exists(self):
		# A declared link icon is the logo; the bigger og:image is NOT the logo -- it
		# is captured as the image instead.
		_p, soup = fixtures.make_page(
			"https://x.example",
			"<html><head>"
			"<link rel='icon' href='/fav.ico' sizes='16x16'>"
			"<meta property='og:image' content='https://x.example/social-1200.png'>"
			"</head><body></body></html>",
		)
		logo = extractors.extract_logo(soup, "https://x.example")
		self.assertEqual(logo.value, "https://x.example/fav.ico")
		self.assertEqual(logo.method, Method.FAVICON)
		image = extractors.extract_image(soup, "https://x.example")
		self.assertEqual(image.value, "https://x.example/social-1200.png")
		self.assertEqual(image.method, Method.META_TAG)

	def test_falls_back_to_favicon_ico_ignoring_og_image(self):
		# No link icon declared -> favicon.ico, even though an og:image exists.
		_p, soup = fixtures.make_page(
			"https://x.example",
			"<html><head>"
			"<meta property='og:image' content='https://x.example/banner.png'>"
			"</head><body></body></html>",
		)
		logo = extractors.extract_logo(soup, "https://x.example")
		self.assertEqual(logo.value, "https://x.example/favicon.ico")
		self.assertEqual(logo.method, Method.FAVICON)

	def test_image_prefers_json_ld_logo_over_og_image(self):
		# extract_image keeps the old "best big image" priority: curated JSON-LD logo
		# outranks a social-share og:image.
		_p, soup = fixtures.make_page(
			"https://x.example",
			"<html><head>"
			'<script type="application/ld+json">'
			'{"@type":"Organization","logo":"https://x.example/brand.png"}</script>'
			"<meta property='og:image' content='https://x.example/banner.png'>"
			"</head><body></body></html>",
		)
		image = extractors.extract_image(soup, "https://x.example")
		self.assertEqual(image.value, "https://x.example/brand.png")
		self.assertEqual(image.method, Method.JSON_LD)

	def test_image_empty_when_none_declared(self):
		_p, soup = fixtures.make_page("https://x.example", "<html><head></head><body></body></html>")
		self.assertEqual(extractors.extract_image(soup, "https://x.example").value, "")

	def test_falls_back_to_favicon_ico(self):
		_p, soup = fixtures.make_page("https://x.example", "<html><head></head><body></body></html>")
		logo = extractors.extract_logo(soup, "https://x.example")
		self.assertEqual(logo.value, "https://x.example/favicon.ico")
		self.assertEqual(logo.method, Method.FAVICON)

	def test_rejects_unsafe_scheme_in_icon(self):
		# A crawled site injecting a javascript: icon href must not be stored as the
		# logo -- it is skipped and the safe favicon.ico fallback is used.
		_p, soup = fixtures.make_page(
			"https://x.example",
			"<html><head><link rel='icon' href='javascript:alert(1)'></head><body></body></html>",
		)
		logo = extractors.extract_logo(soup, "https://x.example")
		self.assertEqual(logo.value, "https://x.example/favicon.ico")

	def test_rejects_unsafe_scheme_in_image(self):
		# javascript:/data: in JSON-LD logo or og:image is dropped, never stored.
		_p, soup = fixtures.make_page(
			"https://x.example",
			"<html><head>"
			'<script type="application/ld+json">{"@type":"Organization","logo":"javascript:evil()"}</script>'
			"<meta property='og:image' content='data:image/svg+xml;base64,PHN2Zz4='>"
			"</head><body></body></html>",
		)
		self.assertEqual(extractors.extract_image(soup, "https://x.example").value, "")


class CompanyNameCleaningTest(UnitTestCase):
	def test_strips_tagline_after_separator(self):
		self.assertEqual(extractors._clean_company_name("PostHog – dev tools"), "PostHog")

	def test_trims_descriptive_sentence_title(self):
		self.assertEqual(
			extractors._clean_company_name(
				"ITC Ltd has diversified presence in FMCG, Paperboards & Packaging"
			),
			"ITC Ltd",
		)
		self.assertEqual(extractors._clean_company_name("Acme Corp provides cloud security"), "Acme Corp")

	def test_brand_at_end_matches_domain(self):
		self.assertEqual(
			extractors._clean_company_name(
				"Innovative IT & Software Development Services | 8848 Digital",
				"https://8848digital.com/",
			),
			"8848 Digital",
		)

	def test_without_domain_hint_falls_back_to_first_chunk(self):
		self.assertEqual(
			extractors._clean_company_name("Innovative IT & Software Development Services | 8848 Digital"),
			"Innovative IT & Software Development Services",
		)


class JsonLdTest(UnitTestCase):
	def test_parses_graph_blocks(self):
		_page, soup = fixtures.make_page(
			"https://x.com",
			'<html><head><script type="application/ld+json">'
			'{"@graph":[{"@type":"Organization","name":"A"},{"@type":"WebSite"}]}'
			"</script></head><body></body></html>",
		)
		blocks = extractors.parse_json_ld(soup)
		self.assertEqual(len(blocks), 2)
		self.assertEqual(blocks[0]["name"], "A")

	def test_ignores_invalid_json(self):
		_page, soup = fixtures.make_page("https://x.com", fixtures.BROKEN_JSON_LD)
		self.assertEqual(extractors.parse_json_ld(soup), [])


class DescriptionSelectionTest(UnitTestCase):
	def test_prefers_about_over_product_homepage(self):
		pages, soups = _pages(
			(
				"https://shop.example",
				"<html><head><meta property='og:description' content='Shop the latest gadgets.'></head><body></body></html>",
			),
			(
				"https://shop.example/about",
				"<html><head><meta name='description' content='Acme builds developer tools for teams.'></head><body></body></html>",
			),
		)
		desc = extractors.select_description(pages, soups)
		self.assertEqual(desc.value, "Acme builds developer tools for teams.")
		self.assertEqual(desc.source, "https://shop.example/about")

	def test_falls_back_to_homepage(self):
		pages, soups = _pages(
			(
				"https://x.example",
				"<html><head><meta property='og:description' content='We make great software.'></head><body></body></html>",
			),
		)
		desc = extractors.select_description(pages, soups)
		self.assertEqual(desc.value, "We make great software.")

	def test_json_ld_is_authoritative(self):
		pages, soups = _pages(
			(
				"https://s.example",
				'<html><head><script type="application/ld+json">{"@type":"Organization","description":"Acme is a developer-tools company."}</script></head><body></body></html>',
			),
			(
				"https://s.example/about",
				"<html><head><meta name='description' content='Some about-page blurb.'></head><body></body></html>",
			),
		)
		desc = extractors.select_description(pages, soups)
		self.assertEqual(desc.method, Method.JSON_LD)

	def test_about_body_paragraph_when_no_meta(self):
		# No description metadata anywhere -> derive it from the About page's body copy.
		about_body = (
			"<main><p>Acme Robotics designs autonomous warehouse robots that help "
			"retailers fulfil orders faster and at lower cost.</p></main>"
		)
		pages, soups = _pages(
			("https://acme.example", "<html><head><title>Acme</title></head><body></body></html>"),
			("https://acme.example/about", f"<html><body>{about_body}</body></html>"),
		)
		desc = extractors.select_description(pages, soups)
		self.assertIn("autonomous warehouse robots", desc.value)
		self.assertEqual(desc.method, Method.BODY_TEXT)
		self.assertEqual(desc.source, "https://acme.example/about")

	def test_head_meta_beats_home_body_paragraph(self):
		# With an og:description present, head meta wins over a homepage body paragraph.
		pages, soups = _pages(
			(
				"https://acme.example",
				"<html><head><meta name='description' content='Acme makes developer tools.'>"
				"</head><body><main><p>Some long homepage marketing paragraph that is "
				"definitely over the minimum length threshold for a body paragraph.</p>"
				"</main></body></html>",
			),
		)
		desc = extractors.select_description(pages, soups)
		self.assertEqual(desc.value, "Acme makes developer tools.")
		self.assertEqual(desc.method, Method.META_TAG)

	def test_industry_keyword_density_steers_about_paragraph_choice(self):
		finance_rules = [
			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
		]
		pages, soups = _pages(
			(
				"https://acmebank.example",
				"<html><head><title>Acme Bank</title></head><body></body></html>",
			),
			(
				"https://acmebank.example/about",
				"<html><body><main>"
				"<p>Putting our long-tenured investment teams on the line to earn the "
				"trust of institutional investors across every market we serve.</p>"
				"<p>Acme Bank is a full-service financial institution serving corporations, "
				"institutions and private clients across commercial and investment banking.</p>"
				"</main></body></html>",
			),
		)
		desc = extractors.select_description(pages, soups, industry_rules=finance_rules)
		self.assertTrue(desc.value.startswith("Acme Bank is a full-service"))


class FirstParagraphTest(UnitTestCase):
	def test_skips_nav_and_returns_first_substantial_paragraph(self):
		html = (
			"<html><body><nav><p>Home About Contact Login Signup Pricing Docs Blog "
			"Resources Careers Partners Support Status Community Downloads Enterprise</p></nav>"
			"<main><p>Short.</p><p>Globex Corp is a logistics platform that moves freight "
			"across the country for enterprise shippers.</p></main></body></html>"
		)
		_page, soup = fixtures.make_page("https://globex.example", html)
		result = extractors.first_paragraph(soup)
		self.assertIn("logistics platform", result)
		self.assertNotIn("Login", result)  # nav paragraph skipped despite being long

	def test_returns_empty_when_no_substantial_paragraph(self):
		_page, soup = fixtures.make_page("https://x.example", "<html><body><p>Hi.</p></body></html>")
		self.assertEqual(extractors.first_paragraph(soup), "")

	def test_prefers_paragraph_with_more_keyword_hits_over_first_qualifying_one(self):
		# A narrow pull-quote with zero industry-keyword hits precedes a genuine
		# descriptive paragraph -- the keyword-denser paragraph should win.
		finance_rules = [
			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
		]
		html = (
			"<html><body><main>"
			"<p>Putting our long-tenured investment teams on the line to earn the "
			"trust of institutional investors across every market we serve.</p>"
			"<p>Acme Bank is a full-service financial institution serving corporations, "
			"institutions and private clients across commercial and investment banking.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmebank.example", html)
		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
		self.assertTrue(result.startswith("Acme Bank is a full-service"))

	def test_falls_back_to_first_paragraph_when_no_keyword_hits(self):
		finance_rules = [
			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
		]
		html = (
			"<html><body><main>"
			"<p>Putting our long-tenured investment teams on the line to earn the "
			"trust of institutional investors across every market we serve.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmebank.example", html)
		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
		self.assertTrue(result.startswith("Putting our long-tenured"))

	def test_falls_back_to_first_paragraph_when_no_industry_rules_given(self):
		# No rules configured at all (industry_rules=None or []) -- with nothing else
		# to separate the candidates, document order still decides.
		html = (
			"<html><body><main>"
			"<p>Putting our long-tenured investment teams on the line to earn the "
			"trust of institutional investors across every market we serve.</p>"
			"<p>Acme Bank is a full-service financial institution serving corporations, "
			"institutions and private clients across commercial and investment banking.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmebank.example", html)
		self.assertTrue(extractors.first_paragraph(soup).startswith("Putting our long-tenured"))
		self.assertTrue(
			extractors.first_paragraph(soup, industry_rules=[]).startswith("Putting our long-tenured")
		)
		self.assertTrue(
			extractors.first_paragraph(soup, industry_rules=None).startswith("Putting our long-tenured")
		)

	def test_company_name_breaks_tie_between_equal_keyword_hits(self):
		# Both paragraphs mention "banking" exactly once (tied on industry hits) --
		# only the one that also names the company should win.
		finance_rules = [fixtures.keyword_rule("Industry", ["banking"], industry="Finance")]
		html = (
			"<html><body><main>"
			"<p>Our roots trace back decades of steady growth in commercial banking "
			"and long-term client relationships across every region we serve.</p>"
			"<p>Acme Bank has led the market in retail banking with award-winning "
			"service for individuals, families and small businesses nationwide.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmebank.example", html)
		result = extractors.first_paragraph(soup, industry_rules=finance_rules, company_name="Acme Bank")
		self.assertTrue(result.startswith("Acme Bank has led the market"))

	def test_leadership_bio_never_wins_over_company_description(self):
		# An About page's founder bio is dense in industry keywords (it lists former
		# employers and sectors), so keyword scoring alone picks it over the real
		# description. Bios must be rejected outright.
		finance_rules = [
			fixtures.keyword_rule("Industry", ["insurance", "asset management"], industry="Finance")
		]
		html = (
			"<html><body><main>"
			"<p>Acme Capital is a registered portfolio manager focused on uncovering "
			"long-term opportunities for clients across public equity markets.</p>"
			"<p>Priya Nair is the Founder and Chief Investment Officer of Acme Capital, "
			"where she previously led insurance and asset management mandates.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmecapital.example", html)
		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
		self.assertTrue(result.startswith("Acme Capital is a registered portfolio manager"))

	def test_call_to_action_paragraph_never_wins(self):
		finance_rules = [fixtures.keyword_rule("Industry", ["hvac"], industry="Construction")]
		html = (
			"<html><body><main>"
			"<p>Kishika Temperatures has been a leader in air conditioning maintenance "
			"and HVAC solutions for over 30 years, serving homes across the region.</p>"
			"<p>Explore our services to see why our HVAC and HVAC-adjacent teams are the "
			"preferred choice for leading developers, hotels and residential projects.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://kishika.example", html)
		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
		self.assertTrue(result.startswith("Kishika Temperatures has been a leader"))

	def test_overview_mentioning_founder_once_is_not_a_profile(self):
		# A company overview may name its founder with a single pronoun. That is
		# not a staff profile and must still win over a generic sector essay.
		html = (
			"<html><body><main>"
			"<p>Over the past decade, advances in warehouse robotics have reshaped how "
			"goods move between suppliers and consumers around the world.</p>"
			"<p>Acme Robotics builds warehouse automation for retailers. Founded by "
			"Jane Doe in 2015, she still leads our engineering team today.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmerobotics.example", html)
		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Robotics builds"))

	def test_company_described_as_founder_of_something_is_not_a_profile(self):
		html = (
			"<html><body><main>"
			"<p>Over the past decade, advances in open hardware have reshaped how "
			"engineers share designs and collaborate across company lines.</p>"
			"<p>Acme Labs is the founder of the Open Hardware Alliance, and we design "
			"reference boards used by hundreds of member companies worldwide.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmelabs.example", html)
		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Labs is the founder"))

	def test_overview_speaking_for_the_company_is_never_a_profile(self):
		# Two pronouns in copy that says "our" is still the company talking about
		# itself, not a staff profile.
		html = (
			"<html><body><main>"
			"<p>Over the past decade, advances in warehouse robotics have reshaped how "
			"goods move between suppliers and consumers around the world.</p>"
			"<p>Acme Robotics builds warehouse automation. Jane Doe founded it in 2015; "
			"she leads our engineering team and her designs power every product.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmerobotics.example", html)
		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Robotics builds"))

	def test_repeated_pronouns_mark_a_profile(self):
		html = (
			"<html><body><main>"
			"<p>Priya Nair joined the firm in 2010. She oversees every client mandate "
			"and her research has been cited across the industry for years.</p>"
			"<p>Acme Capital is a registered portfolio manager focused on uncovering "
			"long-term opportunities for clients across public equity markets.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmecapital.example", html)
		self.assertTrue(extractors.first_paragraph(soup).startswith("Acme Capital is a registered"))

	def test_falls_back_to_rejected_paragraphs_when_nothing_else_qualifies(self):
		# A page of nothing but bios must still yield a paragraph -- rejecting every
		# candidate should not silently drop the description.
		html = (
			"<html><body><main>"
			"<p>Priya Nair is the Founder of Acme Capital, where she leads the firm's "
			"investment committee and oversees portfolio construction end to end.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmecapital.example", html)
		self.assertTrue(extractors.first_paragraph(soup).startswith("Priya Nair is the Founder"))

	def test_speaking_for_the_company_breaks_a_zero_hit_tie(self):
		# Neither paragraph scores an industry hit: a generic essay about the sector
		# sits first, the company's own copy second. Document order must not decide.
		finance_rules = [fixtures.keyword_rule("Industry", ["banking"], industry="Finance")]
		html = (
			"<html><body><main>"
			"<p>Over the past decade, advances in concrete technology have allowed "
			"engineers to design mega structures that redefine infrastructure.</p>"
			"<p>Capital RMC, established in 2016, stands at the forefront of the "
			"construction sector, and we deliver world-class ready-mix concrete.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://capitalrmc.example", html)
		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
		self.assertTrue(result.startswith("Capital RMC, established in 2016"))

	def test_speaking_for_the_company_cannot_outrank_more_industry_hits(self):
		finance_rules = [
			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
		]
		html = (
			"<html><body><main>"
			"<p>A full-service financial institution built on decades of trusted "
			"commercial and retail banking relationships across every market.</p>"
			"<p>We are proud to serve local businesses across the region with "
			"award-winning service and community-first values that endure.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmebank.example", html)
		result = extractors.first_paragraph(soup, industry_rules=finance_rules)
		self.assertTrue(result.startswith("A full-service financial institution"))

	def test_company_name_mentions_cannot_outrank_more_industry_hits(self):
		# The first paragraph scores 2 industry hits and never names the company; the
		# second scores only 1 industry hit despite repeating the company name three
		# times. Industry hits must still decide -- company name is tiebreak-only.
		finance_rules = [
			fixtures.keyword_rule("Industry", ["financial institution", "banking"], industry="Finance")
		]
		html = (
			"<html><body><main>"
			"<p>A full-service financial institution built on decades of trusted "
			"commercial and retail banking relationships across every market.</p>"
			"<p>Acme Bank, Acme Bank, Acme Bank -- proud to serve local businesses "
			"with award-winning retail banking and community-first values.</p>"
			"</main></body></html>"
		)
		_page, soup = fixtures.make_page("https://acmebank.example", html)
		result = extractors.first_paragraph(soup, industry_rules=finance_rules, company_name="Acme Bank")
		self.assertTrue(result.startswith("A full-service financial institution"))


class AboutPageDetectionTest(UnitTestCase):
	def test_canonical_about_slugs_match(self):
		for path in (
			"/about",
			"/about-us",
			"/about_us",
			"/aboutus",
			"/company",
			"/our-story",
			"/en/about-us",
		):
			self.assertTrue(extractors._is_about_page(f"https://x.example{path}"), path)

	def test_prefix_match_no_longer_qualifies(self):
		# Regression guard: "about-our-reporting" (an ESG page) starts with "about-"
		# but is not itself a canonical About-page slug.
		self.assertFalse(extractors._is_about_page("https://x.example/esg-topics/about-our-reporting"))
		self.assertFalse(extractors._is_about_page("https://x.example/about-cookies"))

	def test_non_about_words_still_rejected(self):
		self.assertFalse(extractors._is_about_page("https://x.example/about/careers"))
		self.assertFalse(extractors._is_about_page("https://x.example/about-us/contact"))


# --------------------------------------------------------------------------- #
# Contacts: emails + phones (mechanics)
# --------------------------------------------------------------------------- #
class EmailExtractionTest(UnitTestCase):
	def setUp(self):
		self.pages, _ = _pages(
			("https://acme.example", fixtures.HOMEPAGE),
			("https://acme.example/about", fixtures.ABOUT),
			("https://acme.example/contact", fixtures.CONTACT),
		)

	def test_emails_deduped_and_sourced(self):
		emails = extractors.extract_emails(self.pages)
		values = {e.value for e in emails}
		self.assertIn("hello@acme.example", values)
		self.assertIn("sales@acme.example", values)
		self.assertEqual(len(values), len(emails))  # deduped

	def test_blocklisted_noise_emails_rejected(self):
		pages, _ = _pages(
			("https://x.com", "<html><body>logo@2x.png test@example.com real@x.com</body></html>"),
		)
		values = {e.value for e in extractors.extract_emails(pages)}
		self.assertIn("real@x.com", values)
		self.assertNotIn("test@example.com", values)


class PhoneExtractionTest(UnitTestCase):
	def setUp(self):
		self.pages, _ = _pages(
			("https://acme.example", fixtures.HOMEPAGE),
			("https://acme.example/contact", fixtures.CONTACT),
		)

	def test_tel_link_marked(self):
		phones = extractors.extract_phones(self.pages)
		self.assertIn(Method.TEL_LINK, {p.method for p in phones})
		self.assertTrue(any("4155550142" in p.value.replace("+", "") for p in phones))

	def test_international_number_captured(self):
		pages, _ = _pages(
			("https://x.example/contact", "<html><body><p>Call us: +612 9003 8888</p></body></html>")
		)
		phones = extractors.extract_phones(pages)
		self.assertTrue(any("61290038888" in p.value.replace("+", "") for p in phones))

	def test_number_without_country_code_rejected(self):
		# Deliberate precision-over-recall call: a number with no "+" is either a
		# local/ambiguous format or noise (reference numbers, stats) that happens to
		# fall in a phone-shaped digit range -- only accept explicit international
		# numbers, even though this means dropping some real local-format numbers.
		pages, _ = _pages(
			(
				"https://x.example/contact",
				"<html><body><p>Local line: 2165703. Toll-free: 1800-266-6066.</p>"
				'<a href="tel:18002666066">Call</a></body></html>',
			)
		)
		phones = extractors.extract_phones(pages)
		self.assertEqual(phones, [])


# --------------------------------------------------------------------------- #
# Social profiles (config-driven via Social rules)
# --------------------------------------------------------------------------- #
class SocialTest(UnitTestCase):
	def test_detects_networks_and_skips_share_links(self):
		page, soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
		company = extractors.extract_company_info(page, soup)
		profiles = extractors.extract_social_profiles(
			[page], {page.url: soup}, fixtures.social_rules(), extra_links=company["social_links"]
		)
		self.assertTrue(profiles["linkedin"].value)
		self.assertTrue(profiles["github"].value)
		self.assertTrue(profiles["instagram"].value)
		# JSON-LD sameAs links win and are marked as such.
		self.assertEqual(profiles["linkedin"].method, Method.JSON_LD)
		# A facebook 'sharer' link must be ignored.
		self.assertEqual(profiles["facebook"].value, "")

	def test_protocol_relative_links_made_absolute(self):
		page, soup = fixtures.make_page(
			"https://bluedart.com",
			"<html><body><a href='//www.linkedin.com/company/bluedart/'>li</a>"
			"<a href='//twitter.com/BlueDart_'>tw</a></body></html>",
		)
		profiles = extractors.extract_social_profiles([page], {page.url: soup}, fixtures.social_rules())
		self.assertEqual(profiles["linkedin"].value, "https://www.linkedin.com/company/bluedart/")
		self.assertEqual(profiles["twitter"].value, "https://twitter.com/BlueDart_")


# --------------------------------------------------------------------------- #
# Industry classification (config-driven via Industry rules)
# --------------------------------------------------------------------------- #
class IndustryTest(UnitTestCase):
	def _classify(self, name, description, extra_html=""):
		page, _ = fixtures.make_page(
			"https://x.example",
			f"<html><head><title>{name}</title></head><body>{extra_html}</body></html>",
		)
		company = {"company_name": name, "description": description}
		return extractors.classify_industry([page], company, fixtures.industry_rules())

	def test_classifies_saas_or_crm_with_confidence(self):
		page, soup = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
		company = extractors.extract_company_info(page, soup)
		industry, conf = extractors.classify_industry([page], company, fixtures.industry_rules())
		self.assertIn(industry, ("SaaS", "CRM"))
		self.assertGreater(conf, 0.0)
		self.assertLessEqual(conf, 1.0)

	def test_empty_site_returns_no_industry(self):
		page, _ = fixtures.make_page("https://e.example", fixtures.EMPTY)
		industry, conf = extractors.classify_industry([page], {}, fixtures.industry_rules())
		self.assertEqual(industry, "")
		self.assertEqual(conf, 0.0)

	def test_no_rules_returns_empty(self):
		page, _ = fixtures.make_page("https://e.example", fixtures.HOMEPAGE)
		self.assertEqual(extractors.classify_industry([page], {}, []), ("", 0.0))

	def test_d2c_brand_maps_to_ecommerce(self):
		self.assertEqual(self._classify("Gymshark", "Shop gym clothing and workout apparel.")[0], "Ecommerce")

	def test_beverage_brand_maps_to_food_and_beverage(self):
		self.assertEqual(
			self._classify("Liquid Death", "A beverage company with sparkling water and iced tea.")[0],
			"Food & Beverage",
		)

	def test_legal_help_beats_stray_keyword(self):
		self.assertEqual(
			self._classify(
				"Rocket Lawyer", "Rocket Lawyer makes legal help simple — legal documents and advice."
			)[0],
			"Legal",
		)

	def test_3d_printing_maps_to_manufacturing(self):
		# A "shop" nav link must not beat the core business signal in the headline.
		self.assertEqual(
			self._classify(
				"Desktop Metal",
				"Metal 3D printing and additive manufacturing for engineers.",
				"<nav>Shop</nav>",
			)[0],
			"Manufacturing",
		)

	def test_about_page_headline_supplements_thin_homepage(self):
		# A homepage with no meta description and abstract brand-voice headings
		# carries no classification signal on its own -- a crawled About page's
		# title/headings should still let the classifier find a match.
		finance_rules = [
			fixtures.keyword_rule(
				"Industry",
				["banking", "financial services"],
				industry="Finance",
				match_scope="Headline",
			)
		]
		home, _ = fixtures.make_page(
			"https://acmebank.example",
			"<html><head><title>Acme | Official Website</title></head>"
			"<body><h1>Acme home</h1><h2>What problem can we solve together?</h2></body></html>",
		)
		about, _ = fixtures.make_page(
			"https://acmebank.example/about",
			"<html><head><title>About Acme</title></head>"
			"<body><h2>A leader in financial services and banking</h2></body></html>",
		)
		company = {"company_name": "Acme", "description": ""}
		industry, conf = extractors.classify_industry([home, about], company, finance_rules)
		self.assertEqual(industry, "Finance")
		self.assertGreater(conf, 0.0)

	def test_homepage_only_still_returns_nothing_when_thin(self):
		finance_rules = [
			fixtures.keyword_rule(
				"Industry",
				["banking", "financial services"],
				industry="Finance",
				match_scope="Headline",
			)
		]
		home, _ = fixtures.make_page(
			"https://acmebank.example",
			"<html><head><title>Acme | Official Website</title></head>"
			"<body><h1>Acme home</h1><h2>What problem can we solve together?</h2></body></html>",
		)
		company = {"company_name": "Acme", "description": ""}
		industry, conf = extractors.classify_industry([home], company, finance_rules)
		self.assertEqual(industry, "")
		self.assertEqual(conf, 0.0)


# --------------------------------------------------------------------------- #
# Generic rule executors directly
# --------------------------------------------------------------------------- #
class RuleExecutorTest(UnitTestCase):
	def test_apply_keyword_rules_weights_and_sums(self):
		rule = fixtures.keyword_rule(
			"Industry", ["saas", "subscription"], industry="SaaS", weight=2.0, match_scope="Headline"
		)
		scores = extractors.apply_keyword_rules({"Headline": "a saas subscription saas"}, [rule])
		# 3 hits * weight 2.0 = 6.0
		self.assertEqual(scores["SaaS"], 6.0)


# --------------------------------------------------------------------------- #
# Readability diagnosis (mechanics)
# --------------------------------------------------------------------------- #
class ReadabilityTest(UnitTestCase):
	def test_blocked_waf_challenge(self):
		page, _ = fixtures.make_page(
			"https://www.pepsi.com",
			'<html><head><script src="/_Incapsula_Resource?SWJIYLWA=1"></script></head><body></body></html>',
		)
		self.assertEqual(extractors.diagnose_readability([page]), "blocked")

	def test_empty_js_only_shell(self):
		page, _ = fixtures.make_page(
			"https://spa.example", "<html><head></head><body><div id='root'></div></body></html>"
		)
		self.assertEqual(extractors.diagnose_readability([page]), "empty")

	def test_unreachable_when_failed(self):
		page = CrawledPage(url="https://x.example", status_code=0, error="timeout after 10s")
		self.assertEqual(extractors.diagnose_readability([page]), "unreachable")

	def test_http_403_is_blocked(self):
		page = CrawledPage(
			url="https://www.pepperfry.com",
			status_code=403,
			html="<html><head><title>Access Denied</title></head></html>",
			text="Access Denied",
			title="Access Denied",
		)
		self.assertEqual(extractors.diagnose_readability([page]), "blocked")

	def test_readable_site_returns_empty_reason(self):
		page, _ = fixtures.make_page("https://acme.example", fixtures.HOMEPAGE)
		self.assertEqual(extractors.diagnose_readability([page]), "")