# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
"""Offline HTML fixtures + in-memory config builders for network-free tests.
The HTML fixtures are deterministic snapshots used by the pure unit tests so the
crawler/extractor mechanics can be exercised without touching the network. The
config builders assemble ``Rule`` / ``Mapping`` / ``EnrichmentConfig`` objects in
memory (no DB) so the rule executors can be tested rule-driven, exactly as the
engine would see them after ``get_config()`` — but without hitting Frappe.
"""
from __future__ import annotations
import re
from bs4 import BeautifulSoup
from crm.domain_enrichment.config import EnrichmentConfig, Mapping, Rule
from crm.domain_enrichment.crawler import extract_content
from crm.domain_enrichment.result import CrawledPage
# --------------------------------------------------------------------------- #
# HTML fixtures
# --------------------------------------------------------------------------- #
HOMEPAGE = """
Acme Analytics | AI powered SaaS
We raised a Series A round and are backed by great investors.
Jane Doe
CEO & Co-Founder
John Smith
CTO
Senior Vice President
VP
About Acme
Director
Reach finance at billing@acme.example.
"""
CONTACT = """
Contact Acme
Contact
Head office: 100 Market St, San Francisco, CA, 94103, United States
Email: sales@acme.example
Phone: +1 (415) 555-0199
"""
WORDPRESS = """
"""
EMPTY = ""
# A JavaScript-only SPA shell (e.g. a Vite/React build). The body is an empty mount
# point; everything meaningful is rendered client-side, so requests + BeautifulSoup
# see only the build-tool-default title and favicon -- no description, og, or JSON-LD.
JS_SPA_SHELL = """
Vite App
"""
BROKEN_JSON_LD = """
Broken Co
Broken
"""
# --------------------------------------------------------------------------- #
# Sitemap fixtures
# --------------------------------------------------------------------------- #
URLSET_XML = """
https://acme.example/abouthttps://acme.example/contacthttps://acme.example/blog/post-1"""
SITEMAP_INDEX_XML = """
https://acme.example/sitemap-pages.xmlhttps://acme.example/sitemap-blog.xml"""
# A sitemap trying to smuggle a DTD entity-expansion payload -- must be rejected
# outright by _parse_sitemap before ET.fromstring ever sees it.
MALICIOUS_DOCTYPE_XML = """
]>
https://acme.example/&xxe;"""
def fake_fetch(responses):
"""Build a ``fetch(url, cfg, session=None, html_only=True)`` stub keyed by exact URL.
``responses`` maps url -> (status, body) for a 200-shaped response. A URL absent
from the map behaves like a 404 (mirrors a missing sitemap/robots.txt), so tests
only need to declare the endpoints they care about.
"""
def fake(url, cfg, session=None, html_only=True):
entry = responses.get(url)
if entry is None:
return 404, "", "not found", url
status, body = entry
return status, body, "", url
return fake
def fake_fetch_with_redirects(responses, redirects):
"""Like ``fake_fetch``, but ``redirects`` maps a requested URL to the resolved
(post-redirect) URL returned as ``final_url`` -- simulates a 301/302 hop without
modeling the actual redirect chain, since callers only ever look at the result.
"""
base = fake_fetch(responses)
def fake(url, cfg, session=None, html_only=True):
status, body, error, final_url = base(url, cfg, session=session, html_only=html_only)
return status, body, error, redirects.get(url, final_url)
return fake
# --------------------------------------------------------------------------- #
# Page helper
# --------------------------------------------------------------------------- #
def make_page(url, html, status_code=200):
"""Build a (CrawledPage, soup) pair from raw HTML, exactly as the crawler does.
``extract_content`` mutates its soup, so it is given a throwaway parse and the
returned ``soup`` is a fresh, unmutated DOM (mirrors ``crawler.crawl_page``).
"""
soup = BeautifulSoup(html, "html.parser")
content = extract_content(BeautifulSoup(html, "html.parser"))
page = CrawledPage(
url=url,
status_code=status_code,
html=html,
text=content["text"],
title=content["title"],
headings=content["headings"],
)
return page, soup
# --------------------------------------------------------------------------- #
# In-memory rule builders (no DB) — mirror what config._build_rules() produces
# --------------------------------------------------------------------------- #
def _compile(pattern, is_regex):
if is_regex:
return re.compile(pattern, re.IGNORECASE)
return re.compile(re.escape(pattern), re.IGNORECASE)
def keyword_rule(rule_type, patterns, *, target_value="", industry="", weight=1.0, match_scope="Full Text"):
"""Build a substring keyword Rule directly (is_regex=0 for each pattern)."""
compiled = [(_compile(p, False), p) for p in patterns]
return Rule(
rule_type=rule_type,
target_value=target_value,
industry=industry,
weight=weight,
match_scope=match_scope,
patterns=compiled,
)
def regex_rule(rule_type, patterns, *, target_value="", weight=1.0, match_scope="HTML"):
"""Build a regex Rule directly (is_regex=1 for each pattern)."""
compiled = [(_compile(p, True), p) for p in patterns]
return Rule(
rule_type=rule_type,
target_value=target_value,
weight=weight,
match_scope=match_scope,
patterns=compiled,
)
def industry_rules():
"""A small set of Industry rules (Headline scope) covering the fixtures."""
return [
keyword_rule(
"Industry",
["saas", "software as a service", "subscription"],
industry="SaaS",
match_scope="Headline",
),
keyword_rule(
"Industry",
["crm", "customer relationship"],
industry="CRM",
match_scope="Headline",
),
keyword_rule(
"Industry",
["shop", "free shipping", "apparel", "ecommerce"],
industry="Ecommerce",
match_scope="Headline",
),
keyword_rule(
"Industry",
["beverage", "sparkling water", "iced tea", "cereal"],
industry="Food & Beverage",
match_scope="Headline",
),
keyword_rule(
"Industry",
["3d printing", "additive manufacturing", "manufacturing"],
industry="Manufacturing",
match_scope="Headline",
),
keyword_rule(
"Industry",
["legal help", "legal documents", "legal advice", "law firm"],
industry="Legal",
match_scope="Headline",
),
keyword_rule(
"Industry",
["media company", "journalists", "newsroom"],
industry="Media",
match_scope="Headline",
),
]
def social_rules():
return [
regex_rule("Social", [r"linkedin\.com/(company|in|school)/"], target_value="linkedin"),
regex_rule("Social", [r"(twitter\.com|x\.com)/[A-Za-z0-9_]+"], target_value="twitter"),
regex_rule("Social", [r"github\.com/[A-Za-z0-9_.-]+"], target_value="github"),
regex_rule("Social", [r"facebook\.com/[A-Za-z0-9_.\-/]+"], target_value="facebook"),
regex_rule("Social", [r"instagram\.com/[A-Za-z0-9_.]+"], target_value="instagram"),
]
def make_config(settings=None, **kwargs) -> EnrichmentConfig:
"""Assemble an in-memory ``EnrichmentConfig`` for the SSRF / engine tests."""
cfg = EnrichmentConfig(settings=settings or {})
for key, value in kwargs.items():
setattr(cfg, key, value)
return cfg
def make_mapping(source_key, target_doctype, target_fieldname, write_policy="Fill if empty", **kwargs):
return Mapping(
source_key=source_key,
target_doctype=target_doctype,
target_fieldname=target_fieldname,
write_policy=write_policy,
create_missing_link=kwargs.get("create_missing_link", 0),
default_values=kwargs.get("default_values", ""),
)
"""
EMPTY = ""
# A JavaScript-only SPA shell (e.g. a Vite/React build). The body is an empty mount
# point; everything meaningful is rendered client-side, so requests + BeautifulSoup
# see only the build-tool-default title and favicon -- no description, og, or JSON-LD.
JS_SPA_SHELL = """