snapshot of https://github.com/frappe/crm.git (develop, 2026-09-18; blobs > 131072B omitted)
Some checks failed
Frontend / Unit Tests & Coverage (push) Has been skipped
Server / Tests (frappe ${{ matrix.frappe-branch }}) (push) Has been skipped
Server / Resolve frappe matrix (push) Failing after 1s
Frontend / Lint (push) Failing after 1s
Frontend / Production Build (push) Failing after 1s
Server / Coverage Wrap Up (push) Has been skipped
Frontend / Coverage Wrap Up (push) Has been skipped
Regenerate POT file (translatable strings) / Regenerate POT file (develop) (push) Failing after 1s
Regenerate POT file (translatable strings) / Regenerate POT file (main-hotfix) (push) Failing after 1s
Some checks failed
Frontend / Unit Tests & Coverage (push) Has been skipped
Server / Tests (frappe ${{ matrix.frappe-branch }}) (push) Has been skipped
Server / Resolve frappe matrix (push) Failing after 1s
Frontend / Lint (push) Failing after 1s
Frontend / Production Build (push) Failing after 1s
Server / Coverage Wrap Up (push) Has been skipped
Frontend / Coverage Wrap Up (push) Has been skipped
Regenerate POT file (translatable strings) / Regenerate POT file (develop) (push) Failing after 1s
Regenerate POT file (translatable strings) / Regenerate POT file (main-hotfix) (push) Failing after 1s
This commit is contained in:
upstream-mirror
319
crm/domain_enrichment/tests/fixtures.py
Normal file
319
crm/domain_enrichment/tests/fixtures.py
Normal file
@@ -0,0 +1,319 @@
|
||||
# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and contributors
|
||||
# For license information, please see license.txt
|
||||
|
||||
"""Offline HTML fixtures + in-memory config builders for network-free tests.
|
||||
|
||||
The HTML fixtures are deterministic snapshots used by the pure unit tests so the
|
||||
crawler/extractor mechanics can be exercised without touching the network. The
|
||||
config builders assemble ``Rule`` / ``Mapping`` / ``EnrichmentConfig`` objects in
|
||||
memory (no DB) so the rule executors can be tested rule-driven, exactly as the
|
||||
engine would see them after ``get_config()`` — but without hitting Frappe.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from crm.domain_enrichment.config import EnrichmentConfig, Mapping, Rule
|
||||
from crm.domain_enrichment.crawler import extract_content
|
||||
from crm.domain_enrichment.result import CrawledPage
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# HTML fixtures
|
||||
# --------------------------------------------------------------------------- #
|
||||
HOMEPAGE = """
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Acme Analytics | AI powered SaaS</title>
|
||||
<meta name="description" content="Acme is an AI powered analytics platform.">
|
||||
<meta property="og:site_name" content="Acme Analytics">
|
||||
<meta property="og:description" content="Real-time analytics for modern teams.">
|
||||
<meta property="og:image" content="https://acme.example/logo.png">
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "Organization",
|
||||
"name": "Acme Analytics Inc.",
|
||||
"description": "AI powered analytics SaaS for product teams.",
|
||||
"logo": {"url": "https://acme.example/ld-logo.png"},
|
||||
"sameAs": [
|
||||
"https://www.linkedin.com/company/acme-analytics",
|
||||
"https://twitter.com/acmeanalytics",
|
||||
"https://github.com/acme"
|
||||
]
|
||||
}
|
||||
</script>
|
||||
<script src="/_next/static/chunk.js"></script>
|
||||
<script src="https://www.googletagmanager.com/gtag/js?id=G-XXededXX"></script>
|
||||
</head>
|
||||
<body>
|
||||
<h1>AI powered analytics</h1>
|
||||
<h2>We are hiring across engineering</h2>
|
||||
<nav>
|
||||
<a href="/about">About</a>
|
||||
<a href="/contact">Contact</a>
|
||||
<a href="/team">Team</a>
|
||||
<a href="/careers">Careers</a>
|
||||
<a href="https://facebook.com/sharer?u=x">Share</a>
|
||||
<a href="https://instagram.com/acmeanalytics">Instagram</a>
|
||||
<a href="https://external-site.com/page">External</a>
|
||||
</nav>
|
||||
<p>Contact us at hello@acme.example or call <a href="tel:+1 415 555 0142">us</a>.</p>
|
||||
<p>Our subscription pricing plans start free. Customer relationship tooling.</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
ABOUT = """
|
||||
<!doctype html>
|
||||
<html><head><title>About Acme</title></head>
|
||||
<body>
|
||||
<h1>About Acme</h1>
|
||||
<p>We raised a Series A round and are backed by great investors.</p>
|
||||
<div class="team">
|
||||
<div class="member"><div class="name">Jane Doe</div><div class="role">CEO & Co-Founder</div></div>
|
||||
<div class="member"><div class="name">John Smith</div><div class="role">CTO</div></div>
|
||||
<div class="member"><div class="name">Senior Vice President</div><div class="role">VP</div></div>
|
||||
<div class="member"><div class="name">About Acme</div><div class="role">Director</div></div>
|
||||
</div>
|
||||
<p>Reach finance at billing@acme.example.</p>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
CONTACT = """
|
||||
<!doctype html>
|
||||
<html><head><title>Contact Acme</title></head>
|
||||
<body>
|
||||
<h1>Contact</h1>
|
||||
<p>Head office: 100 Market St, San Francisco, CA, 94103, United States</p>
|
||||
<p>Email: sales@acme.example</p>
|
||||
<p>Phone: +1 (415) 555-0199</p>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
WORDPRESS = """
|
||||
<html><head><meta name="generator" content="WordPress 6.4"></head>
|
||||
<body><img src="/wp-content/uploads/x.png"><script src="/wp-json/index.js"></script></body></html>
|
||||
"""
|
||||
|
||||
EMPTY = "<html><head></head><body></body></html>"
|
||||
|
||||
# A JavaScript-only SPA shell (e.g. a Vite/React build). The body is an empty mount
|
||||
# point; everything meaningful is rendered client-side, so requests + BeautifulSoup
|
||||
# see only the build-tool-default title and favicon -- no description, og, or JSON-LD.
|
||||
JS_SPA_SHELL = """
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Vite App</title>
|
||||
<script type="module" crossorigin src="/assets/index-5d175413.js"></script>
|
||||
<link rel="stylesheet" href="/assets/index-57b7cd59.css">
|
||||
<script type="module">window.onerror = () => {};</script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
BROKEN_JSON_LD = """
|
||||
<html><head>
|
||||
<script type="application/ld+json">{ this is not valid json ]</script>
|
||||
<title>Broken Co</title>
|
||||
</head><body><h1>Broken</h1></body></html>
|
||||
"""
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Sitemap fixtures
|
||||
# --------------------------------------------------------------------------- #
|
||||
URLSET_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<url><loc>https://acme.example/about</loc></url>
|
||||
<url><loc>https://acme.example/contact</loc></url>
|
||||
<url><loc>https://acme.example/blog/post-1</loc></url>
|
||||
</urlset>"""
|
||||
|
||||
SITEMAP_INDEX_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<sitemap><loc>https://acme.example/sitemap-pages.xml</loc></sitemap>
|
||||
<sitemap><loc>https://acme.example/sitemap-blog.xml</loc></sitemap>
|
||||
</sitemapindex>"""
|
||||
|
||||
# A sitemap trying to smuggle a DTD entity-expansion payload -- must be rejected
|
||||
# outright by _parse_sitemap before ET.fromstring ever sees it.
|
||||
MALICIOUS_DOCTYPE_XML = """<?xml version="1.0"?>
|
||||
<!DOCTYPE foo [<!ENTITY xxe "boom">]>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<url><loc>https://acme.example/&xxe;</loc></url>
|
||||
</urlset>"""
|
||||
|
||||
|
||||
def fake_fetch(responses):
|
||||
"""Build a ``fetch(url, cfg, session=None, html_only=True)`` stub keyed by exact URL.
|
||||
|
||||
``responses`` maps url -> (status, body) for a 200-shaped response. A URL absent
|
||||
from the map behaves like a 404 (mirrors a missing sitemap/robots.txt), so tests
|
||||
only need to declare the endpoints they care about.
|
||||
"""
|
||||
|
||||
def fake(url, cfg, session=None, html_only=True):
|
||||
entry = responses.get(url)
|
||||
if entry is None:
|
||||
return 404, "", "not found", url
|
||||
status, body = entry
|
||||
return status, body, "", url
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
def fake_fetch_with_redirects(responses, redirects):
|
||||
"""Like ``fake_fetch``, but ``redirects`` maps a requested URL to the resolved
|
||||
(post-redirect) URL returned as ``final_url`` -- simulates a 301/302 hop without
|
||||
modeling the actual redirect chain, since callers only ever look at the result.
|
||||
"""
|
||||
base = fake_fetch(responses)
|
||||
|
||||
def fake(url, cfg, session=None, html_only=True):
|
||||
status, body, error, final_url = base(url, cfg, session=session, html_only=html_only)
|
||||
return status, body, error, redirects.get(url, final_url)
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Page helper
|
||||
# --------------------------------------------------------------------------- #
|
||||
def make_page(url, html, status_code=200):
|
||||
"""Build a (CrawledPage, soup) pair from raw HTML, exactly as the crawler does.
|
||||
|
||||
``extract_content`` mutates its soup, so it is given a throwaway parse and the
|
||||
returned ``soup`` is a fresh, unmutated DOM (mirrors ``crawler.crawl_page``).
|
||||
"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
content = extract_content(BeautifulSoup(html, "html.parser"))
|
||||
page = CrawledPage(
|
||||
url=url,
|
||||
status_code=status_code,
|
||||
html=html,
|
||||
text=content["text"],
|
||||
title=content["title"],
|
||||
headings=content["headings"],
|
||||
)
|
||||
return page, soup
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# In-memory rule builders (no DB) — mirror what config._build_rules() produces
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _compile(pattern, is_regex):
|
||||
if is_regex:
|
||||
return re.compile(pattern, re.IGNORECASE)
|
||||
return re.compile(re.escape(pattern), re.IGNORECASE)
|
||||
|
||||
|
||||
def keyword_rule(rule_type, patterns, *, target_value="", industry="", weight=1.0, match_scope="Full Text"):
|
||||
"""Build a substring keyword Rule directly (is_regex=0 for each pattern)."""
|
||||
compiled = [(_compile(p, False), p) for p in patterns]
|
||||
return Rule(
|
||||
rule_type=rule_type,
|
||||
target_value=target_value,
|
||||
industry=industry,
|
||||
weight=weight,
|
||||
match_scope=match_scope,
|
||||
patterns=compiled,
|
||||
)
|
||||
|
||||
|
||||
def regex_rule(rule_type, patterns, *, target_value="", weight=1.0, match_scope="HTML"):
|
||||
"""Build a regex Rule directly (is_regex=1 for each pattern)."""
|
||||
compiled = [(_compile(p, True), p) for p in patterns]
|
||||
return Rule(
|
||||
rule_type=rule_type,
|
||||
target_value=target_value,
|
||||
weight=weight,
|
||||
match_scope=match_scope,
|
||||
patterns=compiled,
|
||||
)
|
||||
|
||||
|
||||
def industry_rules():
|
||||
"""A small set of Industry rules (Headline scope) covering the fixtures."""
|
||||
return [
|
||||
keyword_rule(
|
||||
"Industry",
|
||||
["saas", "software as a service", "subscription"],
|
||||
industry="SaaS",
|
||||
match_scope="Headline",
|
||||
),
|
||||
keyword_rule(
|
||||
"Industry",
|
||||
["crm", "customer relationship"],
|
||||
industry="CRM",
|
||||
match_scope="Headline",
|
||||
),
|
||||
keyword_rule(
|
||||
"Industry",
|
||||
["shop", "free shipping", "apparel", "ecommerce"],
|
||||
industry="Ecommerce",
|
||||
match_scope="Headline",
|
||||
),
|
||||
keyword_rule(
|
||||
"Industry",
|
||||
["beverage", "sparkling water", "iced tea", "cereal"],
|
||||
industry="Food & Beverage",
|
||||
match_scope="Headline",
|
||||
),
|
||||
keyword_rule(
|
||||
"Industry",
|
||||
["3d printing", "additive manufacturing", "manufacturing"],
|
||||
industry="Manufacturing",
|
||||
match_scope="Headline",
|
||||
),
|
||||
keyword_rule(
|
||||
"Industry",
|
||||
["legal help", "legal documents", "legal advice", "law firm"],
|
||||
industry="Legal",
|
||||
match_scope="Headline",
|
||||
),
|
||||
keyword_rule(
|
||||
"Industry",
|
||||
["media company", "journalists", "newsroom"],
|
||||
industry="Media",
|
||||
match_scope="Headline",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def social_rules():
|
||||
return [
|
||||
regex_rule("Social", [r"linkedin\.com/(company|in|school)/"], target_value="linkedin"),
|
||||
regex_rule("Social", [r"(twitter\.com|x\.com)/[A-Za-z0-9_]+"], target_value="twitter"),
|
||||
regex_rule("Social", [r"github\.com/[A-Za-z0-9_.-]+"], target_value="github"),
|
||||
regex_rule("Social", [r"facebook\.com/[A-Za-z0-9_.\-/]+"], target_value="facebook"),
|
||||
regex_rule("Social", [r"instagram\.com/[A-Za-z0-9_.]+"], target_value="instagram"),
|
||||
]
|
||||
|
||||
|
||||
def make_config(settings=None, **kwargs) -> EnrichmentConfig:
|
||||
"""Assemble an in-memory ``EnrichmentConfig`` for the SSRF / engine tests."""
|
||||
cfg = EnrichmentConfig(settings=settings or {})
|
||||
for key, value in kwargs.items():
|
||||
setattr(cfg, key, value)
|
||||
return cfg
|
||||
|
||||
|
||||
def make_mapping(source_key, target_doctype, target_fieldname, write_policy="Fill if empty", **kwargs):
|
||||
return Mapping(
|
||||
source_key=source_key,
|
||||
target_doctype=target_doctype,
|
||||
target_fieldname=target_fieldname,
|
||||
write_policy=write_policy,
|
||||
create_missing_link=kwargs.get("create_missing_link", 0),
|
||||
default_values=kwargs.get("default_values", ""),
|
||||
)
|
||||
Reference in New Issue
Block a user