Initial commit: Sibilla — self-hosted Perplexity-like AI search engine
- Flask web app with Perplexity-style search + synthesis pipeline - Degoog → SearXNG → DuckDuckGo cascading search backends - Gemini LLM for query expansion + answer synthesis with citations - MCP server exposing search tools via stdio - 57 unit tests covering all modules
This commit is contained in:
Sibilla committed
1 parent
ea09bb4f4f
commit
f3fe13c60e
14 files changed
+2099
-31
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,135 @@
|
||||
"""Tests for sibilla.ai — GeminiClient with mocked API."""
|
||||
|
||||
import pytest
|
||||
from unittest import mock
|
||||
from sibilla.ai import GeminiClient, SynthesisResult
|
||||
|
||||
|
||||
class TestSynthesisResult:
|
||||
def test_defaults(self):
|
||||
sr = SynthesisResult(answer="")
|
||||
assert sr.answer == ""
|
||||
assert sr.model == ""
|
||||
assert sr.tokens_used == 0
|
||||
assert sr.error is None
|
||||
|
||||
|
||||
class TestGeminiClient:
|
||||
def test_no_api_key_returns_error(self):
|
||||
client = GeminiClient(api_key="")
|
||||
result = client.synthesize("system", "user")
|
||||
assert result.answer == ""
|
||||
assert "GOOGLE_API_KEY" in result.error
|
||||
|
||||
def test_lazy_client_raises_without_key(self):
|
||||
client = GeminiClient(api_key="")
|
||||
with pytest.raises(ValueError, match="GOOGLE_API_KEY"):
|
||||
_ = client.client
|
||||
|
||||
def test_synthesize_success(self):
|
||||
client = GeminiClient(api_key="fake-key")
|
||||
mock_genai = mock.MagicMock()
|
||||
client._client = mock_genai
|
||||
|
||||
mock_part = mock.MagicMock()
|
||||
mock_part.text = "This is the answer."
|
||||
mock_content = mock.MagicMock()
|
||||
mock_content.parts = [mock_part]
|
||||
mock_candidate = mock.MagicMock()
|
||||
mock_candidate.content = mock_content
|
||||
mock_usage = mock.MagicMock()
|
||||
mock_usage.total_token_count = 42
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.candidates = [mock_candidate]
|
||||
mock_response.usage_metadata = mock_usage
|
||||
|
||||
mock_genai.models.generate_content.return_value = mock_response
|
||||
|
||||
result = client.synthesize("system prompt", "user query")
|
||||
|
||||
assert result.answer == "This is the answer."
|
||||
assert result.tokens_used == 42
|
||||
assert result.model == "gemini-2.5-flash"
|
||||
assert result.error is None
|
||||
|
||||
def test_synthesize_no_candidates(self):
|
||||
client = GeminiClient(api_key="fake-key")
|
||||
mock_genai = mock.MagicMock()
|
||||
client._client = mock_genai
|
||||
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.candidates = []
|
||||
mock_response.usage_metadata = None
|
||||
mock_genai.models.generate_content.return_value = mock_response
|
||||
|
||||
result = client.synthesize("s", "u")
|
||||
assert result.answer == ""
|
||||
assert result.error is None
|
||||
|
||||
def test_synthesize_api_exception(self):
|
||||
client = GeminiClient(api_key="fake-key")
|
||||
mock_genai = mock.MagicMock()
|
||||
client._client = mock_genai
|
||||
|
||||
mock_genai.models.generate_content.side_effect = RuntimeError("API down")
|
||||
|
||||
result = client.synthesize("s", "u")
|
||||
assert result.error == "API down"
|
||||
assert result.answer == ""
|
||||
|
||||
def test_rewrite_query_no_api_key(self):
|
||||
client = GeminiClient(api_key="")
|
||||
variants = client.rewrite_query("test query", num_variants=3)
|
||||
assert variants == ["test query"]
|
||||
|
||||
def test_rewrite_query_success(self):
|
||||
client = GeminiClient(api_key="fake-key")
|
||||
mock_genai = mock.MagicMock()
|
||||
client._client = mock_genai
|
||||
|
||||
mock_part = mock.MagicMock()
|
||||
mock_part.text = "rewritten query 1\nrewritten query 2\nrewritten query 3"
|
||||
mock_content = mock.MagicMock()
|
||||
mock_content.parts = [mock_part]
|
||||
mock_candidate = mock.MagicMock()
|
||||
mock_candidate.content = mock_content
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.candidates = [mock_candidate]
|
||||
mock_genai.models.generate_content.return_value = mock_response
|
||||
|
||||
variants = client.rewrite_query("original", num_variants=3)
|
||||
|
||||
assert len(variants) == 4
|
||||
assert variants[0] == "original"
|
||||
assert "rewritten query 1" in variants
|
||||
|
||||
def test_rewrite_query_strips_bullets(self):
|
||||
client = GeminiClient(api_key="fake-key")
|
||||
mock_genai = mock.MagicMock()
|
||||
client._client = mock_genai
|
||||
|
||||
mock_part = mock.MagicMock()
|
||||
mock_part.text = "- bullet query\n• dot query"
|
||||
mock_content = mock.MagicMock()
|
||||
mock_content.parts = [mock_part]
|
||||
mock_candidate = mock.MagicMock()
|
||||
mock_candidate.content = mock_content
|
||||
mock_response = mock.MagicMock()
|
||||
mock_response.candidates = [mock_candidate]
|
||||
mock_genai.models.generate_content.return_value = mock_response
|
||||
|
||||
variants = client.rewrite_query("original", num_variants=2)
|
||||
|
||||
assert "bullet query" in variants
|
||||
assert "dot query" in variants
|
||||
assert not any(v.startswith("-") for v in variants)
|
||||
|
||||
def test_rewrite_query_exception_fallback(self):
|
||||
client = GeminiClient(api_key="fake-key")
|
||||
mock_genai = mock.MagicMock()
|
||||
client._client = mock_genai
|
||||
|
||||
mock_genai.models.generate_content.side_effect = RuntimeError("boom")
|
||||
|
||||
variants = client.rewrite_query("original", num_variants=2)
|
||||
assert variants == ["original"]
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Tests for sibilla.app — Flask routes and search pipeline."""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
from unittest import mock
|
||||
from sibilla.app import create_app
|
||||
from sibilla.search import SearchResult, SearchResponse
|
||||
from sibilla.ai import SynthesisResult
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
"""Create a Flask app in testing mode with a fake Gemini API key."""
|
||||
app = create_app(testing=True)
|
||||
# Point templates to the project-level templates/ directory
|
||||
app.template_folder = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "templates"
|
||||
)
|
||||
app.config.update({
|
||||
"TESTING": True,
|
||||
"GOOGLE_API_KEY": "fake-test-key",
|
||||
"DEGOOG_URL": "http://degoog.test:4444",
|
||||
"SEARXNG_URL": "http://searxng.test:8080",
|
||||
"SIBILLA_DDG_FALLBACK": True,
|
||||
})
|
||||
return app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app):
|
||||
return app.test_client()
|
||||
|
||||
|
||||
class TestHealthEndpoint:
|
||||
def test_health_returns_ok(self, client):
|
||||
resp = client.get("/api/health")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["status"] == "ok"
|
||||
assert "version" in data
|
||||
assert data["gemini_configured"] is True
|
||||
assert "degoog" in data["degoog_url"]
|
||||
|
||||
def test_health_gemini_not_configured(self, app):
|
||||
app.config["GOOGLE_API_KEY"] = ""
|
||||
client = app.test_client()
|
||||
resp = client.get("/api/health")
|
||||
data = resp.get_json()
|
||||
assert data["gemini_configured"] is False
|
||||
|
||||
|
||||
class TestIndexRoute:
|
||||
def test_index_returns_html(self, client):
|
||||
resp = client.get("/")
|
||||
assert resp.status_code == 200
|
||||
assert "text/html" in resp.content_type
|
||||
html = resp.data.decode("utf-8")
|
||||
assert "<title>Sibilla" in html
|
||||
assert 'id="search-form"' in html
|
||||
assert 'id="query"' in html
|
||||
assert 'id="response-area"' in html
|
||||
|
||||
|
||||
class TestAskEndpoint:
|
||||
def test_empty_query_returns_400(self, client):
|
||||
resp = client.post("/api/ask", json={"query": ""})
|
||||
assert resp.status_code == 400
|
||||
assert "non può essere vuota" in resp.get_json()["error"]
|
||||
|
||||
def test_ask_with_degoog_success(self, app):
|
||||
"""Full pipeline test with mocked Degoog search and Gemini synthesis."""
|
||||
client = app.test_client()
|
||||
|
||||
# Mock DegoogClient.search to return results
|
||||
def mock_degoog_search(self, query, num_results=8, language="it"):
|
||||
return SearchResponse(
|
||||
query=query,
|
||||
results=[
|
||||
SearchResult(title="R1", url="https://a.com", snippet="Content A"),
|
||||
SearchResult(title="R2", url="https://b.com", snippet="Content B"),
|
||||
],
|
||||
engine="degoog",
|
||||
)
|
||||
|
||||
# Mock Gemini client to return a synthesized answer
|
||||
def mock_synthesize(self, system_prompt, user_prompt, temperature=0.4, max_tokens=4096):
|
||||
return SynthesisResult(
|
||||
answer="Ecco la risposta sintetizzata [1][2].",
|
||||
model="gemini-2.5-flash",
|
||||
tokens_used=100,
|
||||
)
|
||||
|
||||
def mock_rewrite_query(self, original_query, num_variants=3):
|
||||
return [original_query, "rewritten variant"]
|
||||
|
||||
with mock.patch(
|
||||
"sibilla.app.DegoogClient.search", mock_degoog_search
|
||||
), mock.patch(
|
||||
"sibilla.app.GeminiClient.synthesize", mock_synthesize
|
||||
), mock.patch(
|
||||
"sibilla.app.GeminiClient.rewrite_query", mock_rewrite_query
|
||||
), mock.patch(
|
||||
"sibilla.app.PageFetcher.fetch_results",
|
||||
lambda self, results, max_results=5: results, # passthrough
|
||||
):
|
||||
resp = client.post("/api/ask", json={"query": "test query"})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert "Ecco la risposta sintetizzata" in data["answer"]
|
||||
assert data["search_engine"] == "degoog"
|
||||
assert len(data["sources"]) == 2
|
||||
assert data["sources"][0]["url"] == "https://a.com"
|
||||
assert data["model"] == "gemini-2.5-flash"
|
||||
assert data["tokens_used"] == 100
|
||||
|
||||
def test_ask_fallback_to_searxng(self, app):
|
||||
"""When Degoog returns no results, fall back to SearXNG."""
|
||||
client = app.test_client()
|
||||
|
||||
def mock_degoog_search_empty(self, query, num_results=8, language="it"):
|
||||
return SearchResponse(query=query, results=[], engine="degoog")
|
||||
|
||||
def mock_searxng_search(self, query, num_results=8, engines=None, language="it"):
|
||||
return SearchResponse(
|
||||
query=query,
|
||||
results=[
|
||||
SearchResult(title="SR1", url="https://s.com", snippet="S content"),
|
||||
],
|
||||
engine="searxng",
|
||||
)
|
||||
|
||||
def mock_synthesize(self, system_prompt, user_prompt, temperature=0.4, max_tokens=4096):
|
||||
return SynthesisResult(answer="Answer from SearXNG sources [1].", model="g", tokens_used=50)
|
||||
|
||||
def mock_rewrite_query(self, original_query, num_variants=3):
|
||||
return [original_query]
|
||||
|
||||
with mock.patch(
|
||||
"sibilla.app.DegoogClient.search", mock_degoog_search_empty
|
||||
), mock.patch(
|
||||
"sibilla.app.SearXNGClient.search", mock_searxng_search
|
||||
), mock.patch(
|
||||
"sibilla.app.GeminiClient.synthesize", mock_synthesize
|
||||
), mock.patch(
|
||||
"sibilla.app.GeminiClient.rewrite_query", mock_rewrite_query
|
||||
), mock.patch(
|
||||
"sibilla.app.PageFetcher.fetch_results",
|
||||
lambda self, results, max_results=5: results,
|
||||
):
|
||||
resp = client.post("/api/ask", json={"query": "fallback test"})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["search_engine"] == "searxng"
|
||||
|
||||
def test_ask_no_results_fallback_to_llm(self, app):
|
||||
"""When all search backends fail, fall back to pure LLM."""
|
||||
client = app.test_client()
|
||||
|
||||
def mock_empty_search(self, *args, **kwargs):
|
||||
return SearchResponse(query="q", results=[], engine="none")
|
||||
|
||||
def mock_synthesize(self, system_prompt, user_prompt, temperature=0.4, max_tokens=4096):
|
||||
return SynthesisResult(answer="Nessun risultato, ma posso aiutarti...", model="g", tokens_used=10)
|
||||
|
||||
def mock_rewrite_query(self, original_query, num_variants=3):
|
||||
return [original_query]
|
||||
|
||||
with mock.patch(
|
||||
"sibilla.app.DegoogClient.search", mock_empty_search
|
||||
), mock.patch(
|
||||
"sibilla.app.SearXNGClient.search", mock_empty_search
|
||||
), mock.patch(
|
||||
"sibilla.app.DuckDuckGoClient.search", mock_empty_search
|
||||
), mock.patch(
|
||||
"sibilla.app.GeminiClient.synthesize", mock_synthesize
|
||||
), mock.patch(
|
||||
"sibilla.app.GeminiClient.rewrite_query", mock_rewrite_query
|
||||
):
|
||||
resp = client.post("/api/ask", json={"query": "impossible query"})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["search_engine"] == "none"
|
||||
assert data["sources"] == []
|
||||
|
||||
def test_ask_all_backends_error(self, app):
|
||||
"""When all backends return errors, return 502."""
|
||||
client = app.test_client()
|
||||
|
||||
def mock_error_search(self, *args, **kwargs):
|
||||
return SearchResponse(query="q", results=[], error="Connection refused", engine="degoog")
|
||||
|
||||
def mock_rewrite_query(self, original_query, num_variants=3):
|
||||
return [original_query]
|
||||
|
||||
with mock.patch(
|
||||
"sibilla.app.DegoogClient.search", mock_error_search
|
||||
), mock.patch(
|
||||
"sibilla.app.SearXNGClient.search", mock_error_search
|
||||
), mock.patch(
|
||||
"sibilla.app.DuckDuckGoClient.search", mock_error_search
|
||||
), mock.patch(
|
||||
"sibilla.app.GeminiClient.rewrite_query", mock_rewrite_query
|
||||
):
|
||||
resp = client.post("/api/ask", json={"query": "doomed query"})
|
||||
|
||||
assert resp.status_code == 502
|
||||
|
||||
def test_ask_respects_num_sources(self, app):
|
||||
client = app.test_client()
|
||||
|
||||
call_count = 0
|
||||
|
||||
def mock_degoog_search(self, query, num_results=8, language="it"):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return SearchResponse(
|
||||
query=query,
|
||||
results=[
|
||||
SearchResult(title=f"R{i}", url=f"https://{i}.com", snippet=f"S{i}")
|
||||
for i in range(10)
|
||||
],
|
||||
engine="degoog",
|
||||
)
|
||||
|
||||
def mock_synthesize(self, system_prompt, user_prompt, temperature=0.4, max_tokens=4096):
|
||||
return SynthesisResult(answer="Ok.", model="g", tokens_used=1)
|
||||
|
||||
def mock_rewrite_query(self, original_query, num_variants=3):
|
||||
return [original_query]
|
||||
|
||||
with mock.patch(
|
||||
"sibilla.app.DegoogClient.search", mock_degoog_search
|
||||
), mock.patch(
|
||||
"sibilla.app.GeminiClient.synthesize", mock_synthesize
|
||||
), mock.patch(
|
||||
"sibilla.app.GeminiClient.rewrite_query", mock_rewrite_query
|
||||
), mock.patch(
|
||||
"sibilla.app.PageFetcher.fetch_results",
|
||||
lambda self, results, max_results=5: results,
|
||||
):
|
||||
resp = client.post("/api/ask", json={"query": "test", "num_sources": 3})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert len(data["sources"]) <= 3
|
||||
|
||||
def test_ask_without_json_body(self, client):
|
||||
resp = client.post("/api/ask", data="not json")
|
||||
assert resp.status_code == 400
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Tests for sibilla.prompts — prompt templates and builders."""
|
||||
|
||||
from sibilla.prompts import (
|
||||
SYNTHESIS_SYSTEM_PROMPT,
|
||||
FALLBACK_PROMPT,
|
||||
QUERY_REWRITE_PROMPT,
|
||||
build_synthesis_prompt,
|
||||
)
|
||||
|
||||
|
||||
class TestPromptTemplates:
|
||||
"""Verify all prompt templates are non-empty and well-formed."""
|
||||
|
||||
def test_synthesis_system_prompt_not_empty(self):
|
||||
assert len(SYNTHESIS_SYSTEM_PROMPT) > 50
|
||||
assert "Sibilla" in SYNTHESIS_SYSTEM_PROMPT
|
||||
assert "[1]" in SYNTHESIS_SYSTEM_PROMPT
|
||||
|
||||
def test_fallback_prompt_not_empty(self):
|
||||
assert len(FALLBACK_PROMPT) > 20
|
||||
assert "Sibilla" in FALLBACK_PROMPT
|
||||
|
||||
def test_query_rewrite_prompt_has_placeholder(self):
|
||||
assert "{num_variants}" in QUERY_REWRITE_PROMPT
|
||||
|
||||
|
||||
class TestBuildSynthesisPrompt:
|
||||
"""Test build_synthesis_prompt with various inputs."""
|
||||
|
||||
def test_minimal_prompt(self):
|
||||
sources = [
|
||||
{"index": 1, "title": "Test", "url": "https://example.com", "content": "Hello world"}
|
||||
]
|
||||
prompt = build_synthesis_prompt("what is this?", sources)
|
||||
|
||||
assert "what is this?" in prompt
|
||||
assert "Test" in prompt
|
||||
assert "https://example.com" in prompt
|
||||
assert "Hello world" in prompt
|
||||
assert "[1]" in prompt
|
||||
|
||||
def test_multiple_sources(self):
|
||||
sources = [
|
||||
{"index": 1, "title": "A", "url": "https://a.com", "content": "Content A"},
|
||||
{"index": 2, "title": "B", "url": "https://b.com", "content": "Content B"},
|
||||
{"index": 3, "title": "C", "url": "https://c.com", "content": "Content C"},
|
||||
]
|
||||
prompt = build_synthesis_prompt("query", sources)
|
||||
|
||||
assert "--- Fonte [1] ---" in prompt
|
||||
assert "--- Fonte [2] ---" in prompt
|
||||
assert "--- Fonte [3] ---" in prompt
|
||||
assert "Content A" in prompt
|
||||
assert "Content B" in prompt
|
||||
assert "Content C" in prompt
|
||||
|
||||
def test_with_date_context(self):
|
||||
sources = [{"index": 1, "title": "X", "url": "https://x.com", "content": "Y"}]
|
||||
prompt = build_synthesis_prompt("q", sources, date_context="Data: 16 Luglio 2026")
|
||||
|
||||
assert "16 Luglio 2026" in prompt
|
||||
assert "Contesto temporale" in prompt
|
||||
|
||||
def test_empty_sources(self):
|
||||
prompt = build_synthesis_prompt("query", [])
|
||||
assert "query" in prompt
|
||||
assert "### Fonti da utilizzare" in prompt
|
||||
|
||||
def test_source_missing_fields(self):
|
||||
sources = [{"index": 1}] # missing title, url, content
|
||||
prompt = build_synthesis_prompt("q", sources)
|
||||
assert "Senza titolo" in prompt # fallback title
|
||||
assert "[1]" in prompt
|
||||
|
||||
def test_source_with_snippet_fallback(self):
|
||||
"""When 'content' is missing, fall back to 'snippet'."""
|
||||
sources = [{"index": 1, "title": "T", "url": "https://t.com", "snippet": "snippet text"}]
|
||||
prompt = build_synthesis_prompt("q", sources)
|
||||
assert "snippet text" in prompt
|
||||
@@ -0,0 +1,398 @@
|
||||
"""Tests for sibilla.search — search clients and page fetcher."""
|
||||
|
||||
from unittest import mock
|
||||
from sibilla.search import (
|
||||
DegoogClient,
|
||||
SearXNGClient,
|
||||
DuckDuckGoClient,
|
||||
PageFetcher,
|
||||
SearchResult,
|
||||
SearchResponse,
|
||||
_clean_html,
|
||||
)
|
||||
|
||||
|
||||
class TestCleanHtml:
|
||||
def test_removes_tags(self):
|
||||
assert _clean_html("<p>Hello <b>world</b></p>") == "Hello world"
|
||||
|
||||
def test_plain_text_unchanged(self):
|
||||
assert _clean_html("Plain text") == "Plain text"
|
||||
|
||||
def test_empty_string(self):
|
||||
assert _clean_html("") == ""
|
||||
|
||||
def test_nested_tags(self):
|
||||
assert _clean_html('<div><span class="x">Text</span></div>') == "Text"
|
||||
|
||||
def test_strips_html_entities(self):
|
||||
# HTML entities stay as-is (not entity-decoded), just tags stripped
|
||||
assert _clean_html("<b>Bold & Brave</b>") == "Bold & Brave"
|
||||
|
||||
|
||||
class TestSearchDataclasses:
|
||||
def test_search_result_defaults(self):
|
||||
sr = SearchResult(title="T", url="https://u.com")
|
||||
assert sr.snippet == ""
|
||||
assert sr.full_content == ""
|
||||
assert sr.fetched is False
|
||||
|
||||
def test_search_response_defaults(self):
|
||||
sr = SearchResponse(query="q")
|
||||
assert sr.results == []
|
||||
assert sr.total_results == 0
|
||||
assert sr.engine == "searxng"
|
||||
assert sr.error is None
|
||||
|
||||
|
||||
# ── DegoogClient tests ────────────────────────────────────────────
|
||||
|
||||
class TestDegoogClient:
|
||||
def test_search_success(self):
|
||||
client = DegoogClient(base_url="http://degoog.test:4444")
|
||||
|
||||
mock_data = {
|
||||
"results": [
|
||||
{"title": "R1", "url": "https://a.com", "snippet": "S1"},
|
||||
{"title": "R2", "url": "https://b.com", "snippet": "S2"},
|
||||
]
|
||||
}
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
mock_get.return_value.json.return_value = mock_data
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
resp = client.search("test query", num_results=5)
|
||||
|
||||
assert len(resp.results) == 2
|
||||
assert resp.results[0].title == "R1"
|
||||
assert resp.results[0].url == "https://a.com"
|
||||
assert resp.results[0].snippet == "S1"
|
||||
assert resp.engine == "degoog"
|
||||
assert resp.error is None
|
||||
|
||||
def test_search_respects_num_results(self):
|
||||
client = DegoogClient(base_url="http://degoog.test:4444")
|
||||
|
||||
mock_data = {
|
||||
"results": [
|
||||
{"title": f"R{i}", "url": f"https://{i}.com", "snippet": f"S{i}"}
|
||||
for i in range(10)
|
||||
]
|
||||
}
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
mock_get.return_value.json.return_value = mock_data
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
resp = client.search("q", num_results=3)
|
||||
|
||||
assert len(resp.results) == 3
|
||||
|
||||
def test_search_empty_results(self):
|
||||
client = DegoogClient(base_url="http://degoog.test:4444")
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
mock_get.return_value.json.return_value = {"results": []}
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
resp = client.search("no results query")
|
||||
|
||||
assert resp.results == []
|
||||
assert resp.error is None
|
||||
|
||||
def test_search_handles_request_exception(self):
|
||||
client = DegoogClient(base_url="http://degoog.test:4444")
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
import requests
|
||||
mock_get.side_effect = requests.RequestException("Connection refused")
|
||||
|
||||
resp = client.search("q")
|
||||
|
||||
assert resp.error is not None
|
||||
assert resp.results == []
|
||||
|
||||
def test_search_cleans_html_in_results(self):
|
||||
"""Verify that _clean_html is applied to title and snippet."""
|
||||
client = DegoogClient(base_url="http://degoog.test:4444")
|
||||
|
||||
mock_data = {
|
||||
"results": [
|
||||
{
|
||||
"title": "<b>Bold Title</b>",
|
||||
"url": "https://x.com",
|
||||
"snippet": "<p>Some <em>text</em></p>",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
mock_get.return_value.json.return_value = mock_data
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
resp = client.search("q")
|
||||
|
||||
assert resp.results[0].title == "Bold Title"
|
||||
assert resp.results[0].snippet == "Some text"
|
||||
|
||||
def test_search_uses_content_fallback_for_snippet(self):
|
||||
"""Degoog may return 'content' instead of 'snippet' (OpenWebUI compat)."""
|
||||
client = DegoogClient(base_url="http://degoog.test:4444")
|
||||
|
||||
mock_data = {
|
||||
"results": [
|
||||
{"title": "R", "url": "https://x.com", "content": "Content field"}
|
||||
]
|
||||
}
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
mock_get.return_value.json.return_value = mock_data
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
resp = client.search("q")
|
||||
|
||||
assert resp.results[0].snippet == "Content field"
|
||||
|
||||
def test_search_query_params(self):
|
||||
client = DegoogClient(base_url="http://degoog.test:4444")
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
mock_get.return_value.json.return_value = {"results": []}
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
client.search("test query", language="en")
|
||||
|
||||
call_args = mock_get.call_args
|
||||
assert call_args[1]["params"]["q"] == "test query"
|
||||
assert call_args[1]["params"]["lang"] == "en"
|
||||
|
||||
|
||||
# ── SearXNGClient tests ───────────────────────────────────────────
|
||||
|
||||
class TestSearXNGClient:
|
||||
def test_search_success(self):
|
||||
client = SearXNGClient(base_url="http://searxng.test:8080")
|
||||
|
||||
mock_data = {
|
||||
"results": [
|
||||
{"title": "R1", "url": "https://a.com", "content": "S1 content"},
|
||||
],
|
||||
"number_of_results": 1,
|
||||
}
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
mock_get.return_value.json.return_value = mock_data
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
resp = client.search("test", num_results=5)
|
||||
|
||||
assert len(resp.results) == 1
|
||||
assert resp.results[0].snippet == "S1 content"
|
||||
assert resp.total_results == 1
|
||||
|
||||
def test_search_handles_error(self):
|
||||
client = SearXNGClient(base_url="http://searxng.test:8080")
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
import requests
|
||||
mock_get.side_effect = requests.RequestException("Timeout")
|
||||
|
||||
resp = client.search("q")
|
||||
|
||||
assert resp.error is not None
|
||||
assert "Timeout" in resp.error
|
||||
|
||||
def test_search_with_engines_param(self):
|
||||
client = SearXNGClient(base_url="http://searxng.test:8080")
|
||||
|
||||
with mock.patch.object(client._session, "get") as mock_get:
|
||||
mock_get.return_value.json.return_value = {"results": []}
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
client.search("q", engines="google,bing")
|
||||
|
||||
call_args = mock_get.call_args
|
||||
assert call_args[1]["params"]["engines"] == "google,bing"
|
||||
|
||||
|
||||
# ── DuckDuckGoClient tests ────────────────────────────────────────
|
||||
|
||||
DUCKDUCKGO_HTML = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div class="result">
|
||||
<div class="result__title">
|
||||
<a href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com&rut=abc">Example Page</a>
|
||||
</div>
|
||||
<div class="result__snippet">This is the snippet text for the first result.</div>
|
||||
</div>
|
||||
<div class="result">
|
||||
<div class="result__title">
|
||||
<a href="https://another-site.org/page">Another Site</a>
|
||||
</div>
|
||||
<div class="result__snippet">Second snippet over here.</div>
|
||||
</div>
|
||||
<div class="result">
|
||||
<div class="result__title">
|
||||
<a href="/l/?uddg=https%3A%2F%2Fthird.com">Third Result</a>
|
||||
</div>
|
||||
<!-- No snippet -->
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
class TestDuckDuckGoClient:
|
||||
def test_extract_redirect(self):
|
||||
client = DuckDuckGoClient()
|
||||
url = "https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com&rut=abc"
|
||||
result = client._extract_redirect(url)
|
||||
assert result == "https://example.com"
|
||||
|
||||
def test_extract_redirect_no_uddg(self):
|
||||
client = DuckDuckGoClient()
|
||||
result = client._extract_redirect("https://example.com/page")
|
||||
assert result == "https://example.com/page"
|
||||
|
||||
def test_search_parses_html_results(self):
|
||||
"""Test that DuckDuckGo HTML results are parsed correctly."""
|
||||
client = DuckDuckGoClient()
|
||||
|
||||
mock_resp = mock.MagicMock()
|
||||
mock_resp.text = DUCKDUCKGO_HTML
|
||||
mock_resp.raise_for_status.return_value = None
|
||||
|
||||
with mock.patch.object(client._session, "post") as mock_post:
|
||||
mock_post.return_value = mock_resp
|
||||
|
||||
resp = client.search("test query", num_results=5)
|
||||
|
||||
assert len(resp.results) == 3
|
||||
assert resp.results[0].title == "Example Page"
|
||||
assert resp.results[0].url == "https://example.com" # redirect extracted
|
||||
assert resp.results[0].snippet == "This is the snippet text for the first result."
|
||||
assert resp.results[1].url == "https://another-site.org/page"
|
||||
assert resp.results[2].snippet == "" # no snippet element
|
||||
assert resp.engine == "duckduckgo"
|
||||
assert resp.error is None
|
||||
|
||||
def test_search_handles_error(self):
|
||||
client = DuckDuckGoClient()
|
||||
|
||||
with mock.patch.object(client._session, "post") as mock_post:
|
||||
import requests
|
||||
mock_post.side_effect = requests.RequestException("Network error")
|
||||
|
||||
resp = client.search("q")
|
||||
|
||||
assert resp.error is not None
|
||||
assert "Network error" in resp.error
|
||||
|
||||
|
||||
# ── PageFetcher tests ─────────────────────────────────────────────
|
||||
|
||||
class TestPageFetcher:
|
||||
def test_fetch_extracts_text(self):
|
||||
fetcher = PageFetcher()
|
||||
|
||||
html = """<html><body>
|
||||
<script>alert('xss')</script>
|
||||
<nav>Menu</nav>
|
||||
<p>Hello world</p>
|
||||
<footer>Footer</footer>
|
||||
</body></html>"""
|
||||
|
||||
with mock.patch.object(fetcher._session, "get") as mock_get:
|
||||
mock_get.return_value.text = html
|
||||
mock_get.return_value.headers = {"Content-Type": "text/html; charset=utf-8"}
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
content = fetcher.fetch("https://example.com")
|
||||
|
||||
assert "Hello world" in content
|
||||
assert "alert" not in content # script removed
|
||||
assert "Footer" not in content # footer removed
|
||||
|
||||
def test_fetch_non_html_returns_empty(self):
|
||||
fetcher = PageFetcher()
|
||||
|
||||
with mock.patch.object(fetcher._session, "get") as mock_get:
|
||||
mock_get.return_value.headers = {"Content-Type": "application/pdf"}
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
content = fetcher.fetch("https://example.com/doc.pdf")
|
||||
|
||||
assert content == ""
|
||||
|
||||
def test_fetch_handles_error(self):
|
||||
fetcher = PageFetcher()
|
||||
|
||||
with mock.patch.object(fetcher._session, "get") as mock_get:
|
||||
import requests
|
||||
mock_get.side_effect = requests.RequestException("Connection refused")
|
||||
|
||||
content = fetcher.fetch("https://down.example.com")
|
||||
|
||||
assert content == ""
|
||||
|
||||
def test_fetch_truncates_to_max_chars(self):
|
||||
fetcher = PageFetcher(max_chars=50)
|
||||
|
||||
long_text = "A" * 200
|
||||
html = f"<html><body><p>{long_text}</p></body></html>"
|
||||
|
||||
with mock.patch.object(fetcher._session, "get") as mock_get:
|
||||
mock_get.return_value.text = html
|
||||
mock_get.return_value.headers = {"Content-Type": "text/html"}
|
||||
mock_get.return_value.raise_for_status.return_value = None
|
||||
|
||||
content = fetcher.fetch("https://example.com")
|
||||
|
||||
assert len(content) <= 53 # 50 + "..." truncation marker
|
||||
|
||||
def test_fetch_results_enriches_objects(self):
|
||||
fetcher = PageFetcher()
|
||||
results = [
|
||||
SearchResult(title="A", url="https://a.com"),
|
||||
SearchResult(title="B", url="https://b.com"),
|
||||
]
|
||||
|
||||
with mock.patch.object(fetcher, "fetch") as mock_fetch:
|
||||
mock_fetch.side_effect = ["Content A", "Content B"]
|
||||
|
||||
enriched = fetcher.fetch_results(results, max_results=2)
|
||||
|
||||
assert enriched[0].full_content == "Content A"
|
||||
assert enriched[0].fetched is True
|
||||
assert enriched[1].full_content == "Content B"
|
||||
assert enriched[1].fetched is True
|
||||
|
||||
def test_fetch_results_respects_max(self):
|
||||
fetcher = PageFetcher()
|
||||
results = [SearchResult(title=f"R{i}", url=f"https://{i}.com") for i in range(5)]
|
||||
|
||||
with mock.patch.object(fetcher, "fetch") as mock_fetch:
|
||||
mock_fetch.return_value = "Content"
|
||||
|
||||
fetcher.fetch_results(results, max_results=2)
|
||||
|
||||
assert mock_fetch.call_count == 2
|
||||
|
||||
def test_fetch_results_skips_already_fetched(self):
|
||||
fetcher = PageFetcher()
|
||||
results = [
|
||||
SearchResult(title="A", url="https://a.com", fetched=True, full_content="Old"),
|
||||
SearchResult(title="B", url="https://b.com"),
|
||||
]
|
||||
|
||||
with mock.patch.object(fetcher, "fetch") as mock_fetch:
|
||||
mock_fetch.return_value = "New"
|
||||
|
||||
fetcher.fetch_results(results, max_results=2)
|
||||
|
||||
# Only the non-fetched result should be fetched
|
||||
assert mock_fetch.call_count == 1
|
||||
assert results[0].full_content == "Old" # unchanged
|
||||
Reference in new issue
Block a user